From 760dcc912dae3dc0cba84ce4a491350e380a5526 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 12 May 2024 22:28:12 -0400 Subject: [PATCH 1/4] Rename some subcommands in line with tmc-clie --- moocfi_cses.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/moocfi_cses.py b/moocfi_cses.py index 77c0a19..8a499c8 100644 --- a/moocfi_cses.py +++ b/moocfi_cses.py @@ -130,16 +130,26 @@ def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: ) subparsers = parser.add_subparsers(required=True) - parser_config = subparsers.add_parser("configure", help="Configure moocfi_cses") - parser_config.set_defaults(cmd="configure") + # login subparser + parser_login = subparsers.add_parser("login", help="Login to mooc.fi CSES") + parser_login.set_defaults(cmd="login") + # list exercises subparser parser_list = subparsers.add_parser("list", help="List exercises") parser_list.set_defaults(cmd="list") parser_list.add_argument( "--filter", - help="List complete, incomplete or all tasks (default: %(default)s)", + help="List only complete or incomplete tasks (default: all)", choices=["complete", "incomplete"], ) + parser_list.add_argument( + "--limit", help="Maximum amount of items to list", type=int + ) + + # show exercise subparser + parser_show = subparsers.add_parser("show", help="Show details of an exercise") + parser_show.set_defaults(cmd="show") + parser_show.add_argument("task_id", help="Numerical task identifier or task name") return parser.parse_args(args) @@ -288,10 +298,22 @@ def parse_task_list(html: str | bytes) -> list[Task]: # TODO: This should be part of a UI class or module -def print_task_list(task_list: list[Task], filter: Optional[str] = None) -> None: +def print_task_list( + task_list: list[Task], filter: Optional[str] = None, limit: Optional[int] = None +) -> None: + count: int = 0 for task in task_list: if not filter or filter == task.state.value: print(f"- {task.id}: {task.name} {TASK_STATE_ICON[task.state]}") + count += 1 + if limit and count >= limit: + return + + +# TODO: Implement function that parser the specific task page into Task object +def parse_task(html: str | bytes, task: Task) -> Task: + task = Task("a", "b", TaskState.COMPLETE) + return task # TODO: Implement function that posts the submit form with the correct file @@ -301,12 +323,6 @@ def submit_task(task_id: str, filename: str) -> None: ... -# TODO: Implement function that parser the specific task page into Task object -def parse_task(html: str | bytes, task: Task) -> Task: - task = Task("a", "b", TaskState.COMPLETE) - return task - - def main() -> None: args = parse_args() @@ -316,7 +332,7 @@ def main() -> None: datefmt="%Y-%m-%d %H:%M:%S", ) - if args.cmd == "configure": + if args.cmd == "login": config = create_config() write_config(args.config, config) return @@ -352,7 +368,7 @@ def main() -> None: if args.cmd == "list": html = session.http_request(base_url) task_list = parse_task_list(html) - print_task_list(task_list, filter=args.filter) + print_task_list(task_list, filter=args.filter, limit=args.limit) if __name__ == "__main__": From 106003ea5bf9b07830c786f7b25f1a04497fc362 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Mon, 13 May 2024 12:32:53 -0400 Subject: [PATCH 2/4] Add the show task functions --- moocfi_cses.py | 67 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/moocfi_cses.py b/moocfi_cses.py index 8a499c8..a6416b5 100644 --- a/moocfi_cses.py +++ b/moocfi_cses.py @@ -10,10 +10,12 @@ from getpass import getpass import logging import json -import urllib.parse +from urllib.parse import urljoin from pathlib import Path from typing import AnyStr, Optional +from xml.etree.ElementTree import Element, tostring +from html2text import html2text import htmlement import requests @@ -34,7 +36,7 @@ def __post_init__(self) -> None: @property def is_logged_in(self) -> bool: - html = self.http_request(self.base_url) + html = self.http_request(urljoin(self.base_url, "list")) login_link = find_link(html, './/a[@class="account"]') login_text = login_link.get("text") or "" return self.username in login_text @@ -53,10 +55,10 @@ def login(self) -> None: if self.is_logged_in: return - res = self.http_session.get(self.base_url) + res = self.http_session.get(urljoin(self.base_url, "list")) login_link = find_link(res.text, './/a[@class="account"]') if login_link: - login_url = urllib.parse.urljoin(res.url, login_link.get("href")) + login_url = urljoin(res.url, login_link.get("href")) else: logging.debug( f"url: {res.url}, status: {res.status_code}\nhtml:\n{res.text}" @@ -78,7 +80,7 @@ def login(self) -> None: login_form["session[password]"] = self.password self.http_session.post( - url=urllib.parse.urljoin(res.url, action), + url=urljoin(res.url, action), headers={"referer": res.url}, data=login_form, ) @@ -149,7 +151,7 @@ def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: # show exercise subparser parser_show = subparsers.add_parser("show", help="Show details of an exercise") parser_show.set_defaults(cmd="show") - parser_show.add_argument("task_id", help="Numerical task identifier or task name") + parser_show.add_argument("task_id", help="Numerical task identifier") return parser.parse_args(args) @@ -262,6 +264,9 @@ class Task: id: str name: str state: TaskState + description: str = "N/A" + code: str = "N/A" + submit_file: str = "N/A" # TODO: this should be part of a client class or module @@ -311,11 +316,48 @@ def print_task_list( # TODO: Implement function that parser the specific task page into Task object -def parse_task(html: str | bytes, task: Task) -> Task: - task = Task("a", "b", TaskState.COMPLETE) +# TODO: should we split up this function in a bunch of smaller ones? or will beautifulsoup make it simpler? +def parse_task(html: str | bytes) -> Task: + root = htmlement.fromstring(html) + task_link_element = root.find('.//div[@class="nav sidebar"]/a') + task_link = task_link_element if task_link_element is not None else Element("a") + task_id = task_link.get("href", "").split("/")[-1] + task_name = task_link.text or "N/A" + task_span_element = task_link.find("span") + task_span = task_span_element if task_span_element is not None else Element("span") + task_span_class = task_span.get("class", "") + desc_div_element = root.find('.//div[@class="md"]') + desc_div = desc_div_element if desc_div_element is not None else Element("div") + description = html2text(tostring(desc_div).decode("utf8")) + code = root.findtext(".//pre", "N/A") + submit_file = next( + iter( + [ + code_element.text + for code_element in root.findall(".//code") + if code_element.text is not None and ".py" in code_element.text + ] + ), + "N/A", + ) + task = Task( + id=task_id, + name=task_name, + state=TaskState.COMPLETE if "full" in task_span_class else TaskState.INCOMPLETE, + description=description.strip(), + code=code, + submit_file=submit_file, + ) + return task +def print_task(task: Task) -> None: + print(f"{task.id}: {task.name} {TASK_STATE_ICON[task.state]}") + print(task.description) + print(f"\nSubmission file name: {task.submit_file}") + + # TODO: Implement function that posts the submit form with the correct file def submit_task(task_id: str, filename: str) -> None: """submit file to the submit form or task_id""" @@ -342,7 +384,7 @@ def main() -> None: # Merge cli args and configfile parameters in one dict config.update((k, v) for k, v in vars(args).items() if v is not None) - base_url = f"https://cses.fi/{config['course']}/list/" + base_url = f"https://cses.fi/{config['course']}/" cookiefile = None cookies = dict() @@ -366,10 +408,15 @@ def main() -> None: write_cookie_file(str(cookiefile), cookies) if args.cmd == "list": - html = session.http_request(base_url) + html = session.http_request(urljoin(base_url, "list")) task_list = parse_task_list(html) print_task_list(task_list, filter=args.filter, limit=args.limit) + if args.cmd == "show": + html = session.http_request(urljoin(base_url, f"task/{args.task_id}")) + task = parse_task(html) + print_task(task) + if __name__ == "__main__": main() From 83a9c0e1219a860c209538042eab9ba0a175fc1e Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Mon, 13 May 2024 12:36:34 -0400 Subject: [PATCH 3/4] Fix tests for refactored base_url --- tests/test_moocfi_cses.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_moocfi_cses.py b/tests/test_moocfi_cses.py index 2b9628c..73906fc 100644 --- a/tests/test_moocfi_cses.py +++ b/tests/test_moocfi_cses.py @@ -49,7 +49,6 @@ class TestParseForm: ) -# TODO: verify these mocked tests are actually accurate # TODO: add tests for unreachable and failing endpoints, 4xx, 5xx, etc @pytest.fixture def mock_session() -> moocfi_cses.Session: @@ -64,7 +63,7 @@ def test_login_successful(mock_session: moocfi_cses.Session) -> None: # Mocking the HTTP response for successful login with requests_mock.Mocker() as m: m.get( - "https://example.com", + "https://example.com/list", text='', ) mock_session.login() @@ -75,7 +74,7 @@ def test_login_failed(mock_session: moocfi_cses.Session) -> None: # Mocking the HTTP response for failed login with requests_mock.Mocker() as m: m.get( - "https://example.com", + "https://example.com/list", text='', ) m.get("https://example.com/account", text="Login required") From 14faf25b9fb913fc4a336dc226c98cd8c1e4aac3 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Mon, 13 May 2024 12:39:04 -0400 Subject: [PATCH 4/4] Use AnyStr instead of str | bytes --- moocfi_cses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moocfi_cses.py b/moocfi_cses.py index a6416b5..d3ac115 100644 --- a/moocfi_cses.py +++ b/moocfi_cses.py @@ -270,7 +270,7 @@ class Task: # TODO: this should be part of a client class or module -def parse_task_list(html: str | bytes) -> list[Task]: +def parse_task_list(html: AnyStr) -> list[Task]: """Parse html to find tasks and their status, return something useful, possibly a specific data class""" content_element = htmlement.fromstring(html).find('.//div[@class="content"]') task_list = list() @@ -317,7 +317,7 @@ def print_task_list( # TODO: Implement function that parser the specific task page into Task object # TODO: should we split up this function in a bunch of smaller ones? or will beautifulsoup make it simpler? -def parse_task(html: str | bytes) -> Task: +def parse_task(html: AnyStr) -> Task: root = htmlement.fromstring(html) task_link_element = root.find('.//div[@class="nav sidebar"]/a') task_link = task_link_element if task_link_element is not None else Element("a")