From 515add1e60aada570101f30d43117ad290d6c142 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sat, 25 May 2024 22:19:38 -0400 Subject: [PATCH 01/11] WIP First draft of the work, currently in a broken state --- pyproject.toml | 2 +- src/tyora/tyora.py | 430 ------------------ src/tyora/__init__.py => tests/test_client.py | 0 tests/test_session.py | 6 +- tests/test_tyora.py | 33 -- tests/test_utils.py | 46 ++ tyora/__init__.py | 0 {src/tyora => tyora}/__main__.py | 0 tyora/client.py | 203 +++++++++ {src/tyora => tyora}/session.py | 55 +-- tyora/tyora.py | 257 +++++++++++ tyora/utils.py | 32 ++ 12 files changed, 550 insertions(+), 514 deletions(-) delete mode 100644 src/tyora/tyora.py rename src/tyora/__init__.py => tests/test_client.py (100%) create mode 100644 tests/test_utils.py create mode 100644 tyora/__init__.py rename {src/tyora => tyora}/__main__.py (100%) create mode 100644 tyora/client.py rename {src/tyora => tyora}/session.py (58%) create mode 100644 tyora/tyora.py create mode 100644 tyora/utils.py diff --git a/pyproject.toml b/pyproject.toml index 6033761..4e1b4fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dev-dependencies = [ allow-direct-references = true [tool.hatch.build.targets.wheel] -packages = ["src/tyora"] +packages = ["tyora"] [tool.pytest.ini_options] addopts = "--cov" diff --git a/src/tyora/tyora.py b/src/tyora/tyora.py deleted file mode 100644 index 472e313..0000000 --- a/src/tyora/tyora.py +++ /dev/null @@ -1,430 +0,0 @@ -import argparse -import importlib.metadata -import json -import logging -import sys -from dataclasses import dataclass -from enum import Enum -from getpass import getpass -from pathlib import Path -from time import sleep -from typing import AnyStr, Optional -from urllib.parse import urljoin -from xml.etree.ElementTree import Element, tostring - -import html5lib -import platformdirs -from html2text import html2text - -from .session import MoocfiCsesSession as Session - -logger = logging.getLogger(name="tyora") -try: - __version__ = importlib.metadata.version("tyora") -except importlib.metadata.PackageNotFoundError: - __version__ = "unknown" - -PROG_NAME = "tyora" -CONF_FILE = platformdirs.user_config_path(PROG_NAME) / "config.json" -STATE_DIR = platformdirs.user_state_path(f"{PROG_NAME}") - - -def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Interact with mooc.fi CSES instance") - parser.add_argument( - "-V", "--version", action="version", version=f"%(prog)s {__version__}" - ) - parser.add_argument("-u", "--username", help="tmc.mooc.fi username") - parser.add_argument("-p", "--password", help="tmc.mooc.fi password") - parser.add_argument( - "--debug", help="set logging level to debug", action="store_true" - ) - parser.add_argument( - "--course", - help="SLUG of the course (default: %(default)s)", - default="dsa24k", - ) - parser.add_argument( - "--config", - help="Location of config file (default: %(default)s)", - default=CONF_FILE, - ) - parser.add_argument( - "--no-state", - help="Don't store cookies or cache (they're used for faster access on the future runs)", - action="store_true", - ) - subparsers = parser.add_subparsers(required=True, title="commands", dest="cmd") - - # login subparser - subparsers.add_parser("login", help="Login to mooc.fi CSES") - - # list exercises subparser - parser_list = subparsers.add_parser("list", help="List exercises") - parser_list.add_argument( - "--filter", - help="List only complete or incomplete tasks (default: all)", - choices=["complete", "incomplete"], - ) - parser_list.add_argument( - "--limit", help="Maximum amount of items to list", type=int - ) - - # show exercise subparser - parser_show = subparsers.add_parser("show", help="Show details of an exercise") - parser_show.add_argument("task_id", help="Numerical task identifier") - - # submit exercise solution subparser - parser_submit = subparsers.add_parser("submit", help="Submit an exercise solution") - parser_submit.add_argument( - "--filename", - help="Filename of the solution to submit (if not given will be guessed from task description)", - ) - parser_submit.add_argument("task_id", help="Numerical task identifier") - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - return parser.parse_args(args) - - -def create_config() -> dict[str, str]: - username = input("Your tmc.mooc.fi username: ") - password = getpass("Your tmc.mooc.fi password: ") - config = { - "username": username, - "password": password, - } - - return config - - -def write_config(configfile: str, config: dict[str, str]) -> None: - file_path = Path(configfile).expanduser() - if file_path.exists(): - # TODO: https://github.com/madeddie/tyora/issues/28 - ... - file_path.parent.mkdir(parents=True, exist_ok=True) # Ensure directory exists - print("Writing config to file") - with open(file_path, "w") as f: - json.dump(config, f) - - -def read_config(configfile: str) -> dict[str, str]: - config = dict() - file_path = Path(configfile).expanduser() - with open(file_path, "r") as f: - config = json.load(f) - for setting in ("username", "password"): - assert setting in config - return config - - -def read_cookie_file(cookiefile: str) -> dict[str, str]: - """ - Reads cookies from a JSON formatted file. - - Args: - cookiefile: str path to the file containing cookies. - - Returns: - A dictionary of cookies. - """ - try: - with open(cookiefile, "r") as f: - return json.load(f) - except (FileNotFoundError, json.decoder.JSONDecodeError) as e: - logger.debug(f"Error reading cookies from {cookiefile}: {e}") - return {} - - -def write_cookie_file(cookiefile: str, cookies: dict[str, str]) -> None: - """ - Writes cookies to a file in JSON format. - - Args: - cookiefile: Path to the file for storing cookies. - cookies: A dictionary of cookies to write. - """ - with open(cookiefile, "w") as f: - json.dump(cookies, f) - - -def find_link(html: AnyStr, xpath: str) -> dict[str, Optional[str]]: - """Search for html link by xpath and return dict with href and text""" - anchor_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) - link_data = dict() - if anchor_element is not None: - link_data["href"] = anchor_element.get("href") - link_data["text"] = anchor_element.text - - return link_data - - -def parse_form(html: AnyStr, xpath: str = ".//form") -> dict: - """Search for the first form in html and return dict with action and all other found inputs""" - form_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) - form_data = dict() - if form_element is not None: - form_data["_action"] = form_element.get("action") - for form_input in form_element.iter("input"): - form_key = form_input.get("name") or "" - form_value = form_input.get("value") or "" - form_data[form_key] = form_value - - return form_data - - -class TaskState(Enum): - COMPLETE = "complete" - INCOMPLETE = "incomplete" - - -TASK_STATE_ICON = { - TaskState.COMPLETE: "✅", - TaskState.INCOMPLETE: "❌", -} - - -@dataclass -class Task: - id: str - name: str - state: TaskState - description: Optional[str] = None - code: Optional[str] = None - submit_file: Optional[str] = None - submit_link: Optional[str] = None - - -def parse_task_list(html: AnyStr) -> list[Task]: - """Parse html to find tasks and their status, return something useful, possibly a specific data class""" - content_element = html5lib.parse(html, namespaceHTMLElements=False).find( - './/div[@class="content"]' - ) - task_list = list() - if content_element is not None: - for item in content_element.findall('.//li[@class="task"]'): - item_id = None - item_name = None - item_class = None - - item_link = item.find("a") - if item_link is not None: - item_name = item_link.text or "" - item_id = item_link.get("href", "").split("/")[-1] - - item_spans = item.findall("span") or [] - item_span = next( - (span for span in item_spans if span.get("class", "") != "detail"), None - ) - if item_span is not None: - item_class = item_span.get("class", "") - - if item_id and item_name and item_class: - task = Task( - id=item_id, - name=item_name, - state=( - TaskState.COMPLETE - if "full" in item_class - else TaskState.INCOMPLETE - ), - ) - task_list.append(task) - - return task_list - - -def print_task_list( - task_list: list[Task], filter: Optional[str] = None, limit: Optional[int] = None -) -> None: - count: int = 0 - for task in task_list: - if not filter or filter == task.state.value: - print(f"- {task.id}: {task.name} {TASK_STATE_ICON[task.state]}") - count += 1 - if limit and count >= limit: - return - - -def parse_task(html: AnyStr) -> Task: - root = html5lib.parse(html, namespaceHTMLElements=False) - task_link_element = root.find('.//div[@class="nav sidebar"]/a') - task_link = task_link_element if task_link_element is not None else Element("a") - task_id = task_link.get("href", "").split("/")[-1] - if not task_id: - raise ValueError("Failed to find task id") - task_name = task_link.text or None - if not task_name: - raise ValueError("Failed to find task name") - task_span_element = task_link.find("span") - task_span = task_span_element if task_span_element is not None else Element("span") - task_span_class = task_span.get("class", "") - desc_div_element = root.find('.//div[@class="md"]') - desc_div = desc_div_element if desc_div_element is not None else Element("div") - description = html2text(tostring(desc_div).decode("utf8")) - code = root.findtext(".//pre", None) - submit_link_element = root.find('.//a[.="Submit"]') - submit_link = ( - submit_link_element.get("href", None) - if submit_link_element is not None - else None - ) - - submit_file = next( - iter( - [ - code_element.text - for code_element in root.findall(".//code") - if code_element.text is not None and ".py" in code_element.text - ] - ), - None, - ) - task = Task( - id=task_id, - name=task_name, - state=TaskState.COMPLETE if "full" in task_span_class else TaskState.INCOMPLETE, - description=description.strip(), - code=code, - submit_file=submit_file, - submit_link=submit_link, - ) - - return task - - -def print_task(task: Task) -> None: - print(f"{task.id}: {task.name} {TASK_STATE_ICON[task.state]}") - print(task.description) - print(f"\nSubmission file name: {task.submit_file}") - - -# def submit_task(task_id: str, filename: str) -> None: -# """submit file to the submit form or task_id""" -# html = session.http_request(urljoin(base_url, f"task/{task_id}")) -# task = parse_task(html) -# answer = input("Do you want to submit this task? (y/n): ") -# if answer in ('y', 'Y'): -# with open(filename, 'r') as f: - - -def parse_submit_result(html: AnyStr) -> dict[str, str]: - root = html5lib.parse(html, namespaceHTMLElements=False) - submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") - submit_status_span_element = submit_status_element.find("td/span") or Element( - "span" - ) - submit_status = submit_status_span_element.text or "" - submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") - submit_result_span_element = submit_result_element.find("td/span") or Element( - "span" - ) - submit_result = submit_result_span_element.text or "" - - return { - "status": submit_status.lower(), - "result": submit_result.lower(), - } - - -def main() -> None: - args = parse_args() - - logging.basicConfig( - level=logging.DEBUG if args.debug else logging.WARNING, - format="%(asctime)s %(levelname)s %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - - if args.cmd == "login": - config = create_config() - write_config(args.config, config) - return - - config = read_config(args.config) - - # Merge cli args and configfile parameters in one dict - config.update((k, v) for k, v in vars(args).items() if v is not None) - - base_url = f"https://cses.fi/{config['course']}/" - - cookiefile = None - cookies = dict() - if not args.no_state: - if not STATE_DIR.exists(): - STATE_DIR.mkdir(parents=True, exist_ok=True) - cookiefile = STATE_DIR / "cookies.txt" - cookies = read_cookie_file(str(cookiefile)) - - session = Session( - username=config["username"], - password=config["password"], - base_url=base_url, - cookies=cookies, - ) - session.login() - - if not args.no_state and cookiefile: - cookies = session.cookies.get_dict() - write_cookie_file(str(cookiefile), cookies) - - if args.cmd == "list": - res = session.get(urljoin(base_url, "list")) - res.raise_for_status() - task_list = parse_task_list(res.text) - print_task_list(task_list, filter=args.filter, limit=args.limit) - - if args.cmd == "show": - res = session.get(urljoin(base_url, f"task/{args.task_id}")) - res.raise_for_status() - try: - task = parse_task(res.text) - except ValueError as e: - logger.debug(f"Error parsing task: {e}") - raise - print_task(task) - - if args.cmd == "submit": - res = session.get(urljoin(base_url, f"task/{args.task_id}")) - res.raise_for_status() - task = parse_task(res.text) - if not task.submit_file and not args.filename: - raise ValueError("No submission filename found") - if not task.submit_link: - raise ValueError("No submission link found") - submit_file = args.filename or task.submit_file or "" - - res = session.get(urljoin(base_url, task.submit_link)) - res.raise_for_status() - submit_form_data = parse_form(res.text) - action = submit_form_data.pop("_action") - - for key, value in submit_form_data.items(): - submit_form_data[key] = (None, value) - submit_form_data["file"] = (submit_file, open(submit_file, "rb")) - submit_form_data["lang"] = (None, "Python3") - submit_form_data["option"] = (None, "CPython3") - - res = session.post(urljoin(base_url, action), files=submit_form_data) - res.raise_for_status() - html = res.text - result_url = res.url - print("Waiting for test results.", end="") - while "Test report" not in html: - print(".", end="") - sleep(1) - res = session.get(result_url) - res.raise_for_status() - - print() - results = parse_submit_result(res.text) - - print(f"Submission status: {results['status']}") - print(f"Submission result: {results['result']}") - - -if __name__ == "__main__": - main() diff --git a/src/tyora/__init__.py b/tests/test_client.py similarity index 100% rename from src/tyora/__init__.py rename to tests/test_client.py diff --git a/tests/test_session.py b/tests/test_session.py index 772ea1c..0e42c63 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -8,8 +8,6 @@ @pytest.fixture def mock_session() -> Session: return Session( - username="test_user@test.com", - password="test_password", base_url="https://example.com", cookies=test_cookies, ) @@ -22,7 +20,7 @@ def test_login_successful(mock_session: Session) -> None: "https://example.com/list", text=open("tests/test_data/session_logged_in.html").read(), ) - mock_session.login() + mock_session.login(username="test_user@test.com", password="test_password") print(mock_session.get("https://example.com/list").text) assert mock_session.is_logged_in @@ -43,7 +41,7 @@ def test_login_failed(mock_session: Session) -> None: text=open("tests/test_data/tmcmoocfi-sessions.html").read(), ) with pytest.raises(ValueError): - mock_session.login() + mock_session.login(username="test_user@test.com", password="test_password") def test_loading_cookies(mock_session: Session) -> None: diff --git a/tests/test_tyora.py b/tests/test_tyora.py index 09c3148..4e9d69c 100644 --- a/tests/test_tyora.py +++ b/tests/test_tyora.py @@ -12,39 +12,6 @@ def test_parse_args_command() -> None: assert args.cmd == "list" -class TestFindLink: - valid_html = ( - "" - 'sometext' - "" - ) - invalid_html = "No links here" - valid_xpath = './/a[@class="someclass"]' - invalid_xpath = './/a[@class="somethingelse"]' - valid_return = {"href": "somelink", "text": "sometext"} - - def test_find_link_success(self) -> None: - assert tyora.find_link(self.valid_html, self.valid_xpath) == self.valid_return - - def test_find_link_bad_xpath(self) -> None: - assert tyora.find_link(self.valid_html, self.invalid_xpath) == {} - - def test_find_link_bad_html(self) -> None: - assert tyora.find_link(self.invalid_html, self.valid_xpath) == {} - - -class TestParseForm: - valid_html = ( - '
' - 'sometext' - "
" - ) - noform_html = "No form here" - noinput_html = ( - '
Nothing here
' - ) - - # TODO: functions that use user input or read or write files def test_create_config() -> None: ... diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..4d94cac --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,46 @@ +from tyora import utils + + +class TestFindLink: + valid_html = ( + "" + 'sometext' + "" + ) + invalid_html = "No links here" + valid_xpath = './/a[@class="someclass"]' + invalid_xpath = './/a[@class="somethingelse"]' + valid_return = {"href": "somelink", "text": "sometext"} + + def test_find_link_success(self) -> None: + assert utils.find_link(self.valid_html, self.valid_xpath) == self.valid_return + + def test_find_link_bad_xpath(self) -> None: + assert utils.find_link(self.valid_html, self.invalid_xpath) == {} + + def test_find_link_bad_html(self) -> None: + assert utils.find_link(self.invalid_html, self.valid_xpath) == {} + + +class TestParseForm: + valid_html = ( + '
' + 'sometext' + "
" + ) + noform_html = "No form here" + noinput_html = ( + '
Nothing here
' + ) + + def test_parse_form_success(self) -> None: + assert utils.parse_form(self.valid_html) == { + "_action": "someaction", + "somename": "somevalue", + } + + def test_parse_form_no_form(self) -> None: + assert utils.parse_form(self.noform_html) == {} + + def test_parse_form_no_input(self) -> None: + assert utils.parse_form(self.noinput_html) == {"_action": "someaction"} diff --git a/tyora/__init__.py b/tyora/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tyora/__main__.py b/tyora/__main__.py similarity index 100% rename from src/tyora/__main__.py rename to tyora/__main__.py diff --git a/tyora/client.py b/tyora/client.py new file mode 100644 index 0000000..e1e27f7 --- /dev/null +++ b/tyora/client.py @@ -0,0 +1,203 @@ +import logging + +from enum import Enum +from dataclasses import dataclass +from typing import AnyStr, Optional +from urllib.parse import urljoin +from xml.etree.ElementTree import Element, tostring + +import html5lib +from html2text import html2text + +from .session import MoocfiCsesSession as Session +from .utils import parse_form + +logger = logging.getLogger(__name__) + + +class TaskState(Enum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + + +@dataclass +class Task: + id: str + name: str + state: TaskState + description: Optional[str] = None + code: Optional[str] = None + submit_file: Optional[str] = None + submit_link: Optional[str] = None + + +class Client: + def __init__(self, session: Session) -> None: + self.session = session + + def login(self, username: str, password: str) -> None: + self.session.login(username, password) + + def get_task_list(self) -> list[Task]: + res = self.session.get(urljoin(self.session.base_url, "list")) + res.raise_for_status() + return parse_task_list(res.text) + + def get_task(self, task_id: str) -> Task: + res = self.session.get(urljoin(self.session.base_url, f"task/{task_id}")) + res.raise_for_status() + try: + task = parse_task(res.text) + except ValueError as e: + logger.debug(f"Error parsing task: {e}") + raise + return task + + def submit_task( + self, task_id: str, submission: bytes, filename: Optional[str] + ) -> str: + task = self.get_task(task_id) + if not task.submit_file and not filename: + raise ValueError("No submission filename found") + if not task.submit_link: + raise ValueError("No submit link found") + submit_file = task.submit_file or filename + + res = self.session.get(urljoin(self.session.base_url, task.submit_link)) + res.raise_for_status() + parsed_form_data = parse_form(res.text) + action = parsed_form_data.pop("_action") + + submit_form_data = dict() + for key, value in parsed_form_data.items(): + submit_form_data[key] = (None, value) + submit_form_data["file"] = (submit_file, submission) + submit_form_data["lang"] = (None, "Python3") + submit_form_data["option"] = (None, "CPython3") + + res = self.session.post( + urljoin(self.session.base_url, action), files=submit_form_data + ) + res.raise_for_status() + + return res.url + + +def parse_task_list(html: AnyStr) -> list[Task]: + """Parse html to find tasks and their status, returns list of Task objects""" + root = html5lib.parse(html, namespaceHTMLElements=False) + task_element_list = root.findall('.//li[@class="task"]') + + task_list = list() + for task_element in task_element_list: + task_id = None + task_name = None + task_state = None + + task_link = task_element.find("a") + if task_link is None: + continue + + task_name = task_link.text + task_id = task_link.get("href", "/").split("/")[-1] + if not task_name or not task_id: + continue + + task_element_spans = task_element.findall("span") + if not task_element_spans: + continue + + task_element_span = next( + (span for span in task_element_spans if span.get("class", "") != "detail"), + None, + ) + if task_element_span is not None: + task_element_class = task_element_span.get("class") or "" + + task_state = ( + TaskState.COMPLETE if "full" in task_element_class else TaskState.INCOMPLETE + ) + + task = Task( + id=task_id, + name=task_name, + state=task_state, + ) + task_list.append(task) + + return task_list + + +def parse_task(html: AnyStr) -> Task: + root = html5lib.parse(html, namespaceHTMLElements=False) + task_link_element = root.find('.//div[@class="nav sidebar"]/a') + task_link = task_link_element if task_link_element is not None else Element("a") + task_id = task_link.get("href", "").split("/")[-1] + if not task_id: + raise ValueError("Failed to find task id") + task_name = task_link.text or None + if not task_name: + raise ValueError("Failed to find task name") + task_span_element = task_link.find("span") + task_span = task_span_element if task_span_element is not None else Element("span") + task_span_class = task_span.get("class", "") + desc_div_element = root.find('.//div[@class="md"]') + desc_div = desc_div_element if desc_div_element is not None else Element("div") + description = html2text(tostring(desc_div).decode("utf8")) + code = root.findtext(".//pre", None) + submit_link_element = root.find('.//a[.="Submit"]') + submit_link = ( + submit_link_element.get("href", None) + if submit_link_element is not None + else None + ) + + submit_file = next( + iter( + [ + code_element.text + for code_element in root.findall(".//code") + if code_element.text is not None and ".py" in code_element.text + ] + ), + None, + ) + task = Task( + id=task_id, + name=task_name, + state=TaskState.COMPLETE if "full" in task_span_class else TaskState.INCOMPLETE, + description=description.strip(), + code=code, + submit_file=submit_file, + submit_link=submit_link, + ) + + return task + + +# def submit_task(task_id: str, filename: str) -> None: +# """submit file to the submit form or task_id""" +# html = session.http_request(urljoin(base_url, f"task/{task_id}")) +# task = parse_task(html) +# answer = input("Do you want to submit this task? (y/n): ") +# if answer in ('y', 'Y'): +# with open(filename, 'r') as f: + + +def parse_submit_result(html: AnyStr) -> dict[str, str]: + root = html5lib.parse(html, namespaceHTMLElements=False) + submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") + submit_status_span_element = submit_status_element.find("td/span") or Element( + "span" + ) + submit_status = submit_status_span_element.text or "" + submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") + submit_result_span_element = submit_result_element.find("td/span") or Element( + "span" + ) + submit_result = submit_result_span_element.text or "" + + return { + "status": submit_status.lower(), + "result": submit_result.lower(), + } diff --git a/src/tyora/session.py b/tyora/session.py similarity index 58% rename from src/tyora/session.py rename to tyora/session.py index 376cc45..871ed87 100644 --- a/src/tyora/session.py +++ b/tyora/session.py @@ -2,13 +2,14 @@ import logging import os import sys -from typing import AnyStr, Optional +from typing import Optional from urllib.parse import urljoin -import html5lib import requests from requests_toolbelt import user_agent +from .utils import find_link, parse_form + logger = logging.getLogger(__name__) try: @@ -18,19 +19,9 @@ class MoocfiCsesSession(requests.Session): - def __init__( - self, - username: str, - password: str, - base_url: str, - cookies: Optional[dict] = None, - *args, - **kwargs, - ): + def __init__(self, base_url: str, cookies: Optional[dict] = None, *args, **kwargs): super().__init__(*args, **kwargs) - self.username = username - self.password = password self.base_url = base_url if cookies: @@ -44,11 +35,10 @@ def __init__( def is_logged_in(self) -> bool: res = self.get(urljoin(self.base_url, "list")) res.raise_for_status() - login_link = find_link(res.text, './/a[@class="account"]') - login_text = login_link.get("text") or "" - return self.username in login_text + logout_link = find_link(res.text, './/a[@title="Log out"]') + return bool(logout_link) - def login(self) -> None: + def login(self, username: str, password: str) -> None: """Log into the site using webscraping Steps: @@ -83,8 +73,8 @@ def login(self) -> None: ) raise ValueError("Failed to find login form") - login_form["session[login]"] = self.username - login_form["session[password]"] = self.password + login_form["session[login]"] = username + login_form["session[password]"] = password self.post( url=urljoin(res.url, action), @@ -97,30 +87,3 @@ def login(self) -> None: f"url: {res.url}, status: {res.status_code}\nhtml:\n{res.text}" ) raise ValueError("Login failed") - - -def find_link(html: AnyStr, xpath: str) -> dict[str, Optional[str]]: - """Search for html link by xpath and return dict with href and text""" - anchor_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) - if anchor_element is None: - return dict() - - link_data = dict() - link_data["href"] = anchor_element.get("href") - link_data["text"] = anchor_element.text - - return link_data - - -def parse_form(html: AnyStr, xpath: str = ".//form") -> dict: - """Search for the first form in html and return dict with action and all other found inputs""" - form_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) - form_data = dict() - if form_element is not None: - form_data["_action"] = form_element.get("action") - for form_input in form_element.iter("input"): - form_key = form_input.get("name") or "" - form_value = form_input.get("value") or "" - form_data[form_key] = form_value - - return form_data diff --git a/tyora/tyora.py b/tyora/tyora.py new file mode 100644 index 0000000..e5d62eb --- /dev/null +++ b/tyora/tyora.py @@ -0,0 +1,257 @@ +import argparse +import importlib.metadata +import json +import logging +import sys +from getpass import getpass +from pathlib import Path +from time import sleep +from typing import Optional +from urllib.parse import urljoin + +import platformdirs + +from .client import ( + Client, + Task, + TaskState, + parse_submit_result, + parse_task, + parse_task_list, +) +from .session import MoocfiCsesSession as Session +from .utils import parse_form + +logger = logging.getLogger(name="tyora") +try: + __version__ = importlib.metadata.version("tyora") +except importlib.metadata.PackageNotFoundError: + __version__ = "unknown" + +PROG_NAME = "tyora" +CONF_FILE = platformdirs.user_config_path(PROG_NAME) / "config.json" +STATE_DIR = platformdirs.user_state_path(f"{PROG_NAME}") + + +def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Interact with mooc.fi CSES instance") + parser.add_argument( + "-V", "--version", action="version", version=f"%(prog)s {__version__}" + ) + parser.add_argument("-u", "--username", help="tmc.mooc.fi username") + parser.add_argument("-p", "--password", help="tmc.mooc.fi password") + parser.add_argument( + "--debug", help="set logging level to debug", action="store_true" + ) + parser.add_argument( + "--course", + help="SLUG of the course (default: %(default)s)", + default="dsa24k", + ) + parser.add_argument( + "--config", + help="Location of config file (default: %(default)s)", + default=CONF_FILE, + ) + parser.add_argument( + "--no-state", + help="Don't store cookies or cache (they're used for faster access on the future runs)", + action="store_true", + ) + subparsers = parser.add_subparsers(required=True, title="commands", dest="cmd") + + # login subparser + subparsers.add_parser("login", help="Login to mooc.fi CSES") + + # list exercises subparser + parser_list = subparsers.add_parser("list", help="List exercises") + parser_list.add_argument( + "--filter", + help="List only complete or incomplete tasks (default: all)", + choices=["complete", "incomplete"], + ) + parser_list.add_argument( + "--limit", help="Maximum amount of items to list", type=int + ) + + # show exercise subparser + parser_show = subparsers.add_parser("show", help="Show details of an exercise") + parser_show.add_argument("task_id", help="Numerical task identifier") + + # submit exercise solution subparser + parser_submit = subparsers.add_parser("submit", help="Submit an exercise solution") + parser_submit.add_argument( + "--filename", + help="Filename of the solution to submit (if not given will be guessed from task description)", + ) + parser_submit.add_argument("task_id", help="Numerical task identifier") + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + return parser.parse_args(args) + + +def create_config() -> dict[str, str]: + username = input("Your tmc.mooc.fi username: ") + password = getpass("Your tmc.mooc.fi password: ") + config = { + "username": username, + "password": password, + } + + return config + + +def write_config(configfile: str, config: dict[str, str]) -> None: + file_path = Path(configfile).expanduser() + if file_path.exists(): + # TODO: https://github.com/madeddie/tyora/issues/28 + ... + # Ensure directory exists + file_path.parent.mkdir(parents=True, exist_ok=True) + print("Writing config to file") + with open(file_path, "w") as f: + json.dump(config, f) + + +def read_config(configfile: str) -> dict[str, str]: + config = dict() + file_path = Path(configfile).expanduser() + with open(file_path, "r") as f: + config = json.load(f) + for setting in ("username", "password"): + assert setting in config + return config + + +def read_cookie_file(cookiefile: str) -> dict[str, str]: + """ + Reads cookies from a JSON formatted file. + + Args: + cookiefile: str path to the file containing cookies. + + Returns: + A dictionary of cookies. + """ + try: + with open(cookiefile, "r") as f: + return json.load(f) + except (FileNotFoundError, json.decoder.JSONDecodeError) as e: + logger.debug(f"Error reading cookies from {cookiefile}: {e}") + return {} + + +def write_cookie_file(cookiefile: str, cookies: dict[str, str]) -> None: + """ + Writes cookies to a file in JSON format. + + Args: + cookiefile: Path to the file for storing cookies. + cookies: A dictionary of cookies to write. + """ + with open(cookiefile, "w") as f: + json.dump(cookies, f) + + +TASK_STATE_ICON = { + TaskState.COMPLETE: "✅", + TaskState.INCOMPLETE: "❌", +} + + +def print_task_list( + task_list: list[Task], filter: Optional[str] = None, limit: Optional[int] = None +) -> None: + count: int = 0 + for task in task_list: + if not filter or filter == task.state.value: + print(f"- {task.id}: {task.name} {TASK_STATE_ICON[task.state]}") + count += 1 + if limit and count >= limit: + return + + +def print_task(task: Task) -> None: + print(f"{task.id}: {task.name} {TASK_STATE_ICON[task.state]}") + print(task.description) + print(f"\nSubmission file name: {task.submit_file}") + + +def main() -> None: + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.WARNING, + format="%(asctime)s %(levelname)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + if args.cmd == "login": + config = create_config() + write_config(args.config, config) + return + + config = read_config(args.config) + + # Merge cli args and configfile parameters in one dict + config.update((k, v) for k, v in vars(args).items() if v is not None) + + base_url = f"https://cses.fi/{config['course']}/" + + cookiefile = None + cookies = dict() + if not args.no_state: + if not STATE_DIR.exists(): + STATE_DIR.mkdir(parents=True, exist_ok=True) + cookiefile = STATE_DIR / "cookies.json" + cookies = read_cookie_file(str(cookiefile)) + + session = Session( + base_url=base_url, + cookies=cookies, + ) + + client = Client(session) + client.login(username=config["username"], password=config["password"]) + + if not args.no_state and cookiefile: + cookies = session.cookies.get_dict() + write_cookie_file(str(cookiefile), cookies) + + if args.cmd == "list": + print_task_list(client.get_task_list(), filter=args.filter, limit=args.limit) + + if args.cmd == "show": + print_task(client.get_task(args.task_id)) + + if args.cmd == "submit": + # TODO allow user to paste the code in or even pipe it in + with open(args.filename, "rb") as f: + submission_code = (f.read(),) + + result_url = client.submit_task( + task_id=args.task_id, + filename=args.filename, + submission=submission_code, + ) + print("Waiting for test results.", end="") + while True: + print(".", end="") + res = session.get(result_url) + res.raise_for_status() + if "Test report" in res.text: + break + sleep(1) + + print() + results = parse_submit_result(res.text) + + print(f"Submission status: {results['status']}") + print(f"Submission result: {results['result']}") + + +if __name__ == "__main__": + main() diff --git a/tyora/utils.py b/tyora/utils.py new file mode 100644 index 0000000..7bc1db5 --- /dev/null +++ b/tyora/utils.py @@ -0,0 +1,32 @@ +from typing import AnyStr, Optional + +import html5lib + + +def find_link(html: AnyStr, xpath: str) -> dict[str, Optional[str]]: + """Search for html link by xpath and return dict with href and text""" + anchor_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) + if anchor_element is None: + return dict() + + link_data = dict() + link_data["href"] = anchor_element.get("href") + link_data["text"] = anchor_element.text + + return link_data + + +def parse_form(html: AnyStr, xpath: str = ".//form") -> dict[str, Optional[str]]: + """Search for the first form in html and return dict with action and all other found inputs""" + form_element = html5lib.parse(html, namespaceHTMLElements=False).find(xpath) + if form_element is None: + return dict() + + form_data = dict() + form_data["_action"] = form_element.get("action") + for form_input in form_element.iter("input"): + form_key = form_input.get("name") or "" + form_value = form_input.get("value") or "" + form_data[form_key] = form_value + + return form_data From 366d792828c91f4f8dfa93c7cee9ca614f964db2 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sat, 25 May 2024 22:32:30 -0400 Subject: [PATCH 02/11] WIP code works again, but submission report is broken, needs tests! --- tyora/client.py | 5 +++-- tyora/tyora.py | 7 ++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tyora/client.py b/tyora/client.py index e1e27f7..2cb7241 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -54,7 +54,7 @@ def get_task(self, task_id: str) -> Task: return task def submit_task( - self, task_id: str, submission: bytes, filename: Optional[str] + self, task_id: str, submission: AnyStr, filename: Optional[str] ) -> str: task = self.get_task(task_id) if not task.submit_file and not filename: @@ -74,7 +74,6 @@ def submit_task( submit_form_data["file"] = (submit_file, submission) submit_form_data["lang"] = (None, "Python3") submit_form_data["option"] = (None, "CPython3") - res = self.session.post( urljoin(self.session.base_url, action), files=submit_form_data ) @@ -184,6 +183,8 @@ def parse_task(html: AnyStr) -> Task: # with open(filename, 'r') as f: +# TODO test with failing results +# Seems to be broken since the switch to html5lib, needs tests! def parse_submit_result(html: AnyStr) -> dict[str, str]: root = html5lib.parse(html, namespaceHTMLElements=False) submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") diff --git a/tyora/tyora.py b/tyora/tyora.py index e5d62eb..1e60ccc 100644 --- a/tyora/tyora.py +++ b/tyora/tyora.py @@ -7,7 +7,6 @@ from pathlib import Path from time import sleep from typing import Optional -from urllib.parse import urljoin import platformdirs @@ -16,11 +15,8 @@ Task, TaskState, parse_submit_result, - parse_task, - parse_task_list, ) from .session import MoocfiCsesSession as Session -from .utils import parse_form logger = logging.getLogger(name="tyora") try: @@ -230,7 +226,7 @@ def main() -> None: if args.cmd == "submit": # TODO allow user to paste the code in or even pipe it in with open(args.filename, "rb") as f: - submission_code = (f.read(),) + submission_code = f.read() result_url = client.submit_task( task_id=args.task_id, @@ -241,6 +237,7 @@ def main() -> None: while True: print(".", end="") res = session.get(result_url) + print(res.text) res.raise_for_status() if "Test report" in res.text: break From 28e91ccc5354a795ac8f5b6dfd7cdc06fc647ac5 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 26 May 2024 22:11:08 -0400 Subject: [PATCH 03/11] Fix or ignore typing errors and warnings --- requirements-dev.lock | 2 +- requirements.lock | 2 +- tests/test_session.py | 1 + tests/test_tyora.py | 3 ++- tyora/client.py | 25 +++++++++++++------------ tyora/session.py | 8 +++++--- tyora/tyora.py | 9 +++++---- tyora/utils.py | 4 ++-- 8 files changed, 30 insertions(+), 24 deletions(-) diff --git a/requirements-dev.lock b/requirements-dev.lock index 4d08fc3..38dc3d2 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -48,7 +48,7 @@ pytest==8.2.1 pytest-cov==5.0.0 pyyaml==6.0.1 # via pre-commit -requests==2.32.1 +requests==2.32.2 # via requests-mock # via requests-toolbelt # via tyora diff --git a/requirements.lock b/requirements.lock index ef46c7e..0d13951 100644 --- a/requirements.lock +++ b/requirements.lock @@ -21,7 +21,7 @@ idna==3.7 # via requests platformdirs==4.2.2 # via tyora -requests==2.32.1 +requests==2.32.2 # via requests-toolbelt # via tyora requests-toolbelt==1.0.0 diff --git a/tests/test_session.py b/tests/test_session.py index 0e42c63..aee6c0a 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,5 +1,6 @@ import pytest import requests_mock + from tyora.session import MoocfiCsesSession as Session test_cookies = {"cookie_a": "value_a", "cookie_b": "value_b"} diff --git a/tests/test_tyora.py b/tests/test_tyora.py index 4e9d69c..5233540 100644 --- a/tests/test_tyora.py +++ b/tests/test_tyora.py @@ -1,10 +1,11 @@ import pytest + from tyora import tyora def test_parse_args_missing_args() -> None: with pytest.raises(SystemExit): - tyora.parse_args() + _ = tyora.parse_args() def test_parse_args_command() -> None: diff --git a/tyora/client.py b/tyora/client.py index 2cb7241..56d6e78 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -1,7 +1,8 @@ -import logging +from __future__ import annotations -from enum import Enum +import logging from dataclasses import dataclass +from enum import Enum from typing import AnyStr, Optional from urllib.parse import urljoin from xml.etree.ElementTree import Element, tostring @@ -54,7 +55,7 @@ def get_task(self, task_id: str) -> Task: return task def submit_task( - self, task_id: str, submission: AnyStr, filename: Optional[str] + self, task_id: str, submission: str, filename: Optional[str] ) -> str: task = self.get_task(task_id) if not task.submit_file and not filename: @@ -68,14 +69,15 @@ def submit_task( parsed_form_data = parse_form(res.text) action = parsed_form_data.pop("_action") - submit_form_data = dict() + submit_form_data: dict[str, tuple[Optional[str], Optional[str]]] = dict() for key, value in parsed_form_data.items(): submit_form_data[key] = (None, value) submit_form_data["file"] = (submit_file, submission) submit_form_data["lang"] = (None, "Python3") submit_form_data["option"] = (None, "CPython3") res = self.session.post( - urljoin(self.session.base_url, action), files=submit_form_data + urljoin(self.session.base_url, action), + files=submit_form_data, # type: ignore[arg-type] ) res.raise_for_status() @@ -84,10 +86,10 @@ def submit_task( def parse_task_list(html: AnyStr) -> list[Task]: """Parse html to find tasks and their status, returns list of Task objects""" - root = html5lib.parse(html, namespaceHTMLElements=False) + root = html5lib.parse(html, namespaceHTMLElements=False) # type: ignore[reportUnknownMemberType] task_element_list = root.findall('.//li[@class="task"]') - task_list = list() + task_list: list[Task] = list() for task_element in task_element_list: task_id = None task_name = None @@ -108,10 +110,9 @@ def parse_task_list(html: AnyStr) -> list[Task]: task_element_span = next( (span for span in task_element_spans if span.get("class", "") != "detail"), - None, + Element("span"), ) - if task_element_span is not None: - task_element_class = task_element_span.get("class") or "" + task_element_class = task_element_span.get("class") or "" task_state = ( TaskState.COMPLETE if "full" in task_element_class else TaskState.INCOMPLETE @@ -128,7 +129,7 @@ def parse_task_list(html: AnyStr) -> list[Task]: def parse_task(html: AnyStr) -> Task: - root = html5lib.parse(html, namespaceHTMLElements=False) + root = html5lib.parse(html, namespaceHTMLElements=False) # type: ignore[reportUnknownMemberType] task_link_element = root.find('.//div[@class="nav sidebar"]/a') task_link = task_link_element if task_link_element is not None else Element("a") task_id = task_link.get("href", "").split("/")[-1] @@ -186,7 +187,7 @@ def parse_task(html: AnyStr) -> Task: # TODO test with failing results # Seems to be broken since the switch to html5lib, needs tests! def parse_submit_result(html: AnyStr) -> dict[str, str]: - root = html5lib.parse(html, namespaceHTMLElements=False) + root = html5lib.parse(html, namespaceHTMLElements=False) # type: ignore[reportUnknownMemberType] submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") submit_status_span_element = submit_status_element.find("td/span") or Element( "span" diff --git a/tyora/session.py b/tyora/session.py index 871ed87..e8659e3 100644 --- a/tyora/session.py +++ b/tyora/session.py @@ -19,7 +19,9 @@ class MoocfiCsesSession(requests.Session): - def __init__(self, base_url: str, cookies: Optional[dict] = None, *args, **kwargs): + def __init__( + self, base_url: str, cookies: Optional[dict[str, str]] = None, *args, **kwargs + ): super().__init__(*args, **kwargs) self.base_url = base_url @@ -66,7 +68,7 @@ def login(self, username: str, password: str) -> None: login_form = parse_form(res.text, ".//form") if login_form: action = login_form.get("_action") - login_form.pop("_action") + _ = login_form.pop("_action") else: logger.debug( f"url: {res.url}, status: {res.status_code}\nhtml:\n{res.text}" @@ -76,7 +78,7 @@ def login(self, username: str, password: str) -> None: login_form["session[login]"] = username login_form["session[password]"] = password - self.post( + _ = self.post( url=urljoin(res.url, action), headers={"referer": res.url}, data=login_form, diff --git a/tyora/tyora.py b/tyora/tyora.py index 1e60ccc..d4a42cd 100644 --- a/tyora/tyora.py +++ b/tyora/tyora.py @@ -113,7 +113,7 @@ def write_config(configfile: str, config: dict[str, str]) -> None: def read_config(configfile: str) -> dict[str, str]: - config = dict() + config: dict[str, str] = dict() file_path = Path(configfile).expanduser() with open(file_path, "r") as f: config = json.load(f) @@ -134,7 +134,8 @@ def read_cookie_file(cookiefile: str) -> dict[str, str]: """ try: with open(cookiefile, "r") as f: - return json.load(f) + cookies: dict[str, str] = json.load(f) + return cookies except (FileNotFoundError, json.decoder.JSONDecodeError) as e: logger.debug(f"Error reading cookies from {cookiefile}: {e}") return {} @@ -198,7 +199,7 @@ def main() -> None: base_url = f"https://cses.fi/{config['course']}/" cookiefile = None - cookies = dict() + cookies: dict[str, str] = dict() if not args.no_state: if not STATE_DIR.exists(): STATE_DIR.mkdir(parents=True, exist_ok=True) @@ -225,7 +226,7 @@ def main() -> None: if args.cmd == "submit": # TODO allow user to paste the code in or even pipe it in - with open(args.filename, "rb") as f: + with open(args.filename, "r") as f: submission_code = f.read() result_url = client.submit_task( diff --git a/tyora/utils.py b/tyora/utils.py index 7bc1db5..b4500fe 100644 --- a/tyora/utils.py +++ b/tyora/utils.py @@ -9,7 +9,7 @@ def find_link(html: AnyStr, xpath: str) -> dict[str, Optional[str]]: if anchor_element is None: return dict() - link_data = dict() + link_data: dict[str, Optional[str]] = dict() link_data["href"] = anchor_element.get("href") link_data["text"] = anchor_element.text @@ -22,7 +22,7 @@ def parse_form(html: AnyStr, xpath: str = ".//form") -> dict[str, Optional[str]] if form_element is None: return dict() - form_data = dict() + form_data: dict[str, Optional[str]] = dict() form_data["_action"] = form_element.get("action") for form_input in form_element.iter("input"): form_key = form_input.get("name") or "" From 4ab8b04dd8a607a5fc2e01b8aa593f96e448c3f5 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 26 May 2024 22:16:50 -0400 Subject: [PATCH 04/11] Sort imports with ruff --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4e1b4fe..0c29bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,6 @@ packages = ["tyora"] [tool.pytest.ini_options] addopts = "--cov" + +[tool.ruff.lint] +extend-select = ["I"] From fd6033f868469926afa0680772b7d9e4a3794fb7 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 26 May 2024 22:24:06 -0400 Subject: [PATCH 05/11] Keep login in the session, not in the client --- tyora/client.py | 3 --- tyora/tyora.py | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tyora/client.py b/tyora/client.py index 56d6e78..5d7d3a7 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -36,9 +36,6 @@ class Client: def __init__(self, session: Session) -> None: self.session = session - def login(self, username: str, password: str) -> None: - self.session.login(username, password) - def get_task_list(self) -> list[Task]: res = self.session.get(urljoin(self.session.base_url, "list")) res.raise_for_status() diff --git a/tyora/tyora.py b/tyora/tyora.py index d4a42cd..9fc4d0e 100644 --- a/tyora/tyora.py +++ b/tyora/tyora.py @@ -210,9 +210,9 @@ def main() -> None: base_url=base_url, cookies=cookies, ) - + # TODO: make logging in optional for list and show commands + session.login(username=config["username"], password=config["password"]) client = Client(session) - client.login(username=config["username"], password=config["password"]) if not args.no_state and cookiefile: cookies = session.cookies.get_dict() From f48706dc922f2fe89f2dfe3b4e51fb96cc742710 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 26 May 2024 22:26:39 -0400 Subject: [PATCH 06/11] Update tyora/client.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tyora/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tyora/client.py b/tyora/client.py index 56d6e78..b2fec42 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -59,9 +59,9 @@ def submit_task( ) -> str: task = self.get_task(task_id) if not task.submit_file and not filename: - raise ValueError("No submission filename found") + raise ValueError("No submission filename found for task ID: " + task_id) if not task.submit_link: - raise ValueError("No submit link found") + raise ValueError("No submit link found for task ID: " + task_id) submit_file = task.submit_file or filename res = self.session.get(urljoin(self.session.base_url, task.submit_link)) @@ -80,7 +80,6 @@ def submit_task( files=submit_form_data, # type: ignore[arg-type] ) res.raise_for_status() - return res.url From 6bc28e836ec924ac69e5a825c96c232fafbf0b3c Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 26 May 2024 22:26:56 -0400 Subject: [PATCH 07/11] Fix PR comments --- tyora/session.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tyora/session.py b/tyora/session.py index e8659e3..29ac205 100644 --- a/tyora/session.py +++ b/tyora/session.py @@ -67,8 +67,7 @@ def login(self, username: str, password: str) -> None: res = self.get(login_url, headers={"referer": res.url}) login_form = parse_form(res.text, ".//form") if login_form: - action = login_form.get("_action") - _ = login_form.pop("_action") + action = login_form.pop("_action") else: logger.debug( f"url: {res.url}, status: {res.status_code}\nhtml:\n{res.text}" From 546c00da5141694bf600601c1129257b2c8876ce Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Mon, 27 May 2024 10:22:22 -0400 Subject: [PATCH 08/11] Add testing for most functions --- tests/test_client.py | 85 ++++++++++++++ .../session_logged_in_some_tasks_done.html | 2 +- .../task_3052_incomplete_no_submit_link.html | 109 ++++++++++++++++++ tests/test_data/task_3055_complete.html | 87 ++++++++++++++ 4 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 tests/test_data/task_3052_incomplete_no_submit_link.html create mode 100644 tests/test_data/task_3055_complete.html diff --git a/tests/test_client.py b/tests/test_client.py index e69de29..e51fa62 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -0,0 +1,85 @@ +import pytest +import requests_mock + +from tyora.client import Client, TaskState +from tyora.session import MoocfiCsesSession as Session + +test_cookies = {"cookie_a": "value_a", "cookie_b": "value_b"} + + +@pytest.fixture +def mock_session() -> Session: + return Session( + base_url="https://example.com", + cookies=test_cookies, + ) + + +def test_client(mock_session: Session) -> None: + client = Client(session=mock_session) + + assert client.session == mock_session + + +def test_client_get_task_list(mock_session: Session) -> None: + client = Client(session=mock_session) + + with requests_mock.Mocker() as m: + m.get( + "https://example.com/list", + text=open("tests/test_data/session_logged_in_some_tasks_done.html").read(), + ) + task_list = client.get_task_list() + assert len(task_list) == 4 + assert task_list[0].id == "3055" + assert task_list[0].name == "Candies" + assert task_list[0].state == TaskState.COMPLETE + assert task_list[1].id == "3049" + assert task_list[1].name == "Inversions" + assert task_list[1].state == TaskState.COMPLETE + assert task_list[2].id == "3054" + assert task_list[2].name == "Same bits" + assert task_list[2].state == TaskState.COMPLETE + assert task_list[3].id == "2643" + assert task_list[3].name == "Repeat" + assert task_list[3].state == TaskState.INCOMPLETE + + +def test_client_get_task_complete(mock_session: Session) -> None: + client = Client(session=mock_session) + + with requests_mock.Mocker() as m: + m.get( + "https://example.com/task/3055", + text=open("tests/test_data/task_3055_complete.html").read(), + ) + task = client.get_task("3055") + assert task.id == "3055" + assert task.name == "Candies" + assert task.state == TaskState.COMPLETE + assert ( + task.description + == 'A gummy candy costs a euros and a chocolate candy costs b euros. What is the\nmaximum number of candies you can buy if you have c euros?\n\nYou may assume that a, b and c are integers in the range 1 \\dots 100.\n\nIn a file `candies.py`, implement the function `count` that returns the\nmaximum number of candies.\n\n \n \n def count(a, b, c):\n # TODO\n \n if __name__ == "__main__":\n print(count(3, 4, 11)) # 3\n print(count(5, 1, 100)) # 100\n print(count(2, 3, 1)) # 0\n print(count(2, 3, 9)) # 4\n \n\n_Explanation_ : In the first test, a gummy candy costs 3 euros and a chocolate\ncandy costs 4 euros. You can buy at most 3 candies with 11 euros. For example,\ntwo gummy candies and one chocolate candy cost a total of 10 euros leaving you\nwith 1 euro.' + ) + assert task.submit_link == "/dsa24k/submit/3055/" + + +def test_client_get_task_incomplete_no_submit_link(mock_session: Session) -> None: + client = Client(session=mock_session) + + with requests_mock.Mocker() as m: + m.get( + "https://example.com/task/3052", + text=open( + "tests/test_data/task_3052_incomplete_no_submit_link.html" + ).read(), + ) + task = client.get_task("3052") + assert task.id == "3052" + assert task.name == "Efficiency test" + assert task.state == TaskState.INCOMPLETE + assert ( + task.description + == "The course material includes two different ways to implement the function\n`count_even`:\n\n \n \n # implementation 1\n def count_even(numbers):\n result = 0\n for x in numbers:\n if x % 2 == 0:\n result += 1\n return result\n \n \n \n # implementation 2\n def count_even(numbers):\n return sum(x % 2 == 0 for x in numbers)\n \n\nCompare the efficiencies of the two implementations using a list that contains\n10^7 randomly chosen numbers.\n\nIn this exercise, you get a point automatically when you submit the test\nresults and the code that you used.\n\nImplementation 1 run time: s\n\nImplementation 2 run time: s\n\nThe code you used in the test:" + ) + assert task.submit_link is None diff --git a/tests/test_data/session_logged_in_some_tasks_done.html b/tests/test_data/session_logged_in_some_tasks_done.html index a22971a..3ddc286 100644 --- a/tests/test_data/session_logged_in_some_tasks_done.html +++ b/tests/test_data/session_logged_in_some_tasks_done.html @@ -37,5 +37,5 @@

Data Structures and Algorithms spring 2024

-CSES - Data Structures and Algorithms spring 2024 - Tasks

General

Deadline: N/A

Week 1

Deadline: 2024-03-10 23:59:59

Week 2

Deadline: 2024-03-10 23:59:59

Week 3

Deadline: 2024-03-10 23:59:59

Week 4

Deadline: 2024-03-10 23:59:59

Week 5

Deadline: 2024-03-10 23:59:59

Week 6

Deadline: 2024-03-10 23:59:59

Week 7

Deadline: 2024-03-10 23:59:59

Week 8

Deadline: 2024-03-10 23:59:59

Week 9

Deadline: 2024-05-12 23:59:59

Week 10

Deadline: 2024-05-12 23:59:59

Week 11

Deadline: 2024-05-12 23:59:59

Week 12

Deadline: 2024-05-12 23:59:59

Week 13

Deadline: 2024-05-12 23:59:59

Week 14

Deadline: 2024-05-12 23:59:59

Week 15

Deadline: 2024-05-12 23:59:59

Week 16

Deadline: 2024-05-12 23:59:59

+CSES - Data Structures and Algorithms spring 2024 - Tasks

General

Deadline: N/A

Week 1

Deadline: 2024-03-10 23:59:59

diff --git a/tests/test_data/task_3052_incomplete_no_submit_link.html b/tests/test_data/task_3052_incomplete_no_submit_link.html new file mode 100644 index 0000000..a9fb706 --- /dev/null +++ b/tests/test_data/task_3052_incomplete_no_submit_link.html @@ -0,0 +1,109 @@ + + + + + + + + + + + + + +
+ +
+
+ +
+
+

Data Structures and Algorithms spring 2024

+

Efficiency test

+ +
+
+
+ +
+ +
+ + +CSES - Efficiency test + + + + +
+ + + +

The course material includes two different ways to implement the function count_even:

+
# implementation 1
+def count_even(numbers):
+    result = 0
+    for x in numbers:
+        if x % 2 == 0:
+            result += 1
+    return result
+
# implementation 2
+def count_even(numbers):
+    return sum(x % 2 == 0 for x in numbers)
+
+

Compare the efficiencies of the two implementations using a list that contains 10^7 randomly chosen numbers.

+

In this exercise, you get a point automatically when you submit the test results and the code that you used.

+

Implementation 1 run time: s

+

Implementation 2 run time: s

+

The code you used in the test:
+

+

+ +
+
+

Week 1

CandiesInversionsSame bitsRepeatEfficiency testSequence of numbersTime complexitiesRectangles

Your submissions

+ +
+
diff --git a/tests/test_data/task_3055_complete.html b/tests/test_data/task_3055_complete.html new file mode 100644 index 0000000..fbcd503 --- /dev/null +++ b/tests/test_data/task_3055_complete.html @@ -0,0 +1,87 @@ + + + + + + + + + + + + + +
+ +
+
+ +
+
+

Data Structures and Algorithms spring 2024

+

Candies

+ +
+
+
+ +
+ +
+ + +CSES - Candies + + + + + + +

A gummy candy costs a euros and a chocolate candy costs b euros. What is the maximum number of candies you can buy if you have c euros?

+

You may assume that a, b and c are integers in the range 1 \dots 100.

+

In a file candies.py, implement the function count that returns the maximum number of candies.

+
def count(a, b, c):
+    # TODO
+
+if __name__ == "__main__":
+    print(count(3, 4, 11)) # 3
+    print(count(5, 1, 100)) # 100
+    print(count(2, 3, 1)) # 0
+    print(count(2, 3, 9)) # 4
+
+

Explanation: In the first test, a gummy candy costs 3 euros and a chocolate candy costs 4 euros. You can buy at most 3 candies with 11 euros. For example, two gummy candies and one chocolate candy cost a total of 10 euros leaving you with 1 euro.

+
+
+

Week 1

CandiesInversionsSame bitsRepeatEfficiency testSequence of numbersTime complexitiesRectangles

Your submissions

+ + 2024-05-26 05:30:47 2024-05-26 05:28:54 2024-04-18 18:02:40
+
From 3c16e7a38705797b3303fecca07de00dc3573177 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Mon, 27 May 2024 10:22:49 -0400 Subject: [PATCH 09/11] Fix code based on failing tests --- tyora/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyora/client.py b/tyora/client.py index aef8825..ef241ae 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -126,7 +126,7 @@ def parse_task_list(html: AnyStr) -> list[Task]: def parse_task(html: AnyStr) -> Task: root = html5lib.parse(html, namespaceHTMLElements=False) # type: ignore[reportUnknownMemberType] - task_link_element = root.find('.//div[@class="nav sidebar"]/a') + task_link_element = root.find('.//div[@class="nav sidebar"]/a[@class="current"]') task_link = task_link_element if task_link_element is not None else Element("a") task_id = task_link.get("href", "").split("/")[-1] if not task_id: From cd2119315007c9dcbd46919e80a29e7e0afd0bd5 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Tue, 28 May 2024 09:59:47 -0400 Subject: [PATCH 10/11] Add test, fix typing, minor cleanup --- pyproject.toml | 3 +++ tests/test_client.py | 14 ++++++++++++++ tests/test_data/submit_3055_success.html | 0 tyora/client.py | 9 --------- tyora/tyora.py | 13 +++++-------- 5 files changed, 22 insertions(+), 17 deletions(-) create mode 100644 tests/test_data/submit_3055_success.html diff --git a/pyproject.toml b/pyproject.toml index 0c29bde..b3e71a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,6 @@ addopts = "--cov" [tool.ruff.lint] extend-select = ["I"] + +[tool.pyright] +reportAny = false diff --git a/tests/test_client.py b/tests/test_client.py index e51fa62..03d5fef 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -83,3 +83,17 @@ def test_client_get_task_incomplete_no_submit_link(mock_session: Session) -> Non == "The course material includes two different ways to implement the function\n`count_even`:\n\n \n \n # implementation 1\n def count_even(numbers):\n result = 0\n for x in numbers:\n if x % 2 == 0:\n result += 1\n return result\n \n \n \n # implementation 2\n def count_even(numbers):\n return sum(x % 2 == 0 for x in numbers)\n \n\nCompare the efficiencies of the two implementations using a list that contains\n10^7 randomly chosen numbers.\n\nIn this exercise, you get a point automatically when you submit the test\nresults and the code that you used.\n\nImplementation 1 run time: s\n\nImplementation 2 run time: s\n\nThe code you used in the test:" ) assert task.submit_link is None + + +def test_client_submit_task(mock_session: Session) -> None: + client = Client(session=mock_session) + + with requests_mock.Mocker() as m: + m.post( + "https://example.com/submit/3055", + text=open("tests/test_data/submit_3055_success.html").read(), + ) + result = client.submit_task( + "3055", "print('Hello, World!')\n", filename="test.py" + ) + assert result == "Success" diff --git a/tests/test_data/submit_3055_success.html b/tests/test_data/submit_3055_success.html new file mode 100644 index 0000000..e69de29 diff --git a/tyora/client.py b/tyora/client.py index ef241ae..8fed3df 100644 --- a/tyora/client.py +++ b/tyora/client.py @@ -171,15 +171,6 @@ def parse_task(html: AnyStr) -> Task: return task -# def submit_task(task_id: str, filename: str) -> None: -# """submit file to the submit form or task_id""" -# html = session.http_request(urljoin(base_url, f"task/{task_id}")) -# task = parse_task(html) -# answer = input("Do you want to submit this task? (y/n): ") -# if answer in ('y', 'Y'): -# with open(filename, 'r') as f: - - # TODO test with failing results # Seems to be broken since the switch to html5lib, needs tests! def parse_submit_result(html: AnyStr) -> dict[str, str]: diff --git a/tyora/tyora.py b/tyora/tyora.py index 9fc4d0e..d6e2403 100644 --- a/tyora/tyora.py +++ b/tyora/tyora.py @@ -6,16 +6,11 @@ from getpass import getpass from pathlib import Path from time import sleep -from typing import Optional +from typing import Optional, no_type_check import platformdirs -from .client import ( - Client, - Task, - TaskState, - parse_submit_result, -) +from .client import Client, Task, TaskState, parse_submit_result from .session import MoocfiCsesSession as Session logger = logging.getLogger(name="tyora") @@ -29,6 +24,8 @@ STATE_DIR = platformdirs.user_state_path(f"{PROG_NAME}") +# Disable typechecking for the argparse function calls since we ignore most returned values +@no_type_check def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Interact with mooc.fi CSES instance") parser.add_argument( @@ -226,7 +223,7 @@ def main() -> None: if args.cmd == "submit": # TODO allow user to paste the code in or even pipe it in - with open(args.filename, "r") as f: + with open(args.filename) as f: submission_code = f.read() result_url = client.submit_task( From 659245b6ad5a2d67a38a1c224eaae3b5b6c7a771 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Tue, 28 May 2024 10:30:16 -0400 Subject: [PATCH 11/11] Fix submit_task tests (incomplete) The flow is not completed in the test, to be revisited --- tests/test_client.py | 14 +- tests/test_data/submit_3055_form.html | 282 ++++++++++++++++++ .../task_3052_incomplete_no_submit_link.html | 2 +- tests/test_data/task_3055_complete.html | 2 +- 4 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 tests/test_data/submit_3055_form.html diff --git a/tests/test_client.py b/tests/test_client.py index 03d5fef..c9c9fab 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -89,11 +89,19 @@ def test_client_submit_task(mock_session: Session) -> None: client = Client(session=mock_session) with requests_mock.Mocker() as m: + m.get( + "https://example.com/dsa24k/submit/3055/", + text=open("tests/test_data/submit_3055_form.html").read(), + ) m.post( - "https://example.com/submit/3055", - text=open("tests/test_data/submit_3055_success.html").read(), + "https://example.com/course/send.php", + headers={"location": "/dsa24k/result/0000/"}, + ) + m.get( + "https://example.com/task/3055", + text=open("tests/test_data/task_3055_complete.html").read(), ) result = client.submit_task( "3055", "print('Hello, World!')\n", filename="test.py" ) - assert result == "Success" + assert result == "https://example.com/course/send.php" diff --git a/tests/test_data/submit_3055_form.html b/tests/test_data/submit_3055_form.html new file mode 100644 index 0000000..3b6c760 --- /dev/null +++ b/tests/test_data/submit_3055_form.html @@ -0,0 +1,282 @@ + + + + + + + + + + + + + +
+ +
+
+ +
+
+

Data Structures and Algorithms spring 2024

+

Candies

+ +
+
+
+ +
+ +
+ + +CSES - Candies - SubmitThe deadline for this task has passed, but you can still submit.
+ + +

Task: +Candies +

+

Code: + +

+

Language: + + +

+ +

+ + +
+ +
+
+

Week 1

CandiesInversionsSame bitsRepeatEfficiency testSequence of numbersTime complexitiesRectangles

Your submissions

+ + 2000-01-01 11:42:69
+
diff --git a/tests/test_data/task_3052_incomplete_no_submit_link.html b/tests/test_data/task_3052_incomplete_no_submit_link.html index a9fb706..917967d 100644 --- a/tests/test_data/task_3052_incomplete_no_submit_link.html +++ b/tests/test_data/task_3052_incomplete_no_submit_link.html @@ -62,7 +62,7 @@

Efficiency test

});
- +

The course material includes two different ways to implement the function count_even:

diff --git a/tests/test_data/task_3055_complete.html b/tests/test_data/task_3055_complete.html index fbcd503..183dbf7 100644 --- a/tests/test_data/task_3055_complete.html +++ b/tests/test_data/task_3055_complete.html @@ -83,5 +83,5 @@

Candies

Week 1

CandiesInversionsSame bitsRepeatEfficiency testSequence of numbersTime complexitiesRectangles

Your submissions

- 2024-05-26 05:30:47 2024-05-26 05:28:54 2024-04-18 18:02:40
+ 2000-01-01 11:42:69