From 271fd04ba3781d1f98543e36c09af2ea8a1cfcb6 Mon Sep 17 00:00:00 2001 From: Artyom Semidolin <43622365+Artanias@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:32:01 +0300 Subject: [PATCH] fix: Review threads and format with black test modules --- docker.mk | 16 +++- src/codeplag/types.py | 3 + src/codeplag/utils.py | 4 + src/webparsers/async_github_parser.py | 34 ++++--- test/auto/functional/test_check.py | 126 ++++++++++++-------------- test/unit/codeplag/test_utils.py | 93 +++++++++---------- 6 files changed, 142 insertions(+), 134 deletions(-) diff --git a/docker.mk b/docker.mk index b349982..e6d3fb9 100644 --- a/docker.mk +++ b/docker.mk @@ -21,11 +21,16 @@ docker-test: docker-test-image "$(TEST_DOCKER_TAG)" docker-autotest: docker-test-image - docker run --rm \ - --volume $(PWD)/test:/usr/src/$(UTIL_NAME)/test \ - --env-file .env \ - "$(TEST_DOCKER_TAG)" bash -c \ - "make && make autotest" + @if [ $(shell find . -maxdepth 1 -type f -name .env | wc --lines) != 1 ]; then \ + echo "Requires '.env' file with provided GitHub token for running autotests."; \ + exit 200; \ + else \ + docker run --rm \ + --volume $(PWD)/test:/usr/src/$(UTIL_NAME)/test \ + --env-file .env \ + "$(TEST_DOCKER_TAG)" bash -c \ + "make && make autotest"; \ + fi docker-build-package: docker-test-image docker run --rm \ @@ -50,6 +55,7 @@ docker-image: docker-base-image docker-test-image ) docker-run: docker-image + @touch .env docker run --rm --tty --interactive \ --env-file .env \ "$(DOCKER_TAG)" diff --git a/src/codeplag/types.py b/src/codeplag/types.py index 677ead4..b9129ba 100644 --- a/src/codeplag/types.py +++ b/src/codeplag/types.py @@ -87,10 +87,13 @@ class CompareInfo(NamedTuple): structure: Optional[StructuresInfo] = None +# TODO: Rework it structure class WorksReport(TypedDict): date: str first_path: str second_path: str + first_modify_date: NotRequired[str] + second_modify_date: NotRequired[str] first_heads: List[str] second_heads: List[str] fast: Dict[str, int] # dict from FastMetrics diff --git a/src/codeplag/utils.py b/src/codeplag/utils.py index 3cd25a0..2a7dad1 100644 --- a/src/codeplag/utils.py +++ b/src/codeplag/utils.py @@ -207,6 +207,10 @@ def save_result(self, fast=fast_metrics._asdict(), structure=struct_info_dict ) + if first_work.modify_date: + report["first_modify_date"] = first_work.modify_date + if second_work.modify_date: + report["second_modify_date"] = second_work.modify_date try: report_file = self.reports / f'{uuid.uuid4().hex}.json' diff --git a/src/webparsers/async_github_parser.py b/src/webparsers/async_github_parser.py index 4fceb97..c4b92e0 100644 --- a/src/webparsers/async_github_parser.py +++ b/src/webparsers/async_github_parser.py @@ -22,22 +22,28 @@ _GH_URL: Final[str] = 'https://github.com/' -""" -import asyncio -async def test(): - async with aiohttp.ClientSession() as session: - gh_parser = AsyncGithubParser(session, token=) - tasks = [] - for _ in range(3): - tasks.append(asyncio.create_task(gh_parser.)) - async for ... in gh_parser.: - ... - return await asyncio.gather(*tasks) -asyncio.run(test()) -""" - class AsyncGithubParser: + """Asynchronous parser which works with GitHub REST API. + + Example: + >>> import asyncio + >>> async def request(): + ... async with aiohttp.ClientSession() as session: + ... gh_parser = AsyncGithubParser(session, token=) + ... tasks = [] + ... for _ in range(3): + ... tasks.append(asyncio.create_task(gh_parser.)) + ... return await asyncio.gather(*tasks) + ... + >>> asyncio.run(test()) + >>> async def loop(): + ... async for ... in gh_parser.: + ... ... + ... + >>> asynio.run(loop()) + """ + USER_INFO = '/users/{username}' USER_REPOS = '/users/{username}/repos{?per_page,page}' diff --git a/test/auto/functional/test_check.py b/test/auto/functional/test_check.py index 7ffb22c..04f8514 100644 --- a/test/auto/functional/test_check.py +++ b/test/auto/functional/test_check.py @@ -11,74 +11,65 @@ from codeplag.types import WorksReport CPP_FILES = [ - 'test/unit/codeplag/cplag/data/sample1.cpp', - 'test/unit/codeplag/cplag/data/sample2.cpp' + "test/unit/codeplag/cplag/data/sample1.cpp", + "test/unit/codeplag/cplag/data/sample2.cpp", ] -PY_FILES = [ - 'src/codeplag/pyplag/astwalkers.py', - 'setup.py' -] -CPP_DIR = 'test/unit/codeplag/cplag/data' -PY_DIRS = [ - 'test/unit/codeplag/cplag', - 'test/unit' -] -REPO_URL = 'https://github.com/OSLL/code-plagiarism' -REPORTS_FOLDER = os.path.abspath('./reports') +PY_FILES = ["src/codeplag/pyplag/astwalkers.py", "setup.py"] +CPP_DIR = "test/unit/codeplag/cplag/data" +PY_DIRS = ["test/unit/codeplag/cplag", "test/unit"] +REPO_URL = "https://github.com/OSLL/code-plagiarism" +REPORTS_FOLDER = os.path.abspath("./reports") CPP_GITHUB_FILES = [ - f'{REPO_URL}/blob/main/test/unit/codeplag/cplag/data/sample3.cpp', - f'{REPO_URL}/blob/main/test/unit/codeplag/cplag/data/sample4.cpp' + f"{REPO_URL}/blob/main/test/unit/codeplag/cplag/data/sample3.cpp", + f"{REPO_URL}/blob/main/test/unit/codeplag/cplag/data/sample4.cpp", ] PY_GITHUB_FILES = [ - f'{REPO_URL}/blob/main/src/codeplag/pyplag/utils.py', - f'{REPO_URL}/blob/main/setup.py' + f"{REPO_URL}/blob/main/src/codeplag/pyplag/utils.py", + f"{REPO_URL}/blob/main/setup.py", ] -CPP_GITHUB_DIR = f'{REPO_URL}/tree/main/test' -PY_GITHUB_DIR = f'{REPO_URL}/blob/main/src/codeplag/pyplag' +CPP_GITHUB_DIR = f"{REPO_URL}/tree/main/test" +PY_GITHUB_DIR = f"{REPO_URL}/blob/main/src/codeplag/pyplag" -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def setup_module(): first_cond = modify_settings(environment=".env").returncode == 0 - second_cond = os.environ.get('ACCESS_TOKEN', '') != '' + second_cond = os.environ.get("ACCESS_TOKEN", "") != "" assert first_cond or second_cond def test_check_util_version(): - result = run_util(['--version']) + result = run_util(["--version"]) assert result.returncode == SUCCESS_CODE - assert f'{UTIL_NAME} {UTIL_VERSION}' in result.stdout.decode('utf-8') + assert f"{UTIL_NAME} {UTIL_VERSION}" in result.stdout.decode("utf-8") @pytest.mark.parametrize( "cmd, out", [ - (['--files', *CPP_FILES], b'Getting works features from files'), + (["--files", *CPP_FILES], b"Getting works features from files"), ( - ['--directories', CPP_DIR], - f'Getting works features from {CPP_DIR}'.encode('utf-8') + ["--directories", CPP_DIR], + f"Getting works features from {CPP_DIR}".encode("utf-8"), ), ( - ['--github-files', *CPP_GITHUB_FILES], - b'Getting works features from GitHub urls' + ["--github-files", *CPP_GITHUB_FILES], + b"Getting works features from GitHub urls", ), ( - ['--github-project-folders', CPP_GITHUB_DIR], - f'Getting works features from {CPP_GITHUB_DIR}'.encode('utf-8') + ["--github-project-folders", CPP_GITHUB_DIR], + f"Getting works features from {CPP_GITHUB_DIR}".encode("utf-8"), ), ( - ['--github-user', 'OSLL', '--repo-regexp', 'code-plag'], - f'Getting works features from {REPO_URL}'.encode('utf-8') - ) - ] + ["--github-user", "OSLL", "--repo-regexp", "code-plag"], + f"Getting works features from {REPO_URL}".encode("utf-8"), + ), + ], ) def test_compare_cpp_files(cmd, out): - result = run_check( - cmd, - extension='cpp' - ) + result = run_check(cmd, extension="cpp") assert result.returncode == SUCCESS_CODE assert out in result.stdout @@ -87,31 +78,28 @@ def test_compare_cpp_files(cmd, out): @pytest.mark.parametrize( "cmd, out", [ + (["--files", *PY_FILES], b"Getting works features from files"), ( - ['--files', *PY_FILES], - b'Getting works features from files' + ["--directories", *PY_DIRS], + f"Getting works features from {PY_DIRS[0]}".encode("utf-8"), ), ( - ['--directories', *PY_DIRS], - f'Getting works features from {PY_DIRS[0]}'.encode('utf-8') + ["--github-files", *PY_GITHUB_FILES], + b"Getting works features from GitHub urls", ), ( - ['--github-files', *PY_GITHUB_FILES], - b'Getting works features from GitHub urls' + ["--github-project-folders", PY_GITHUB_DIR], + f"Getting works features from {PY_GITHUB_DIR}".encode("utf-8"), ), ( - ['--github-project-folders', PY_GITHUB_DIR], - f'Getting works features from {PY_GITHUB_DIR}'.encode('utf-8') + ["--github-user", "OSLL", "--repo-regexp", "code-plag"], + f"Getting works features from {REPO_URL}".encode("utf-8"), ), ( - ['--github-user', 'OSLL', '--repo-regexp', 'code-plag'], - f'Getting works features from {REPO_URL}'.encode('utf-8') + ["--directories", *PY_DIRS, "--mode", "one_to_one"], + f"Getting works features from {PY_DIRS[0]}".encode("utf-8"), ), - ( - ['--directories', *PY_DIRS, '--mode', 'one_to_one'], - f'Getting works features from {PY_DIRS[0]}'.encode('utf-8'), - ) - ] + ], ) def test_compare_py_files(cmd, out): result = run_check(cmd) @@ -126,24 +114,30 @@ def test_save_reports(): assert os.path.exists(REPORTS_FOLDER) assert modify_settings(REPORTS_FOLDER).returncode == 0 - assert run_check( - [ - '--directories', - './test/auto', - '--files', - './test/auto/utils.py', - './test/auto/test_bugs.py' - ] - ).returncode == 0 + assert ( + run_check( + [ + "--directories", + "./test/auto", + "--files", + "./test/auto/utils.py", + "./test/auto/test_bugs.py", + ] + ).returncode + == 0 + ) reports_files = os.listdir(REPORTS_FOLDER) assert len(reports_files) > 0 for file in reports_files: - assert re.search('.*[.]json$', file) - filepath = f'{REPORTS_FOLDER}/{file}' - with open(filepath, 'r') as f: + assert re.search(".*[.]json$", file) + filepath = f"{REPORTS_FOLDER}/{file}" + with open(filepath, "r") as f: report = json.loads(f.read()) - for key in WorksReport.__annotations__.keys(): + for key in set( + WorksReport.__annotations__.keys() + - ["first_modify_date", "second_modify_date"] + ): assert key in report shutil.rmtree(REPORTS_FOLDER) diff --git a/test/unit/codeplag/test_utils.py b/test/unit/codeplag/test_utils.py index 8e10869..983119f 100644 --- a/test/unit/codeplag/test_utils.py +++ b/test/unit/codeplag/test_utils.py @@ -7,6 +7,7 @@ from pytest_mock import MockerFixture from codeplag.pyplag.utils import get_ast_from_filename, get_features_from_ast +from codeplag.types import WorksReport from codeplag.utils import ( CodeplagEngine, calc_iterations, @@ -16,9 +17,9 @@ ) CWD = Path(os.path.dirname(os.path.abspath(__file__))) -FILEPATH1 = CWD / './data/test1.py' -FILEPATH2 = CWD / './data/test2.py' -FILEPATH3 = CWD / './data/test3.py' +FILEPATH1 = CWD / "./data/test1.py" +FILEPATH2 = CWD / "./data/test2.py" +FILEPATH3 = CWD / "./data/test3.py" def test_fast_compare_normal(): @@ -34,15 +35,11 @@ def test_fast_compare_normal(): assert metrics.jakkar == pytest.approx(0.737, 0.001) assert metrics.operators == pytest.approx(0.667, 0.001) - assert metrics.keywords == 1. + assert metrics.keywords == 1.0 assert metrics.literals == 0.75 assert metrics.weighted_average == pytest.approx(0.774, 0.001) - metrics2 = fast_compare( - features1, - features2, - weights=(0.5, 0.6, 0.7, 0.8) - ) + metrics2 = fast_compare(features1, features2, weights=(0.5, 0.6, 0.7, 0.8)) assert metrics2.weighted_average == pytest.approx(0.796, 0.001) @@ -63,14 +60,14 @@ def test_compare_works(): assert compare_info.fast.jakkar == pytest.approx(0.737, 0.001) assert compare_info.fast.operators == pytest.approx(0.667, 0.001) - assert compare_info.fast.keywords == 1. + assert compare_info.fast.keywords == 1.0 assert compare_info.fast.literals == 0.75 assert compare_info.fast.weighted_average == pytest.approx(0.774, 0.001) assert compare_info.structure is not None assert compare_info.structure.similarity == pytest.approx(0.823, 0.001) assert compare_info.structure.compliance_matrix.tolist() == [ [[19, 24], [7, 21]], - [[5, 27], [8, 9]] + [[5, 27], [8, 9]], ] compare_info2 = compare_works(features1, features3) @@ -86,17 +83,14 @@ def test_compare_works(): @pytest.fixture def mock_default_logger(mocker: MockerFixture): mocked_logger = mocker.Mock() - mocker.patch.object(logging, 'getLogger', return_value=mocked_logger) + mocker.patch.object(logging, "getLogger", return_value=mocked_logger) return mocked_logger def test_save_result(mocker, mock_default_logger): - parsed_args = {"extension": 'py', 'root': 'check'} - code_engine = CodeplagEngine( - mock_default_logger, - parsed_args - ) + parsed_args = {"extension": "py", "root": "check"} + code_engine = CodeplagEngine(mock_default_logger, parsed_args) tree1 = get_ast_from_filename(FILEPATH1) tree2 = get_ast_from_filename(FILEPATH2) assert tree1 is not None @@ -107,13 +101,10 @@ def test_save_result(mocker, mock_default_logger): compare_info = compare_works(features1, features2) assert compare_info.structure is not None - mocker.patch.object(Path, 'open', side_effect=FileNotFoundError) - code_engine.reports = Path('/bad_dir') + mocker.patch.object(Path, "open", side_effect=FileNotFoundError) + code_engine.reports = Path("/bad_dir") code_engine.save_result( - features1, - features2, - compare_info.fast, - compare_info.structure + features1, features2, compare_info.fast, compare_info.structure ) assert not Path.open.called assert mock_default_logger.error.call_args == call( @@ -121,38 +112,42 @@ def test_save_result(mocker, mock_default_logger): ) mocker.patch.object(Path, "open", side_effect=PermissionError) - code_engine.reports = Path('/etc') + code_engine.reports = Path("/etc") code_engine.save_result( - features1, - features2, - compare_info.fast, - compare_info.structure + features1, features2, compare_info.fast, compare_info.structure ) Path.open.assert_called_once() assert mock_default_logger.error.call_args == call( - 'Not enough rights to write reports to the folder.' + "Not enough rights to write reports to the folder." ) - mocker.patch.object(Path, "open") - code_engine.reports = Path('./src') + # First ok test + mock_write_config = mocker.patch("codeplag.utils.write_config") + code_engine.reports = Path("./src") code_engine.save_result( - features1, - features2, - compare_info.fast, - compare_info.structure + features1, features2, compare_info.fast, compare_info.structure + ) + mock_write_config.assert_called_once() + assert set(mock_write_config.call_args[0][1].keys()) == set( + WorksReport.__annotations__.keys() - ["first_modify_date", "second_modify_date"] ) - Path.open.assert_called_once() + # Second ok test + mock_write_config.reset_mock() + features1.modify_date = "2023-07-14T11:22:30Z" + features2.modify_date = "2023-07-09T14:35:07Z" + code_engine.save_result( + features1, features2, compare_info.fast, compare_info.structure + ) + mock_write_config.assert_called_once() + assert ( + mock_write_config.call_args[0][1].keys() == WorksReport.__annotations__.keys() + ) @pytest.mark.parametrize( "args, result", - [ - ([4, 10], 0.4), - ([10, 0], 0.0), - ([5, 10, 15, 0], 0.5), - ([5, 6, 1, 4], 0.875) - ] + [([4, 10], 0.4), ([10, 0], 0.0), ([5, 10, 15, 0], 0.5), ([5, 6, 1, 4], 0.875)], ) def test_calc_progress(args, result): assert calc_progress(*args) == result @@ -161,13 +156,13 @@ def test_calc_progress(args, result): @pytest.mark.parametrize( "count, mode, expected", [ - (10, 'many_to_many', 45), - (3, 'one_to_one', 3), - (10, 'bad_mode', 0), - (-100, 'many_to_many', 0), - (0, 'one_to_one', 0), - (-10, 'bad_mode', 0) - ] + (10, "many_to_many", 45), + (3, "one_to_one", 3), + (10, "bad_mode", 0), + (-100, "many_to_many", 0), + (0, "one_to_one", 0), + (-10, "bad_mode", 0), + ], ) def test_calc_iterations(count, mode, expected): assert calc_iterations(count, mode) == expected