diff --git a/.bettercodehub.yml b/.bettercodehub.yml index 7b2376a..21ded3a 100644 --- a/.bettercodehub.yml +++ b/.bettercodehub.yml @@ -1 +1,4 @@ -component_depth: 2 +component_depth: 1 + +languages: +- python diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fed3772..deee219 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9] + python-version: [3.9] steps: - uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 5291175..785641e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ######################################################################################## -FROM python:3.8-slim-buster +FROM python:3.9-slim-buster ######################################################################################## LABEL maintainer="Software Improvement Group Research " diff --git a/README.md b/README.md index 1d6af29..205ca39 100644 --- a/README.md +++ b/README.md @@ -29,27 +29,29 @@ The currently supported forges are "mvn", "debian", and "PyPI". The plugin will raise an exception if the `forge` in the message is not supported or empty. #### Maven -The default topic to consume: `fasten.RepoCloner.out` +The default topic to consume: `fasten.SyncJava.out` -An example message: +An example message produced by the SyncJava plugin, which merges out messages from RepoCloner and JavaCGOpal: ```json { "input": {}, "host": "fasten-repo-cloner-56dcf76495-bn4c2", "created_at": 1602739158, - "plugin_name": "RepoCloner", - "payload": { - "repoUrl": "", - "date": 1291905586, - "forge": "mvn", - "groupId": "fasten-project", - "artifactId": "fasten", - "version": "1.0.0", - "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar", - "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten", - "repoType": "git", - "commitTag": "v1.0.0" + "plugin_name": "SyncJava", + "fasten.RepoCloner.out" : { + "payload": { + "repoUrl": "", + "date": 1291905586, + "forge": "mvn", + "groupId": "fasten-project", + "artifactId": "fasten", + "version": "1.0.0", + "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar", + "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten", + "repoType": "git", + "commitTag": "v1.0.0" + } } } ``` diff --git a/entrypoint.py b/entrypoint.py index fbfdcab..16cae79 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -23,7 +23,7 @@ plugin_name = 'RapidPlugin' plugin_description = 'A FASTEN plug-in to populate risk related metadata for a product.' -plugin_version = '1.0.0' +plugin_version = '1.1.0' def get_args_parser(): diff --git a/integration_tests/mocks.py b/integration_tests/mocks.py index 4e70919..498407f 100644 --- a/integration_tests/mocks.py +++ b/integration_tests/mocks.py @@ -14,7 +14,7 @@ # from fasten.plugins.kafka import KafkaPlugin -from rapidplugin.kafka_non_blocking import KafkaPluginNonBlocking +from fasten.plugins.kafka import KafkaPluginNonBlocking class MockConsumer(KafkaPluginNonBlocking): diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 784309b..d3aa094 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -17,7 +17,6 @@ from time import sleep from integration_tests.mocks import MockConsumer from integration_tests.mocks import MockProducer -from rapidplugin.tests.sources import fix_sourcePath @pytest.fixture() @@ -111,6 +110,17 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, "repoType": "hg", "commitTag": "1.0.0" }, + { + "fasten.RepoCloner.out": { + "payload": + { + "forge": "debian", + "product": "d1", + "version": "1.0.0", + "sourcePath": "/home/plugin/rapidplugin/tests/resources/debian/d1" + } + } + }, { "forge": "debian", "product": "d1", diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 37a3b4a..eead982 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -19,7 +19,7 @@ import lizard from rapidplugin.domain.package import Package, File, Function -from rapidplugin.utils.utils import MavenUtils +from rapidplugin.utils.utils import MavenUtils, KafkaUtils logger = logging.getLogger(__name__) @@ -45,6 +45,9 @@ def analyze(self, payload): m.update(metadata) m.update(function.metadata()) m.update(function.metrics()) + old_f = m['filename'] + new_f = KafkaUtils.relativize_filename(old_f) + m.update({'filename': new_f}) out_payloads.append(m) logger.debug("callable: {}".format(m) + '\n') return out_payloads diff --git a/rapidplugin/kafka_non_blocking.py b/rapidplugin/kafka_non_blocking.py deleted file mode 100644 index c6c6226..0000000 --- a/rapidplugin/kafka_non_blocking.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2020 Software Improvement Group -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import json -from time import sleep -from kafka import KafkaConsumer -from kafka.errors import NoBrokersAvailable -from fasten.plugins.kafka import KafkaPlugin - - -class KafkaPluginNonBlocking(KafkaPlugin): - - consumer_timeout_ms = None - - def set_consumer(self): - """Set consumer to read (non-blocking) from consume_topic. - """ - try: - assert self.consume_topic is not None - assert self.bootstrap_servers is not None - assert self.group_id is not None - assert self.consumer_timeout_ms is not None - except (AssertionError, NameError) as e: - self.err(("You should have set consume_topic, bootstrap_servers, " - "group_id, and consumer_timeout_ms")) - raise e - self.consumer = KafkaConsumer( - self.consume_topic, - bootstrap_servers=self.bootstrap_servers.split(','), - auto_offset_reset='earliest', - enable_auto_commit=True, - max_poll_records=1, - group_id=self.group_id, - value_deserializer=lambda x: json.loads(x.decode('utf-8')), - consumer_timeout_ms=self.consumer_timeout_ms - ) - - def set_consumer_with_retry(self): - while True: - try: - self.set_consumer() - self.set_producer() - except NoBrokersAvailable: - self.err("Could not connect consumer to Kafka, re-trying...") - else: - self.log("Connected consumer to Kafka successfully.") - break - sleep(self.consumption_delay_sec) - - def set_producer_with_retry(self): - while True: - try: - self.set_producer() - except NoBrokersAvailable: - self.err("Could not connect producer to Kafka, re-trying...") - else: - self.log("Connected producer to Kafka successfully.") - break - sleep(self.consumption_delay_sec) - - def skip_messages(self): - assert self.consumer is not None, "Consumer needs to be set before messages can ben consumed." - for m in self.consumer: - self.consumer.commit() diff --git a/rapidplugin/rapid_plugin.py b/rapidplugin/rapid_plugin.py index 115ad33..4cf7711 100644 --- a/rapidplugin/rapid_plugin.py +++ b/rapidplugin/rapid_plugin.py @@ -15,7 +15,7 @@ import datetime from time import sleep -from rapidplugin.kafka_non_blocking import KafkaPluginNonBlocking +from fasten.plugins.kafka import KafkaPluginNonBlocking from rapidplugin.analysis.lizard_analyzer import LizardAnalyzer from rapidplugin.utils.utils import KafkaUtils @@ -104,6 +104,7 @@ def consume(self, record): Arguments: record (JSON): message from self.consume_topic ''' + record = KafkaUtils.extract_from_sync(record) payload = record['payload'] if 'payload' in record else record in_payload = KafkaUtils.tailor_input(payload) try: diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py index 2028bd3..d442af9 100644 --- a/rapidplugin/tests/test_analyzer.py +++ b/rapidplugin/tests/test_analyzer.py @@ -101,6 +101,12 @@ def analyzer(sources): (pypi_message, 2, 1, 5) ] +# List of (payload, filename) pairs +FUNCTION_FILENAME_DATA = [ + (mvn_message_with_hg_repo, "m3.java"), + (debian_message, "d1.c"), + (pypi_message, "p1.py") +] @pytest.mark.parametrize('record,fc', FUNCTION_COUNT_DATA) def test_function_count(analyzer, sources, record, fc: int): @@ -123,3 +129,10 @@ def test_function_metrics(analyzer, sources, record, nloc: int, complexity: int, assert metrics['nloc'] == nloc assert metrics['complexity'] == complexity assert metrics['token_count'] == token_count + + +@pytest.mark.parametrize('record,filename', FUNCTION_FILENAME_DATA) +def test_function_filename(analyzer, sources, record, filename): + out_payloads = analyzer.analyze(fix_sourcePath(record, sources)) + metadata = out_payloads[0] + assert metadata['filename'] == filename diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index b1d21d9..cef0a06 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -59,13 +59,13 @@ def test_checkout_git(repo_path, repo_type, commit_tag, sources_dir, repos): repo.git.add(".") repo.git.commit(m="first commit.") repo.create_tag('1.0.0') - with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: + with MavenUtils.checkout_version(sources_dir,repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) as source_path: assert sorted(os.listdir(Path(source_path))) == sorted(['1.0.0.zip', 'm1.java']) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', [("maven/hg/m3", "hg", "1.0.0")]) def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) - with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: + with MavenUtils.checkout_version(sources_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) as source_path: assert sorted(os.listdir(source_path)) == sorted(['m3.java', '.hg_archival.txt']) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', @@ -78,7 +78,8 @@ def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos): def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) with pytest.raises(Exception) as e: - MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) + MavenUtils.checkout_version(sources_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) + print(str(e)) PAYLOAD_TAILOR_DATA = [ ({"product": "a"}, {"product": "a"}), @@ -91,3 +92,9 @@ def test_tailor_input(in_payload, out_payload): tailored = KafkaUtils.tailor_input(in_payload) assert tailored == out_payload +@pytest.mark.parametrize('old, new', [ + ('/private/var/folders/wr/qmdcbfsj4v98v8rwb11zjr5c0000gn/T/pytest-of-cgao/pytest-7/sources0/tmppsmmokh_/d1.c', 'd1.c') +]) +def test_relativize_filename(old, new): + new_file_name = KafkaUtils.relativize_filename(old) + assert new_file_name == new diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 9d2c153..b092854 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -22,6 +22,7 @@ from tempfile import TemporaryDirectory import os import shutil +import re class MavenUtils: @@ -48,6 +49,7 @@ def get_source_path(payload, base_dir): @staticmethod def get_source_mvn(payload, base_dir): + source_config = {} if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] if sources_url != "": @@ -57,7 +59,7 @@ def get_source_mvn(payload, base_dir): repo_path = payload['repoPath'] repo_type = payload['repoType'] commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) + source_path = MavenUtils.checkout_version(base_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) return source_path @staticmethod @@ -90,22 +92,28 @@ def download_jar(url, base_dir): return tmp @staticmethod - def checkout_version(repo_path, repo_type, version_tag, base_dir): + def checkout_version(base_dir, **source_config): + repo_type = source_config['repo_type'] + repo_path = source_config['repo_path'] + version_tag = source_config['version_tag'] assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) assert repo_path != "", "Empty repo_path." assert version_tag != "", "Empty version_tag." tmp = TemporaryDirectory(dir=base_dir) tmp_path = Path(tmp.name) if repo_type == "git": - MavenUtils.git_checkout(repo_path, version_tag, tmp_path) + MavenUtils.git_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path) elif repo_type == "svn": - MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) + MavenUtils.svn_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path) elif repo_type == "hg": - MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) + MavenUtils.hg_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path) return tmp @staticmethod - def git_checkout(repo_path, version_tag, tmp_path): + def git_checkout(**source_config): + repo_path = source_config['repo_path'] + version_tag = source_config['version_tag'] + tmp_path = source_config['tmp_path'] repo = Repo(repo_path) # assert repo.tag(version_tag) is None, "Tag: '{}' does not exist.".format(version_tag) archive_name = version_tag+".zip" @@ -115,14 +123,17 @@ def git_checkout(repo_path, version_tag, tmp_path): zipObj.extractall(tmp_path) @staticmethod - def svn_checkout(repo_path, version_tag, tmp_path): + def svn_checkout(**source_config): raise NotImplementedError("Svn repo not supported.") # 'svn export' does not support tag # r = LocalClient(repo_path) # r.export(tmp_path, version_tag) @staticmethod - def hg_checkout(repo_path, version_tag, tmp_path): + def hg_checkout(**source_config): + repo_path = source_config['repo_path'] + version_tag = source_config['version_tag'] + tmp_path = source_config['tmp_path'] wd = os.getcwd() os.chdir(repo_path) cmd = [ @@ -180,4 +191,27 @@ def tailor_input(payload): payload[key] = tailor[key] return payload + @staticmethod + def extract_from_sync(payload): + """ + Extract content of RepoCloner in the synchronized topic. + :param payload: payload of fasten.SyncJava.out, see + https://github.com/fasten-project/synchronize-javacg + :return: payload of RepoCloner, see + https://github.com/fasten-project/fasten/wiki/Kafka-Topics#fastenrepocloner + """ + extract = payload['fasten.RepoCloner.out'] if 'fasten.RepoCloner.out' in payload else payload + return extract + @staticmethod + def relativize_filename(filename): + """ + Extract the relative path of the source code file. + :param filename: absolute path included by Lizard tool, + e.g. 'work_directory/tmppsmmokh_/d1.c' + :return: filename relative to the temporal source directory, + e.g. 'd1.c' + + """ + regex = re.compile('(/tmp).{8}/') + return regex.split(filename)[2] diff --git a/setup.py b/setup.py index 9b8547e..e5cbcf4 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='quality-analyzer', - version='1.0.0', + version='1.1.0', description='FASTEN RAPID Plugin', long_description=long_description, long_description_content_type='text/markdown', @@ -16,11 +16,11 @@ author='', author_email='', classifiers=[ - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], keywords='', packages=find_packages(), - python_requires='>=3.8', + python_requires='>=3.9', install_requires=[ 'fasten', 'lizard', diff --git a/tox.ini b/tox.ini index 2b6d46c..44f23e2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py39 [testenv] passenv = * @@ -14,7 +14,7 @@ commands = flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics docker build -t rapidplugin-test . - docker run --entrypoint python rapidplugin-test -m pytest rapidplugin/tests + docker run --rm --entrypoint python rapidplugin-test -m pytest rapidplugin/tests docker-compose -f integration_tests/kafka-compose.yml up -d docker-compose -f integration_tests/kafka-compose.yml run wait-for-kafka pytest integration_tests