From dadd63f14597d7c8c5cfa55b41f4a9bd6156c1d6 Mon Sep 17 00:00:00 2001 From: Magiel Bruntink Date: Mon, 6 Dec 2021 21:35:01 +0100 Subject: [PATCH] Added snappy compression and made all tests run in Docker. --- Dockerfile | 21 +++++++++-- .../kafka-compose.yml => docker-compose.yml | 36 +++++++++++-------- entrypoint.py | 4 +-- integration_tests/README.md | 35 ------------------ integration_tests/test_plugin.py | 8 ++--- rapidplugin/analysis/graph_query.py | 2 +- rapidplugin/analysis/lizard_analyzer.py | 2 +- rapidplugin/config.py | 2 +- rapidplugin/domain/callgraph.py | 2 +- rapidplugin/domain/package.py | 2 +- rapidplugin/rapid_plugin.py | 2 +- rapidplugin/tests/sources.py | 15 ++++++++ rapidplugin/tests/test_analyzer.py | 3 +- rapidplugin/tests/test_package.py | 2 +- rapidplugin/tests/test_utils.py | 2 +- rapidplugin/utils/utils.py | 2 +- setup.py | 23 ++++++++++-- tox.ini | 25 ++++++++++--- 18 files changed, 113 insertions(+), 75 deletions(-) rename integration_tests/kafka-compose.yml => docker-compose.yml (61%) delete mode 100644 integration_tests/README.md diff --git a/Dockerfile b/Dockerfile index efe4d0a..06aff04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,20 @@ +# Copyright 2021 Software Improvement Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ######################################################################################## -FROM python:3.9-slim-buster +FROM python:3.9-slim-bullseye ######################################################################################## LABEL maintainer="Software Improvement Group Research " @@ -9,7 +24,8 @@ RUN apt-get update \ && apt-get install -y -qq \ git \ subversion \ - mercurial + mercurial \ + libsnappy-dev RUN git config --global user.email "research@softwareimprovementgroup.com" \ && git config --global user.name "Software Improvement Group Research" @@ -17,6 +33,7 @@ RUN git config --global user.email "research@softwareimprovementgroup.com" \ WORKDIR /plugin COPY rapidplugin rapidplugin/ +COPY integration_tests/*.py integration_tests/ COPY entrypoint.py . COPY setup.py . COPY requirements.txt . diff --git a/integration_tests/kafka-compose.yml b/docker-compose.yml similarity index 61% rename from integration_tests/kafka-compose.yml rename to docker-compose.yml index 51f4d7b..8ced598 100644 --- a/integration_tests/kafka-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,35 +16,43 @@ version: '2' services: zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" + image: zookeeper:3.7.0 + kafka: - image: wurstmeister/kafka - ports: - - "9092:9092" + depends_on: + - zookeeper + image: wurstmeister/kafka:2.13-2.7.1 environment: KAFKA_ADVERTISED_HOST_NAME: localhost KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_MESSAGE_MAX_BYTES: 2097152000 - KAFKA_LISTENERS: INSIDE://kafka:29092,OUTSIDE://kafka:9092 - KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:29092,OUTSIDE://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_LISTENERS: INSIDE://kafka:9092 + KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE KAFKA_CREATE_TOPICS: "fasten.MetadataDBJavaExtension.out:1:1,fasten.RapidPlugin.callable.out:1:1,fasten.RapidPlugin.callable.err:1:1,fasten.RapidPlugin.callable.log:1:1" - volumes: - - /var/run/docker.sock:/var/run/docker.sock + KAFKA_COMPRESSION_TYPE: snappy rapidplugin: + depends_on: + - kafka + image: rapidplugin-test + entrypoint: ['python', '/plugin/entrypoint.py', '--group_id=RapidPlugin-TEST', '--bootstrap_servers=kafka:9092', '--sources_dir=/plugin/rapidplugin/tests/resources'] + + rapidplugin-unit-tests: + image: rapidplugin-test + entrypoint: ['python', '-m', 'pytest', 'rapidplugin/tests'] + + rapidplugin-integration-tests: image: rapidplugin-test - entrypoint: ['python', '/plugin/entrypoint.py', '--group_id=RapidPlugin-TEST', '--bootstrap_servers=kafka:29092', '--sources_dir=/plugin/rapidplugin/tests/resources'] + entrypoint: ['python', '-m', 'pytest', 'integration_tests'] wait-for-kafka: image: waisbrot/wait links: - kafka environment: - - TARGETS=kafka:29092 + - TARGETS=kafka:9092 networks: default: diff --git a/entrypoint.py b/entrypoint.py index 69eb180..eed6b91 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ plugin_name = 'RapidPlugin' plugin_description = 'A FASTEN plug-in to populate risk related metadata for a product.' -plugin_version = '1.2.0' +plugin_version = '1.2.1' def get_args_parser(): diff --git a/integration_tests/README.md b/integration_tests/README.md deleted file mode 100644 index 8a4b1f9..0000000 --- a/integration_tests/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Prerequisites - -``` -docker -docker-compose -``` - -# Setup - -To setup a Kafka cluster, execute the following in ```kafka-docker/``` - -``` console -docker-compose -f docker-compose-single-broker.yml up -d -``` - -The Kafka cluster will exist on the Docker network ```kafka-docker_default```, where the broker will be reachable at ```kafka:29092```. - -From outside the Docker network, the broker is reachable at ```localhost:9092```. - -# Testing - -To test putting data into a topic, run the following: -``` console -docker run -i --rm --network=kafka-docker_default \ - edenhill/kafkacat:1.6.0 kafkacat \ - -b kafka:29092 -t fasten.RepoCloner.out -P \ - < rapidplugin/integration_tests/resources/fasten.RepoCloner.out-testdata.txt -``` - -Run the following to verify that the data was inserted: -``` console -docker run -it --rm --network=kafka-docker_default \ - edenhill/kafkacat:1.6.0 kafkacat \ - -b kafka:29092 -t fasten.RepoCloner.out -C -``` diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 47c9bf8..567e157 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -21,7 +21,7 @@ @pytest.fixture() def mock_in(): - mock = MockProducer('localhost:9092', + mock = MockProducer('kafka:9092', 'fasten.MetadataDBJavaExtension.out') yield mock mock.free_resource() @@ -30,7 +30,7 @@ def mock_in(): @pytest.fixture() def mock_out(): mock = MockConsumer('MockConsumerOut', - 'localhost:9092', + 'kafka:9092', 'fasten.RapidPlugin.callable.out') mock.skip_messages() yield mock @@ -40,7 +40,7 @@ def mock_out(): @pytest.fixture() def mock_log(): mock = MockConsumer('MockConsumerLog', - 'localhost:9092', + 'kafka:9092', 'fasten.RapidPlugin.log') mock.skip_messages() yield mock @@ -50,7 +50,7 @@ def mock_log(): @pytest.fixture() def mock_err(): mock = MockConsumer('MockConsumerErr', - 'localhost:9092', + 'kafka:9092', 'fasten.RapidPlugin.err') mock.skip_messages() yield mock diff --git a/rapidplugin/analysis/graph_query.py b/rapidplugin/analysis/graph_query.py index a8edbae..ceda7c5 100644 --- a/rapidplugin/analysis/graph_query.py +++ b/rapidplugin/analysis/graph_query.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 87a6331..1236e4c 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/config.py b/rapidplugin/config.py index 6033806..cae52cb 100644 --- a/rapidplugin/config.py +++ b/rapidplugin/config.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/domain/callgraph.py b/rapidplugin/domain/callgraph.py index 8c86c54..99f67bd 100644 --- a/rapidplugin/domain/callgraph.py +++ b/rapidplugin/domain/callgraph.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/domain/package.py b/rapidplugin/domain/package.py index ffcad20..7b268d7 100644 --- a/rapidplugin/domain/package.py +++ b/rapidplugin/domain/package.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/rapid_plugin.py b/rapidplugin/rapid_plugin.py index 5d822e1..0b03142 100644 --- a/rapidplugin/rapid_plugin.py +++ b/rapidplugin/rapid_plugin.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/tests/sources.py b/rapidplugin/tests/sources.py index 5136c72..5441a9a 100644 --- a/rapidplugin/tests/sources.py +++ b/rapidplugin/tests/sources.py @@ -1,3 +1,18 @@ +# Copyright 2021 Software Improvement Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import os import shutil import pytest diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py index 850af00..163f99a 100644 --- a/rapidplugin/tests/test_analyzer.py +++ b/rapidplugin/tests/test_analyzer.py @@ -1,5 +1,4 @@ - -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/tests/test_package.py b/rapidplugin/tests/test_package.py index d57b16d..6972710 100644 --- a/rapidplugin/tests/test_package.py +++ b/rapidplugin/tests/test_package.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 213e778..877dd59 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 37fb112..f80be45 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 Software Improvement Group +# Copyright 2021 Software Improvement Group # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/setup.py b/setup.py index 28fe62e..3e9edee 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,19 @@ +# Copyright 2021 Software Improvement Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + from os import path from setuptools import setup, find_packages @@ -8,7 +24,7 @@ setup( name='quality-analyzer', - version='1.2.0', + version='1.2.1', description='FASTEN RAPID Plugin', long_description=long_description, long_description_content_type='text/markdown', @@ -26,6 +42,9 @@ 'lizard', 'requests', 'gitpython', - 'svn' + 'svn', + 'svn', + 'kafka-python==2.0.2', + 'python-snappy' ] ) diff --git a/tox.ini b/tox.ini index 44f23e2..52c57a4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,18 @@ +# Copyright 2021 Software Improvement Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + [tox] envlist = py39 @@ -14,10 +29,10 @@ commands = flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics docker build -t rapidplugin-test . - docker run --rm --entrypoint python rapidplugin-test -m pytest rapidplugin/tests - docker-compose -f integration_tests/kafka-compose.yml up -d - docker-compose -f integration_tests/kafka-compose.yml run wait-for-kafka - pytest integration_tests + docker-compose run rapidplugin-unit-tests + docker-compose up -d rapidplugin + docker-compose run wait-for-kafka + docker-compose run rapidplugin-integration-tests commands_post = - docker-compose -f integration_tests/kafka-compose.yml down + docker-compose down