Added snappy compression and made all tests run in Docker.

fasten-project · Dec 6, 2021 · dadd63f · dadd63f
1 parent 52886ae
commit dadd63f
Show file tree

Hide file tree

Showing 18 changed files with 113 additions and 75 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,20 @@
+# Copyright 2021 Software Improvement Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ########################################################################################
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bullseye
 ########################################################################################
 LABEL maintainer="Software Improvement Group Research <[email protected]>"
 
@@ -9,14 +24,16 @@ RUN apt-get update \
  && apt-get install -y -qq \
         git \
         subversion \
-        mercurial
+        mercurial \
+        libsnappy-dev
 
 RUN git config --global user.email "[email protected]" \
  && git config --global user.name "Software Improvement Group Research"
 
 WORKDIR /plugin
 
 COPY rapidplugin rapidplugin/
+COPY integration_tests/*.py integration_tests/
 COPY entrypoint.py .
 COPY setup.py .
 COPY requirements.txt .

diff --git a/integration_tests/kafka-compose.yml → docker-compose.yml b/integration_tests/kafka-compose.yml → docker-compose.yml
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,35 +16,43 @@
 version: '2'
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    image: zookeeper:3.7.0
+
   kafka:
-    image: wurstmeister/kafka
-    ports:
-      - "9092:9092"
+    depends_on:
+      - zookeeper
+    image: wurstmeister/kafka:2.13-2.7.1
     environment:
       KAFKA_ADVERTISED_HOST_NAME: localhost
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_MESSAGE_MAX_BYTES: 2097152000
-      KAFKA_LISTENERS: INSIDE://kafka:29092,OUTSIDE://kafka:9092
-      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:29092,OUTSIDE://localhost:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_LISTENERS: INSIDE://kafka:9092
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
       KAFKA_CREATE_TOPICS: "fasten.MetadataDBJavaExtension.out:1:1,fasten.RapidPlugin.callable.out:1:1,fasten.RapidPlugin.callable.err:1:1,fasten.RapidPlugin.callable.log:1:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_COMPRESSION_TYPE: snappy
 
   rapidplugin:
+    depends_on:
+      - kafka
+    image: rapidplugin-test
+    entrypoint: ['python', '/plugin/entrypoint.py', '--group_id=RapidPlugin-TEST', '--bootstrap_servers=kafka:9092', '--sources_dir=/plugin/rapidplugin/tests/resources']
+
+  rapidplugin-unit-tests:
+    image: rapidplugin-test
+    entrypoint: ['python', '-m', 'pytest', 'rapidplugin/tests']
+
+  rapidplugin-integration-tests:
     image: rapidplugin-test
-    entrypoint: ['python', '/plugin/entrypoint.py', '--group_id=RapidPlugin-TEST', '--bootstrap_servers=kafka:29092', '--sources_dir=/plugin/rapidplugin/tests/resources']
+    entrypoint: ['python', '-m', 'pytest', 'integration_tests']
 
   wait-for-kafka:
     image: waisbrot/wait
     links:
       - kafka
     environment:
-      - TARGETS=kafka:29092
+      - TARGETS=kafka:9092
 
 networks:
   default:

diff --git a/entrypoint.py b/entrypoint.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
 
 plugin_name = 'RapidPlugin'
 plugin_description = 'A FASTEN plug-in to populate risk related metadata for a product.'
-plugin_version = '1.2.0'
+plugin_version = '1.2.1'
 
 
 def get_args_parser():

diff --git a/integration_tests/README.md b/integration_tests/README.md
diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py
@@ -21,7 +21,7 @@
 
 @pytest.fixture()
 def mock_in():
-    mock = MockProducer('localhost:9092',
+    mock = MockProducer('kafka:9092',
                         'fasten.MetadataDBJavaExtension.out')
     yield mock
     mock.free_resource()
@@ -30,7 +30,7 @@ def mock_in():
 @pytest.fixture()
 def mock_out():
     mock = MockConsumer('MockConsumerOut',
-                        'localhost:9092',
+                        'kafka:9092',
                         'fasten.RapidPlugin.callable.out')
     mock.skip_messages()
     yield mock
@@ -40,7 +40,7 @@ def mock_out():
 @pytest.fixture()
 def mock_log():
     mock = MockConsumer('MockConsumerLog',
-                        'localhost:9092',
+                        'kafka:9092',
                         'fasten.RapidPlugin.log')
     mock.skip_messages()
     yield mock
@@ -50,7 +50,7 @@ def mock_log():
 @pytest.fixture()
 def mock_err():
     mock = MockConsumer('MockConsumerErr',
-                        'localhost:9092',
+                        'kafka:9092',
                         'fasten.RapidPlugin.err')
     mock.skip_messages()
     yield mock

diff --git a/rapidplugin/analysis/graph_query.py b/rapidplugin/analysis/graph_query.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/config.py b/rapidplugin/config.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/domain/callgraph.py b/rapidplugin/domain/callgraph.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/domain/package.py b/rapidplugin/domain/package.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/rapid_plugin.py b/rapidplugin/rapid_plugin.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/tests/sources.py b/rapidplugin/tests/sources.py
@@ -1,3 +1,18 @@
+# Copyright 2021 Software Improvement Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
 import shutil
 import pytest

diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py
@@ -1,5 +1,4 @@
-
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/tests/test_package.py b/rapidplugin/tests/test_package.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Software Improvement Group
+# Copyright 2021 Software Improvement Group
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/setup.py b/setup.py
@@ -1,3 +1,19 @@
+# Copyright 2021 Software Improvement Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
 from os import path
 from setuptools import setup, find_packages
 
@@ -8,7 +24,7 @@
 
 setup(
     name='quality-analyzer',
-    version='1.2.0',
+    version='1.2.1',
     description='FASTEN RAPID Plugin',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -26,6 +42,9 @@
         'lizard',
         'requests',
         'gitpython',
-        'svn'
+        'svn',
+        'svn',
+        'kafka-python==2.0.2',
+        'python-snappy'
     ]
 )
diff --git a/tox.ini b/tox.ini
@@ -1,3 +1,18 @@
+# Copyright 2021 Software Improvement Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 [tox]
 envlist = py39
 
@@ -14,10 +29,10 @@ commands =
     flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     docker build -t rapidplugin-test .
-    docker run --rm --entrypoint python rapidplugin-test -m pytest rapidplugin/tests
-    docker-compose -f integration_tests/kafka-compose.yml up -d
-    docker-compose -f integration_tests/kafka-compose.yml run wait-for-kafka
-    pytest integration_tests
+    docker-compose run rapidplugin-unit-tests
+    docker-compose up -d rapidplugin
+    docker-compose run wait-for-kafka
+    docker-compose run rapidplugin-integration-tests
 
 commands_post =
-    docker-compose -f integration_tests/kafka-compose.yml down
+    docker-compose down