Merge pull request #856 from arenadata/develop

Release 2021.05.25

.dockerignore

@@ -1,32 +1,12 @@
 */.git*
-*@tmp/
-.dockerignore
-.git*
-.git/
-.pytest_cache
-Dockerfile*
-Jenkinsfile
-Makefile
-README.md
-allure-*/
-build-adcm.sh
-ui-tests.sh
-func-tests.sh
-data/
-docs_adcm/
-go/Makefile
-go/pkg/
-go/src/
-html/
-latex/
-make_html.sh
-make_pdf.sh
-pdf/
-pylintrc
-requirements-test.txt
-runserver.sh
-server.sh
-tests/
-var/cluster.db
-web
+*
+!COPYRIGHT
+!LICENSE
+!conf
+!config.json
+!go/bin/runstatus
+!os
+!python
+!requirements*
 !web/build_static.sh
+!wwwroot

Makefile

@@ -1,7 +1,12 @@
 # Set number of threads
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-ADCMBASE_IMAGE ?= arenadata/adcmbase
-ADCMBASE_TAG ?= 20200812154141
+
+ADCMBASE_IMAGE ?= hub.arenadata.io/adcm/base
+ADCMBASE_TAG ?= 20210317134752
+
+APP_IMAGE ?= hub.adsw.io/adcm/adcm
+APP_TAG ?= $(subst /,_,$(BRANCH_NAME))
+
 SELENOID_HOST ?= 10.92.2.65
 SELENOID_PORT ?= 4444
 
@@ -29,11 +34,16 @@ buildjs: ## Build client side js/html/css in directory wwwroot
 	@docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./build.sh
 
 buildbase: ## Build base image for ADCM's container. That is alpine with all packages.
-	@docker build --pull=true -f Dockerfile_base --no-cache=true -t $(ADCMBASE_IMAGE):$$(date '+%Y%m%d%H%M%S') -t $(ADCMBASE_IMAGE):latest .
+	cd assemble/base && docker build --pull=true --no-cache=true \
+	-t $(ADCMBASE_IMAGE):$$(date '+%Y%m%d%H%M%S') -t $(ADCMBASE_IMAGE):latest \
+	.
 
 build: describe buildss buildjs ## Build final docker image and all depended targets except baseimage.
-	@docker build --no-cache=true --pull=true -t ci.arenadata.io/adcm:$(subst /,_,$(BRANCH_NAME)) .
-
+	@docker build --no-cache=true \
+	-f assemble/app/Dockerfile \
+	-t $(APP_IMAGE):$(APP_TAG) \
+	--build-arg  ADCMBASE_IMAGE=$(ADCMBASE_IMAGE) --build-arg  ADCMBASE_TAG=$(ADCMBASE_TAG) \
+	.
 
 ##################################################
 #                 T E S T S
@@ -49,29 +59,29 @@ unittests: ## Run unittests
 pytest: ## Run functional tests
 	docker pull ci.arenadata.io/functest:3.8.6.slim.buster-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host -v $(CURDIR)/:/adcm -w /adcm/ \
-	-e BUILD_TAG=${BUILD_TAG} -e ADCM_TAG=$(subst /,_,$(BRANCH_NAME)) -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
+	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
 	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	ci.arenadata.io/functest:3.8.6.slim.buster-x64 /bin/sh -e ./pytest.sh
+	ci.arenadata.io/functest:3.8.6.slim.buster-x64 /bin/sh -e \
+	./pytest.sh --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
 
 pytest_release: ## Run functional tests on release
 	docker pull ci.arenadata.io/functest:3.8.6.slim.buster.firefox-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host -v $(CURDIR)/:/adcm -w /adcm/ \
-	-e BUILD_TAG=${BUILD_TAG} -e ADCM_TAG=$(subst /,_,$(BRANCH_NAME)) -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
+	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
 	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	ci.arenadata.io/functest:3.8.6.slim.buster.firefox-x64 /bin/sh -e ./pytest.sh --firefox
+	ci.arenadata.io/functest:3.8.6.slim.buster.firefox-x64 /bin/sh -e \
+	./pytest.sh --firefox --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
 
 ng_tests: ## Run Angular tests
 	docker pull ci.arenadata.io/functest:3.8.6.slim.buster-x64
-	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web/src ci.arenadata.io/functest:3.8.6.slim.buster-x64 /bin/sh -c "export CHROME_BIN=/usr/bin/google-chrome; npm install && ng test --watch=false"
+	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web ci.arenadata.io/functest:3.8.6.slim.buster-x64 ./ng_test.sh
 
 linters : ## Run linters
 	docker pull ci.arenadata.io/pr-builder:3-x64
 	docker run -i --rm -v $(CURDIR)/:/source -w /source ci.arenadata.io/pr-builder:3-x64 /linters.sh shellcheck pylint pep8
 
 npm_check: ## Run npm-check
-	docker pull ci.arenadata.io/functest:3.8.6.slim.buster-x64
-	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web/src ci.arenadata.io/functest:3.8.6.slim.buster-x64 \
-	/bin/bash -c 'npm i --production && { ignore=`cat ../../.npmcheckignore | grep -v "#"`\; npm-check --production --skip-unused --ignore $$ignore || true; } && npm audit'
+	docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./npm_check.sh
 
 django_tests : ## Run django tests.
 	docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)

Dockerfile → assemble/app/Dockerfile

@@ -1,8 +1,10 @@
-FROM arenadata/adcmbase:20200812154141
+ARG ADCMBASE_IMAGE
+ARG ADCMBASE_TAG
+FROM $ADCMBASE_IMAGE:$ADCMBASE_TAG
 
 COPY . /adcm/
 
-RUN cp -r /adcm/os/* / && rm -rf /adcm/os; cp -r /adcm/python/ansible/* /usr/local/lib/python3.8/site-packages/ansible/ && rm -rf /adcm/python/ansible && rmdir /var/log/nginx;
+RUN cp -r /adcm/os/* / && rm -rf /adcm/os; cp -r /adcm/python/ansible/* /usr/local/lib/python3.?/site-packages/ansible/ && rm -rf /adcm/python/ansible && rmdir /var/log/nginx;
 
 # Secret_key is mandatory for build_static procedure,
 # but should not be hardcoded in the image.

assemble/base/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9-alpine
+
+COPY requirements-base.txt /
+COPY build.sh /
+
+RUN /build.sh
+
+CMD ["/bin/sh"]

assemble/base/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+set -eu
+
+apk add --update --no-cache --virtual .pynacl_deps build-base python3-dev libffi-dev openssl-dev linux-headers libxslt-dev rust cargo
+apk add --update --no-cache libffi openssl libxslt libstdc++
+apk add --update --no-cache nginx sshpass runit openssh-keygen openssh-client git dcron logrotate curl rsync
+pip install --no-cache-dir -r /requirements-base.txt
+rm /requirements-base.txt
+apk del git
+apk del .pynacl_deps
+rm /etc/nginx/conf.d/default.conf

assemble/base/requirements-base.txt

@@ -0,0 +1,26 @@
+# ansible==2.8.3
+git+https://github.com/arenadata/[email protected]
+coreapi
+django
+django-cors-headers
+django-filter
+django-rest-swagger
+djangorestframework
+django-background-tasks
+social-auth-app-django
+git+git://github.com/arenadata/django-generate-secret-key.git
+jinja2
+markdown
+mitogen
+pyyaml
+ruyaml
+toml
+uwsgi
+version_utils
+ruyaml
+yspec
+# Custom bundle libs
+apache-libcloud
+jmespath
+lxml
+pycrypto

conf/adcm/config.yaml

@@ -2,19 +2,13 @@
 
   type: adcm
   name: ADCM
-  version: 1.2
+  version: 1.3
 
   config:
     - name: "global"
       display_name: "Global Options"
       type: "group"
       subs:
-        - name: "send_stats"
-          display_name: "Send Anonymous Statistics"
-          description: |
-            We will send anonymous statistic about number of bundles your use and number of hosts and clusters, but without any config or names.
-          type: boolean
-          default: true
         - name: "adcm_url"
           display_name: "ADCM's URL"
           description: |

config.json

@@ -1,4 +1,4 @@
 {
-	"version": "2020.04.24.11",
-	"commit_id": "c827ccf"
+	"version": "2021.05.12.14",
+	"commit_id": "20b2b8b3"
 }

pylintrc

@@ -51,7 +51,7 @@ confidence=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 
-disable=missing-docstring,invalid-name,no-self-use,abstract-method,unused-argument,no-else-return,duplicate-code,fixme,no-member,too-few-public-methods,wrong-import-order,useless-import-alias
+disable=missing-docstring,invalid-name,no-self-use,abstract-method,unused-argument,no-else-return,duplicate-code,fixme,no-member,too-few-public-methods,wrong-import-order,useless-import-alias,consider-using-with
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

pytest.sh

@@ -18,9 +18,9 @@ pip3 install -r requirements-test.txt
 find . -name "*.pyc" -type f -delete
 find . -name "__pycache__" -type d -delete
 { # try
-    pytest tests/ui_tests tests/functional -s -v -n auto --maxfail 30 \
-    --showlocals --alluredir ./allure-results/ --durations=20 \
-    --reruns 2 --remote-executor-host "$SELENOID_HOST" --timeout=360 "$@" &&
+    pytest tests/functional tests/ui_tests -s -v -n auto --maxfail 30 \
+    --showlocals --alluredir ./allure-results/ --durations=20 -p allure_pytest \
+    --reruns 2 --remote-executor-host "$SELENOID_HOST" --timeout=1080 "$@" &&
     chmod -R o+xw allure-results
 } || { # catch
     chmod -R o+xw allure-results

python/ansible/plugins/action/adcm_config.py

@@ -19,8 +19,16 @@
 sys.path.append('/adcm/python')
 
 import adcm.init_django
-from cm.ansible_plugin import ContextActionModule
-import cm.adcm_config
+from cm.ansible_plugin import (
+    ContextActionModule,
+    set_cluster_config,
+    set_service_config,
+    set_service_config_by_id,
+    set_host_config,
+    set_provider_config,
+    set_component_config,
+    set_component_config_by_name,
+)
 
 
 ANSIBLE_METADATA = {'metadata_version': '1.1', 'supported_by': 'Arenadata'}
@@ -83,12 +91,12 @@
 
 class ActionModule(ContextActionModule):
 
-    _VALID_ARGS = frozenset(('type', 'key', 'value', 'service_name', 'host_id'))
+    _VALID_ARGS = frozenset(('type', 'key', 'value', 'service_name', 'component_name', 'host_id'))
     _MANDATORY_ARGS = ('type', 'key', 'value')
 
     def _do_cluster(self, task_vars, context):
         res = self._wrap_call(
-            cm.adcm_config.set_cluster_config,
+            set_cluster_config,
             context['cluster_id'],
             self._task.args["key"],
             self._task.args["value"]
@@ -98,7 +106,7 @@ def _do_cluster(self, task_vars, context):
 
     def _do_service_by_name(self, task_vars, context):
         res = self._wrap_call(
-            cm.adcm_config.set_service_config,
+            set_service_config,
             context['cluster_id'],
             self._task.args["service_name"],
             self._task.args["key"],
@@ -109,7 +117,7 @@ def _do_service_by_name(self, task_vars, context):
 
     def _do_service(self, task_vars, context):
         res = self._wrap_call(
-            cm.adcm_config.set_service_config_by_id,
+            set_service_config_by_id,
             context['cluster_id'],
             context['service_id'],
             self._task.args["key"],
@@ -120,7 +128,7 @@ def _do_service(self, task_vars, context):
 
     def _do_host(self, task_vars, context):
         res = self._wrap_call(
-            cm.adcm_config.set_host_config,
+            set_host_config,
             context['host_id'],
             self._task.args["key"],
             self._task.args["value"]
@@ -131,7 +139,7 @@ def _do_host(self, task_vars, context):
     def _do_host_from_provider(self, task_vars, context):
         # TODO: Check that host is in provider
         res = self._wrap_call(
-            cm.adcm_config.set_host_config,
+            set_host_config,
             self._task.args['host_id'],
             self._task.args["key"],
             self._task.args["value"]
@@ -141,10 +149,33 @@ def _do_host_from_provider(self, task_vars, context):
 
     def _do_provider(self, task_vars, context):
         res = self._wrap_call(
-            cm.adcm_config.set_provider_config,
+            set_provider_config,
             context['provider_id'],
             self._task.args["key"],
             self._task.args["value"]
         )
         res['value'] = self._task.args["value"]
         return res
+
+    def _do_component_by_name(self, task_vars, context):
+        res = self._wrap_call(
+            set_component_config_by_name,
+            context['cluster_id'],
+            context['service_id'],
+            self._task.args['component_name'],
+            self._task.args.get('service_name', None),
+            self._task.args['key'],
+            self._task.args['value']
+        )
+        res['value'] = self._task.args['value']
+        return res
+
+    def _do_component(self, task_vars, context):
+        res = self._wrap_call(
+            set_component_config,
+            context['component_id'],
+            self._task.args['key'],
+            self._task.args['value']
+        )
+        res['value'] = self._task.args['value']
+        return res

python/ansible/plugins/action/adcm_hc.py

@@ -54,8 +54,7 @@
 
 sys.path.append('/adcm/python')
 import adcm.init_django
-import cm.api
-from cm.ansible_plugin import get_object_id_from_context
+from cm.ansible_plugin import get_object_id_from_context, change_hc
 from cm.errors import AdcmEx
 from cm.logger import log
 
@@ -87,7 +86,7 @@ def run(self, tmp=None, task_vars=None):
                 raise AnsibleError('Invalid operation arguments: %s' % op)
 
         try:
-            cm.api.change_hc(job_id, cluster_id, ops)
+            change_hc(job_id, cluster_id, ops)
         except AdcmEx as e:
             raise AnsibleError(e.code + ": " + e.msg) from e