diff --git a/Dockerfile b/Dockerfile index c27ed1bc47..10d7770f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM arenadata/adcmbase:20200121195750 +FROM arenadata/adcmbase:20200203174702 COPY . /adcm/ -RUN cp -r /adcm/os/* / && rm -rf /adcm/os; cp -r /adcm/ansible/* /usr/local/lib/python3.8/site-packages/ansible/ && rm -rf /adcm/ansible && rmdir /var/log/nginx; +RUN cp -r /adcm/os/* / && rm -rf /adcm/os; cp -r /adcm/python/ansible/* /usr/local/lib/python3.8/site-packages/ansible/ && rm -rf /adcm/python/ansible && rmdir /var/log/nginx; # Secret_key is mandatory for build_static procedure, # but should not be hardcoded in the image. diff --git a/Makefile b/Makefile index c1d6fe481c..19fe847ccc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Set number of threads BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) ADCMBASE_IMAGE ?= arenadata/adcmbase -ADCMBASE_TAG ?= 20200121195750 +ADCMBASE_TAG ?= 20200203174702 # Default target @@ -64,4 +64,4 @@ npm_check: ## Run npm-check django_tests : ## Run django tests. docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) - docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) python manage.py test cm + docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) python python/manage.py test cm diff --git a/conf/nginx/adcm.conf b/conf/nginx/adcm.conf index 71b6c918d3..4fe2fe7ea4 100644 --- a/conf/nginx/adcm.conf +++ b/conf/nginx/adcm.conf @@ -32,6 +32,9 @@ server { root /adcm/wwwroot; location / { + if ($arg_nocache) { + add_header Cache-Control no-cache; + } try_files $uri $uri/ /index.html; } diff --git a/data/download/adb.2.6.tar b/data/download/adb.2.6.tar new file mode 100644 index 0000000000..eb35425f1b Binary files /dev/null and b/data/download/adb.2.6.tar differ diff --git a/data/download/ssh.1.3.tar b/data/download/ssh.1.3.tar new file mode 100644 index 0000000000..d08651250f Binary files /dev/null and b/data/download/ssh.1.3.tar differ diff --git a/os/etc/service/init/run b/os/etc/service/init/run index fb4af0cf8b..f7098c72fd 100755 --- a/os/etc/service/init/run +++ b/os/etc/service/init/run @@ -38,13 +38,13 @@ key=/root/.ssh/id_rsa if [ ! -f "${adcmroot}"/var/cluster.db ]; then echo "Assume first run" - "${adcmroot}"/manage.py generate_secret_key "${adcmsecretfile}" - "${adcmroot}"/manage.py migrate + "${adcmroot}"/python/manage.py generate_secret_key "${adcmsecretfile}" + "${adcmroot}"/python/manage.py migrate else - "${adcmroot}"/manage.py migrate + "${adcmroot}"/python/manage.py migrate fi -"${adcmroot}"/init_db.py +"${adcmroot}"/python/init_db.py touch "${initreadyfile}" # That is sleep forever case because I have no idea how to force diff --git a/os/etc/service/wsgi/run b/os/etc/service/wsgi/run index 71227fbe9c..ca2c9be4cf 100755 --- a/os/etc/service/wsgi/run +++ b/os/etc/service/wsgi/run @@ -19,5 +19,5 @@ echo "Run main wsgi application ..." exec 1>"${adcmlog}/service_wsgi.out" exec 2>"${adcmlog}/service_wsgi.err" -cd "${adcmroot}" +cd "${adcmroot}/python" uwsgi --socket "${wsgisocketfile}" --pidfile "/run/uwsgi.pid" --module adcm.wsgi --chmod-socket=777 --logger file:logfile="${adcmlog}/wsgi.log",maxsize=2000000 diff --git a/adcm/__init__.py b/python/adcm/__init__.py similarity index 100% rename from adcm/__init__.py rename to python/adcm/__init__.py diff --git a/adcm/dev.py b/python/adcm/dev.py similarity index 100% rename from adcm/dev.py rename to python/adcm/dev.py diff --git a/adcm/init_django.py b/python/adcm/init_django.py similarity index 100% rename from adcm/init_django.py rename to python/adcm/init_django.py diff --git a/adcm/settings.py b/python/adcm/settings.py similarity index 98% rename from adcm/settings.py rename to python/adcm/settings.py index 579da6011d..700fb5ca58 100644 --- a/adcm/settings.py +++ b/python/adcm/settings.py @@ -23,12 +23,13 @@ import json import os +from os.path import dirname from django.core.management.utils import get_random_secret_key # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = dirname(dirname(dirname(os.path.abspath(__file__)))) CONF_DIR = BASE_DIR + '/data/conf/' SECRET_KEY_FILE = CONF_DIR + '/secret_key.txt' CONFIG_FILE = BASE_DIR + '/config.json' diff --git a/adcm/urls.py b/python/adcm/urls.py similarity index 100% rename from adcm/urls.py rename to python/adcm/urls.py diff --git a/adcm/wsgi.py b/python/adcm/wsgi.py similarity index 100% rename from adcm/wsgi.py rename to python/adcm/wsgi.py diff --git a/ansible/plugins/action/adcm_add_host.py b/python/ansible/plugins/action/adcm_add_host.py similarity index 98% rename from ansible/plugins/action/adcm_add_host.py rename to python/ansible/plugins/action/adcm_add_host.py index 2569e9f4a1..9cc043007e 100644 --- a/ansible/plugins/action/adcm_add_host.py +++ b/python/ansible/plugins/action/adcm_add_host.py @@ -52,7 +52,7 @@ from ansible.errors import AnsibleError from ansible.plugins.action import ActionBase -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django import cm.api from cm.ansible_plugin import get_context_id diff --git a/ansible/plugins/action/adcm_check.py b/python/ansible/plugins/action/adcm_check.py similarity index 65% rename from ansible/plugins/action/adcm_check.py rename to python/ansible/plugins/action/adcm_check.py index 5832728460..a0b30b2365 100644 --- a/ansible/plugins/action/adcm_check.py +++ b/python/ansible/plugins/action/adcm_check.py @@ -11,9 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=wrong-import-position,unused-import +# pylint: disable=wrong-import-position, unused-import, import-error from __future__ import absolute_import, division, print_function + __metaclass__ = type ANSIBLE_METADATA = {'metadata_version': '1.1', 'supported_by': 'Arenadata'} @@ -58,7 +59,7 @@ import sys from ansible.plugins.action import ActionBase -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django import cm.job from cm.errors import AdcmEx @@ -66,29 +67,43 @@ class ActionModule(ActionBase): - TRANSFERS_FILES = False - _VALID_ARGS = frozenset(('title', 'result', 'msg')) + _VALID_ARGS = frozenset(('title', 'result', 'msg', 'fail_msg', 'success_msg')) def run(self, tmp=None, task_vars=None): job_id = None if task_vars is not None and 'job' in task_vars or 'id' in task_vars['job']: job_id = task_vars['job']['id'] - result = super(ActionModule, self).run(tmp, task_vars) + old_optional_condition = 'msg' in self._task.args + new_optional_condition = 'fail_msg' in self._task.args and 'success_msg' in self._task.args + optional_condition = old_optional_condition or new_optional_condition + required_condition = ( + 'title' in self._task.args and 'result' in self._task.args and optional_condition + ) + + if not required_condition: + return { + "failed": True, + "msg": ("title, result and msg, fail_msg or success" + "_msg are mandatory args of adcm_check") + } - if ('title' not in self._task.args or - 'result' not in self._task.args or - 'msg' not in self._task.args): - return {"failed": True, "msg": "title, result and msg are mandatory args of adcm_check"} result = super(ActionModule, self).run(tmp, task_vars) title = self._task.args['title'] result = self._task.args['result'] - msg = self._task.args['msg'] + msg = self._task.args.get('msg', '') + fail_msg = self._task.args.get('fail_msg', '') + success_msg = self._task.args.get('success_msg', '') - log.debug('ansible adcm_check: %s, %s, %s, %s', job_id, title, result, msg) + log.debug('ansible adcm_check: %s, %s, %s, %s, %s, %s', + job_id, title, result, msg, fail_msg, success_msg) try: + if result: + msg = success_msg if success_msg else msg + else: + msg = fail_msg if fail_msg else msg cm.job.log_check(job_id, title, result, msg) except AdcmEx as e: return {"failed": True, "msg": e.code + ":" + e.msg} diff --git a/ansible/plugins/action/adcm_config.py b/python/ansible/plugins/action/adcm_config.py similarity index 99% rename from ansible/plugins/action/adcm_config.py rename to python/ansible/plugins/action/adcm_config.py index f0a11216ad..f33f318c84 100644 --- a/ansible/plugins/action/adcm_config.py +++ b/python/ansible/plugins/action/adcm_config.py @@ -16,7 +16,7 @@ __metaclass__ = type import sys -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django from cm.ansible_plugin import ContextActionModule diff --git a/ansible/plugins/action/adcm_delete_host.py b/python/ansible/plugins/action/adcm_delete_host.py similarity index 98% rename from ansible/plugins/action/adcm_delete_host.py rename to python/ansible/plugins/action/adcm_delete_host.py index 7105b28286..54ed7d2de5 100644 --- a/ansible/plugins/action/adcm_delete_host.py +++ b/python/ansible/plugins/action/adcm_delete_host.py @@ -40,7 +40,7 @@ from ansible.errors import AnsibleError from ansible.plugins.action import ActionBase -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django import cm.api from cm.ansible_plugin import get_context_id diff --git a/python/ansible/plugins/action/adcm_delete_service.py b/python/ansible/plugins/action/adcm_delete_service.py new file mode 100644 index 0000000000..68854e8019 --- /dev/null +++ b/python/ansible/plugins/action/adcm_delete_service.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=wrong-import-position,unused-import + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'supported_by': 'Arenadata'} + +DOCUMENTATION = r''' +--- +module: adcm_delete_service +short_description: delete service from cluster in ADCM DB +description: + - The C(adcm_delete_service) module is intended to delete service from ADCM DB. + This module should be run in service context. Service Id is taken from context. +options: +''' + +EXAMPLES = r''' + - name: delete service from cluster + adcm_delete_service: +''' + +RETURN = r''' +''' + +import sys +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +sys.path.append('/adcm/python') +import adcm.init_django +import cm.api +from cm.ansible_plugin import get_context_id +from cm.errors import AdcmEx +from cm.logger import log + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(()) + + def run(self, tmp=None, task_vars=None): + msg = 'You can delete service only in service context' + service_id = get_context_id(task_vars, 'service', 'service_id', msg) + log.info('ansible module adcm_delete_service: service #%s', service_id) + + try: + cm.api.delete_service_by_id(service_id) + except AdcmEx as e: + raise AnsibleError(e.code + ":" + e.msg) + + return {"failed": False, "changed": True} diff --git a/ansible/plugins/action/adcm_state.py b/python/ansible/plugins/action/adcm_state.py similarity index 99% rename from ansible/plugins/action/adcm_state.py rename to python/ansible/plugins/action/adcm_state.py index 23fac72f64..274685e992 100644 --- a/ansible/plugins/action/adcm_state.py +++ b/python/ansible/plugins/action/adcm_state.py @@ -15,7 +15,7 @@ __metaclass__ = type import sys -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django from cm.ansible_plugin import ContextActionModule diff --git a/ansible/plugins/lookup/adcm_config.py b/python/ansible/plugins/lookup/adcm_config.py similarity index 99% rename from ansible/plugins/lookup/adcm_config.py rename to python/ansible/plugins/lookup/adcm_config.py index f89e6e44db..01d8f13025 100644 --- a/ansible/plugins/lookup/adcm_config.py +++ b/python/ansible/plugins/lookup/adcm_config.py @@ -23,7 +23,7 @@ from ansible.utils.display import Display # pylint: disable=ungrouped-imports display = Display() -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django import cm.adcm_config from cm.logger import log diff --git a/ansible/plugins/lookup/adcm_state.py b/python/ansible/plugins/lookup/adcm_state.py similarity index 99% rename from ansible/plugins/lookup/adcm_state.py rename to python/ansible/plugins/lookup/adcm_state.py index d082f52be9..c4aa34d164 100644 --- a/ansible/plugins/lookup/adcm_state.py +++ b/python/ansible/plugins/lookup/adcm_state.py @@ -23,7 +23,7 @@ display = Display() import sys -sys.path.append('/adcm') +sys.path.append('/adcm/python') import adcm.init_django import cm.api import cm.status_api diff --git a/api/__init__.py b/python/api/__init__.py similarity index 100% rename from api/__init__.py rename to python/api/__init__.py diff --git a/api/api_views.py b/python/api/api_views.py similarity index 100% rename from api/api_views.py rename to python/api/api_views.py diff --git a/api/apps.py b/python/api/apps.py similarity index 100% rename from api/apps.py rename to python/api/apps.py diff --git a/api/cluster_serial.py b/python/api/cluster_serial.py similarity index 99% rename from api/cluster_serial.py rename to python/api/cluster_serial.py index 2560e2d3a8..fb80fe4a5a 100644 --- a/api/cluster_serial.py +++ b/python/api/cluster_serial.py @@ -423,7 +423,7 @@ def get_kwargs(self, obj): id = serializers.IntegerField(read_only=True) name = serializers.SerializerMethodField() - component_prototype_id = serializers.SerializerMethodField() + prototype_id = serializers.SerializerMethodField() display_name = serializers.SerializerMethodField() description = serializers.SerializerMethodField() url = MyUrlField(read_only=True, view_name='service-component-details') @@ -431,7 +431,7 @@ def get_kwargs(self, obj): def get_name(self, obj): return obj.component.name - def get_component_prototype_id(self, obj): + def get_prototype_id(self, obj): return obj.component.id def get_display_name(self, obj): diff --git a/api/cluster_views.py b/python/api/cluster_views.py similarity index 100% rename from api/cluster_views.py rename to python/api/cluster_views.py diff --git a/api/docs.py b/python/api/docs.py similarity index 100% rename from api/docs.py rename to python/api/docs.py diff --git a/api/serializers.py b/python/api/serializers.py similarity index 99% rename from api/serializers.py rename to python/api/serializers.py index db7363b7ee..f6bd28e1a9 100644 --- a/api/serializers.py +++ b/python/api/serializers.py @@ -629,6 +629,7 @@ class ProviderActionShort(ActionShort): class UpgradeSerializer(serializers.Serializer): id = serializers.IntegerField(read_only=True) name = serializers.CharField(required=False) + bundle_id = serializers.IntegerField(read_only=True) description = serializers.CharField(required=False) min_version = serializers.CharField(required=False) max_version = serializers.CharField(required=False) @@ -636,6 +637,7 @@ class UpgradeSerializer(serializers.Serializer): max_strict = serializers.BooleanField(required=False) upgradable = serializers.BooleanField(required=False) license = serializers.CharField(required=False) + license_url = hlink('bundle-license', 'bundle_id', 'bundle_id') from_edition = JSONField(required=False) state_available = JSONField(required=False) state_on_success = serializers.CharField(required=False) diff --git a/api/stack_serial.py b/python/api/stack_serial.py similarity index 100% rename from api/stack_serial.py rename to python/api/stack_serial.py diff --git a/api/stack_views.py b/python/api/stack_views.py similarity index 100% rename from api/stack_views.py rename to python/api/stack_views.py diff --git a/api/templates/docs-html/document.html b/python/api/templates/docs-html/document.html similarity index 100% rename from api/templates/docs-html/document.html rename to python/api/templates/docs-html/document.html diff --git a/api/templates/docs-html/index.html b/python/api/templates/docs-html/index.html similarity index 100% rename from api/templates/docs-html/index.html rename to python/api/templates/docs-html/index.html diff --git a/api/templates/docs-html/link.html b/python/api/templates/docs-html/link.html similarity index 100% rename from api/templates/docs-html/link.html rename to python/api/templates/docs-html/link.html diff --git a/api/templates/docs-md/document.md b/python/api/templates/docs-md/document.md similarity index 100% rename from api/templates/docs-md/document.md rename to python/api/templates/docs-md/document.md diff --git a/api/templates/docs-md/index.md b/python/api/templates/docs-md/index.md similarity index 61% rename from api/templates/docs-md/index.md rename to python/api/templates/docs-md/index.md index 80d4d478ef..6c5c397ce5 100644 --- a/api/templates/docs-md/index.md +++ b/python/api/templates/docs-md/index.md @@ -1,3 +1,3 @@ -{% load staticfiles %} +{% load static %} {% include "docs-md/document.md" %} diff --git a/api/templates/docs-md/link.md b/python/api/templates/docs-md/link.md similarity index 100% rename from api/templates/docs-md/link.md rename to python/api/templates/docs-md/link.md diff --git a/api/urls.py b/python/api/urls.py similarity index 100% rename from api/urls.py rename to python/api/urls.py diff --git a/api/views.py b/python/api/views.py similarity index 100% rename from api/views.py rename to python/api/views.py diff --git a/cm/__init__.py b/python/cm/__init__.py similarity index 100% rename from cm/__init__.py rename to python/cm/__init__.py diff --git a/cm/adcm_config.py b/python/cm/adcm_config.py similarity index 100% rename from cm/adcm_config.py rename to python/cm/adcm_config.py diff --git a/cm/admin.py b/python/cm/admin.py similarity index 100% rename from cm/admin.py rename to python/cm/admin.py diff --git a/cm/ansible_plugin.py b/python/cm/ansible_plugin.py similarity index 100% rename from cm/ansible_plugin.py rename to python/cm/ansible_plugin.py diff --git a/cm/api.py b/python/cm/api.py similarity index 98% rename from cm/api.py rename to python/cm/api.py index b705bd86e3..eb347d2889 100644 --- a/cm/api.py +++ b/python/cm/api.py @@ -139,6 +139,11 @@ def delete_host(host): def delete_host_by_id(host_id): + """ + Host deleting + + This is intended for use in adcm_delete_host ansible plugin + """ try: host = Host.objects.get(id=host_id) except Host.DoesNotExist: @@ -146,6 +151,21 @@ def delete_host_by_id(host_id): delete_host(host) +def delete_service_by_id(service_id): + """ + Unconditional removal of service from cluster + + This is intended for use in adcm_delete_service ansible plugin + """ + try: + service = ClusterObject.objects.get(id=service_id) + except ClusterObject.DoesNotExist: + err('SERVICE_NOT_FOUND', 'Service with id #{} is not found'.format(service_id)) + cm.status_api.post_event('delete', 'service', service.id) + service.delete() + cm.status_api.load_service_map() + + def delete_service(service): if HostComponent.objects.filter(cluster=service.cluster, service=service): err('SERVICE_CONFLICT', 'Service #{} has component(s) on host(s)'.format(service.id)) diff --git a/cm/apps.py b/python/cm/apps.py similarity index 100% rename from cm/apps.py rename to python/cm/apps.py diff --git a/cm/bundle.py b/python/cm/bundle.py similarity index 100% rename from cm/bundle.py rename to python/cm/bundle.py diff --git a/cm/config.py b/python/cm/config.py similarity index 91% rename from cm/config.py rename to python/cm/config.py index 55bee7d3f0..2f5e880a3d 100644 --- a/cm/config.py +++ b/python/cm/config.py @@ -12,14 +12,16 @@ import json import os +from os.path import dirname - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = dirname(dirname(dirname(os.path.abspath(__file__)))) BASE_DIR = os.environ.get('ADCM_BASE_DIR', BASE_DIR) STACK_DIR = BASE_DIR STACK_DIR = os.environ.get('ADCM_STACK_DIR', STACK_DIR) +CODE_DIR = os.path.join(BASE_DIR, 'python') + LOG_DIR = os.path.join(BASE_DIR, 'data', 'log') RUN_DIR = os.path.join(BASE_DIR, 'data', 'run') diff --git a/cm/daemon.py b/python/cm/daemon.py similarity index 100% rename from cm/daemon.py rename to python/cm/daemon.py diff --git a/cm/errors.py b/python/cm/errors.py similarity index 100% rename from cm/errors.py rename to python/cm/errors.py diff --git a/cm/inventory.py b/python/cm/inventory.py similarity index 97% rename from cm/inventory.py rename to python/cm/inventory.py index 4e5ec91dbc..96481530c4 100644 --- a/cm/inventory.py +++ b/python/cm/inventory.py @@ -174,9 +174,10 @@ def get_provider_hosts(provider_id): def get_host(host_id): host = Host.objects.get(id=host_id) - groups = {'HOST': {'hosts': get_hosts([host])}} - groups.update(get_provider_hosts(host.provider.id)) - groups['PROVIDER']['vars'] = get_provider_config(host.provider.id) + groups = {'HOST': { + 'hosts': get_hosts([host]), + 'vars': get_provider_config(host.provider.id) + }} return groups diff --git a/cm/issue.py b/python/cm/issue.py similarity index 100% rename from cm/issue.py rename to python/cm/issue.py diff --git a/cm/job.py b/python/cm/job.py similarity index 98% rename from cm/job.py rename to python/cm/job.py index 6a09c00b56..46896cf503 100644 --- a/cm/job.py +++ b/python/cm/job.py @@ -31,7 +31,7 @@ from cm.logger import log from cm.models import ( Cluster, Action, SubAction, TaskLog, JobLog, CheckLog, Host, ADCM, - ClusterObject, HostComponent, ServiceComponent, HostProvider, + ClusterObject, HostComponent, ServiceComponent, HostProvider, DummyData, ) @@ -634,13 +634,16 @@ def set_task_status(task, status): def get_task_obj(context, obj_id): - if context == 'service': - obj = ClusterObject.objects.get(id=obj_id) - elif context == 'host': + def get_obj_safe(model, obj_id): try: - obj = Host.objects.get(id=obj_id) - except Host.DoesNotExist: + return model.objects.get(id=obj_id) + except model.DoesNotExist: return None + + if context == 'service': + obj = get_obj_safe(ClusterObject, obj_id) + elif context == 'host': + obj = get_obj_safe(Host, obj_id) elif context == 'cluster': obj = Cluster.objects.get(id=obj_id) elif context == 'provider': @@ -715,6 +718,7 @@ def finish_task(task, job, status): obj = get_task_obj(action.prototype.type, task.object_id) state = get_state(action, job, status) with transaction.atomic(): + DummyData.objects.filter(id=1).update(date=timezone.now()) if state is not None: set_action_state(action, task, obj, state) unlock_objects(obj) @@ -801,7 +805,7 @@ def check_all_status(): def run_task(task, args=''): err_file = open(os.path.join(config.LOG_DIR, 'task_runner.err'), 'a+') proc = subprocess.Popen([ - os.path.join(config.BASE_DIR, 'task_runner.py'), + os.path.join(config.CODE_DIR, 'task_runner.py'), str(task.id), args ], stderr=err_file) diff --git a/cm/logger.py b/python/cm/logger.py similarity index 100% rename from cm/logger.py rename to python/cm/logger.py diff --git a/cm/migrations/0001_initial.py b/python/cm/migrations/0001_initial.py similarity index 100% rename from cm/migrations/0001_initial.py rename to python/cm/migrations/0001_initial.py diff --git a/cm/migrations/0002_auto_20180801_1117.py b/python/cm/migrations/0002_auto_20180801_1117.py similarity index 100% rename from cm/migrations/0002_auto_20180801_1117.py rename to python/cm/migrations/0002_auto_20180801_1117.py diff --git a/cm/migrations/0003_auto_20180829_1020.py b/python/cm/migrations/0003_auto_20180829_1020.py similarity index 100% rename from cm/migrations/0003_auto_20180829_1020.py rename to python/cm/migrations/0003_auto_20180829_1020.py diff --git a/cm/migrations/0004_auto_20180914_1042.py b/python/cm/migrations/0004_auto_20180914_1042.py similarity index 100% rename from cm/migrations/0004_auto_20180914_1042.py rename to python/cm/migrations/0004_auto_20180914_1042.py diff --git a/cm/migrations/0005_auto_20180928_0945.py b/python/cm/migrations/0005_auto_20180928_0945.py similarity index 100% rename from cm/migrations/0005_auto_20180928_0945.py rename to python/cm/migrations/0005_auto_20180928_0945.py diff --git a/cm/migrations/0006_auto_20181009_1135.py b/python/cm/migrations/0006_auto_20181009_1135.py similarity index 100% rename from cm/migrations/0006_auto_20181009_1135.py rename to python/cm/migrations/0006_auto_20181009_1135.py diff --git a/cm/migrations/0007_auto_20181023_1048.py b/python/cm/migrations/0007_auto_20181023_1048.py similarity index 100% rename from cm/migrations/0007_auto_20181023_1048.py rename to python/cm/migrations/0007_auto_20181023_1048.py diff --git a/cm/migrations/0008_auto_20181107_1216.py b/python/cm/migrations/0008_auto_20181107_1216.py similarity index 100% rename from cm/migrations/0008_auto_20181107_1216.py rename to python/cm/migrations/0008_auto_20181107_1216.py diff --git a/cm/migrations/0009_auto_20181113_1112.py b/python/cm/migrations/0009_auto_20181113_1112.py similarity index 100% rename from cm/migrations/0009_auto_20181113_1112.py rename to python/cm/migrations/0009_auto_20181113_1112.py diff --git a/cm/migrations/0010_auto_20181212_1213.py b/python/cm/migrations/0010_auto_20181212_1213.py similarity index 100% rename from cm/migrations/0010_auto_20181212_1213.py rename to python/cm/migrations/0010_auto_20181212_1213.py diff --git a/cm/migrations/0011_auto_20181220_1327.py b/python/cm/migrations/0011_auto_20181220_1327.py similarity index 100% rename from cm/migrations/0011_auto_20181220_1327.py rename to python/cm/migrations/0011_auto_20181220_1327.py diff --git a/cm/migrations/0012_auto_20181226_0926.py b/python/cm/migrations/0012_auto_20181226_0926.py similarity index 100% rename from cm/migrations/0012_auto_20181226_0926.py rename to python/cm/migrations/0012_auto_20181226_0926.py diff --git a/cm/migrations/0013_auto_20190116_1143.py b/python/cm/migrations/0013_auto_20190116_1143.py similarity index 100% rename from cm/migrations/0013_auto_20190116_1143.py rename to python/cm/migrations/0013_auto_20190116_1143.py diff --git a/cm/migrations/0014_auto_20190124_1344.py b/python/cm/migrations/0014_auto_20190124_1344.py similarity index 100% rename from cm/migrations/0014_auto_20190124_1344.py rename to python/cm/migrations/0014_auto_20190124_1344.py diff --git a/cm/migrations/0015_auto_20190128_1142.py b/python/cm/migrations/0015_auto_20190128_1142.py similarity index 100% rename from cm/migrations/0015_auto_20190128_1142.py rename to python/cm/migrations/0015_auto_20190128_1142.py diff --git a/cm/migrations/0016_auto_20190218_1214.py b/python/cm/migrations/0016_auto_20190218_1214.py similarity index 100% rename from cm/migrations/0016_auto_20190218_1214.py rename to python/cm/migrations/0016_auto_20190218_1214.py diff --git a/cm/migrations/0017_auto_20190220_1137.py b/python/cm/migrations/0017_auto_20190220_1137.py similarity index 100% rename from cm/migrations/0017_auto_20190220_1137.py rename to python/cm/migrations/0017_auto_20190220_1137.py diff --git a/cm/migrations/0018_auto_20190220_1101.py b/python/cm/migrations/0018_auto_20190220_1101.py similarity index 100% rename from cm/migrations/0018_auto_20190220_1101.py rename to python/cm/migrations/0018_auto_20190220_1101.py diff --git a/cm/migrations/0019_auto_20190314_1321.py b/python/cm/migrations/0019_auto_20190314_1321.py similarity index 100% rename from cm/migrations/0019_auto_20190314_1321.py rename to python/cm/migrations/0019_auto_20190314_1321.py diff --git a/cm/migrations/0020_auto_20190321_1126.py b/python/cm/migrations/0020_auto_20190321_1126.py similarity index 100% rename from cm/migrations/0020_auto_20190321_1126.py rename to python/cm/migrations/0020_auto_20190321_1126.py diff --git a/cm/migrations/0021_auto_20190607_1027.py b/python/cm/migrations/0021_auto_20190607_1027.py similarity index 100% rename from cm/migrations/0021_auto_20190607_1027.py rename to python/cm/migrations/0021_auto_20190607_1027.py diff --git a/cm/migrations/0022_auto_20190620_1251.py b/python/cm/migrations/0022_auto_20190620_1251.py similarity index 100% rename from cm/migrations/0022_auto_20190620_1251.py rename to python/cm/migrations/0022_auto_20190620_1251.py diff --git a/cm/migrations/0023_auto_20190624_1441.py b/python/cm/migrations/0023_auto_20190624_1441.py similarity index 100% rename from cm/migrations/0023_auto_20190624_1441.py rename to python/cm/migrations/0023_auto_20190624_1441.py diff --git a/cm/migrations/0024_auto_20190715_1548.py b/python/cm/migrations/0024_auto_20190715_1548.py similarity index 100% rename from cm/migrations/0024_auto_20190715_1548.py rename to python/cm/migrations/0024_auto_20190715_1548.py diff --git a/cm/migrations/0025_auto_20190719_1036.py b/python/cm/migrations/0025_auto_20190719_1036.py similarity index 100% rename from cm/migrations/0025_auto_20190719_1036.py rename to python/cm/migrations/0025_auto_20190719_1036.py diff --git a/cm/migrations/0026_auto_20190805_1442.py b/python/cm/migrations/0026_auto_20190805_1442.py similarity index 100% rename from cm/migrations/0026_auto_20190805_1442.py rename to python/cm/migrations/0026_auto_20190805_1442.py diff --git a/cm/migrations/0027_auto_20190809_1134.py b/python/cm/migrations/0027_auto_20190809_1134.py similarity index 100% rename from cm/migrations/0027_auto_20190809_1134.py rename to python/cm/migrations/0027_auto_20190809_1134.py diff --git a/cm/migrations/0028_auto_20190813_1005.py b/python/cm/migrations/0028_auto_20190813_1005.py similarity index 100% rename from cm/migrations/0028_auto_20190813_1005.py rename to python/cm/migrations/0028_auto_20190813_1005.py diff --git a/cm/migrations/0029_auto_20190814_1306.py b/python/cm/migrations/0029_auto_20190814_1306.py similarity index 100% rename from cm/migrations/0029_auto_20190814_1306.py rename to python/cm/migrations/0029_auto_20190814_1306.py diff --git a/cm/migrations/0030_auto_20190820_1600.py b/python/cm/migrations/0030_auto_20190820_1600.py similarity index 100% rename from cm/migrations/0030_auto_20190820_1600.py rename to python/cm/migrations/0030_auto_20190820_1600.py diff --git a/cm/migrations/0031_auto_20190926_1600.py b/python/cm/migrations/0031_auto_20190926_1600.py similarity index 100% rename from cm/migrations/0031_auto_20190926_1600.py rename to python/cm/migrations/0031_auto_20190926_1600.py diff --git a/cm/migrations/0032_auto_20191015_1600.py b/python/cm/migrations/0032_auto_20191015_1600.py similarity index 100% rename from cm/migrations/0032_auto_20191015_1600.py rename to python/cm/migrations/0032_auto_20191015_1600.py diff --git a/cm/migrations/0033_auto_20191023_1459.py b/python/cm/migrations/0033_auto_20191023_1459.py similarity index 100% rename from cm/migrations/0033_auto_20191023_1459.py rename to python/cm/migrations/0033_auto_20191023_1459.py diff --git a/cm/migrations/0034_auto_20191029_1041.py b/python/cm/migrations/0034_auto_20191029_1041.py similarity index 100% rename from cm/migrations/0034_auto_20191029_1041.py rename to python/cm/migrations/0034_auto_20191029_1041.py diff --git a/cm/migrations/0035_auto_20191031_1600.py b/python/cm/migrations/0035_auto_20191031_1600.py similarity index 100% rename from cm/migrations/0035_auto_20191031_1600.py rename to python/cm/migrations/0035_auto_20191031_1600.py diff --git a/cm/migrations/0036_auto_20191111_1109.py b/python/cm/migrations/0036_auto_20191111_1109.py similarity index 100% rename from cm/migrations/0036_auto_20191111_1109.py rename to python/cm/migrations/0036_auto_20191111_1109.py diff --git a/cm/migrations/0037_auto_20191120_1600.py b/python/cm/migrations/0037_auto_20191120_1600.py similarity index 100% rename from cm/migrations/0037_auto_20191120_1600.py rename to python/cm/migrations/0037_auto_20191120_1600.py diff --git a/cm/migrations/0038_auto_20191119_1200.py b/python/cm/migrations/0038_auto_20191119_1200.py similarity index 100% rename from cm/migrations/0038_auto_20191119_1200.py rename to python/cm/migrations/0038_auto_20191119_1200.py diff --git a/cm/migrations/0039_auto_20191203_0909.py b/python/cm/migrations/0039_auto_20191203_0909.py similarity index 100% rename from cm/migrations/0039_auto_20191203_0909.py rename to python/cm/migrations/0039_auto_20191203_0909.py diff --git a/cm/migrations/0040_auto_20191218_1358.py b/python/cm/migrations/0040_auto_20191218_1358.py similarity index 100% rename from cm/migrations/0040_auto_20191218_1358.py rename to python/cm/migrations/0040_auto_20191218_1358.py diff --git a/cm/migrations/0041_auto_20191220_1338.py b/python/cm/migrations/0041_auto_20191220_1338.py similarity index 100% rename from cm/migrations/0041_auto_20191220_1338.py rename to python/cm/migrations/0041_auto_20191220_1338.py diff --git a/cm/migrations/0043_auto_20200109_1600.py b/python/cm/migrations/0043_auto_20200109_1600.py similarity index 100% rename from cm/migrations/0043_auto_20200109_1600.py rename to python/cm/migrations/0043_auto_20200109_1600.py diff --git a/cm/migrations/0044_auto_20200115_1058.py b/python/cm/migrations/0044_auto_20200115_1058.py similarity index 100% rename from cm/migrations/0044_auto_20200115_1058.py rename to python/cm/migrations/0044_auto_20200115_1058.py diff --git a/cm/migrations/0045_auto_20200117_1253.py b/python/cm/migrations/0045_auto_20200117_1253.py similarity index 100% rename from cm/migrations/0045_auto_20200117_1253.py rename to python/cm/migrations/0045_auto_20200117_1253.py diff --git a/cm/migrations/0046_auto_20200120_1417.py b/python/cm/migrations/0046_auto_20200120_1417.py similarity index 100% rename from cm/migrations/0046_auto_20200120_1417.py rename to python/cm/migrations/0046_auto_20200120_1417.py diff --git a/cm/migrations/__init__.py b/python/cm/migrations/__init__.py similarity index 100% rename from cm/migrations/__init__.py rename to python/cm/migrations/__init__.py diff --git a/cm/models.py b/python/cm/models.py similarity index 100% rename from cm/models.py rename to python/cm/models.py diff --git a/cm/stack.py b/python/cm/stack.py similarity index 100% rename from cm/stack.py rename to python/cm/stack.py diff --git a/cm/static/adcm.css b/python/cm/static/adcm.css similarity index 100% rename from cm/static/adcm.css rename to python/cm/static/adcm.css diff --git a/cm/static/bootstrap.min.css b/python/cm/static/bootstrap.min.css similarity index 100% rename from cm/static/bootstrap.min.css rename to python/cm/static/bootstrap.min.css diff --git a/cm/static/starter-template.css b/python/cm/static/starter-template.css similarity index 100% rename from cm/static/starter-template.css rename to python/cm/static/starter-template.css diff --git a/cm/status_api.py b/python/cm/status_api.py similarity index 100% rename from cm/status_api.py rename to python/cm/status_api.py diff --git a/cm/templates/base.html b/python/cm/templates/base.html similarity index 100% rename from cm/templates/base.html rename to python/cm/templates/base.html diff --git a/cm/templates/error.html b/python/cm/templates/error.html similarity index 100% rename from cm/templates/error.html rename to python/cm/templates/error.html diff --git a/cm/templates/index.html b/python/cm/templates/index.html similarity index 100% rename from cm/templates/index.html rename to python/cm/templates/index.html diff --git a/cm/templates/token.html b/python/cm/templates/token.html similarity index 100% rename from cm/templates/token.html rename to python/cm/templates/token.html diff --git a/cm/templates/ws.html b/python/cm/templates/ws.html similarity index 100% rename from cm/templates/ws.html rename to python/cm/templates/ws.html diff --git a/cm/tests.py b/python/cm/tests.py similarity index 100% rename from cm/tests.py rename to python/cm/tests.py diff --git a/cm/tests_adcm_config.py b/python/cm/tests_adcm_config.py similarity index 100% rename from cm/tests_adcm_config.py rename to python/cm/tests_adcm_config.py diff --git a/cm/tests_api.py b/python/cm/tests_api.py similarity index 100% rename from cm/tests_api.py rename to python/cm/tests_api.py diff --git a/cm/tests_inventory.py b/python/cm/tests_inventory.py similarity index 96% rename from cm/tests_inventory.py rename to python/cm/tests_inventory.py index 582f3655c6..8fec38ad39 100644 --- a/cm/tests_inventory.py +++ b/python/cm/tests_inventory.py @@ -174,9 +174,6 @@ def test_get_host(self, mock_get_hosts, mock_get_provider_hosts): groups = cm.inventory.get_host(host.id) test_groups = { 'HOST': { - 'hosts': [] - }, - 'PROVIDER': { 'hosts': [], 'vars': { 'provider': { @@ -190,7 +187,6 @@ def test_get_host(self, mock_get_hosts, mock_get_provider_hosts): } self.assertDictEqual(groups, test_groups) mock_get_hosts.assert_called_once_with([host]) - mock_get_provider_hosts.assert_called_once_with(host.provider.id) @patch('json.dump') @patch('cm.inventory.open') @@ -233,13 +229,6 @@ def test_prepare_job_inventory(self, mock_open, mock_dump): 'all': { 'children': { 'HOST': { - 'hosts': { - '': { - 'adcm_hostid': 1 - } - } - }, - 'PROVIDER': { 'hosts': { '': { 'adcm_hostid': 1 diff --git a/cm/tests_job.py b/python/cm/tests_job.py similarity index 100% rename from cm/tests_job.py rename to python/cm/tests_job.py diff --git a/cm/tests_runner.py b/python/cm/tests_runner.py similarity index 99% rename from cm/tests_runner.py rename to python/cm/tests_runner.py index 309e0e96ae..b05180921f 100644 --- a/cm/tests_runner.py +++ b/python/cm/tests_runner.py @@ -86,7 +86,7 @@ def test_run_job(self, mock_subprocess_popen): mock_subprocess_popen.return_value = process_mock code = task_runner.run_job(1, 1, '', '') cmd = [ - '{}/job_runner.py'.format(config.BASE_DIR), + '{}/job_runner.py'.format(config.CODE_DIR), str(1) ] mock_subprocess_popen.assert_called_once_with(cmd, stdout='', stderr='') diff --git a/cm/upgrade.py b/python/cm/upgrade.py similarity index 89% rename from cm/upgrade.py rename to python/cm/upgrade.py index a2a3d80ec9..af9313a3bf 100644 --- a/cm/upgrade.py +++ b/python/cm/upgrade.py @@ -10,17 +10,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import functools +import json -from version_utils import rpm from django.db import transaction +from version_utils import rpm -import cm.status_api +import cm.issue import cm.config as config -from cm.logger import log # pylint: disable=unused-import +import cm.status_api from cm.adcm_config import proto_ref, obj_ref, switch_config from cm.errors import raise_AdcmEx as err +from cm.logger import log # pylint: disable=unused-import from cm.models import Prototype, Component, Host, HostComponent, ServiceComponent from cm.models import PrototypeImport, ClusterBind, ClusterObject, Upgrade @@ -78,46 +79,46 @@ def check_upgrade_version(obj, upgrade): if upgrade.min_strict: if rpm.compare_versions(proto.version, upgrade.min_version) <= 0: msg = '{} version {} is less than or equal to upgrade min version {}' - return (False, msg.format(proto.type, proto.version, upgrade.min_version)) + return False, msg.format(proto.type, proto.version, upgrade.min_version) else: if rpm.compare_versions(proto.version, upgrade.min_version) < 0: msg = '{} version {} is less than upgrade min version {}' - return (False, msg.format(proto.type, proto.version, upgrade.min_version)) + return False, msg.format(proto.type, proto.version, upgrade.min_version) if upgrade.max_strict: if rpm.compare_versions(proto.version, upgrade.max_version) >= 0: msg = '{} version {} is more than or equal to upgrade max version {}' - return (False, msg.format(proto.type, proto.version, upgrade.max_version)) + return False, msg.format(proto.type, proto.version, upgrade.max_version) else: if rpm.compare_versions(proto.version, upgrade.max_version) > 0: msg = '{} version {} is more than upgrade max version {}' - return (False, msg.format(proto.type, proto.version, upgrade.max_version)) - return (True, '') + return False, msg.format(proto.type, proto.version, upgrade.max_version) + return True, '' def check_upgrade_edition(obj, upgrade): if not upgrade.from_edition: - return (True, '') + return True, '' from_edition = json.loads(upgrade.from_edition) if obj.prototype.bundle.edition not in from_edition: msg = 'bundle edition "{}" is not in upgrade list: {}' - return (False, msg.format(obj.prototype.bundle.edition, from_edition)) - return (True, '') + return False, msg.format(obj.prototype.bundle.edition, from_edition) + return True, '' def check_upgrade_state(obj, upgrade): if obj.state == config.Job.LOCKED: - return (False, 'object is locked') + return False, 'object is locked' if upgrade.state_available: available = json.loads(upgrade.state_available) if obj.state in available: - return (True, '') + return True, '' elif available == 'any': - return (True, '') + return True, '' else: msg = '{} state "{}" is not in available states list: {}' - return (False, msg.format(obj.prototype.type, obj.state, available)) + return False, msg.format(obj.prototype.type, obj.state, available) else: - return (False, 'no available states') + return False, 'no available states' def check_upgrade_import(obj, upgrade): # pylint: disable=too-many-branches @@ -134,7 +135,7 @@ def get_import(cbind): # pylint: disable=redefined-outer-name return cbind.cluster if obj.prototype.type != 'cluster': - return (True, '') + return True, '' for cbind in ClusterBind.objects.filter(cluster=obj): export = get_export(cbind) @@ -145,12 +146,12 @@ def get_import(cbind): # pylint: disable=redefined-outer-name ) except Prototype.DoesNotExist: msg = 'Upgrade does not have new version of {} required for import' - return (False, msg.format(proto_ref(impr_obj.prototype))) + return False, msg.format(proto_ref(impr_obj.prototype)) try: pi = PrototypeImport.objects.get(prototype=proto, name=export.prototype.name) except PrototypeImport.DoesNotExist: msg = 'New version of {} does not have import "{}"' - return (False, msg.format(proto_ref(proto), export.prototype.name)) + return False, msg.format(proto_ref(proto), export.prototype.name) if not version_in(export.prototype.version, pi): msg = 'Import "{}" of {} versions ({}, {}) does not match export version: {} ({})' return (False, msg.format( @@ -166,7 +167,7 @@ def get_import(cbind): # pylint: disable=redefined-outer-name ) except Prototype.DoesNotExist: msg = 'Upgrade does not have new version of {} required for export' - return (False, msg.format(proto_ref(export))) + return False, msg.format(proto_ref(export.prototype)) import_obj = get_import(cbind) pi = PrototypeImport.objects.get(prototype=import_obj.prototype, name=export.prototype.name) if not version_in(proto.version, pi): @@ -175,13 +176,13 @@ def get_import(cbind): # pylint: disable=redefined-outer-name proto_ref(proto), pi.min_version, pi.max_version, obj_ref(import_obj) )) - return (True, '') + return True, '' def check_upgrade(obj, upgrade): issue = cm.issue.get_issue(obj) if not cm.issue.issue_to_bool(issue): - return (False, '{} has issue: {}'.format(obj_ref(obj), issue)) + return False, '{} has issue: {}'.format(obj_ref(obj), issue) check_list = [ check_upgrade_version, check_upgrade_edition, check_upgrade_state, check_upgrade_import @@ -189,8 +190,8 @@ def check_upgrade(obj, upgrade): for func in check_list: ok, msg = func(obj, upgrade) if not ok: - return (False, msg) - return (True, '') + return False, msg + return True, '' def switch_hc(obj, upgrade): diff --git a/cm/views.py b/python/cm/views.py similarity index 100% rename from cm/views.py rename to python/cm/views.py diff --git a/drf_docs.py b/python/drf_docs.py similarity index 100% rename from drf_docs.py rename to python/drf_docs.py diff --git a/init_db.py b/python/init_db.py similarity index 100% rename from init_db.py rename to python/init_db.py diff --git a/job_runner.py b/python/job_runner.py similarity index 100% rename from job_runner.py rename to python/job_runner.py diff --git a/manage.py b/python/manage.py similarity index 100% rename from manage.py rename to python/manage.py diff --git a/server.sh b/python/server.sh similarity index 100% rename from server.sh rename to python/server.sh diff --git a/task_runner.py b/python/task_runner.py similarity index 97% rename from task_runner.py rename to python/task_runner.py index 2d0804780f..8d1ea8dbc8 100755 --- a/task_runner.py +++ b/python/task_runner.py @@ -71,7 +71,7 @@ def terminate_task(signum, frame): def open_file(root, tag, task_id): - fname = "{}/{}-{}.txt".format(root, task_id, tag) + fname = os.path.join(root, f'{task_id}-{tag}.txt') f = open(fname, 'w') return f @@ -80,7 +80,7 @@ def run_job(task_id, job_id, out_file, err_file): log.debug("run job #%s of task #%s", job_id, task_id) try: proc = subprocess.Popen([ - '{}/job_runner.py'.format(config.BASE_DIR), + os.path.join(config.CODE_DIR, 'job_runner.py'), str(job_id) ], stdout=out_file, stderr=err_file) res = proc.wait() diff --git a/requirements.txt b/requirements.txt index 3c4b8f5e7f..2e504f9b3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # ansible==2.8.3 -git+https://github.com/arenadata/ansible.git@v2.8.7-p1 +git+https://github.com/arenadata/ansible.git@v2.8.8-p1 coreapi -django < 3.0 +django django-cors-headers django-filter django-rest-swagger @@ -9,7 +9,7 @@ djangorestframework social-auth-app-django git+git://github.com/arenadata/django-generate-secret-key.git jinja2 -markdown==2.6.11 # Due to bug in djangorestframework +markdown pyyaml toml uwsgi diff --git a/tests/base/run_test.sh b/tests/base/run_test.sh index 93568ab570..00daf2de8f 100755 --- a/tests/base/run_test.sh +++ b/tests/base/run_test.sh @@ -13,14 +13,17 @@ port=8040 -export DJANGO_SETTINGS_MODULE=tests.base.settings -export ADCM_STACK_DIR="tests/base" +# shellcheck disable=SC2086,SC2155,SC2046 +export PYTHONPATH="$(dirname `pwd`)" +export DJANGO_SETTINGS_MODULE=base.settings +export ADCM_STACK_DIR="../tests/base" +BASE_DIR="../../python" init_db() { rm -f db_test.db - ../../manage.py migrate - ../../init_db.py + ${BASE_DIR}/manage.py migrate + ${BASE_DIR}/init_db.py cp db_test.db fixture.db } @@ -28,14 +31,14 @@ init_db() { run_debug() { rm -f db_test.db cp fixture.db db_test.db - ../../manage.py runserver 8040 + ${BASE_DIR}/manage.py runserver 8040 } run_test() { rm -f db_test.db cp fixture.db db_test.db - cd ../.. || exit 1 + cd ${BASE_DIR} || exit 1 ./server.sh start "${port}" echo "tests/base/test_api.py ${1} ${2}" TEST_CASE="" @@ -48,9 +51,9 @@ run_test() { fi; done if [[ "$TEST_CASE" != '' ]]; then - ./tests/base/test_api.py "$TEST_CASE" + ../tests/base/test_api.py "$TEST_CASE" else - ./tests/base/test_api.py + ../tests/base/test_api.py fi; ./server.sh stop } diff --git a/tests/base/settings.py b/tests/base/settings.py index 67ce13ab87..a054180b24 100644 --- a/tests/base/settings.py +++ b/tests/base/settings.py @@ -22,9 +22,10 @@ """ import os +from os.path import dirname # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = dirname(dirname(dirname(os.path.abspath(__file__)))) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -107,7 +108,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'base/db_test.db'), + 'NAME': os.path.join(BASE_DIR, 'tests/base/db_test.db'), # 'ENGINE': 'django.db.backends.postgresql', # 'NAME': 'adcm', # 'HOST': 'localhost', diff --git a/tests/conftest.py b/tests/conftest.py index 2238eb49ed..e08d189778 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,4 +9,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import sys + + pytest_plugins = "adcm_pytest_plugin" + +# We have a number of calls from functional or ui_tests to cm module, +# so we need a way to extend PYTHONPATH at test time. +testdir = os.path.dirname(__file__) +rootdir = os.path.dirname(testdir) +pythondir = os.path.abspath(os.path.join(rootdir, 'python')) +sys.path.append(pythondir) diff --git a/tests/functional/test_full_upgrade.py b/tests/functional/test_full_upgrade.py new file mode 100644 index 0000000000..ac33ea405c --- /dev/null +++ b/tests/functional/test_full_upgrade.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=W0611, W0621 +from adcm_client.objects import ADCMClient +from adcm_pytest_plugin.utils import get_data_dir + + +def test_full_upgrade_hostprovider_first(sdk_client_fs: ADCMClient): + """Create cluster and hostprovider with host and components + and upgrade cluster and host with provider after that + and check that all was upgraded. + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'cluster')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_cluster')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="zookeeper") + comp = service.component(name='master') + hp_bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'hostprovider')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_hostprovider')) + hostprovider = hp_bundle.provider_create("test") + host = hostprovider.host_create(fqdn="localhost") + cluster.host_add(host) + cluster.hostcomponent_set((host, comp)) + upgr_hp = hostprovider.upgrade(name='upgrade to 2.0') + upgr_hp.do() + upgr_cl = cluster.upgrade(name='upgrade to 1.6') + upgr_cl.do() + cluster.reread() + service.reread() + hostprovider.reread() + host.reread() + assert cluster.prototype().version == '1.6' + assert service.prototype().version == '3.4.11' + assert hostprovider.prototype().version == '2.0' + assert host.prototype().version == '00.10' + + +def test_full_upgrade_cluster_first(sdk_client_fs: ADCMClient): + """Create cluster and hostprovider with host and components + and upgrade cluster and host with provider after that + and check that all was upgraded. + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'cluster')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_cluster')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="zookeeper") + comp = service.component(name='master') + hp_bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'hostprovider')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_hostprovider')) + hostprovider = hp_bundle.provider_create("test") + host = hostprovider.host_create(fqdn="localhost") + cluster.host_add(host) + cluster.hostcomponent_set((host, comp)) + upgr_cl = cluster.upgrade(name='upgrade to 1.6') + upgr_cl.do() + upgr_hp = hostprovider.upgrade(name='upgrade to 2.0') + upgr_hp.do() + cluster.reread() + service.reread() + hostprovider.reread() + host.reread() + assert cluster.prototype().version == '1.6' + assert service.prototype().version == '3.4.11' + assert hostprovider.prototype().version == '2.0' + assert host.prototype().version == '00.10' diff --git a/tests/functional/test_full_upgrade_data/cluster/config.yaml b/tests/functional/test_full_upgrade_data/cluster/config.yaml new file mode 100644 index 0000000000..b3c21b9c0c --- /dev/null +++ b/tests/functional/test_full_upgrade_data/cluster/config.yaml @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.5 + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + + +- type: service + name: zookeeper + version: '3.4.10' + config: + required_service: + type: integer + required: true + default: 10 + + int_key_service: + type: integer + required: false + default: 150 + + components: + master: + display_name: "Master Node" + description: "This node control all data nodes (see below)" + constraint: [1,2] diff --git a/tests/functional/test_full_upgrade_data/hostprovider/config.yaml b/tests/functional/test_full_upgrade_data/hostprovider/config.yaml new file mode 100644 index 0000000000..c508d55bf7 --- /dev/null +++ b/tests/functional/test_full_upgrade_data/hostprovider/config.yaml @@ -0,0 +1,57 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- + + type: provider + name: sample hostprovider + version: 1.0 + config: + required: + type: integer + required: yes + default: 400 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: NO + default: 60 + + fkey: + type: float + required: false + default: 1.5 + + bool: + type: boolean + required : no + default: false + +- type: host + name: vHost + version: '00.09' + + config: + str_param: + type: string + required: false + default: '123' + description: must be changed to bar() + int: + type: integer + required: false + default: 2 + description: must be changed to 5 diff --git a/tests/functional/test_full_upgrade_data/upgradable_cluster/config.yaml b/tests/functional/test_full_upgrade_data/upgradable_cluster/config.yaml new file mode 100644 index 0000000000..07eb615038 --- /dev/null +++ b/tests/functional/test_full_upgrade_data/upgradable_cluster/config.yaml @@ -0,0 +1,65 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + +- type: service + name: zookeeper + version: '3.4.11' + config: + required_service: + type: integer + required: true + default: 10 + + int_key_service: + type: integer + required: false + default: 150 + components: + master: + display_name: "Master Node" + description: "This node control all data nodes (see below)" + constraint: [1,2] diff --git a/tests/functional/test_full_upgrade_data/upgradable_hostprovider/config.yaml b/tests/functional/test_full_upgrade_data/upgradable_hostprovider/config.yaml new file mode 100644 index 0000000000..28dccb41c5 --- /dev/null +++ b/tests/functional/test_full_upgrade_data/upgradable_hostprovider/config.yaml @@ -0,0 +1,78 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- + + type: provider + name: sample hostprovider + version: 2.0 + upgrade: + - + versions: + min: 0.4 + max: 1.9 + description: New cool upgrade + name: upgrade to 2.0 + states: + available: any + on_success: started + - + versions: + min: 1.0 + max: 2.9 + description: Super new upgrade + name: upgrade 2 + states: + available: [started] + on_success: ver2.4 + config: + required: + type: integer + required: yes + default: 400 + max: 500 + min: 200 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: NO + default: 60 + + fkey: + type: float + required: false + default: 1.5 + + bool: + type: boolean + required : no + default: false + +- type: host + name: vHost + version: '00.10' + + config: + str_param: + type: string + required: false + default: '123' + description: must be changed to bar() + int: + type: integer + required: false + default: 2 + description: must be changed to 5 diff --git a/tests/functional/test_upgrade_cluster.py b/tests/functional/test_upgrade_cluster.py index 6dbd16826c..3edad5f202 100644 --- a/tests/functional/test_upgrade_cluster.py +++ b/tests/functional/test_upgrade_cluster.py @@ -19,6 +19,54 @@ from tests.library.errorcodes import UPGRADE_ERROR +def test_upgrade_with_two_clusters(sdk_client_fs: ADCMClient): + """Upgrade cluster when we have two created clusters from one bundle + Scenario: + 1. Create two clusters from one bundle + 2. Upload upgradable bundle + 3. Upgrade first cluster + 4. Check that only first cluster was upgraded + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'cluster')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_cluster')) + cluster_first = bundle.cluster_create("test") + cluster_second = bundle.cluster_create("test2") + service = cluster_first.service_add(name="zookeeper") + upgr_cl = cluster_first.upgrade(name='upgrade to 1.6') + upgr_cl.do() + cluster_first.reread() + service.reread() + cluster_second.reread() + assert cluster_first.prototype().version == '1.6' + assert service.prototype().version == '3.4.11' + assert cluster_second.prototype().version == '1.5' + + +def test_check_prototype(sdk_client_fs: ADCMClient): + """Check prototype for service and cluster after upgrade + :param sdk_client_fs: + :return: + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'cluster')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_cluster')) + cluster = bundle.cluster_create("test") + cl_id_before = cluster.id + service = cluster.service_add(name="zookeeper") + serv_id_before = service.id + cluster_proto_before = cluster.prototype() + service_proto_before = service.prototype() + upgr = cluster.upgrade(name='upgrade to 1.6') + upgr.do() + cluster.reread() + service.reread() + cluster_proto_after = cluster.prototype() + service_proto_after = service.prototype() + assert cl_id_before == cluster.id + assert serv_id_before == service.id + assert cluster_proto_before.id != cluster_proto_after.id + assert service_proto_before.id != service_proto_after.id + + def test_check_config(sdk_client_fs: ADCMClient): """Check default service and cluster config fields after upgrade :return: diff --git a/tests/functional/test_upgrade_cluster_data/cluster/config.yaml b/tests/functional/test_upgrade_cluster_data/cluster/config.yaml index f72ffb2b2b..c6bbd38dd2 100644 --- a/tests/functional/test_upgrade_cluster_data/cluster/config.yaml +++ b/tests/functional/test_upgrade_cluster_data/cluster/config.yaml @@ -41,4 +41,4 @@ int_key_service: type: integer required: false - default: 150 \ No newline at end of file + default: 150 diff --git a/tests/functional/test_upgrade_cluster_imports.py b/tests/functional/test_upgrade_cluster_imports.py new file mode 100644 index 0000000000..ccdd60572b --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports.py @@ -0,0 +1,181 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=W0611, W0621 +import coreapi +import pytest + + +from adcm_client.objects import ADCMClient +from adcm_pytest_plugin.utils import get_data_dir, parametrize_by_data_subdirs +from tests.library import errorcodes as err + + +def test_upgrade_cluster_with_import(sdk_client_fs: ADCMClient): + """Scenario: + 1. Create cluster for upgrade with exports + 2. Create upgradable cluster with imports + 3. Bind service and cluster + 4. Upgrade cluster + 5. Check that cluster was upgraded + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir( + __file__, 'upgrade_cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(get_data_dir( + __file__, 'upgradable_cluster_with_import')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_config_before = cluster.config() + service_config_before = service.config() + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster.upgrade(name='upgrade to 1.6') + upgr.do() + cluster.reread() + service.reread() + cluster_config_after = cluster.config() + service_config_after = service.config() + assert cluster.prototype().version == '1.6' + assert service.prototype().version == '2.2' + for variable in cluster_config_before: + assert cluster_config_before[variable] == cluster_config_after[variable] + for variable in service_config_before: + assert service_config_before[variable] == service_config_after[variable] + + +def test_upgrade_cluster_with_export(sdk_client_fs: ADCMClient): + """Scenario: + 1. Create cluster for upgrade with export + 2. Create cluster for upgrade with import + 3. Load upgradable bundle with import + 4. Bind service and cluster + 5. Upgrade cluster with import + 6. Check that cluster was upgraded + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(get_data_dir( + __file__, 'upgrade_cluster_with_import')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_cluster_with_import')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster_import.upgrade(name='upgrade to 1.6') + id_before = cluster_import.prototype_id + upgr.do() + cluster_import.reread() + assert cluster_import.prototype().version == '1.6' + assert cluster_import.prototype_id != id_before + + +@parametrize_by_data_subdirs(__file__, "upgradable_cluster_with_strict_incorrect_version") +def test_incorrect_import_strict_version(sdk_client_fs: ADCMClient, path): + """Upgrade cluster with service incorrect strict version + Scenario: + 1. Create cluster for upgrade with exports + 2. Create upgradable cluster with import and incorrect strict version + 3. Create service + 4. Import service from cluster with export to cluster from step 2 (with import) + 4. Upgrade cluster from step 1 + 5. Check that cluster was not upgraded because incorrect version for service + in cluster with import + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgrade_cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(path) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster.upgrade(name='upgrade to 1.6') + with pytest.raises(coreapi.exceptions.ErrorMessage) as e: + upgr.do() + err.UPGRADE_ERROR.equal(e) + + +@parametrize_by_data_subdirs(__file__, "upgradable_cluster_with_incorrect_version") +def test_incorrect_import_version(sdk_client_fs: ADCMClient, path): + """Upgrade cluster with service incorrect version + Scenario: + 1. Create cluster for upgrade with exports + 2. Create upgradable cluster with import and incorrect version + 3. Create service + 4. Import service from cluster with export to cluster from step 2 (with import) + 4. Upgrade cluster from step 1 + 5. Check that cluster was not upgraded because incorrect version for service + in cluster with import + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgrade_cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(path) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster.upgrade(name='upgrade to 1.6') + with pytest.raises(coreapi.exceptions.ErrorMessage) as e: + upgr.do() + err.UPGRADE_ERROR.equal(e) + + +def test_upgrade_cluster_without_service_config_in_import(sdk_client_fs: ADCMClient): + """Upgrade cluster with service when in new cluster when + we haven't some service configuration variables + Scenario: + 1. Create cluster for upgrade with export + 2. Create upgradable cluster with import and without config in import + 3. Bind service from cluster with export to cluster with import + 4. Upgrade cluster with export + 5. Check upgrade error + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgrade_cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(get_data_dir( + __file__, 'upgradable_cluster_without_service')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster.upgrade(name='upgrade to 1.6') + with pytest.raises(coreapi.exceptions.ErrorMessage) as e: + upgr.do() + err.UPGRADE_ERROR.equal(e) + + +def test_upgrade_cluster_with_new_configuration_variables(sdk_client_fs: ADCMClient): + """Upgrade to cluster with new configuration variables + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgrade_cluster_with_export')) + bundle_import = sdk_client_fs.upload_from_fs(get_data_dir( + __file__, 'upgradable_cluster_with_import_new_config_vars')) + cluster = bundle.cluster_create("test") + service = cluster.service_add(name="hadoop") + cluster_config_before = cluster.config() + service_config_before = service.config() + cluster_import = bundle_import.cluster_create("cluster_import") + cluster_import.bind(service) + cluster_import.bind(cluster) + upgr = cluster.upgrade(name='upgrade to 1.6') + upgr.do() + cluster.reread() + service.reread() + cluster_config_after = cluster.config() + service_config_after = service.config() + assert cluster.prototype().version == '1.6' + assert service.prototype().version == '2.2' + assert len(cluster_config_after) == 4, cluster_config_after + assert len(service_config_after) == 3, service_config_after + for variable in cluster_config_before: + assert cluster_config_before[variable] == cluster_config_after[variable] + for variable in service_config_before: + assert service_config_before[variable] == service_config_after[variable] diff --git a/tests/functional/test_upgrade_cluster_imports_data/cluster_with_export/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/cluster_with_export/config.yaml new file mode 100644 index 0000000000..912072acdc --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/cluster_with_export/config.yaml @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.0 + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + + export: + - required + - str-key + - int_key + +- type: service + name: hadoop + version: 2.1 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 + + export: + - core-site + - quorum diff --git a/tests/functional/test_upgrade_cluster_imports_data/config_generator_import.py b/tests/functional/test_upgrade_cluster_imports_data/config_generator_import.py new file mode 100644 index 0000000000..a35bb8a34d --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/config_generator_import.py @@ -0,0 +1,114 @@ +import os + +SERVICE_VERSIONS = (('service-less', '2.3', '2.4'), ("service-greater", '1', '2')) + +CLUSTER_VERSIONS = (('cluster-less', '1.7', '2.4'), ("cluster-greater", '0.5', '0.9')) +TEMPLATE_SERVICE = """ +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: {0} + max: {1} + ADH: + versions: + min: 0.1 + max: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 +""" + +TEMPLATE_CLUSTER = """ +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 1.5 + max: 2.5 + ADH: + versions: + min: {0} + max: {1} + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 +""" + +for t in SERVICE_VERSIONS: + d_name = "upgradable_cluster_with_incorrect_version/{}".format(t[0]) + os.makedirs(d_name) + with open("{}/config.yaml".format(d_name), "w+") as f: + f.write(TEMPLATE_SERVICE.format(t[1], t[2])) + +for t in CLUSTER_VERSIONS: + d_name = "upgradable_cluster_with_incorrect_version/{}".format(t[0]) + os.makedirs(d_name) + with open("{}/config.yaml".format(d_name), "w+") as f: + f.write(TEMPLATE_CLUSTER.format(t[1], t[2])) diff --git a/tests/functional/test_upgrade_cluster_imports_data/config_generator_strict_import.py b/tests/functional/test_upgrade_cluster_imports_data/config_generator_strict_import.py new file mode 100644 index 0000000000..625be2332b --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/config_generator_strict_import.py @@ -0,0 +1,122 @@ +import os + +SERVICE_VERSIONS = (("service-less-equal", "2.2", "3.0"), + ("service-greater-equal", "1.5", "2.2"), + ("service-equal", "2.2", '2.2'), + ('service-less', '2.3', '2.4'), + ("service-greater", '1', '2')) + +CLUSTER_VERSIONS = (("cluster-less-equal", "1.6", "2.0"), + ("cluster-greater-equal", "0.9", "1.6"), + ("cluster-equal", "1.6", '1.6'), + ('cluster-less', '1.7', '2.4'), + ("cluster-greater", '0.5', '0.9')) +TEMPLATE_SERVICE = """ +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: {0} + max_strict: {1} + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 +""" + +TEMPLATE_CLUSTER = """ +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: {0} + max_strict: {1} + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 +""" + +for t in SERVICE_VERSIONS: + d_name = "upgradable_cluster_with_strict_incorrect_version/{}".format(t[0]) + os.makedirs(d_name) + with open("{}/config.yaml".format(d_name), "w+") as f: + f.write(TEMPLATE_SERVICE.format(t[1], t[2])) + +for t in CLUSTER_VERSIONS: + d_name = "upgradable_cluster_with_strict_incorrect_version/{}".format(t[0]) + os.makedirs(d_name) + with open("{}/config.yaml".format(d_name), "w+") as f: + f.write(TEMPLATE_CLUSTER.format(t[1], t[2])) diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import/config.yaml new file mode 100644 index 0000000000..e5e70867b5 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import/config.yaml @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + import: + hadoop: + versions: + min: 1.8 + max: 2.5 + ADH: + versions: + min: 1.0 + max: 2.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import_new_config_vars/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import_new_config_vars/config.yaml new file mode 100644 index 0000000000..fe36260806 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_import_new_config_vars/config.yaml @@ -0,0 +1,79 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + str-key2: + default: value + type: string + required: false + import: + hadoop: + versions: + min: 1.8 + max: 2.5 + ADH: + versions: + min: 1.0 + max: 2.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 + serv_int_key: + type: integer + required: false + default: 150 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-greater/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-greater/config.yaml new file mode 100644 index 0000000000..31d21f6a0d --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-greater/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 1.5 + max: 2.5 + ADH: + versions: + min: 0.5 + max: 0.9 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-less/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-less/config.yaml new file mode 100644 index 0000000000..e942b949b9 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/cluster-less/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 1.5 + max: 2.5 + ADH: + versions: + min: 1.7 + max: 2.4 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-greater/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-greater/config.yaml new file mode 100644 index 0000000000..cc145b14b9 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-greater/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 1 + max: 2 + ADH: + versions: + min: 0.1 + max: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-less/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-less/config.yaml new file mode 100644 index 0000000000..ff485db112 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_incorrect_version/service-less/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 2.3 + max: 2.4 + ADH: + versions: + min: 0.1 + max: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-equal/config.yaml new file mode 100644 index 0000000000..a3a010746e --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: 1.6 + max_strict: 1.6 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater-equal/config.yaml new file mode 100644 index 0000000000..eda0de0fa6 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: 0.9 + max_strict: 1.6 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater/config.yaml new file mode 100644 index 0000000000..3001ec1938 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-greater/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: 0.5 + max_strict: 0.9 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less-equal/config.yaml new file mode 100644 index 0000000000..fcef1a88c8 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: 1.6 + max_strict: 2.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less/config.yaml new file mode 100644 index 0000000000..26f5937b4f --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/cluster-less/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.5 + ADH: + versions: + min_strict: 1.7 + max_strict: 2.4 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-equal/config.yaml new file mode 100644 index 0000000000..4449c6e97b --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 2.2 + max_strict: 2.2 + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater-equal/config.yaml new file mode 100644 index 0000000000..8b98108150 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1.5 + max_strict: 2.2 + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater/config.yaml new file mode 100644 index 0000000000..093625f115 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-greater/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 1 + max_strict: 2 + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less-equal/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less-equal/config.yaml new file mode 100644 index 0000000000..b56423c875 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less-equal/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 2.2 + max_strict: 3.0 + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less/config.yaml new file mode 100644 index 0000000000..cba198a38f --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_with_strict_incorrect_version/service-less/config.yaml @@ -0,0 +1,47 @@ + +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min_strict: 2.3 + max_strict: 2.4 + ADH: + versions: + min_strict: 0.1 + max_strict: 4.0 + +- type: service + name: hadoop + version: 2.2 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_without_service/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_without_service/config.yaml new file mode 100644 index 0000000000..429a62000e --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgradable_cluster_without_service/config.yaml @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- + type: cluster + name: ADH + version: 1.6 + upgrade: + - versions: + min: 0.4 + max: 1.5 + name: upgrade to 1.6 + description: New cool upgrade + states: + available: any + on_success: upgradable + - versions: + min: 1.0 + max: 1.8 + description: Super new upgrade + name: upgrade 2 + states: + available: [created, installed, upgradable] + on_success: upgradated + import: + hadoop: + versions: + min: 1.8 + max: 2.5 + ADH: + versions: + min: 1.0 + max: 2.0 diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_export/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_export/config.yaml new file mode 100644 index 0000000000..41fe71b414 --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_export/config.yaml @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- type: service + name: hadoop + version: 2.1 + + config: + core-site: + param1: + type: string + required: false + param2: + type: integer + required: false + quorum: + type: integer + default: 3 + + export: + - core-site + - quorum + +- type: cluster + name: ADH + description: Arenadata Nothing + version: 1.0 + + config: + required: + type: integer + required: true + default: 15 + str-key: + default: value + type: string + required: false + + int_key: + type: integer + required: false + default: 150 + + export: + - required + - str-key + - int_key diff --git a/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_import/config.yaml b/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_import/config.yaml new file mode 100644 index 0000000000..107f50160b --- /dev/null +++ b/tests/functional/test_upgrade_cluster_imports_data/upgrade_cluster_with_import/config.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- type: cluster + name: ADH + description: Arenadata Nothing + version: 1.1 + + import: + hadoop: + versions: + min: 1.8 + max: 2.5 + ADH: + versions: + min: 1.0 + max: 2.0 diff --git a/tests/functional/test_upgrade_hostprovider.py b/tests/functional/test_upgrade_hostprovider.py index 6397621cc8..a74791800f 100644 --- a/tests/functional/test_upgrade_hostprovider.py +++ b/tests/functional/test_upgrade_hostprovider.py @@ -19,6 +19,89 @@ from tests.library.errorcodes import UPGRADE_ERROR +# pylint: disable=too-many-locals +def test_upgrade_with_two_hostproviders(sdk_client_fs: ADCMClient): + """Upgrade hostprovider when we have two created hostproviders with hosts from one bundle + Scenario: + 1. Create two hostproviders from one bundle + 2. Upload upgradable bundle + 3. Create host for each hostprovider + 4. Upgrade first hostprovider + 5. Check that only first hostprovider and hosts was upgraded + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'hostprovider')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_hostprovider')) + hostprovider_first = bundle.provider_create("hp_first") + hostprovider_first_proto_before = hostprovider_first.prototype() + hostprovider_first_id_before = hostprovider_first.id + + hostprovider_second = bundle.provider_create("hp_second") + hostprovider_second_proto_before = hostprovider_second.prototype() + hostprovider_second_id_before = hostprovider_second.id + hp1_host1 = hostprovider_first.host_create(fqdn="localhost") + hp1_host1_id_before = hp1_host1.id + hp1_host1_proto_before = hp1_host1.prototype() + hp1_host2 = hostprovider_first.host_create(fqdn="localhost2") + hp1_host3 = hostprovider_first.host_create(fqdn="localhost3") + hp2_host1 = hostprovider_second.host_create(fqdn="hp2-localhost") + hp2_host1_proto_before = hp2_host1.prototype() + hp2_host1_id_before = hp2_host1.id + hp2_host2 = hostprovider_second.host_create(fqdn="hp2-localhost2") + hp2_host3 = hostprovider_second.host_create(fqdn="hp2-localhost3") + upgr = hostprovider_first.upgrade(name='upgrade to 2.0') + upgr.do() + hostprovider_first.reread() + hostprovider_second.reread() + hp1_host1.reread() + hp1_host2.reread() + hp1_host3.reread() + hp2_host1.reread() + hp2_host2.reread() + hp2_host3.reread() + hp_first_proto_after = hostprovider_first.prototype() + hp1_host_proto_after = hp1_host1.prototype() + hp_second_proto_after = hostprovider_second.prototype() + hp2_host1_proto_after = hp2_host1.prototype() + assert hostprovider_first.prototype().version == '2.0' + assert hp1_host1.prototype().version == '00.10' + assert hostprovider_second.prototype().version == '1.0' + assert hp2_host1.prototype().version == '00.09' + assert hostprovider_first_id_before == hostprovider_first.id + assert hp1_host1_id_before == hp1_host1.id + assert hostprovider_first_proto_before.id != hp_first_proto_after.id + assert hp1_host1_proto_before.id != hp1_host_proto_after.id + + assert hostprovider_second_id_before == hostprovider_second.id + assert hp2_host1_id_before == hp2_host1.id + assert hostprovider_second_proto_before.id == hp_second_proto_after.id + assert hp2_host1_proto_before.id == hp2_host1_proto_after.id + + +def test_check_prototype(sdk_client_fs: ADCMClient): + """Check prototype for provider and host after upgrade + :param sdk_client_fs: + :return: + """ + bundle = sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'hostprovider')) + sdk_client_fs.upload_from_fs(get_data_dir(__file__, 'upgradable_hostprovider')) + hostprovider = bundle.provider_create("test") + host = hostprovider.host_create(fqdn="localhost") + hostprovider_proto_before = hostprovider.prototype() + hp_id_before = hostprovider.id + host_proto_before = host.prototype() + ht_id_before = host.id + upgr = hostprovider.upgrade(name='upgrade to 2.0') + upgr.do() + hostprovider.reread() + host.reread() + hostprovider_proto_after = hostprovider.prototype() + host_proto_after = host.prototype() + assert hp_id_before == hostprovider.id + assert ht_id_before == host.id + assert hostprovider_proto_before.id != hostprovider_proto_after.id + assert host_proto_before.id != host_proto_after.id + + def test_check_config(sdk_client_fs: ADCMClient): """Check default host and hostprovider config fields after upgrade :return: @@ -36,6 +119,7 @@ def test_check_config(sdk_client_fs: ADCMClient): hostprovider_config_after = hostprovider.config() host_config_after = host.config() assert hostprovider.prototype().version == '2.0' + assert host.prototype().version == '00.10' for variable in hostprovider_config_before: assert hostprovider_config_before[variable] == hostprovider_config_after[variable] for variable in host_config_before: diff --git a/tests/functional/test_upgrade_hostprovider_data/hostprovider/config.yaml b/tests/functional/test_upgrade_hostprovider_data/hostprovider/config.yaml index 1bc01f2ec6..2fed33c993 100644 --- a/tests/functional/test_upgrade_hostprovider_data/hostprovider/config.yaml +++ b/tests/functional/test_upgrade_hostprovider_data/hostprovider/config.yaml @@ -14,7 +14,7 @@ type: provider name: sample hostprovider - version: 1.0 + version: '1.0' config: required: type: integer @@ -42,7 +42,7 @@ - type: host name: vHost - version: 00.09 + version: '00.09' config: str_param: @@ -54,4 +54,4 @@ type: integer required: false default: 2 - description: must be changed to 5 \ No newline at end of file + description: must be changed to 5 diff --git a/tests/functional/test_upgrade_hostprovider_data/upgradable_hostprovider/config.yaml b/tests/functional/test_upgrade_hostprovider_data/upgradable_hostprovider/config.yaml index 166af71e85..bee71b995f 100644 --- a/tests/functional/test_upgrade_hostprovider_data/upgradable_hostprovider/config.yaml +++ b/tests/functional/test_upgrade_hostprovider_data/upgradable_hostprovider/config.yaml @@ -14,7 +14,7 @@ type: provider name: sample hostprovider - version: 2.0 + version: '2.0' upgrade: - versions: @@ -63,7 +63,7 @@ - type: host name: vHost - version: 00.09 + version: '00.10' config: str_param: @@ -75,4 +75,4 @@ type: integer required: false default: 2 - description: must be changed to 5 \ No newline at end of file + description: must be changed to 5 diff --git a/web/.prettierrc b/web/.prettierrc index 33ab25c7ab..798083de4d 100644 --- a/web/.prettierrc +++ b/web/.prettierrc @@ -2,5 +2,5 @@ "tabWidth": 2, "useTabs": false, "singleQuote": true, - "printWidth": 160 + "printWidth": 180 } diff --git a/web/build_static.sh b/web/build_static.sh index 6ae1c7476b..3e1ba7b0a5 100755 --- a/web/build_static.sh +++ b/web/build_static.sh @@ -21,7 +21,7 @@ cd "${workdir}/../" # So we temporary make that dir to remove after operation. mkdir -p data/log -python manage.py collectstatic --noinput +python python/manage.py collectstatic --noinput cp ./wwwroot/static/rest_framework/css/* ./wwwroot/static/rest_framework/docs/css/ diff --git a/web/package.json b/web/package.json index d545762391..1674082ad1 100644 --- a/web/package.json +++ b/web/package.json @@ -39,7 +39,7 @@ "@ngrx/schematics": "^8.6.0", "@types/jasmine": "~3.5.0", "@types/jasminewd2": "~2.0.8", - "@types/node": "~13.1.5", + "@types/node": "~13.5.3", "codelyzer": "^5.2.1", "eslint": "^6.8.0", "jasmine-core": "~3.5.0", @@ -52,7 +52,7 @@ "karma-jasmine-html-reporter": "^1.5.1", "node-sass": "^4.13.0", "protractor": "~5.4.2", - "ts-node": "~8.5.4", + "ts-node": "~8.6.2", "tslint": "~5.20.1", "typescript": "3.5.3" } diff --git a/web/src/app/app.component.ts b/web/src/app/app.component.ts index 27977914f6..0ed7dcc5d6 100644 --- a/web/src/app/app.component.ts +++ b/web/src/app/app.component.ts @@ -13,23 +13,11 @@ import { Component, OnInit } from '@angular/core'; import { MatDialog } from '@angular/material/dialog'; import { MatSnackBar } from '@angular/material/snack-bar'; import { NavigationStart, Router } from '@angular/router'; -import { ConfigService, IConfig, Message, MessageService } from '@app/core'; -import { - getConnectStatus, - getFirstAdminLogin, - getMessage, - getRoot, - isAuthenticated, - loadProfile, - loadRoot, - loadStack, - rootError, - socketInit, - State -} from '@app/core/store'; +import { ConfigService, Message, MessageService } from '@app/core'; +import { getConnectStatus, getFirstAdminLogin, getMessage, getRoot, isAuthenticated, loadProfile, loadRoot, loadStack, rootError, socketInit, State } from '@app/core/store'; import { select, Store } from '@ngrx/store'; -import { combineLatest, Observable, of } from 'rxjs'; -import { filter, tap } from 'rxjs/operators'; +import { combineLatest } from 'rxjs'; +import { filter, tap, switchMap } from 'rxjs/operators'; @Component({ selector: 'app-root', @@ -44,9 +32,9 @@ import { filter, tap } from 'rxjs/operators';