Merge branch 'develop' into 'master'

Release 2022.12.06

See merge request arenadata/development/adcm!2301

.dockerignore

@@ -1,13 +1,6 @@
-*/.git*
-*
-!COPYRIGHT
-!LICENSE
-!assemble/app/build_venv.sh
-!conf
-!config.json
-!go/bin/runstatus
-!os
-!python
-!requirements*
-!web/build_static.sh
-!wwwroot
+**/*.git*
+**/.*
+data
+venv
+web
+Makefile

.gitlab-ci.yml

@@ -0,0 +1,17 @@
+include:
+  - project: "arenadata/infrastructure/code/ci/gitlab_ci_files"
+    ref: master
+    file: "/development/adcm.yml"
+
+
+Linters:
+  script:
+    - apk update && apk upgrade && apk add build-base linux-headers openssl libc6-compat openldap-dev python3-dev py3-pip
+    - pip install -r requirements-venv-2.9.txt
+    - pip install autoflake black flake8 isort pylint
+    - black --check license_checker.py python tests
+    - autoflake --check --quiet -r --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python tests
+    - isort --check license_checker.py python tests
+    - python3 license_checker.py --folders python go
+    - flake8 --max-line-length=120 tests/functional tests/ui_tests
+    - pylint --rcfile pyproject.toml --recursive y python tests

.pre-commit-config.yaml

@@ -6,6 +6,9 @@ repos:
         additional_dependencies: [ "click==8.0.4" ]
         language_version: python3
         files: "^python/"
+        args: [
+          "--config", "pyproject.toml",
+        ]
 
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1

Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.10-alpine
+RUN apk update && \
+    apk upgrade && \
+    apk add --virtual .build-deps \
+        build-base \
+        linux-headers && \
+    apk add \
+        bash \
+        openssl \
+        libc6-compat \
+        openldap-dev \
+        git \
+        runit \
+        nginx \
+        openssh-client \
+        logrotate
+COPY requirements*.txt /adcm/
+RUN pip install --upgrade pip &&  \
+    pip install --no-cache-dir -r /adcm/requirements-venv-default.txt && \
+    python -m venv /adcm/venv/2.9 && \
+    . /adcm/venv/2.9/bin/activate && \
+    pip install --no-cache-dir -r /adcm/requirements-venv-2.9.txt && \
+    deactivate && \
+    python -m venv /adcm/venv/default &&  \
+    . /adcm/venv/default/bin/activate && \
+    pip install --no-cache-dir -r /adcm/requirements-venv-default.txt && \
+    deactivate
+RUN apk del .build-deps
+COPY . /adcm
+RUN mkdir -p /adcm/data/log && \
+    mkdir -p /usr/share/ansible/plugins/modules && \
+    cp -r /adcm/os/* / && \
+    cp /adcm/os/etc/crontabs/root /var/spool/cron/crontabs/root && \
+    cp -r /adcm/python/ansible/* adcm/venv/default/lib/python3.10/site-packages/ansible/ && \
+    cp -r /adcm/python/ansible/* adcm/venv/2.9/lib/python3.10/site-packages/ansible/ && \
+    python /adcm/python/manage.py collectstatic --noinput && \
+    cp -r /adcm/wwwroot/static/rest_framework/css/* /adcm/wwwroot/static/rest_framework/docs/css/
+EXPOSE 8000
+CMD ["/etc/startup.sh"]

Makefile

@@ -1,99 +1,63 @@
-# Set number of threads
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-
-ADCMBASE_IMAGE ?= hub.arenadata.io/adcm/base
-ADCMTEST_IMAGE ?= hub.arenadata.io/adcm/test
-ADCMBASE_TAG ?= 20220929145118
 APP_IMAGE ?= hub.adsw.io/adcm/adcm
 APP_TAG ?= $(subst /,_,$(BRANCH_NAME))
-
 SELENOID_HOST ?= 10.92.2.65
 SELENOID_PORT ?= 4444
 
-
-# Default target
 .PHONY: help
 
-help: ## Shows that help
+help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-clean: ## Cleanup. Just a cleanup.
-	@docker run -i --rm  -v $(CURDIR):/code -w /code  busybox:latest /bin/sh -c "rm -rf /code/web/node_modules/ /code/web/package-lock.json /code/wwwroot /code/.version /code/go/bin /code/go/pkg /code/go/src/github.com"
-
-##################################################
-#                 B U I L D
-##################################################
-
-describe: ## Create .version file with output of describe
-	./gues_version.sh
+describe:
+	@echo '{"version": "$(shell date '+%Y.%m.%d.%H')","commit_id": "$(shell git log --pretty=format:'%h' -n 1)"}' > config.json
+	cp config.json web/src/assets/config.json
 
-buildss: ## Build status server
-	@docker run -i --rm -v $(CURDIR)/go:/code -w /code  golang:1.15-alpine3.13 sh -c "apk --update add make git && make && rm -f /code/adcm/go.sum"
+buildss:
+	@docker run -i --rm -v $(CURDIR)/go:/code -w /code golang sh -c "make"
 
-buildjs: ## Build client side js/html/css in directory wwwroot
+buildjs:
 	@docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:16-alpine ./build.sh
 
-build: describe buildss buildjs ## Build final docker image and all depended targets except baseimage.
-	@docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
-	@docker build --no-cache=true \
-	-f assemble/app/Dockerfile \
-	-t $(APP_IMAGE):$(APP_TAG) \
-	--build-arg  ADCMBASE_IMAGE=$(ADCMBASE_IMAGE) --build-arg  ADCMBASE_TAG=$(ADCMBASE_TAG) \
-	.
+build_base:
+	@docker build . -t $(APP_IMAGE):$(APP_TAG)
 
-##################################################
-#                 T E S T S
-##################################################
+build: describe buildss buildjs build_base
 
-testpyreqs: ## Install test prereqs into user's pip target dir
-	pip install --user -r requirements-test.txt
+unittests: build_base
+	docker run -e DJANGO_SETTINGS_MODULE=adcm.settings -i --rm -v $(CURDIR)/data:/adcm/data $(APP_IMAGE):$(APP_TAG) \
+	sh -c "pip install --no-cache -r /adcm/requirements.txt && /adcm/python/manage.py test /adcm/python -v 2"
 
-test_image:
-	docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
-
-unittests: test_image ## Run unittests
-		docker run -e DJANGO_SETTINGS_MODULE=adcm.settings -i --rm -v $(CURDIR)/python:/adcm/python -v $(CURDIR)/data:/adcm/data -v $(CURDIR)/requirements.txt:/adcm/requirements.txt -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /venv.sh reqs_and_run default /adcm/requirements.txt python python/manage.py test python -v 2
-
-pytest: ## Run functional tests
-	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster-x64
+pytest:
+	docker pull hub.adsw.io/library/functest:3.10.6.slim.buster-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host \
 	-v $(CURDIR)/:/adcm -w /adcm/ \
 	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
-	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	hub.adsw.io/library/functest:3.8.6.slim.buster-x64 /bin/sh -e \
-	./pytest.sh -m "not full and not extra_rbac and not ldap" \
-	--adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
+	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" -e ALLURE_TESTPLAN_PATH="${ALLURE_TESTPLAN_PATH}" \
+	hub.adsw.io/library/functest:3.10.6.slim.buster-x64 /bin/sh -e \
+	./pytest.sh ${PYTEST_MARK_KEY} ${PYTEST_MARK_VALUE} ${PYTEST_EXPRESSION_KEY} ${PYTEST_EXPRESSION_VALUE} \
+	--adcm-image="hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))" \
 
-pytest_release: ## Run functional tests on release
-	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster.firefox-x64
+pytest_release:
+	docker pull hub.adsw.io/library/functest:3.10.6.slim.buster.firefox-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host \
 	-v $(CURDIR)/:/adcm -v ${LDAP_CONF_FILE}:${LDAP_CONF_FILE} -w /adcm/ \
 	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
-	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	hub.adsw.io/library/functest:3.8.6.slim.buster.firefox-x64 /bin/sh -e \
-	./pytest.sh --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))' \
-	--ldap-conf ${LDAP_CONF_FILE}
+	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" -e ALLURE_TESTPLAN_PATH="${ALLURE_TESTPLAN_PATH}" \
+	hub.adsw.io/library/functest:3.10.6.slim.buster.firefox-x64 /bin/sh -e \
+	./pytest.sh --adcm-image="hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))" --ldap-conf ${LDAP_CONF_FILE} \
+	${PYTEST_MARK_KEY} ${PYTEST_MARK_VALUE} ${PYTEST_EXPRESSION_KEY} ${PYTEST_EXPRESSION_VALUE}
+
 
-ng_tests: ## Run Angular tests
+ng_tests:
 	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster_node16-x64
 	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web hub.adsw.io/library/functest:3.8.6.slim.buster_node16-x64 ./ng_test.sh
 
-linters: test_image ## Run linters
-	docker run -i --rm -e PYTHONPATH="/source/tests" -v $(CURDIR)/:/source -w /source $(ADCMTEST_IMAGE):$(ADCMBASE_TAG) \
-        /bin/sh -eol pipefail -c "/linters.sh shellcheck && \
-			/venv.sh run default pip install -U -r requirements.txt -r requirements-test.txt && \
-			/venv.sh run default pylint --rcfile pyproject.toml --recursive y python && \
-			/linters.sh -b ./tests -f ../tests pylint && \
-			/linters.sh -f ./tests black && \
-			/linters.sh -f ./tests/functional flake8_pytest_style && \
-			/linters.sh -f ./tests/ui_tests flake8_pytest_style"
-
-npm_check: ## Run npm-check
+npm_check:
 	docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:16-alpine ./npm_check.sh
 
-##################################################
-#                 U T I L S
-##################################################
-
-base_shell: ## Just mount a dir to base image and run bash on it over docker run
-	docker run -e DJANGO_SETTINGS_MODULE=adcm.settings -it --rm -v $(CURDIR)/python:/adcm/python -v $(CURDIR)/data:/adcm/data -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /bin/bash -l
+pretty:
+	black license_checker.py python tests
+	autoflake -r -i --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python tests
+	isort license_checker.py python tests
+	python license_checker.py --fix --folders python go

conf/adcm/config.yaml

@@ -2,7 +2,7 @@
 
   type: adcm
   name: ADCM
-  version: 2.2
+  version: 2.3
 
   actions:
     run_ldap_sync:
@@ -52,8 +52,6 @@
     - name: "yandex_oauth"
       display_name: "Yandex Auth"
       type: "group"
-      ui_options:
-        invisible: true
       subs:
         - name: "client_id"
           type: string
@@ -63,6 +61,8 @@
           required: false
           ui_options:
             no_confirm: true
+      ui_options:
+        invisible: true
     - name: "job_log"
       display_name: "Job Log"
       type: "group"