Merge branch 'release' into 'master'

Release

See merge request arenadata/development/adcm!2485

.gitignore

@@ -9,6 +9,7 @@
 .venv
 .version
 .vscode
+config.json
 /allure-results/
 /cluster.db
 /coverage.xml
@@ -27,4 +28,6 @@
 /tests/db_test.db
 /tests/fixture.db
 /web/node_modules/
+/web/yarn.lock
+/web/src/assets/config.json
 /wwwroot

.gitlab-ci.yml

@@ -2,16 +2,3 @@ include:
   - project: "arenadata/infrastructure/code/ci/gitlab_ci_files"
     ref: master
     file: "/development/adcm.yml"
-
-
-Linters:
-  script:
-    - apk update && apk upgrade && apk add build-base linux-headers openssl libc6-compat openldap-dev python3-dev py3-pip
-    - pip install -r requirements-venv-2.9.txt
-    - pip install autoflake black flake8 isort pylint
-    - black --check license_checker.py python tests
-    - autoflake --check --quiet -r --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python tests
-    - isort --check license_checker.py python tests
-    - python3 license_checker.py --folders python go
-    - flake8 --max-line-length=120 tests/functional tests/ui_tests
-    - pylint --rcfile pyproject.toml --recursive y python tests

.pre-commit-config.yaml

@@ -16,7 +16,6 @@ repos:
       - id: isort
         args: [
           "--profile", "black",
-          "--src-path", "python",
           "--skip-glob", "python/ansible/plugins",
           "--skip", "python/init_db.py",
           "--skip", "python/task_runner.py",

CODEOWNERS

@@ -0,0 +1,27 @@
+/conf/ @aas @k.shestakov @d.skrynnik @tsd
+/go/ @aas @aer @d.skrynnik
+/os/ @aas @k.shestakov @d.skrynnik @tsd
+/python/ @aas @k.shestakov @d.skrynnik @tsd
+/spec/ @aas @ke
+/tests/ @aas @aer @v.chudasov
+/web/ @v.remizov @d.bardin @k.fedorenko
+.dockerignore @aas @k.shestakov
+.gitignore @aas @k.shestakov
+.gitlab-ci.yaml @aas @k.shestakov
+.pre-commit-config.yaml @aas @k.shestakov
+CODEOWNERS @aas @v.remizov
+config.json @aas @k.shestakov
+COPYRIGHT @aas @ke
+docker-compose.yml @aas @k.shestakov
+Dockerfile @aas @k.shestakov
+init-user-db.sh @aas @k.shestakov
+LICENSE @aas @ke
+license_checker.py @aas @k.shestakov
+Makefile @aas @k.shestakov @aer
+pyproject.toml @aas @k.shestakov @aer
+pytest.sh @aas @k.shestakov @aer @v.chudasov
+README.md @aas @k.shestakov @aer @v.chudasov @d.skrynnik @tsd @ke @k.fedorenko
+requirements.txt @aas @k.shestakov
+requirements-test.txt @aas @k.shestakov @aer @v.chudasov
+requirements-venv-2.9.txt @aas @k.shestakov
+requirements-venv-default.txt @aas @k.shestakov

Dockerfile

@@ -6,20 +6,21 @@ RUN apk update && \
         linux-headers && \
     apk add \
         bash \
-        openssl \
+        curl \
+        git \
         libc6-compat \
         libffi \
-        libxslt \
         libstdc++ \
-        openldap-dev \
-        git \
-        runit \
+        libxslt \
+        logrotate \
+        musl-dev \
         nginx \
+        openldap-dev \
         openssh-client \
         openssh-keygen \
-        logrotate \
-        curl \
+        openssl \
         rsync \
+        runit \
         sshpass
 COPY requirements*.txt /adcm/
 RUN pip install --upgrade pip &&  \

Makefile

@@ -24,10 +24,19 @@ build_base:
 
 build: describe buildss buildjs build_base
 
-unittests: build_base
-	docker run -e DJANGO_SETTINGS_MODULE=adcm.settings -i --rm -v $(CURDIR)/data:/adcm/data $(APP_IMAGE):$(APP_TAG) \
+unittests_sqlite: describe
+	docker run -i --rm -v $(CURDIR)/data:/adcm/data -e DJANGO_SETTINGS_MODULE=adcm.settings $(APP_IMAGE):$(APP_TAG) \
 	sh -c "pip install --no-cache -r /adcm/requirements.txt && /adcm/python/manage.py test /adcm/python -v 2"
 
+unittests_postgresql: describe
+	docker network create adcm
+	docker run -d -e POSTGRES_PASSWORD="postgres" --network=adcm --name postgres postgres:14
+	docker run -i --rm --network=adcm -v $(CURDIR)/data:/adcm/data -e DJANGO_SETTINGS_MODULE=adcm.settings \
+	-e DB_HOST="postgres" -e DB_PORT=5432 -e DB_NAME="postgres" -e DB_USER="postgres" -e DB_PASS="postgres" \
+	$(APP_IMAGE):$(APP_TAG) \
+	sh -c "pip install --no-cache -r /adcm/requirements.txt && /adcm/python/manage.py test /adcm/python -v 2"
+	docker stop postgres && docker rm postgres && docker network rm adcm
+
 pytest:
 	docker pull hub.adsw.io/library/functest:3.10.6.slim.buster-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host \
@@ -61,3 +70,11 @@ pretty:
 	autoflake -r -i --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python tests
 	isort license_checker.py python tests
 	python license_checker.py --fix --folders python go
+
+lint:
+	black --check license_checker.py python tests
+	autoflake --check --quiet -r --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python tests
+	isort --check license_checker.py python tests
+	python license_checker.py --folders python go
+	flake8 --max-line-length=120 tests/functional tests/ui_tests
+	pylint --rcfile pyproject.toml --recursive y python tests

README.md

@@ -1,10 +1,6 @@
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/arenadata/adcm.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/arenadata/adcm/alerts/)
-[![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/arenadata/adcm.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/arenadata/adcm/context:javascript)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/arenadata/adcm.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/arenadata/adcm/context:python)
-
 # Arenadata Cluster Manager
 
-That is Arenadata Cluster Manager Project (aka Chapelnik)
+That is Arenadata Cluster Manager Project aka Chapelnik
 
 # Documentation
 
@@ -86,7 +82,7 @@ After this you will see invocation of black and pylint on every commit.
 
 ## Link ADWP_UI packages
 
-If you need to debug packages from ADWP_UI, you should do next:
+If you need to debug packages from ADWP_UI, you should do:
 
 In ADWP_UI repository:
 ```sh
@@ -102,4 +98,65 @@ cd web
 sudo rm -rf ./node_modules OR rmdir -force ./node_modules(WIN)
 yarn link "@adwp-ui/widgets"
 yarn install
-```
+```
+
+## Running ADCM using SQLite
+
+1. Start container:
+
+    ```shell
+    docker run -d --restart=always -p 8000:8000 -v /opt/adcm:/adcm/data --name adcm hub.arenadata.io/adcm/adcm:latest
+    ```
+
+    Use `-v /opt/adcm:/adcm/data:Z` for SELinux
+
+## Running ADCM using client PostgreSQL DB
+
+1. Start container:
+   ```shell
+   docker run -d --restart=always -p 8000:8000 -v /opt/adcm:/adcm/data 
+   -e DB_HOST="DATABASE_HOSTNAME_OR_IP_ADDRESS" -e DB_PORT="DATABASE_TCP_PORT" 
+   -e DB_USER="DATABASE_USERNAME" -e DB_NAME="DATABASE_NAME" 
+   -e DB_PASS="DATABASE_USER_PASSWORD" --name adcm hub.arenadata.io/adcm/adcm:latest
+   ```
+   Use `-v /opt/adcm:/adcm/data:Z` for SELinux
+   Target PostgreSQL DB must not have DB with name `DATABASE_NAME`
+
+## Migrate SQLite -> client PostgreSQL
+>__NOTE__: `adcm` is the ADCM's container name. 
+1. Dump SQLite DB to file:
+   ```shell
+   docker exec -it adcm /adcm/python/manage.py dumpdata --natural-foreign --natural-primary -o /adcm/data/var/data.json
+   ```
+2. Stop container:
+   ```shell
+   docker stop adcm
+   docker rm adcm
+   ```
+3. Start container in `MIGRATION_MODE`:
+   ```shell
+   docker run -d --restart=always -p 8000:8000 -v /opt/adcm:/adcm/data 
+   -e DB_HOST="DATABASE_HOSTNAME_OR_IP_ADDRESS" -e DB_PORT="DATABASE_TCP_PORT" 
+   -e DB_USER="DATABASE_USERNAME" -e DB_NAME="DATABASE_NAME" 
+   -e DB_PASS="DATABASE_USER_PASSWORD" -e MIGRATION_MODE=1
+   --name adcm hub.arenadata.io/adcm/adcm:latest
+   ```
+   Use `-v /opt/adcm:/adcm/data:Z` for SELinux
+   Target PostgreSQL DB must not have DB with name `DATABASE_NAME`
+4. Load dumped SQLite DB data to PostgreSQL
+   ```shell
+   docker exec -it adcm /adcm/python/manage.py loaddata /adcm/data/var/data.json
+   ```
+5. Stop container:
+   ```shell
+   docker stop adcm
+   docker rm adcm
+   ```
+6. Start container:
+   ```shell
+   docker run -d --restart=always -p 8000:8000 -v /opt/adcm:/adcm/data 
+   -e DB_HOST="DATABASE_HOSTNAME_OR_IP_ADDRESS" -e DB_PORT="DATABASE_TCP_PORT" 
+   -e DB_USER="DATABASE_USERNAME" -e DB_NAME="DATABASE_NAME" 
+   -e DB_PASS="DATABASE_USER_PASSWORD" -e MIGRATION_MODE=0
+   --name adcm hub.arenadata.io/adcm/adcm:latest
+   ```

conf/adcm/config.yaml

@@ -2,7 +2,7 @@
 
   type: adcm
   name: ADCM
-  version: 2.3
+  version: 2.4
 
   actions:
     run_ldap_sync:
@@ -107,6 +107,8 @@
       display_name: "Nginx Server Logrotate"
       type: "group"
       activatable: true
+      ui_options:
+        invisible: true
       subs:
         - name: size
           display_name: "Max file size"

config.json

@@ -1,4 +1 @@
-{
-	"version": "2021.05.12.14",
-	"commit_id": "20b2b8b3"
-}
+{"version": "2023.01.26.11","commit_id": "4c01e5bbd"}

go/Makefile

@@ -13,4 +13,4 @@ lint:
 	cd $(ADCM); golangci-lint run
 
 build:
-	cd $(ADCM); CGO_ENABLED=0 go mod tidy && go build -o ../bin/runstatus runstatus.go
+	cd $(ADCM); CGO_ENABLED=0 go mod tidy && CGO_ENABLED=0 go build -o ../bin/runstatus runstatus.go

license_checker.py

@@ -17,32 +17,32 @@
 
 APACHE_LICENCE_PY = [
     '# Licensed under the Apache License, Version 2.0 (the "License");\n',
-    '# you may not use this file except in compliance with the License.\n',
-    '# You may obtain a copy of the License at\n',
-    '#\n',
-    '#      http://www.apache.org/licenses/LICENSE-2.0\n',
-    '#\n',
-    '# Unless required by applicable law or agreed to in writing, software\n',
+    "# you may not use this file except in compliance with the License.\n",
+    "# You may obtain a copy of the License at\n",
+    "#\n",
+    "#      http://www.apache.org/licenses/LICENSE-2.0\n",
+    "#\n",
+    "# Unless required by applicable law or agreed to in writing, software\n",
     '# distributed under the License is distributed on an "AS IS" BASIS,\n',
-    '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n',
-    '# See the License for the specific language governing permissions and\n',
-    '# limitations under the License.\n',
-    '\n',
+    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "# See the License for the specific language governing permissions and\n",
+    "# limitations under the License.\n",
+    "\n",
 ]
 
 APACHE_LICENCE_GO = [
     '// Licensed under the Apache License, Version 2.0 (the "License");\n',
-    '// you may not use this file except in compliance with the License.\n',
-    '// You may obtain a copy of the License at\n',
-    '//\n',
-    '//      http://www.apache.org/licenses/LICENSE-2.0\n',
-    '//\n',
-    '// Unless required by applicable law or agreed to in writing, software\n',
+    "// you may not use this file except in compliance with the License.\n",
+    "// You may obtain a copy of the License at\n",
+    "//\n",
+    "//      http://www.apache.org/licenses/LICENSE-2.0\n",
+    "//\n",
+    "// Unless required by applicable law or agreed to in writing, software\n",
     '// distributed under the License is distributed on an "AS IS" BASIS,\n',
-    '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n',
-    '// See the License for the specific language governing permissions and\n',
-    '// limitations under the License.\n',
-    '\n',
+    "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "// See the License for the specific language governing permissions and\n",
+    "// limitations under the License.\n",
+    "\n",
 ]
 
 
@@ -112,15 +112,18 @@ def main():
 
     for folder in args.folders:
         number_of_fixed, number_of_skipped = check_and_fix_files(
-            number_of_fixed, number_of_skipped, args.fix, Path(folder)
+            number_of_fixed,
+            number_of_skipped,
+            args.fix,
+            Path(folder),
         )
 
     if number_of_fixed == number_of_skipped == 0:
         sys.stdout.write("Licence is present in all python and go files \n")
         sys.exit(0)
 
     sys.stdout.write(
-        f"Updating licence skipped in {number_of_skipped} files." f" Licence was updated in {number_of_fixed} files \n"
+        f"Updating licence skipped in {number_of_skipped} files." f" Licence was updated in {number_of_fixed} files \n",
     )
 
     if args.fix:

os/etc/nginx/http.d/proxy.inc

@@ -13,6 +13,14 @@
         try_files $uri $uri/ /index.html;
     }
 
+    location /swagger {
+        uwsgi_pass  django;
+        uwsgi_read_timeout 300s;
+        uwsgi_send_timeout 300s;
+        include     /adcm/conf/nginx/uwsgi_params;
+        proxy_set_header Host $http_host;
+    }
+
     location /api {
         uwsgi_pass  django;
         uwsgi_read_timeout 300s;
@@ -40,7 +48,7 @@
     }
 
     location /status/ {
-        # That takes so many lines in acess log for statuses,
+        # That takes so many lines in access log for statuses,
         # so it is better to disable access for that url
         access_log /var/log/nginx/access.log combined if=$abnormal;
         proxy_pass http://127.0.0.1:8020/;

os/etc/sv/cron/run

@@ -15,5 +15,7 @@
 
 waitforwsgi
 
-echo "Run cron ..."
-crond -f -l 2
+if [ -z "$MIGRATION_MODE" ] || [ "$MIGRATION_MODE" -ne 1 ]; then
+    echo "Run cron ..."
+    crond -f -l 2
+fi

os/etc/sv/init/run

@@ -41,12 +41,6 @@ echo "Root keys generation"
 key=/root/.ssh/id_rsa
 [ ! -f ${key} ] && ssh-keygen -f ${key} -t rsa -N ""
 
-if [ ! -f "${adcmvar}"/cluster.db ]; then
-    echo "Assume first run, generate Django secret key"
-    "${adcmroot}"/python/manage.py generate_secret_key
-    mv "${adcmroot}/python/secretkey.txt" "${adcmsecretfile}"
-fi
-
 if [ ! \( -f "${adcmuserconf}/ssl/cert.pem" -a -f "${adcmuserconf}/ssl/key.pem" \) ]; then
     mkdir -p "${adcmuserconf}/ssl"
     cd "${adcmuserconf}/ssl"
@@ -56,8 +50,12 @@ fi
 
 "${adcmroot}"/python/backupdb.py
 "${adcmroot}"/python/manage.py migrate
-"${adcmroot}"/python/init_db.py
-"${adcmroot}"/python/manage.py upgraderole
+
+if [ -z "$MIGRATION_MODE" ] || [ "$MIGRATION_MODE" -ne 1 ]; then
+    "${adcmroot}"/python/init_db.py
+    "${adcmroot}"/python/manage.py upgraderole
+fi
+
 touch "${initreadyfile}"
 
 # That is sleep forever case because I have no idea how to force