From f03105a14f574669353eac01c4c2ef8cdbd376f5 Mon Sep 17 00:00:00 2001 From: Valeria Bulanova Date: Tue, 16 Jul 2024 17:15:23 +0300 Subject: [PATCH 1/4] dev(connectors): BI-5528 File apps --- app/app_configs/api.yaml | 8 + app/app_configs/file_api.yaml | 31 +++ app/app_configs/file_worker.yaml | 21 ++ app/dl_control_api/pyproject.toml | 1 + app/dl_data_api/pyproject.toml | 1 + app/dl_file_uploader_api/Dockerfile | 43 ++++ app/dl_file_uploader_api/LICENSE | 201 ++++++++++++++++++ app/dl_file_uploader_api/README.md | 1 + .../dl_file_uploader_api/__init__.py | 6 + .../dl_file_uploader_api/app.py | 40 ++++ .../dl_file_uploader_api/app_factory.py | 25 +++ .../dl_file_uploader_api/app_settings.py | 5 + .../docker/gunicorn_config.py | 22 ++ .../docker/gunicorn_logging.ini | 27 +++ .../etc/service/dl_file_uploader_api/run | 18 ++ app/dl_file_uploader_api/pyproject.toml | 37 ++++ app/dl_file_uploader_worker/Dockerfile | 32 +++ app/dl_file_uploader_worker/README.md | 1 + .../dl_file_uploader_worker/__init__.py | 6 + .../dl_file_uploader_worker/app.py | 58 +++++ .../dl_file_uploader_worker/app_factory.py | 14 ++ .../dl_file_uploader_worker/app_settings.py | 5 + .../etc/service/file_secure_reader/run | 29 +++ .../etc/service/file_uploader_worker/run | 7 + app/dl_file_uploader_worker/pyproject.toml | 43 ++++ docker_build/bake_apps.hcl | 24 +++ lib/dl_api_lib/dl_api_lib/app_settings.py | 18 ++ metapkg/poetry.lock | 48 ++++- metapkg/pyproject.toml | 6 + 29 files changed, 777 insertions(+), 1 deletion(-) create mode 100644 app/app_configs/file_api.yaml create mode 100644 app/app_configs/file_worker.yaml create mode 100644 app/dl_file_uploader_api/Dockerfile create mode 100644 app/dl_file_uploader_api/LICENSE create mode 100644 app/dl_file_uploader_api/README.md create mode 100644 app/dl_file_uploader_api/dl_file_uploader_api/__init__.py create mode 100644 app/dl_file_uploader_api/dl_file_uploader_api/app.py create mode 100644 app/dl_file_uploader_api/dl_file_uploader_api/app_factory.py create mode 100644 app/dl_file_uploader_api/dl_file_uploader_api/app_settings.py create mode 100644 app/dl_file_uploader_api/docker/gunicorn_config.py create mode 100644 app/dl_file_uploader_api/docker/gunicorn_logging.ini create mode 100755 app/dl_file_uploader_api/etc/service/dl_file_uploader_api/run create mode 100644 app/dl_file_uploader_api/pyproject.toml create mode 100644 app/dl_file_uploader_worker/Dockerfile create mode 100644 app/dl_file_uploader_worker/README.md create mode 100644 app/dl_file_uploader_worker/dl_file_uploader_worker/__init__.py create mode 100644 app/dl_file_uploader_worker/dl_file_uploader_worker/app.py create mode 100644 app/dl_file_uploader_worker/dl_file_uploader_worker/app_factory.py create mode 100644 app/dl_file_uploader_worker/dl_file_uploader_worker/app_settings.py create mode 100755 app/dl_file_uploader_worker/etc/service/file_secure_reader/run create mode 100755 app/dl_file_uploader_worker/etc/service/file_uploader_worker/run create mode 100644 app/dl_file_uploader_worker/pyproject.toml diff --git a/app/app_configs/api.yaml b/app/app_configs/api.yaml index a002e33b6..13d399f0f 100644 --- a/app/app_configs/api.yaml +++ b/app/app_configs/api.yaml @@ -6,6 +6,10 @@ CONNECTORS: CONN_CHYT_PUBLIC_CLIQUES: [] YDB: ENABLE_AUTH_TYPE_PICKER: true + FILE: + CONN_FILE_CH_HOST: http://db-clickhouse + CONN_FILE_CH_PORT: 8123 + CONN_FILE_CH_USERNAME: default CONNECTOR_AVAILABILITY: icon_src: icon_type: data @@ -26,6 +30,8 @@ CONNECTOR_AVAILABILITY: availability: free - conn_type: mssql availability: free + - availability: free + conn_type: file visible_connectors: [] CORS_ALLOWED_HEADERS: - Content-Type @@ -48,3 +54,5 @@ DL_CRY_ACTUAL_KEY_ID: '0' DL_CRY_KEYMAP: {} SENTRY_DSN_DATASET_API: '' SENTRY_ENABLED: false +S3_ENDPOINT_URL: http://s3-storage:8000 +FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader diff --git a/app/app_configs/file_api.yaml b/app/app_configs/file_api.yaml new file mode 100644 index 000000000..b1d97a591 --- /dev/null +++ b/app/app_configs/file_api.yaml @@ -0,0 +1,31 @@ +--- +CORS_ALLOWED_HEADERS: +- Content-Type +- X-Request-ID +- X-DL-TenantId +- X-CSRF-Token +- X-DL-Allow-Superuser +- X-DL-Sudo +CORS_ALLOWED_ORIGINS: +- '*' +CORS_EXPOSE_HEADERS: +- X-Request-ID +CSRF_HEADER_NAME: X-CSRF-Token +CSRF_METHODS: +- POST +- PUT +- DELETE +CSRF_TIME_LIMIT: 43200 +FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader +FILE_UPLOADER_S3_TMP_BUCKET_NAME: dl-file-uploader-tmp +REDIS_FILE_UPLOADER_DATA_DB: 9 +REDIS_FILE_UPLOADER_TASKS_DB: 11 +REDIS_PERSISTENT_CLUSTER_NAME: '' +REDIS_PERSISTENT_HOSTS: +- redis +REDIS_PERSISTENT_PORT: 6379 +REDIS_PERSISTENT_MODE: 2 +REDIS_PERSISTENT_SSL: null +S3_ENDPOINT_URL: http://s3-storage:8000 +SENTRY_DSN_FILE_UPLOADER_API: '' +SENTRY_ENABLED: false diff --git a/app/app_configs/file_worker.yaml b/app/app_configs/file_worker.yaml new file mode 100644 index 000000000..bf03a5fc9 --- /dev/null +++ b/app/app_configs/file_worker.yaml @@ -0,0 +1,21 @@ +--- +CONNECTORS: + FILE: + CONN_FILE_CH_HOST: http://db-clickhouse + CONN_FILE_CH_PORT: 8123 + CONN_FILE_CH_USERNAME: default +FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader +FILE_UPLOADER_S3_TMP_BUCKET_NAME: dl-file-uploader-tmp +REDIS_FILE_UPLOADER_DATA_DB: 9 +REDIS_FILE_UPLOADER_TASKS_DB: 11 +REDIS_PERSISTENT_CLUSTER_NAME: '' +REDIS_PERSISTENT_HOSTS: +- redis +REDIS_PERSISTENT_MODE: 2 +REDIS_PERSISTENT_PORT: 6379 +REDIS_PERSISTENT_SSL: null +S3_ENDPOINT_URL: http://s3-storage:8000 +SENTRY_DSN_FILE_UPLOADER_WORKER: '' +SENTRY_ENABLED: false +US_BASE_URL: http://us:8083 + diff --git a/app/dl_control_api/pyproject.toml b/app/dl_control_api/pyproject.toml index de30f64b9..78f1bd053 100644 --- a/app/dl_control_api/pyproject.toml +++ b/app/dl_control_api/pyproject.toml @@ -19,6 +19,7 @@ dl-sqlalchemy-clickhouse = {path = "../../lib/dl_sqlalchemy_clickhouse"} dl-connector-postgresql = {path = "../../lib/dl_connector_postgresql"} dl-connector-clickhouse = {path = "../../lib/dl_connector_clickhouse"} dl-connector-ydb = {path = "../../lib/dl_connector_ydb"} +dl-connector-bundle-chs3 = {path = "../../lib/dl_connector_bundle_chs3"} dl-connector-chyt = {path = "../../lib/dl_connector_chyt"} dl-connector-mysql = {path = "../../lib/dl_connector_mysql"} dl-connector-mssql = {path = "../../lib/dl_connector_mssql"} diff --git a/app/dl_data_api/pyproject.toml b/app/dl_data_api/pyproject.toml index 65dab1d40..af47fe86b 100644 --- a/app/dl_data_api/pyproject.toml +++ b/app/dl_data_api/pyproject.toml @@ -17,6 +17,7 @@ dl-sqlalchemy-postgres = {path = "../../lib/dl_sqlalchemy_postgres"} dl-utils = {path = "../../lib/dl_utils"} dl-sqlalchemy-common = {path = "../../lib/dl_sqlalchemy_common"} dl-sqlalchemy-clickhouse = {path = "../../lib/dl_sqlalchemy_clickhouse"} +dl-connector-bundle-chs3 = {path = "../../lib/dl_connector_bundle_chs3"} dl-connector-postgresql = {path = "../../lib/dl_connector_postgresql"} dl-connector-clickhouse = {path = "../../lib/dl_connector_clickhouse"} dl-connector-chyt = {path = "../../lib/dl_connector_chyt"} diff --git a/app/dl_file_uploader_api/Dockerfile b/app/dl_file_uploader_api/Dockerfile new file mode 100644 index 000000000..783e3881b --- /dev/null +++ b/app/dl_file_uploader_api/Dockerfile @@ -0,0 +1,43 @@ +#syntax=docker/dockerfile:1.4 +FROM bake_ctx_base_img + +ENV APP_DIR="app/dl_file_uploader_api" + +RUN mkdir -p /code +COPY /etc /etc + +ENV GUNICORN_LOG_CONFIG /code/gunicorn_logging.ini +ENV GUNICORN_CONFIG /code/gunicorn_config.py +COPY docker/gunicorn_logging.ini ${GUNICORN_LOG_CONFIG} +COPY docker/gunicorn_config.py ${GUNICORN_CONFIG} + +# Libs & metapkg +COPY --from=bake_ctx_src_lib / /src +COPY --from=bake_ctx_metapkg / /src + +# Sources copy +RUN mkdir -p /src/$APP_DIR +COPY /pyproject.toml /src/$APP_DIR/pyproject.toml +COPY /dl_file_uploader_api /src/$APP_DIR/dl_file_uploader_api +COPY /README.md /src/$APP_DIR/README.md + +# Installation +WORKDIR /src/metapkg +RUN poetry export --only app_dl_os_file_uploader_api --without-hashes --format=requirements.txt > requirements.txt +RUN pip install -r requirements.txt + +# Configs +ENV CONFIG_PATH="/etc/backend-configs/file_api.yaml" +COPY --from=bake_ctx_app_configs /app_configs/file_api.yaml "$CONFIG_PATH" + +# Setting up the runtime user +ARG USER=app +ARG GID=1000 +ARG UID=1000 +RUN groupadd -r -g ${GID} ${USER} && \ + useradd -mr -g ${USER} -u ${UID} -s /bin/bash ${USER} +USER ${USER} + +EXPOSE 8080 + +ENTRYPOINT ["/etc/service/dl_file_uploader_api/run"] diff --git a/app/dl_file_uploader_api/LICENSE b/app/dl_file_uploader_api/LICENSE new file mode 100644 index 000000000..74ba5f6c7 --- /dev/null +++ b/app/dl_file_uploader_api/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 YANDEX LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/app/dl_file_uploader_api/README.md b/app/dl_file_uploader_api/README.md new file mode 100644 index 000000000..5b57ed0a5 --- /dev/null +++ b/app/dl_file_uploader_api/README.md @@ -0,0 +1 @@ +# datalens-file-uploader-api diff --git a/app/dl_file_uploader_api/dl_file_uploader_api/__init__.py b/app/dl_file_uploader_api/dl_file_uploader_api/__init__.py new file mode 100644 index 000000000..94ca957a8 --- /dev/null +++ b/app/dl_file_uploader_api/dl_file_uploader_api/__init__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +import dl_version + + +app_version = f"dl_api:{dl_version.__version__}" diff --git a/app/dl_file_uploader_api/dl_file_uploader_api/app.py b/app/dl_file_uploader_api/dl_file_uploader_api/app.py new file mode 100644 index 000000000..00d441e71 --- /dev/null +++ b/app/dl_file_uploader_api/dl_file_uploader_api/app.py @@ -0,0 +1,40 @@ +import logging +import os + +from aiohttp import web + +from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback +from dl_core.logging_config import configure_logging +from dl_file_uploader_api import app_version +from dl_file_uploader_api.app_factory import StandaloneFileUploaderApiAppFactory +from dl_file_uploader_api.app_settings import FileUploaderAPISettingsOS + + +LOGGER = logging.getLogger(__name__) + + +async def create_gunicorn_app() -> web.Application: + settings = load_settings_from_env_with_fallback(FileUploaderAPISettingsOS) + configure_logging(app_name="dl_file_uploader_api") + try: + LOGGER.info("Creating application instance...") + app_factory = StandaloneFileUploaderApiAppFactory(settings=settings) + app = app_factory.create_app(app_version) + LOGGER.info("Application instance was created") + return app + except Exception: + LOGGER.exception("Exception during app creation") + raise + + +def main() -> None: + current_app = create_gunicorn_app() + web.run_app( + current_app, + host=os.environ["APP_HOST"], + port=int(os.environ["APP_PORT"]), + ) + + +if __name__ == "__main__": + main() diff --git a/app/dl_file_uploader_api/dl_file_uploader_api/app_factory.py b/app/dl_file_uploader_api/dl_file_uploader_api/app_factory.py new file mode 100644 index 000000000..429b581e7 --- /dev/null +++ b/app/dl_file_uploader_api/dl_file_uploader_api/app_factory.py @@ -0,0 +1,25 @@ +import attr + +from dl_api_commons.aio.middlewares.auth_trust_middleware import auth_trust_middleware +from dl_api_commons.aio.middlewares.csrf import CSRFMiddleware +from dl_api_commons.aio.typing import AIOHTTPMiddleware +from dl_file_uploader_api_lib.app import FileUploaderApiAppFactory +from dl_file_uploader_api_lib.settings import FileUploaderAPISettings + + +class CSRFMiddlewareOS(CSRFMiddleware): + USER_ID_COOKIES = ("yandexuid",) + + +@attr.s(kw_only=True) +class StandaloneFileUploaderApiAppFactory(FileUploaderApiAppFactory[FileUploaderAPISettings]): + CSRF_MIDDLEWARE_CLS = CSRFMiddlewareOS + + def get_auth_middlewares(self) -> list[AIOHTTPMiddleware]: + auth_mw_list: list[AIOHTTPMiddleware] = [ + auth_trust_middleware( + fake_user_id="_user_id_", + fake_user_name="_user_name_", + ) + ] + return auth_mw_list diff --git a/app/dl_file_uploader_api/dl_file_uploader_api/app_settings.py b/app/dl_file_uploader_api/dl_file_uploader_api/app_settings.py new file mode 100644 index 000000000..1be78ad3d --- /dev/null +++ b/app/dl_file_uploader_api/dl_file_uploader_api/app_settings.py @@ -0,0 +1,5 @@ +from dl_file_uploader_api_lib.settings import FileUploaderAPISettings + + +class FileUploaderAPISettingsOS(FileUploaderAPISettings): + pass diff --git a/app/dl_file_uploader_api/docker/gunicorn_config.py b/app/dl_file_uploader_api/docker/gunicorn_config.py new file mode 100644 index 000000000..bf9b95035 --- /dev/null +++ b/app/dl_file_uploader_api/docker/gunicorn_config.py @@ -0,0 +1,22 @@ +# higher than the supposed default nginx default of 75 seconds +keepalive = 80 +timeout = 90 + +# # Configuration in commandline: +# import os +# log_config = os.environ['GUNICORN_LOG_CONFIG'] +# bind = "[::]:" + os.environ.get('GUNICORN_LISTEN_PORT', '8080') +# max_requests = int(os.environ.get('GUNICORN_MAX_RQ', '5000')) +# max_requests_jitter = int(os.environ.get('GUNICORN_MAX_RQ_JITTER', '500')) +# worker_class = 'aiohttp.GunicornWebWorker' +# workers = int(os.environ.get('GUNICORN_WORKERS_COUNT', '1')) + +# # Other notable configuration: +# worker_connections = 1000 +# threads = 2 +# proc_name = 'app01' +# backlog = 2048 +# accesslog = 'access.log' +# errorlog = 'error.log' +# user = 'ubuntu' +# group = 'ubuntu' diff --git a/app/dl_file_uploader_api/docker/gunicorn_logging.ini b/app/dl_file_uploader_api/docker/gunicorn_logging.ini new file mode 100644 index 000000000..9781f52a1 --- /dev/null +++ b/app/dl_file_uploader_api/docker/gunicorn_logging.ini @@ -0,0 +1,27 @@ +[formatters] +keys = json + +[handlers] +keys = stdout + +[loggers] +keys = gunicorn,root + +[formatter_json] +class = dl_core.logging_config.StdoutFormatter + +[handler_stdout] +class = logging.StreamHandler +formatter = json +stream = sys.stdout +level = DEBUG + +[logger_root] +level = DEBUG +handlers = stdout + +[logger_gunicorn] +qualname=gunicorn +propagate=0 +level = INFO +handlers = stdout diff --git a/app/dl_file_uploader_api/etc/service/dl_file_uploader_api/run b/app/dl_file_uploader_api/etc/service/dl_file_uploader_api/run new file mode 100755 index 000000000..755cd8942 --- /dev/null +++ b/app/dl_file_uploader_api/etc/service/dl_file_uploader_api/run @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +set -x # debug. +set -eu + +HTTP_BIND_PORT="8080" +if [ "${BISYS_USE_IPV6:-1}" = "1" ]; then HTTP_BIND_ADDR="[::]"; else HTTP_BIND_ADDR="0.0.0.0"; fi + +GUNICORN_LISTEN_PORT="${BI_API_GUNICORN_LISTEN_PORT:-${GUNICORN_LISTEN_PORT:-${HTTP_BIND_PORT}}}" + +exec gunicorn dl_file_uploader_api.app:create_gunicorn_app \ + --config "${BI_API_GUNICORN_CONFIG:-${GUNICORN_CONFIG}}" \ + --log-config "${BI_API_GUNICORN_LOG_CONFIG:-${GUNICORN_LOG_CONFIG}}" \ + --bind "${HTTP_BIND_ADDR}:${GUNICORN_LISTEN_PORT}" \ + --max-requests "${BI_API_GUNICORN_MAX_RQ:-${GUNICORN_MAX_RQ:-5000}}" \ + --max-requests-jitter "${BI_API_GUNICORN_MAX_RQ_JITTER:-${GUNICORN_MAX_RQ_JITTER:-500}}" \ + --workers "${BI_API_GUNICORN_WORKERS_COUNT:-${GUNICORN_WORKERS_COUNT:-1}}" \ + --worker-class aiohttp.GunicornWebWorker diff --git a/app/dl_file_uploader_api/pyproject.toml b/app/dl_file_uploader_api/pyproject.toml new file mode 100644 index 000000000..aed95b081 --- /dev/null +++ b/app/dl_file_uploader_api/pyproject.toml @@ -0,0 +1,37 @@ +[tool.poetry] +name = "dl-file-uploader-api" +version = "0.0.0" +description = "" +authors = ["DataLens Team "] +packages = [{include = "dl_file_uploader_api"}] +license = "Apache 2.0" +readme = "README.md" + +[tool.poetry.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +gunicorn = ">=21.2.0" +python = ">=3.10, <3.13" +dl-version = {path = "../../lib/dl_version"} +dl-core = {path = "../../lib/dl_core"} +dl-file-uploader-api-lib = {path = "../../lib/dl_file_uploader_api_lib"} + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra" +testpaths = [] + + + +[datalens_ci] +skip_test = true + +[tool.mypy] +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +strict_optional = true diff --git a/app/dl_file_uploader_worker/Dockerfile b/app/dl_file_uploader_worker/Dockerfile new file mode 100644 index 000000000..80df69336 --- /dev/null +++ b/app/dl_file_uploader_worker/Dockerfile @@ -0,0 +1,32 @@ +#syntax=docker/dockerfile:1.4 +FROM bake_ctx_base_img + +ENV APP_DIR="app/dl_file_uploader_worker" + +# Runit +COPY /etc /etc + +# Libs & metapkg +COPY --from=bake_ctx_src_lib / /src +COPY --from=bake_ctx_metapkg / /src + +# Sources copy +RUN mkdir -p /src/$APP_DIR +COPY /pyproject.toml /src/$APP_DIR/pyproject.toml +COPY /dl_file_uploader_worker /src/$APP_DIR/dl_file_uploader_worker +COPY /README.md /src/$APP_DIR/README.md + +# Installation +WORKDIR /src/metapkg +RUN poetry export --only app_dl_os_file_uploader_worker --without-hashes --format=requirements.txt > requirements.txt +RUN pip install -r requirements.txt + +# Entrypoint +WORKDIR /src/$APP_DIR + +# Configs +ENV CONFIG_PATH="/etc/backend-configs/file_worker.yaml" +COPY --from=bake_ctx_app_configs /app_configs/file_worker.yaml "$CONFIG_PATH" + +ENTRYPOINT ["/etc/service/file_uploader_worker/run"] + diff --git a/app/dl_file_uploader_worker/README.md b/app/dl_file_uploader_worker/README.md new file mode 100644 index 000000000..4d5a07c4f --- /dev/null +++ b/app/dl_file_uploader_worker/README.md @@ -0,0 +1 @@ +# datalens-file-uploader-worker diff --git a/app/dl_file_uploader_worker/dl_file_uploader_worker/__init__.py b/app/dl_file_uploader_worker/dl_file_uploader_worker/__init__.py new file mode 100644 index 000000000..cb0d0b690 --- /dev/null +++ b/app/dl_file_uploader_worker/dl_file_uploader_worker/__init__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +import dl_version + + +app_version = f"dl_api:{dl_version.__version__}" \ No newline at end of file diff --git a/app/dl_file_uploader_worker/dl_file_uploader_worker/app.py b/app/dl_file_uploader_worker/dl_file_uploader_worker/app.py new file mode 100644 index 000000000..e3e1c5dce --- /dev/null +++ b/app/dl_file_uploader_worker/dl_file_uploader_worker/app.py @@ -0,0 +1,58 @@ +import asyncio +import logging + +from dl_api_commons.sentry_config import ( + SentryConfig, + configure_sentry, +) +from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback +from dl_configs.utils import get_root_certificates +from dl_core.logging_config import configure_logging +from dl_file_uploader_worker import app_version +from dl_file_uploader_worker.app_factory import StandaloneFileUploaderWorkerFactory +from dl_file_uploader_worker.app_settings import FileUploaderWorkerSettingsOS +from dl_task_processor.worker import HealthChecker + + +LOGGER = logging.getLogger(__name__) + + +def load_settings() -> FileUploaderWorkerSettingsOS: + settings = load_settings_from_env_with_fallback(FileUploaderWorkerSettingsOS) + return settings + + +def run_standalone_worker() -> None: + loop = asyncio.get_event_loop() + settings = load_settings() + ca_data = get_root_certificates(path=settings.CA_FILE_PATH) + worker = StandaloneFileUploaderWorkerFactory(settings=settings, ca_data=ca_data).create_worker() + configure_logging( + app_name="dl_file_uploader_worker", + ) + if settings.SENTRY_DSN is not None: + configure_sentry(SentryConfig(dsn=settings.SENTRY_DSN, release=app_version)) + worker_task = loop.create_task(worker.start()) + try: + loop.run_forever() + except asyncio.CancelledError: # pragma: no cover + # happens on shutdown, fine + pass + finally: + worker_task.cancel() + loop.run_until_complete(worker.stop()) + loop.close() + + +def run_health_check() -> None: + loop = asyncio.get_event_loop() + settings = load_settings() + configure_logging( + app_name="dl_file_uploader_worker_health_check", + ) + ca_data = get_root_certificates(path=settings.CA_FILE_PATH) + if settings.SENTRY_DSN is not None: + configure_sentry(SentryConfig(dsn=settings.SENTRY_DSN, release=app_version)) + worker = StandaloneFileUploaderWorkerFactory(settings=settings, ca_data=ca_data).create_worker() + health_checker = HealthChecker(worker) + loop.run_until_complete(health_checker.check()) diff --git a/app/dl_file_uploader_worker/dl_file_uploader_worker/app_factory.py b/app/dl_file_uploader_worker/dl_file_uploader_worker/app_factory.py new file mode 100644 index 000000000..01c4af7c7 --- /dev/null +++ b/app/dl_file_uploader_worker/dl_file_uploader_worker/app_factory.py @@ -0,0 +1,14 @@ +import attr + +from dl_api_commons.tenant_resolver import ( + CommonTenantResolver, + TenantResolver, +) +from dl_file_uploader_worker.app_settings import FileUploaderWorkerSettingsOS +from dl_file_uploader_worker_lib.app import FileUploaderWorkerFactory + + +@attr.s(kw_only=True) +class StandaloneFileUploaderWorkerFactory(FileUploaderWorkerFactory[FileUploaderWorkerSettingsOS]): + def _get_tenant_resolver(self) -> TenantResolver: + return CommonTenantResolver() diff --git a/app/dl_file_uploader_worker/dl_file_uploader_worker/app_settings.py b/app/dl_file_uploader_worker/dl_file_uploader_worker/app_settings.py new file mode 100644 index 000000000..cf4ab215e --- /dev/null +++ b/app/dl_file_uploader_worker/dl_file_uploader_worker/app_settings.py @@ -0,0 +1,5 @@ +from dl_file_uploader_worker_lib.settings import FileUploaderWorkerSettings + + +class FileUploaderWorkerSettingsOS(FileUploaderWorkerSettings): + pass diff --git a/app/dl_file_uploader_worker/etc/service/file_secure_reader/run b/app/dl_file_uploader_worker/etc/service/file_secure_reader/run new file mode 100755 index 000000000..2bf6305c9 --- /dev/null +++ b/app/dl_file_uploader_worker/etc/service/file_secure_reader/run @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +set -x # debug. +set -eu + +if [ x"${RUN_FILE_SECURE_READER:-1}" != x"1" ]; then + sleep 9000 + exit 0 +fi + +mkdir -p /var/sockets +chown 1000:1000 /var/sockets + +socket="${SECREADER_GUNICORN_SOCKET_PATH:-/var/sockets/reader.sock}" +workers="${SECREADER_GUNICORN_WORKERS_COUNT:-1}" +max_rq="${SECREADER_GUNICORN_MAX_RQ:-5000}" +max_rq_jitter="${SECREADER_GUNICORN_MAX_RQ_JITTER:-500}" + +portoctl exec self/secure_reader \ + command='gunicorn \ + dl_file_uploader_worker.app:create_secure_reader_gunicorn_app \ + --bind unix:'$socket' \ + --max-requests '$max_rq' \ + --max-requests-jitter '$max_rq_jitter' \ + --workers='$workers' \ + --worker-class aiohttp.GunicornWebWorker' \ + bind='/var/sockets /var/sockets rw' \ + net=none enable_porto=false \ + user=1000 group=1000 capabilities= hostname=secure-reader diff --git a/app/dl_file_uploader_worker/etc/service/file_uploader_worker/run b/app/dl_file_uploader_worker/etc/service/file_uploader_worker/run new file mode 100755 index 000000000..8d5883b7a --- /dev/null +++ b/app/dl_file_uploader_worker/etc/service/file_uploader_worker/run @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +set -x # debug. +set -eu + +exec \ + file-uploader-worker diff --git a/app/dl_file_uploader_worker/pyproject.toml b/app/dl_file_uploader_worker/pyproject.toml new file mode 100644 index 000000000..f087adcf2 --- /dev/null +++ b/app/dl_file_uploader_worker/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "dl-file-uploader-worker" +version = "0.0.0" +description = "" +authors = ["DataLens Team "] +packages = [{include = "dl_file_uploader_worker"}] +license = "Apache 2.0" +readme = "README.md" + +[tool.poetry.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +gunicorn = ">=21.2.0" +ipython = ">=8.10.0" +python = ">=3.10, <3.13" +dl-core = {path = "../../lib/dl_core"} +dl-file-uploader-worker-lib = {path = "../../lib/dl_file_uploader_worker_lib"} +dl-file-secure-reader-lib = {path = "../../lib/dl_file_secure_reader_lib"} +dl-task-processor = {path = "../../lib/dl_task_processor"} +dl-version = {path = "../../lib/dl_version"} + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra" +testpaths = [] + +[datalens_ci] +skip_test = true + + +[tool.poetry.scripts] +file-uploader-worker = "dl_file_uploader_worker.app:run_standalone_worker" +file-uploader-worker-health-check = "dl_file_uploader_worker.app:run_health_check" + +[tool.mypy] +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +strict_optional = true diff --git a/docker_build/bake_apps.hcl b/docker_build/bake_apps.hcl index b5517cc81..493ac2103 100644 --- a/docker_build/bake_apps.hcl +++ b/docker_build/bake_apps.hcl @@ -21,3 +21,27 @@ target "dl_data_api" { context = "${DL_B_PROJECT_ROOT}/app/dl_data_api" dockerfile = "Dockerfile" } + +target "dl_file_uploader_api" { + pull = false + contexts = { + bake_ctx_base_img = "target:base_jammy_db" + bake_ctx_src_lib = "target:dl_src_lib" + bake_ctx_metapkg = "target:dl_src_metapkg" + bake_ctx_app_configs = "target:dl_app_configs" + } + context = "${DL_B_PROJECT_ROOT}/app/dl_file_uploader_api" + dockerfile = "Dockerfile" +} + +target "dl_file_uploader_worker" { + pull = false + contexts = { + bake_ctx_base_img = "target:base_jammy_db" + bake_ctx_src_lib = "target:dl_src_lib" + bake_ctx_metapkg = "target:dl_src_metapkg" + bake_ctx_app_configs = "target:dl_app_configs" + } + context = "${DL_B_PROJECT_ROOT}/app/dl_file_uploader_worker" + dockerfile = "Dockerfile" +} diff --git a/lib/dl_api_lib/dl_api_lib/app_settings.py b/lib/dl_api_lib/dl_api_lib/app_settings.py index 629793f37..374964dcf 100644 --- a/lib/dl_api_lib/dl_api_lib/app_settings.py +++ b/lib/dl_api_lib/dl_api_lib/app_settings.py @@ -298,6 +298,14 @@ def jaeger_service_name(self) -> str: app_prefix: ClassVar[str] = "y" +class FileUploaderWorkerAppSettings(AppSettings): + pass + + +class FileUploaderApiAppSettings(AppSettings): + pass + + @attr.s(frozen=True, kw_only=True) class ControlApiAppTestingsSettings: us_auth_mode_override: Optional[USAuthMode] = attr.ib(default=None) @@ -345,3 +353,13 @@ class ControlApiAppSettingsOS(AppSettingsOS, ControlApiAppSettings): @attr.s(frozen=True) class DataApiAppSettingsOS(AppSettingsOS, DataApiAppSettings): ... + + +@attr.s(frozen=True) +class FileUploaderApiAppSettingsOS(AppSettingsOS, FileUploaderApiAppSettings): + ... + + +@attr.s(frozen=True) +class FileUploaderWorkerAppSettingsOS(AppSettingsOS, FileUploaderWorkerAppSettings): + ... diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index 3ed016e9d..4e75a8e11 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1820,6 +1820,7 @@ dl-api-lib = {path = "../../lib/dl_api_lib"} dl-app-tools = {path = "../../lib/dl_app_tools"} dl-cache-engine = {path = "../../lib/dl_cache_engine"} dl-configs = {path = "../../lib/dl_configs"} +dl-connector-bundle-chs3 = {path = "../../lib/dl_connector_bundle_chs3"} dl-connector-chyt = {path = "../../lib/dl_connector_chyt"} dl-connector-clickhouse = {path = "../../lib/dl_connector_clickhouse"} dl-connector-greenplum = {path = "../../lib/dl_connector_greenplum"} @@ -1972,6 +1973,7 @@ dl-api-lib = {path = "../../lib/dl_api_lib"} dl-app-tools = {path = "../../lib/dl_app_tools"} dl-cache-engine = {path = "../../lib/dl_cache_engine"} dl-configs = {path = "../../lib/dl_configs"} +dl-connector-bundle-chs3 = {path = "../../lib/dl_connector_bundle_chs3"} dl-connector-chyt = {path = "../../lib/dl_connector_chyt"} dl-connector-clickhouse = {path = "../../lib/dl_connector_clickhouse"} dl-connector-greenplum = {path = "../../lib/dl_connector_greenplum"} @@ -2032,6 +2034,27 @@ openpyxl = ">=3.0.10" type = "directory" url = "../lib/dl_file_secure_reader_lib" +[[package]] +name = "dl-file-uploader-api" +version = "0.0.0" +description = "" +optional = false +python-versions = ">=3.10, <3.13" +files = [] +develop = false + +[package.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +dl-core = {path = "../../lib/dl_core"} +dl-file-uploader-api-lib = {path = "../../lib/dl_file_uploader_api_lib"} +dl-version = {path = "../../lib/dl_version"} +gunicorn = ">=21.2.0" + +[package.source] +type = "directory" +url = "../app/dl_file_uploader_api" + [[package]] name = "dl-file-uploader-api-lib" version = "0.0.0" @@ -2122,6 +2145,30 @@ dl-utils = {path = "../dl_utils"} type = "directory" url = "../lib/dl_file_uploader_task_interface" +[[package]] +name = "dl-file-uploader-worker" +version = "0.0.0" +description = "" +optional = false +python-versions = ">=3.10, <3.13" +files = [] +develop = false + +[package.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +dl-core = {path = "../../lib/dl_core"} +dl-file-secure-reader-lib = {path = "../../lib/dl_file_secure_reader_lib"} +dl-file-uploader-worker-lib = {path = "../../lib/dl_file_uploader_worker_lib"} +dl-task-processor = {path = "../../lib/dl_task_processor"} +dl-version = {path = "../../lib/dl_version"} +gunicorn = ">=21.2.0" +ipython = ">=8.10.0" + +[package.source] +type = "directory" +url = "../app/dl_file_uploader_worker" + [[package]] name = "dl-file-uploader-worker-lib" version = "0.0.0" @@ -5541,7 +5588,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index c58c1d54d..d7186736d 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -200,6 +200,12 @@ dl-control-api = {path = "../app/dl_control_api", develop = true} [tool.poetry.group.app_dl_os_data_api.dependencies] dl-data-api = {path = "../app/dl_data_api", develop = true} +[tool.poetry.group.app_dl_os_file_uploader_api.dependencies] +dl-file-uploader-api = {path = "../app/dl_file_uploader_api"} + +[tool.poetry.group.app_dl_os_file_uploader_worker.dependencies] +dl-file-uploader-worker = {path = "../app/dl_file_uploader_worker"} + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From cccbda4b1a11041a116cb1e64ce0e18fade8c452 Mon Sep 17 00:00:00 2001 From: Valeria Bulanova Date: Tue, 13 Aug 2024 14:15:49 +0300 Subject: [PATCH 2/4] Fixes --- .../dl_file_uploader_api/app.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/dl_file_uploader_api/dl_file_uploader_api/app.py b/app/dl_file_uploader_api/dl_file_uploader_api/app.py index 00d441e71..f5c2147bf 100644 --- a/app/dl_file_uploader_api/dl_file_uploader_api/app.py +++ b/app/dl_file_uploader_api/dl_file_uploader_api/app.py @@ -1,7 +1,9 @@ import logging import os +import time from aiohttp import web +import boto3 from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback from dl_core.logging_config import configure_logging @@ -20,6 +22,7 @@ async def create_gunicorn_app() -> web.Application: LOGGER.info("Creating application instance...") app_factory = StandaloneFileUploaderApiAppFactory(settings=settings) app = app_factory.create_app(app_version) + ensure_buckets(settings) LOGGER.info("Application instance was created") return app except Exception: @@ -27,6 +30,24 @@ async def create_gunicorn_app() -> web.Application: raise +def ensure_buckets(settings: FileUploaderAPISettingsOS): + s3 = boto3.resource( + service_name="s3", + endpoint_url=settings.S3.ENDPOINT_URL, + aws_access_key_id=settings.S3.ACCESS_KEY_ID, + aws_secret_access_key=settings.S3.SECRET_ACCESS_KEY, + ) + for bucket_name in [settings.S3_TMP_BUCKET_NAME, settings.S3_PERSISTENT_BUCKET_NAME]: + bucket = s3.Bucket(bucket_name) + + if bucket.creation_date is None: + try: + s3.create_bucket(Bucket=bucket_name) + LOGGER.info(f"Bucket {bucket_name} was created") + except Exception: + LOGGER.info(f"Error while creating bucket {bucket_name}") + + def main() -> None: current_app = create_gunicorn_app() web.run_app( From 30134389efb6f8fa49e64f37ab95757f85cb88bf Mon Sep 17 00:00:00 2001 From: Valeria Bulanova Date: Tue, 13 Aug 2024 17:57:38 +0300 Subject: [PATCH 3/4] Fix --- app/app_configs/api.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/app_configs/api.yaml b/app/app_configs/api.yaml index 13d399f0f..dc857f55a 100644 --- a/app/app_configs/api.yaml +++ b/app/app_configs/api.yaml @@ -56,3 +56,19 @@ SENTRY_DSN_DATASET_API: '' SENTRY_ENABLED: false S3_ENDPOINT_URL: http://s3-storage:8000 FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader +BI_API_CONNECTOR_WHITELIST: +- clickhouse +- postgresql +- chyt +- ydb +- mysql +- greenplum +- file +CORE_CONNECTOR_WHITELIST: +- clickhouse +- postgresql +- chyt +- ydb +- mysql +- greenplum +- file \ No newline at end of file From 1474b11e9b4c741551e71bc7117a2218f262df8d Mon Sep 17 00:00:00 2001 From: Valeria Bulanova Date: Fri, 16 Aug 2024 19:47:31 +0500 Subject: [PATCH 4/4] Fix --- app/app_configs/api.yaml | 15 ++++++++++++--- app/app_configs/file_worker.yaml | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/app_configs/api.yaml b/app/app_configs/api.yaml index dc857f55a..12c979728 100644 --- a/app/app_configs/api.yaml +++ b/app/app_configs/api.yaml @@ -7,9 +7,10 @@ CONNECTORS: YDB: ENABLE_AUTH_TYPE_PICKER: true FILE: - CONN_FILE_CH_HOST: http://db-clickhouse + CONN_FILE_CH_HOST: db-clickhouse CONN_FILE_CH_PORT: 8123 - CONN_FILE_CH_USERNAME: default + CONN_FILE_CH_USERNAME: datalens + CONN_FILE_CH_SECURE: 0 CONNECTOR_AVAILABILITY: icon_src: icon_type: data @@ -71,4 +72,12 @@ CORE_CONNECTOR_WHITELIST: - ydb - mysql - greenplum -- file \ No newline at end of file +- file +FILE_UPLOADER_BASE_URL: http://file-uploader-api:8080 +REDIS_FILE_UPLOADER_TASKS_DB: 11 +REDIS_PERSISTENT_CLUSTER_NAME: '_' +REDIS_PERSISTENT_HOSTS: +- redis +REDIS_PERSISTENT_PORT: 6379 +REDIS_PERSISTENT_MODE: 2 +REDIS_PERSISTENT_SSL: null diff --git a/app/app_configs/file_worker.yaml b/app/app_configs/file_worker.yaml index bf03a5fc9..a79a98206 100644 --- a/app/app_configs/file_worker.yaml +++ b/app/app_configs/file_worker.yaml @@ -1,9 +1,10 @@ --- CONNECTORS: FILE: - CONN_FILE_CH_HOST: http://db-clickhouse + CONN_FILE_CH_HOST: db-clickhouse CONN_FILE_CH_PORT: 8123 - CONN_FILE_CH_USERNAME: default + CONN_FILE_CH_USERNAME: datalens + CONN_FILE_CH_SECURE: 0 FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader FILE_UPLOADER_S3_TMP_BUCKET_NAME: dl-file-uploader-tmp REDIS_FILE_UPLOADER_DATA_DB: 9