From ee10e6e9993b603035d2852c3f39876b3bdeed42 Mon Sep 17 00:00:00 2001 From: Valeria Bulanova Date: Thu, 25 Jul 2024 18:29:57 +0300 Subject: [PATCH] Fixes --- app/app_configs/api.yaml | 6 +-- app/app_configs/file_api.yaml | 32 +++++++++++++++ app/app_configs/file_worker.yaml | 21 ++++++++++ app/dl_file_uploader_api/Dockerfile | 4 +- app/dl_file_uploader_api/pyproject.toml | 2 +- app/dl_file_uploader_worker/Dockerfile | 4 ++ app/dl_file_uploader_worker/pyproject.toml | 2 +- metapkg/poetry.lock | 48 +++++++++++++++++++++- 8 files changed, 108 insertions(+), 11 deletions(-) create mode 100644 app/app_configs/file_api.yaml create mode 100644 app/app_configs/file_worker.yaml diff --git a/app/app_configs/api.yaml b/app/app_configs/api.yaml index 70ff7aaa8..25ed16f75 100644 --- a/app/app_configs/api.yaml +++ b/app/app_configs/api.yaml @@ -45,8 +45,4 @@ CSRF_TIME_LIMIT: 43200 DL_CRY_ACTUAL_KEY_ID: '0' DL_CRY_KEYMAP: {} SENTRY_DSN_DATASET_API: '' -SENTRY_ENABLED: false -REDIS_FILE_UPLOADER_DATA_DB: 9 -REDIS_FILE_UPLOADER_TASKS_DB: 11 -FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: bi-file-uploader-testing -FILE_UPLOADER_S3_TMP_BUCKET_NAME: bi-file-uploader-tmp-testing \ No newline at end of file +SENTRY_ENABLED: false \ No newline at end of file diff --git a/app/app_configs/file_api.yaml b/app/app_configs/file_api.yaml new file mode 100644 index 000000000..d7093b422 --- /dev/null +++ b/app/app_configs/file_api.yaml @@ -0,0 +1,32 @@ +--- +CORS_ALLOWED_HEADERS: +- Content-Type +- X-Request-ID +- X-DL-TenantId +- X-CSRF-Token +- X-DL-Allow-Superuser +- X-DL-Sudo +CORS_ALLOWED_ORIGINS: +- '*' +CORS_EXPOSE_HEADERS: +- X-Request-ID +CSRF_HEADER_NAME: X-CSRF-Token +CSRF_METHODS: +- POST +- PUT +- DELETE +CSRF_TIME_LIMIT: 43200 +FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader +FILE_UPLOADER_S3_TMP_BUCKET_NAME: dl-file-uploader-tmp +REDIS_FILE_UPLOADER_DATA_DB: 9 +REDIS_FILE_UPLOADER_TASKS_DB: 11 +REDIS_PERSISTENT_CLUSTER_NAME: opensource +REDIS_PERSISTENT_HOSTS: +- http://redis:6379 +REDIS_PERSISTENT_MODE: 1 +REDIS_PERSISTENT_PORT: 26379 +REDIS_PERSISTENT_SSL: null +S3_ENDPOINT_URL: http://s3-storage:8000 +SENTRY_DSN_FILE_UPLOADER_API: '' +SENTRY_ENABLED: false + diff --git a/app/app_configs/file_worker.yaml b/app/app_configs/file_worker.yaml new file mode 100644 index 000000000..b6161d9cd --- /dev/null +++ b/app/app_configs/file_worker.yaml @@ -0,0 +1,21 @@ +--- +CONNECTORS: + FILE: + CONN_FILE_CH_HOST: http://db-clickhouse + CONN_FILE_CH_PORT: 8123 + CONN_FILE_CH_USERNAME: default +FILE_UPLOADER_S3_PERSISTENT_BUCKET_NAME: dl-file-uploader +FILE_UPLOADER_S3_TMP_BUCKET_NAME: dl-file-uploader-tmp +REDIS_FILE_UPLOADER_DATA_DB: 9 +REDIS_FILE_UPLOADER_TASKS_DB: 11 +REDIS_PERSISTENT_CLUSTER_NAME: opensource +REDIS_PERSISTENT_HOSTS: +- http://redis +REDIS_PERSISTENT_MODE: 1 +REDIS_PERSISTENT_PORT: 6379 +REDIS_PERSISTENT_SSL: null +S3_ENDPOINT_URL: http://s3-storage:8000 +SENTRY_DSN_FILE_UPLOADER_WORKER: '' +SENTRY_ENABLED: false +US_BASE_URL: http://us:8083 + diff --git a/app/dl_file_uploader_api/Dockerfile b/app/dl_file_uploader_api/Dockerfile index 4f54568f1..783e3881b 100644 --- a/app/dl_file_uploader_api/Dockerfile +++ b/app/dl_file_uploader_api/Dockerfile @@ -27,8 +27,8 @@ RUN poetry export --only app_dl_os_file_uploader_api --without-hashes --format=r RUN pip install -r requirements.txt # Configs -ENV CONFIG_PATH="/etc/backend-configs/api.yaml" -COPY --from=bake_ctx_app_configs /app_configs/api.yaml "$CONFIG_PATH" +ENV CONFIG_PATH="/etc/backend-configs/file_api.yaml" +COPY --from=bake_ctx_app_configs /app_configs/file_api.yaml "$CONFIG_PATH" # Setting up the runtime user ARG USER=app diff --git a/app/dl_file_uploader_api/pyproject.toml b/app/dl_file_uploader_api/pyproject.toml index e47764302..3191298d9 100644 --- a/app/dl_file_uploader_api/pyproject.toml +++ b/app/dl_file_uploader_api/pyproject.toml @@ -11,7 +11,7 @@ readme = "README.md" aiohttp = ">=3.9.1" attrs = ">=22.2.0" gunicorn = ">=21.2.0" -python = ">=3.10, <3.12" +python = ">=3.10, <3.13" dl-version = {path = "../../lib/dl_version"} dl-core = {path = "../../lib/dl_core"} dl-file-uploader-api-lib = {path = "../../lib/dl_file_uploader_api_lib"} diff --git a/app/dl_file_uploader_worker/Dockerfile b/app/dl_file_uploader_worker/Dockerfile index 0374517f2..afd9a0ac5 100644 --- a/app/dl_file_uploader_worker/Dockerfile +++ b/app/dl_file_uploader_worker/Dockerfile @@ -24,5 +24,9 @@ RUN pip install -r requirements.txt # Entrypoint WORKDIR /src/$APP_DIR +# Configs +ENV CONFIG_PATH="/etc/backend-configs/file_worker.yaml" +COPY --from=bake_ctx_app_configs /app_configs/file_worker.yaml "$CONFIG_PATH" + ENTRYPOINT ["/etc/service/dl_file_uploader_worker/run"] diff --git a/app/dl_file_uploader_worker/pyproject.toml b/app/dl_file_uploader_worker/pyproject.toml index 84907b98b..76b91ff62 100644 --- a/app/dl_file_uploader_worker/pyproject.toml +++ b/app/dl_file_uploader_worker/pyproject.toml @@ -12,7 +12,7 @@ aiohttp = ">=3.9.1" attrs = ">=22.2.0" gunicorn = ">=21.2.0" ipython = ">=8.10.0" -python = ">=3.10, <3.12" +python = ">=3.10, <3.13" dl-core = {path = "../../lib/dl_core"} dl-file-uploader-worker-lib = {path = "../../lib/dl_file_uploader_worker_lib"} dl-file-secure-reader-lib = {path = "../../lib/dl_file_secure_reader_lib"} diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index b69d9f90c..0dc9c60d1 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -964,6 +964,51 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "datalens-file-uploader-api" +version = "0.0.0" +description = "" +optional = false +python-versions = ">=3.10, <3.13" +files = [] +develop = false + +[package.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +dl-core = {path = "../../lib/dl_core"} +dl-file-uploader-api-lib = {path = "../../lib/dl_file_uploader_api_lib"} +dl-version = {path = "../../lib/dl_version"} +gunicorn = ">=21.2.0" + +[package.source] +type = "directory" +url = "../app/dl_file_uploader_api" + +[[package]] +name = "datalens-file-uploader-worker" +version = "0.0.0" +description = "" +optional = false +python-versions = ">=3.10, <3.13" +files = [] +develop = false + +[package.dependencies] +aiohttp = ">=3.9.1" +attrs = ">=22.2.0" +dl-core = {path = "../../lib/dl_core"} +dl-file-secure-reader-lib = {path = "../../lib/dl_file_secure_reader_lib"} +dl-file-uploader-worker-lib = {path = "../../lib/dl_file_uploader_worker_lib"} +dl-task-processor = {path = "../../lib/dl_task_processor"} +dl-version = {path = "../../lib/dl_version"} +gunicorn = ">=21.2.0" +ipython = ">=8.10.0" + +[package.source] +type = "directory" +url = "../app/dl_file_uploader_worker" + [[package]] name = "decorator" version = "5.1.1" @@ -5495,7 +5540,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -6944,4 +6988,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.13" -content-hash = "687fcd96fbbbb795f5f4c860914da11ac8538e7c8675a2fc5671341f123cc7eb" +content-hash = "1d9d0a22eb6fe74c07f9abdca2fc0ac3513f2af6adbc0b1b5c24b79e61f957a4"