From 7e075b566a219338fc822567b795058693f53bf7 Mon Sep 17 00:00:00 2001 From: sambles Date: Fri, 2 Feb 2024 14:21:13 +0000 Subject: [PATCH 1/5] Remove pip from 2nd stage server build (#962) --- Dockerfile.api_server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.api_server b/Dockerfile.api_server index 7e4d93037..cb835195a 100755 --- a/Dockerfile.api_server +++ b/Dockerfile.api_server @@ -20,7 +20,7 @@ USER server FROM ubuntu:22.04 RUN apt-get update \ && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends sudo python3 python3-pip curl libmariadbclient-dev-compat \ + && apt-get install -y --no-install-recommends sudo python3 python3-pkg-resources curl libmariadbclient-dev-compat \ && rm -rf /var/lib/apt/lists/ RUN adduser --home /home/server --shell /bin/bash --disabled-password --gecos "" server From eee5638060988b9f35ffc7e7f97a8f29c9d01478 Mon Sep 17 00:00:00 2001 From: sambles Date: Fri, 2 Feb 2024 14:21:30 +0000 Subject: [PATCH 2/5] Fix Azure support for workers (platform 1) (#939) * Fix celery DB password speical chars * Fix share-fs copy - chmod error --- src/conf/celeryconf.py | 5 +++-- src/model_execution_worker/storage_manager.py | 2 +- src/model_execution_worker/tasks.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/conf/celeryconf.py b/src/conf/celeryconf.py index 4c1f410d5..1cc201e41 100644 --- a/src/conf/celeryconf.py +++ b/src/conf/celeryconf.py @@ -1,3 +1,4 @@ +import urllib from src.conf.iniconf import settings #: Celery config - ignore result? @@ -21,8 +22,8 @@ else: CELERY_RESULT_BACKEND = '{DB_ENGINE}://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}'.format( DB_ENGINE=settings.get('celery', 'db_engine'), - DB_USER=settings.get('celery', 'db_user'), - DB_PASS=settings.get('celery', 'db_pass'), + DB_USER=urllib.parse.quote(settings.get('celery', 'db_user')), + DB_PASS=urllib.parse.quote(settings.get('celery', 'db_pass')), DB_HOST=settings.get('celery', 'db_host'), DB_PORT=settings.get('celery', 'db_port'), DB_NAME=settings.get('celery', 'db_name', fallback='celery'), diff --git a/src/model_execution_worker/storage_manager.py b/src/model_execution_worker/storage_manager.py index f9ab8320e..ebc916cbc 100755 --- a/src/model_execution_worker/storage_manager.py +++ b/src/model_execution_worker/storage_manager.py @@ -130,7 +130,7 @@ def _store_file(self, file_path, suffix=None): self.media_root, self._get_unique_filename(ext)) self.logger.info('Store file: {} -> {}'.format(file_path, stored_fp)) - return shutil.copy(file_path, stored_fp) + return shutil.copyfile(file_path, stored_fp) def _store_dir(self, directory_path, suffix=None, arcname=None): """ Compress and store a directory diff --git a/src/model_execution_worker/tasks.py b/src/model_execution_worker/tasks.py index 0087fcbbb..33e8d4ed6 100755 --- a/src/model_execution_worker/tasks.py +++ b/src/model_execution_worker/tasks.py @@ -639,7 +639,7 @@ def prepare_complex_model_file_inputs(complex_model_files, run_directory): to_path = os.path.join(run_directory, orig_fn) if os.name == 'nt': logging.info(f'complex_model_file: copy {from_path} to {to_path}') - shutil.copy(from_path, to_path) + shutil.copyfile(from_path, to_path) else: logging.info(f'complex_model_file: link {from_path} to {to_path}') os.symlink(from_path, to_path) From a3ae6ec5385a4e4dd7191aca2bb03646ef6aa265 Mon Sep 17 00:00:00 2001 From: awsbuild Date: Fri, 1 Mar 2024 09:26:11 +0000 Subject: [PATCH 3/5] Set version 1.27.8 --- VERSION | 2 +- requirements-worker.txt | 6 ++---- requirements.txt | 6 ++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index 127aeda7e..31a078060 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.27.7 +1.27.8 diff --git a/requirements-worker.txt b/requirements-worker.txt index 957af9d95..d20d9f58b 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -142,10 +142,8 @@ numpy==1.23.5 # scikit-learn # scipy # shapely -oasislmf[extra]==1.27.7 - # via - # -r requirements-worker.in - # oasislmf +oasislmf[extra]==1.27.8 + # via -r requirements-worker.in ods-tools==3.0.8 # via oasislmf packaging==23.1 diff --git a/requirements.txt b/requirements.txt index b77bb52de..9d42b6795 100644 --- a/requirements.txt +++ b/requirements.txt @@ -287,10 +287,8 @@ numpy==1.23.5 # scikit-learn # scipy # shapely -oasislmf[extra]==1.27.7 - # via - # -r ./requirements-worker.in - # oasislmf +oasislmf[extra]==1.27.8 + # via -r ./requirements-worker.in ods-tools==3.0.8 # via # -r ./requirements-server.in From 2fce375aaf8b9f078e9fa2f831b8d601c7bb7535 Mon Sep 17 00:00:00 2001 From: sambles Date: Fri, 1 Mar 2024 10:14:57 +0000 Subject: [PATCH 4/5] Updated Package Requirements: cryptography==42.0.4 pyopenssl django==3.2.23 pyarrow==14.0.1 (#988) Co-authored-by: awsbuild --- requirements-server.txt | 8 ++++---- requirements-worker.txt | 4 ++-- requirements.txt | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements-server.txt b/requirements-server.txt index 679507348..8b57ef3d3 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -71,7 +71,7 @@ coreschema==0.0.4 # via # coreapi # drf-yasg -cryptography==41.0.2 +cryptography==42.0.4 # via # autobahn # azure-storage-blob @@ -79,7 +79,7 @@ cryptography==41.0.2 # service-identity daphne==4.0.0 # via -r requirements-server.in -django==3.2.20 +django==3.2.23 # via # -r requirements-server.in # channels @@ -179,7 +179,7 @@ prompt-toolkit==3.0.38 # via click-repl psycopg2-binary==2.9.6 # via -r requirements-server.in -pyarrow==12.0.0 +pyarrow==14.0.1 # via -r requirements-server.in pyasn1==0.5.0 # via @@ -193,7 +193,7 @@ pyjwt==2.7.0 # via djangorestframework-simplejwt pymysql==1.0.3 # via -r requirements-server.in -pyopenssl==23.2.0 +pyopenssl==24.0.0 # via twisted pyrsistent==0.19.3 # via jsonschema diff --git a/requirements-worker.txt b/requirements-worker.txt index d20d9f58b..d8914635d 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -75,7 +75,7 @@ cookiecutter==2.1.1 # via oasislmf cramjam==2.6.2 # via fastparquet -cryptography==41.0.2 +cryptography==42.0.4 # via azure-storage-blob exceptiongroup==1.1.3 # via pytest @@ -166,7 +166,7 @@ prompt-toolkit==3.0.38 # via click-repl psycopg2-binary==2.9.6 # via -r requirements-worker.in -pyarrow==12.0.0 +pyarrow==14.0.1 # via oasislmf pycparser==2.21 # via cffi diff --git a/requirements.txt b/requirements.txt index 9d42b6795..6bf4eb4f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -128,7 +128,7 @@ coverage[toml]==7.2.5 # pytest-cov cramjam==2.6.2 # via fastparquet -cryptography==41.0.2 +cryptography==42.0.4 # via # autobahn # azure-storage-blob @@ -138,7 +138,7 @@ daphne==4.0.0 # via -r ./requirements-server.in distlib==0.3.6 # via virtualenv -django==3.2.20 +django==3.2.23 # via # -r ./requirements-server.in # channels @@ -330,7 +330,7 @@ psycopg2-binary==2.9.6 # via # -r ./requirements-server.in # -r ./requirements-worker.in -pyarrow==12.0.0 +pyarrow==14.0.1 # via # -r ./requirements-server.in # oasislmf @@ -352,7 +352,7 @@ pymysql==1.0.3 # via # -r ./requirements-server.in # -r ./requirements-worker.in -pyopenssl==23.2.0 +pyopenssl==24.0.0 # via # -r requirements.in # twisted From abb28c49648ccda71231a77e2411822b0327b7ca Mon Sep 17 00:00:00 2001 From: awsbuild Date: Fri, 1 Mar 2024 10:30:10 +0000 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 44856bfa1..f566a145b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,8 @@ OasisPlatform Changelog ======================= +.. _`1.27.8`: https://github.com/OasisLMF/OasisPlatform/compare/1.27.7...1.27.8 + .. _`1.27.7`: https://github.com/OasisLMF/OasisPlatform/compare/1.27.6...1.27.7 `1.27.6`_