Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix/903 portfolio linking sub directories #950

Merged
merged 10 commits into from
Jan 19, 2024
Merged

Fix/903 portfolio linking sub directories #950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4cb90b2
Fix share-fs portfolio link + remove sub-dir on store
sambles Jan 18, 2024
685803c
Update v2 file linking
sambles Jan 18, 2024
bbc9001
PEP
sambles Jan 18, 2024
bda0240
Add missing S3 storage check
sambles Jan 18, 2024
90c5199
Update image testing - S3 storage
sambles Jan 18, 2024
9dbce99
retest
sambles Jan 18, 2024
2a7bcd5
trigger test again
sambles Jan 18, 2024
c173f35
Name S3 checks by version
sambles Jan 18, 2024
0cc765a
Test fix for input tar extract
sambles Jan 19, 2024
3d14fb1
try fix again
sambles Jan 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/test-images.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,38 @@ jobs:
pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml "
storage_suffix: '-all-checks'

storage_s3_v1:
name: V1 Storage Compatibility (S3)
secrets: inherit
needs: [setup]
uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
with:
piwind_branch: ${{ needs.setup.outputs.piwind_branch }}
server_image: ${{ needs.setup.outputs.build_server_img }}
server_tag: ${{ needs.setup.outputs.build_server_tag }}
worker_image: ${{ needs.setup.outputs.build_worker_img }}
worker_tag: ${{ needs.setup.outputs.build_worker_tag }}
worker_api_ver: 'v1'
debug_mode: 1
pytest_opts: "--docker-compose=./docker/plat2-v2.s3.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
storage_suffix: '_s3'

storage_s3_v2:
name: V2 Storage Compatibility (S3)
secrets: inherit
needs: [setup]
uses: OasisLMF/OasisPiWind/.github/workflows/integration.yml@main
with:
piwind_branch: ${{ needs.setup.outputs.piwind_branch }}
server_image: ${{ needs.setup.outputs.build_server_img }}
server_tag: ${{ needs.setup.outputs.build_server_tag }}
worker_image: ${{ needs.setup.outputs.build_worker_img }}
worker_tag: ${{ needs.setup.outputs.build_worker_tag }}
worker_api_ver: 'v2'
debug_mode: 1
pytest_opts: "--docker-compose=./docker/plat2-v2.s3.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
storage_suffix: '_s3'

worker_debian:
name: Worker Debian
secrets: inherit
Expand Down Expand Up @@ -259,3 +291,4 @@ jobs:
debug_mode: 1
pytest_opts: "--docker-compose=./docker/plat2-v2.docker-compose.yml ${{ needs.setup.outputs.pytest_opts }}"
storage_suffix: "_worker-${{ needs.setup.outputs.release_stable_28 }}"

16 changes: 10 additions & 6 deletions src/model_execution_worker/storage_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def _fetch_file(self, reference, output_path, subdir):
raise MissingInputsException(fpath)

def filepath(self, reference):
""" return the absolute filepath
""" return the absolute filepath
"""
fpath = os.path.join(
self.media_root,
Expand All @@ -204,11 +204,15 @@ def extract(self, archive_fp, directory, storage_subdir=''):
temp_dir = tempfile.TemporaryDirectory()
try:
temp_dir_path = temp_dir.__enter__()
local_archive_path = self.get(
archive_fp,
os.path.join(temp_dir_path, os.path.basename(archive_fp)),
subdir=storage_subdir
)
if os.path.isfile(archive_fp):
local_archive_path = os.path.abspath(archive_fp)
else:
local_archive_path = self.get(
archive_fp,
os.path.join(temp_dir_path, os.path.basename(archive_fp)),
subdir=storage_subdir
)

with tarfile.open(local_archive_path) as f:
f.extractall(directory)
finally:
Expand Down
67 changes: 45 additions & 22 deletions src/server/oasisapi/portfolios/v1_api/serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from os import path
import mimetypes

from drf_yasg.utils import swagger_serializer_method
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
Expand Down Expand Up @@ -260,29 +262,51 @@ def validate(self, attrs):
raise serializers.ValidationError(errors)
return super(PortfolioStorageSerializer, self).validate(attrs)

def inferr_content_type(self, stored_filename):
inferred_type = mimetypes.MimeTypes().guess_type(stored_filename)[0]
if not inferred_type and stored_filename.lower().endswith('parquet'):
# mimetypes dosn't work for parquet so handle that here
inferred_type = 'application/octet-stream'
if not inferred_type:
inferred_type = default_storage.default_content_type
return inferred_type

def get_content_type(self, stored_filename):
try: # fetch content_type stored in Django's DB
return RelatedFile.objects.get(file=path.basename(stored_filename)).content_type
except ObjectDoesNotExist:
try: # Find content_type from S3 Object header
object_header = default_storage.connection.meta.client.head_object(
Bucket=default_storage.bucket_name,
Key=stored_filename)
return object_header['ContentType']
except ClientError:
# fallback to the default content_type
return default_storage.default_content_type
# Find content_type from S3 Object header
if hasattr(default_storage, 'bucket'):
try:
object_header = default_storage.connection.meta.client.head_object(
Bucket=default_storage.bucket_name,
Key=stored_filename)
return object_header['ContentType']
except S3_ClientError:
return self.inferr_content_type(stored_filename)

# Find content_type from Blob Storage
elif hasattr(default_storage, 'azure_container'):
blob_client = default_storage.client.get_blob_client(stored_filename)
blob_properties = blob_client.get_blob_properties()
return blob_properties.content_settings.content_type

else:
return self.inferr_content_type(stored_filename)

def update(self, instance, validated_data):
files_for_removal = list()

for field in validated_data:
content_type = self.get_content_type(validated_data[field])
old_file_name = validated_data[field]
content_type = self.get_content_type(old_file_name)
fname = path.basename(old_file_name)
new_file_name = default_storage.get_alternative_name(fname, '')

# S3 storage - File copy needed
if hasattr(default_storage, 'bucket'):
fname = path.basename(validated_data[field])
new_file = ContentFile(b'')
new_file.name = default_storage.get_alternative_name(fname, '')
new_file.name = new_file_name
new_related_file = RelatedFile.objects.create(
file=new_file,
filename=fname,
Expand All @@ -292,18 +316,17 @@ def update(self, instance, validated_data):
)
bucket = default_storage.bucket
stored_file = default_storage.open(new_related_file.file.name)
stored_file.obj.copy({"Bucket": bucket.name, "Key": validated_data[field]})
stored_file.obj.copy({"Bucket": bucket.name, "Key": old_file_name})
stored_file.obj.wait_until_exists()

elif hasattr(default_storage, 'azure_container'):
# https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-copy?tabs=python
fname = path.basename(validated_data[field])
new_file_name = default_storage.get_alternative_name(validated_data[field], '')
new_blobname = path.basename(new_file_name)
new_file_name = default_storage.get_alternative_name(old_file_name, '')
new_blobname = '/'.join([default_storage.location, path.basename(new_file_name)])

# Copies a blob asynchronously.
source_blob = default_storage.client.get_blob_client(validated_data[field])
dest_blob = default_storage.client.get_blob_client(new_file_name)
source_blob = default_storage.client.get_blob_client(old_file_name)
dest_blob = default_storage.client.get_blob_client(new_blobname)

try:
lease = BlobLeaseClient(source_blob)
Expand All @@ -316,9 +339,9 @@ def update(self, instance, validated_data):
lease.break_lease()
raise e

stored_blob = default_storage.open(new_blobname)
stored_blob = default_storage.open(new_file_name)
new_related_file = RelatedFile.objects.create(
file=File(stored_blob, name=new_blobname),
file=File(stored_blob, name=new_file_name),
filename=fname,
content_type=content_type,
creator=self.context['request'].user,
Expand All @@ -327,11 +350,11 @@ def update(self, instance, validated_data):

# Shared-fs
else:
stored_file = default_storage.open(validated_data[field])
new_file = File(stored_file, name=validated_data[field])
stored_file = default_storage.open(old_file_name)
new_file = File(stored_file, name=new_file_name)
new_related_file = RelatedFile.objects.create(
file=new_file,
filename=validated_data[field],
filename=fname,
content_type=content_type,
creator=self.context['request'].user,
store_as_filename=True,
Expand Down
20 changes: 10 additions & 10 deletions src/server/oasisapi/portfolios/v2_api/serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,13 +324,14 @@ def update(self, instance, validated_data):
files_for_removal = list()
user = self.context['request'].user
for field in validated_data:
content_type = self.get_content_type(validated_data[field])
old_file_name = validated_data[field]
content_type = self.get_content_type(old_file_name)
fname = path.basename(old_file_name)
new_file_name = default_storage.get_alternative_name(fname, '')

# S3 storage - File copy needed
if hasattr(default_storage, 'bucket'):
fname = path.basename(validated_data[field])
new_file = ContentFile(b'')
new_file.name = default_storage.get_alternative_name(fname, '')
new_related_file = RelatedFile.objects.create(
file=new_file,
filename=fname,
Expand All @@ -347,12 +348,11 @@ def update(self, instance, validated_data):

elif hasattr(default_storage, 'azure_container'):
# https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-copy?tabs=python
fname = path.basename(validated_data[field])
new_file_name = default_storage.get_alternative_name(validated_data[field], '')
new_file_name = default_storage.get_alternative_name(old_file_name, '')
new_blobname = '/'.join([default_storage.location, path.basename(new_file_name)])

# Copies a blob asynchronously.
source_blob = default_storage.client.get_blob_client(validated_data[field])
source_blob = default_storage.client.get_blob_client(old_file_name)
dest_blob = default_storage.client.get_blob_client(new_blobname)

try:
Expand All @@ -371,17 +371,17 @@ def update(self, instance, validated_data):
file=File(stored_blob, name=new_file_name),
filename=fname,
content_type=content_type,
creator=self.context['request'].user,
creator=user,
store_as_filename=True,
)

# Shared-fs
else:
stored_file = default_storage.open(validated_data[field])
new_file = File(stored_file, name=validated_data[field])
stored_file = default_storage.open(old_file_name)
new_file = File(stored_file, name=new_file_name)
new_related_file = RelatedFile.objects.create(
file=new_file,
filename=validated_data[field],
filename=fname,
content_type=content_type,
creator=user,
store_as_filename=True,
Expand Down
Loading