From 136a4833c4256b1d20fabda8dc0cf617f8677817 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Wed, 7 Apr 2021 15:01:37 +0200 Subject: [PATCH 1/3] Added -s parameter to curl for upload forms in storage/xfer-external API --- src/common/cscs_api_common.py | 7 +++++-- src/storage/s3v2OS.py | 2 +- src/storage/s3v4OS.py | 2 +- src/storage/swiftOS.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/common/cscs_api_common.py b/src/common/cscs_api_common.py index b55fe4b0..9f8d5c89 100644 --- a/src/common/cscs_api_common.py +++ b/src/common/cscs_api_common.py @@ -346,7 +346,6 @@ def exec_remote_command(auth_header, system_name, system_addr, action, file_tran logging.info(f"stdout: ({stdout_errno}) --> {stdout_errda}") logging.info(f"sdtout: ({stdout_errno}) --> {outlines}") - # TODO: change precedence of error, because in /xfer-external/download this gives error and it s not an error if stderr_errno == 0: if stderr_errda and not (in_str(stderr_errda,"Could not chdir to home directory") or in_str(stderr_errda,"scancel: Terminating job")): result = {"error": 1, "msg": stderr_errda} @@ -359,13 +358,17 @@ def exec_remote_command(auth_header, system_name, system_addr, action, file_tran else: result = {"error": 0, "msg": outlines} elif stderr_errno > 0: - result = {"error": stderr_errno, "msg": stderr_errda or stdout_errda} + if stderr_errno == 7: + result = {"error": 7, "msg": "Failed to connect to staging area server"} + else: + result = {"error": stderr_errno, "msg": stderr_errda or stdout_errda} elif len(stderr_errda) > 0: result = {"error": 1, "msg": stderr_errda} elif stdout_errno == -2: result = {"error": -2, "msg": "Receive ready timeout exceeded"} elif stderr_errno == -1: result = {"error": -1, "msg": "No exit status was provided by the server"} + # first if paramiko exception raise diff --git a/src/storage/s3v2OS.py b/src/storage/s3v2OS.py index 3db9d5d2..9304a6c6 100644 --- a/src/storage/s3v2OS.py +++ b/src/storage/s3v2OS.py @@ -328,7 +328,7 @@ def create_upload_form(self, sourcepath, containername, prefix, ttl, max_file_si "headers": {} } - command = f"curl -i -X {httpVerb} '{url}?AWSAccessKeyId={self.user}&Signature={sig}&Expires={expires}' -T {sourcepath}" + command = f"curl -s -i -X {httpVerb} '{url}?AWSAccessKeyId={self.user}&Signature={sig}&Expires={expires}' -T {sourcepath}" retval["command"] = command diff --git a/src/storage/s3v4OS.py b/src/storage/s3v4OS.py index e906beb6..5154f6d8 100644 --- a/src/storage/s3v4OS.py +++ b/src/storage/s3v4OS.py @@ -401,7 +401,7 @@ def create_upload_form(self, sourcepath, containername, prefix, ttl, max_file_si "headers": {} } - command = f"curl -i -X {httpVerb} {endpoint_url}/{containername}" + command = f"curl -s -i -X {httpVerb} {endpoint_url}/{containername}" for k,v in retval["parameters"]["data"].items(): command += f" -F '{k}={v}'" diff --git a/src/storage/swiftOS.py b/src/storage/swiftOS.py index b332db54..7338349e 100644 --- a/src/storage/swiftOS.py +++ b/src/storage/swiftOS.py @@ -245,7 +245,7 @@ def create_upload_form(self,sourcepath,containername,prefix,ttl,max_file_size): signature = hmac.new(secret, hmac_body, sha1).hexdigest() # added OBJECT_PREFIX as dir_[task_id] in order to become unique the upload instead of user/filename - command = f"curl -i {swift_url}/{swift_version}/{swift_account}/{containername}/{prefix}/" \ + command = f"curl -s -i {swift_url}/{swift_version}/{swift_account}/{containername}/{prefix}/" \ f" -X POST " \ f"-F max_file_size={max_file_size} -F max_file_count={max_file_count} " \ f"-F expires={expires} -F signature={signature} " \ From 23d9de79de700c2ca728652cbe39986fa6a8fa1e Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Wed, 7 Apr 2021 15:21:40 +0200 Subject: [PATCH 2/3] updated version on openapi spec --- doc/openapi/firecrest-api.yaml | 2 +- doc/openapi/firecrest-developers-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index 3a8d3210..2133f93f 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.2-beta1 + version: 1.7.3-beta1 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index 40aad4d1..fed8b0cf 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.2-beta1 + version: 1.7.3-beta1 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a From ce500141af028b81cef49b9d0b758c36a3cc10c9 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Wed, 7 Apr 2021 19:18:42 +0200 Subject: [PATCH 3/3] Added --show-error option to curl cmd --- src/storage/s3v2OS.py | 2 +- src/storage/s3v4OS.py | 2 +- src/storage/swiftOS.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/s3v2OS.py b/src/storage/s3v2OS.py index 9304a6c6..e8efa969 100644 --- a/src/storage/s3v2OS.py +++ b/src/storage/s3v2OS.py @@ -328,7 +328,7 @@ def create_upload_form(self, sourcepath, containername, prefix, ttl, max_file_si "headers": {} } - command = f"curl -s -i -X {httpVerb} '{url}?AWSAccessKeyId={self.user}&Signature={sig}&Expires={expires}' -T {sourcepath}" + command = f"curl --show-error -s -i -X {httpVerb} '{url}?AWSAccessKeyId={self.user}&Signature={sig}&Expires={expires}' -T {sourcepath}" retval["command"] = command diff --git a/src/storage/s3v4OS.py b/src/storage/s3v4OS.py index 5154f6d8..d1a540b0 100644 --- a/src/storage/s3v4OS.py +++ b/src/storage/s3v4OS.py @@ -401,7 +401,7 @@ def create_upload_form(self, sourcepath, containername, prefix, ttl, max_file_si "headers": {} } - command = f"curl -s -i -X {httpVerb} {endpoint_url}/{containername}" + command = f"curl --show-error -s -i -X {httpVerb} {endpoint_url}/{containername}" for k,v in retval["parameters"]["data"].items(): command += f" -F '{k}={v}'" diff --git a/src/storage/swiftOS.py b/src/storage/swiftOS.py index 7338349e..9ad491c5 100644 --- a/src/storage/swiftOS.py +++ b/src/storage/swiftOS.py @@ -245,7 +245,7 @@ def create_upload_form(self,sourcepath,containername,prefix,ttl,max_file_size): signature = hmac.new(secret, hmac_body, sha1).hexdigest() # added OBJECT_PREFIX as dir_[task_id] in order to become unique the upload instead of user/filename - command = f"curl -s -i {swift_url}/{swift_version}/{swift_account}/{containername}/{prefix}/" \ + command = f"curl --show-error -s -i {swift_url}/{swift_version}/{swift_account}/{containername}/{prefix}/" \ f" -X POST " \ f"-F max_file_size={max_file_size} -F max_file_count={max_file_count} " \ f"-F expires={expires} -F signature={signature} " \