From ddc8a6367c87b0ff3938ef7b59c40484a0247c13 Mon Sep 17 00:00:00 2001 From: jo-pol Date: Thu, 21 Dec 2023 14:27:46 +0100 Subject: [PATCH] log response content in case of HTTPError --- src/datastation/common/utils.py | 7 ++++ src/datastation/dataverse/banner_api.py | 6 +-- src/datastation/dataverse/dataset_api.py | 37 +++++++++--------- src/datastation/dataverse/dataverse_api.py | 4 +- src/datastation/dataverse/file_api.py | 2 +- src/datastation/dataverse/metrics_api.py | 2 +- src/datastation/dataverse/search_api.py | 2 +- src/datastation/dv_api.py | 44 +++++++++++----------- 8 files changed, 56 insertions(+), 48 deletions(-) diff --git a/src/datastation/common/utils.py b/src/datastation/common/utils.py index b818e0f..09f18a8 100644 --- a/src/datastation/common/utils.py +++ b/src/datastation/common/utils.py @@ -1,4 +1,5 @@ import json as jsonlib +import logging import os import shutil import argparse @@ -19,6 +20,12 @@ def add_batch_processor_args(parser, report: bool = True): dest='report_file') +def raise_for_status(r): + if r.status_code >= 400: + logging.error(f"{r.status_code} {r.reason} {r.json()}") + r.raise_for_status() + + def positive_int_argument_converter(value): try: ivalue = int(value) diff --git a/src/datastation/dataverse/banner_api.py b/src/datastation/dataverse/banner_api.py index 81231da..4a81fef 100644 --- a/src/datastation/dataverse/banner_api.py +++ b/src/datastation/dataverse/banner_api.py @@ -18,7 +18,7 @@ def list(self, dry_run: bool = False): print(f"Would have sent the following request: {url}") return r = requests.get(url, headers=headers, params={'unblock-key': self.unblock_key}) - r.raise_for_status() + raise_for_status(r)() return r def add(self, msg: str, dismissible_by_user: bool = False, lang: str = 'en', dry_run: bool = False): @@ -39,7 +39,7 @@ def add(self, msg: str, dismissible_by_user: bool = False, lang: str = 'en', dry print(json.dumps(banner, indent=4)) return r = requests.post(url, headers=headers, params={'unblock-key': self.unblock_key}, json=banner) - r.raise_for_status() + raise_for_status(r)() return r def remove(self, banner_id: int, dry_run: bool = False): @@ -50,5 +50,5 @@ def remove(self, banner_id: int, dry_run: bool = False): print(f"Would have sent the following request: {url}") return r = requests.delete(url, headers=headers, params={'unblock-key': self.unblock_key}) - r.raise_for_status() + raise_for_status(r)() return r diff --git a/src/datastation/dataverse/dataset_api.py b/src/datastation/dataverse/dataset_api.py index b3b3ded..7547a45 100644 --- a/src/datastation/dataverse/dataset_api.py +++ b/src/datastation/dataverse/dataset_api.py @@ -1,9 +1,10 @@ import json +import logging import time import requests -from datastation.common.utils import print_dry_run_message +from datastation.common.utils import print_dry_run_message, raise_for_status class DatasetApi: @@ -28,7 +29,7 @@ def get(self, version=":latest", dry_run=False): return None dv_resp = requests.get(url, headers=headers, params=params) - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data @@ -44,7 +45,7 @@ def edit_metadata(self, data: dict, dry_run=False, replace: bool = False): return None else: r = requests.put(url, headers=headers, params=params, data=data) - r.raise_for_status() + raise_for_status(r) return r def get_role_assignments(self, dry_run=False): @@ -56,7 +57,7 @@ def get_role_assignments(self, dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json()['data'] def add_role_assignment(self, assignee, role, dry_run=False): @@ -70,7 +71,7 @@ def add_role_assignment(self, assignee, role, dry_run=False): return None else: r = requests.post(url, headers=headers, params=params, json=role_assignment) - r.raise_for_status() + raise_for_status(r) return r def remove_role_assignment(self, assignment_id, dry_run=False): @@ -82,7 +83,7 @@ def remove_role_assignment(self, assignment_id, dry_run=False): return None else: r = requests.delete(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r def is_draft(self, dry_run=False): @@ -94,7 +95,7 @@ def is_draft(self, dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json()['data']['latestVersion']['versionState'] == 'DRAFT' def delete_draft(self, dry_run=False): @@ -106,7 +107,7 @@ def delete_draft(self, dry_run=False): return None else: r = requests.delete(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def destroy(self, dry_run=False): @@ -122,7 +123,7 @@ def destroy(self, dry_run=False): print_dry_run_message(method='DELETE', url=url, headers=headers, params=params) return None r = requests.delete(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def get_metadata(self, version=':latest', dry_run=False): @@ -139,7 +140,7 @@ def get_metadata(self, version=':latest', dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json()['data'] def get_metadata_export(self, exporter='dataverse_json', dry_run=False): @@ -155,7 +156,7 @@ def get_metadata_export(self, exporter='dataverse_json', dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.text def get_locks(self, lock_type=None, dry_run=False): @@ -169,7 +170,7 @@ def get_locks(self, lock_type=None, dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json()['data'] def add_lock(self, lock_type, dry_run=False): @@ -181,7 +182,7 @@ def add_lock(self, lock_type, dry_run=False): return None else: r = requests.post(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def remove_lock(self, lock_type=None, dry_run=False): @@ -195,7 +196,7 @@ def remove_lock(self, lock_type=None, dry_run=False): return None else: r = requests.delete(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def remove_all_locks(self, dry_run=False): @@ -209,7 +210,7 @@ def publish(self, update_type='major', dry_run=False): print_dry_run_message(method='POST', url=url, headers=headers, params=params) return None r = requests.post(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def reindex(self, dry_run=False): @@ -223,7 +224,7 @@ def reindex(self, dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def modify_registration_metadata(self, dry_run=False): @@ -235,7 +236,7 @@ def modify_registration_metadata(self, dry_run=False): return None else: r = requests.post(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json() def get_files(self, version=':latest', dry_run=False): @@ -247,7 +248,7 @@ def get_files(self, version=':latest', dry_run=False): return None else: r = requests.get(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r) return r.json()['data'] def await_unlock(self, lock_type=None, sleep_time=5, max_tries=10): diff --git a/src/datastation/dataverse/dataverse_api.py b/src/datastation/dataverse/dataverse_api.py index 39d6f25..9d60978 100644 --- a/src/datastation/dataverse/dataverse_api.py +++ b/src/datastation/dataverse/dataverse_api.py @@ -18,7 +18,7 @@ def get_contents(self, alias="root", dry_run=False): return None dv_resp = requests.get(url, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()["data"] return resp_data @@ -32,5 +32,5 @@ def get_storage_size(self, alias="root", dry_run=False): return None else: r = requests.get(url, headers=headers) - r.raise_for_status() + raise_for_status(r)() return r.json()['data']['message'] diff --git a/src/datastation/dataverse/file_api.py b/src/datastation/dataverse/file_api.py index ed17b1d..70ac817 100644 --- a/src/datastation/dataverse/file_api.py +++ b/src/datastation/dataverse/file_api.py @@ -20,5 +20,5 @@ def reingest(self, dry_run=False): print_dry_run_message(method='POST', url=url, headers=headers, params=params) return None r = requests.post(url, headers=headers, params=params) - r.raise_for_status() + raise_for_status(r)() return r diff --git a/src/datastation/dataverse/metrics_api.py b/src/datastation/dataverse/metrics_api.py index 1f90ba5..dde1dfb 100644 --- a/src/datastation/dataverse/metrics_api.py +++ b/src/datastation/dataverse/metrics_api.py @@ -14,5 +14,5 @@ def get_tree(self, dry_run: bool = False): print(f"Would have sent the following request: {url}") return r = requests.get(url) - r.raise_for_status() + raise_for_status(r)() return r.json()['data'] diff --git a/src/datastation/dataverse/search_api.py b/src/datastation/dataverse/search_api.py index b3c3af1..d9a7e84 100644 --- a/src/datastation/dataverse/search_api.py +++ b/src/datastation/dataverse/search_api.py @@ -54,7 +54,7 @@ def search(self, query="*", subtree="root", object_type="dataset", dry_run=False while True: dv_resp = requests.get(self.url, headers=headers, params=params) - dv_resp.raise_for_status() + raise_for_status(dv_resp) data = dv_resp.json()["data"] items = data["items"] diff --git a/src/datastation/dv_api.py b/src/datastation/dv_api.py index 37c4f7f..62f0c52 100644 --- a/src/datastation/dv_api.py +++ b/src/datastation/dv_api.py @@ -42,7 +42,7 @@ def search(server_url, subtree, start=0, rows=10): # print("Status code: {}".format(dv_resp.status_code)) # print("Json: {}".format(dv_resp.json())) # the json result is a dictionary... so we could check for something in it - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] # print(json.dumps(resp_data, indent=2)) return resp_data @@ -57,7 +57,7 @@ def get_dataset_metadata_export(server_url, pid, exporter = 'dataverse_json', re # print("Status code: {}".format(dv_resp.status_code)) # print("Json: {}".format(dv_resp.json())) # the json result is a dictionary... so we could check for something in it - dv_resp.raise_for_status() + raise_for_status(dv_resp) # assume json, but not all exporters have that! if response_is_json: resp_data = dv_resp.json() # Note that the response json has no wrapper around the data @@ -75,7 +75,7 @@ def get_dataset_metadata(server_url, api_token, pid): # print("Status code: {}".format(dv_resp.status_code)) # print("Json: {}".format(dv_resp.json())) # the json result is a dictionary... so we could check for something in it - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data @@ -88,7 +88,7 @@ def replace_dataset_metadatafield(server_url, api_token, pid, field): server_url + '/api/datasets/:persistentId/editMetadata?persistentId=' + pid + '&replace=true', data=json.dumps(field, ensure_ascii=False), headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) except requests.exceptions.RequestException as re: print("RequestException: ", re) raise @@ -103,7 +103,7 @@ def get_dataset_roleassigments(server_url, api_token, pid): dv_resp = requests.get(server_url + '/api/datasets/:persistentId/assignments', params=params, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) except requests.exceptions.RequestException as re: print("RequestException: ", re) raise @@ -116,7 +116,7 @@ def delete_dataset_role_assignment(server_url, api_token, pid, assignment_id): dv_resp = requests.delete(server_url + '/api/datasets/:persistentId/assignments/' + str(assignment_id) + '?persistentId=' + pid, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) def add_dataset_role_assignment(server_url, api_token, pid, assignment): @@ -126,7 +126,7 @@ def add_dataset_role_assignment(server_url, api_token, pid, assignment): headers=headers, data=json.dumps(assignment, ensure_ascii=False), params=params) - dv_resp.raise_for_status() + raise_for_status(dv_resp) def get_dataset_locks(server_url: str, pid: str): @@ -135,7 +135,7 @@ def get_dataset_locks(server_url: str, pid: str): # print("Status code: {}".format(dv_resp.status_code)) # print("Json: {}".format(dv_resp.json())) # the json result is a dictionary... so we could check for something in it - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data @@ -145,14 +145,14 @@ def get_dataset_files(server_url: str, pid: str, version=':latest'): # print("Status code: {}".format(dv_resp.status_code)) # print("Json: {}".format(dv_resp.json())) # the json result is a dictionary... so we could check for something in it - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data def reingest_file(server_url: str, api_token: str, file_id: str): headers = {'X-Dataverse-key': api_token, 'Content-Type': 'application/json'} dv_resp = requests.post(server_url + '/api/files/' + file_id + '/reingest', headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data @@ -161,21 +161,21 @@ def create_dataset_lock(server_url, api_token, pid, lock_type): headers = {'X-Dataverse-key': api_token} dv_resp = requests.post("{}/api/datasets/:persistentId/lock/{}?persistentId={}" .format(server_url, lock_type, pid), headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) def delete_dataset_locks_all(server_url, api_token, pid): headers = {'X-Dataverse-key': api_token} dv_resp = requests.delete(server_url + '/api/datasets/:persistentId/locks?persistentId=' + pid, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) def delete_dataset_lock(server_url, api_token, pid, lock_type): headers = {'X-Dataverse-key': api_token} dv_resp = requests.delete("{}/api/datasets/:persistentId/locks?persistentId={}&type={}" .format(server_url, pid, lock_type), headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) def publish_dataset(server_url, api_token, pid, version_upgrade_type="major"): @@ -184,14 +184,14 @@ def publish_dataset(server_url, api_token, pid, version_upgrade_type="major"): dv_resp = requests.post(server_url + '/api/datasets/:persistentId/actions/:publish?persistentId=' + pid + '&type=' + version_upgrade_type, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) # This is via the admin api and does not use the token, # but instead will need to be run on localhost or via an SSH tunnel for instance! def reindex_dataset(server_url, pid): dv_resp = requests.get(server_url + '/api/admin/index/dataset?persistentId=' + pid) - dv_resp.raise_for_status() + raise_for_status(dv_resp) resp_data = dv_resp.json()['data'] return resp_data @@ -202,7 +202,7 @@ def delete_dataset_draft(server_url, api_token, pid): headers = {'X-Dataverse-key': api_token} dv_resp = requests.delete(server_url + '/api/datasets/:persistentId/versions/:draft?persistentId=' + pid, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) # Warning: this deletes also a PUBLISHED dataset! @@ -212,7 +212,7 @@ def destroy_dataset(server_url, api_token, pid): headers = {'X-Dataverse-key': api_token} dv_resp = requests.delete(server_url + '/api/datasets/:persistentId/destroy/?persistentId=' + pid, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) # Remember to get info on the OAI endpoint you can do: @@ -230,7 +230,7 @@ def get_oai_records(server_url, format, set=None): dv_resp = requests.get(server_url + '/oai', params=params) - dv_resp.raise_for_status() + raise_for_status(dv_resp) # assume XML xml_doc = etree.fromstring(dv_resp.content) # alternatively we could use the parse directly and not requests.get @@ -244,7 +244,7 @@ def get_oai_records_resume(server_url, token): dv_resp = requests.get(server_url + '/oai', params=params) - dv_resp.raise_for_status() + raise_for_status(dv_resp) # assume XML xml_doc = etree.fromstring(dv_resp.content) return xml_doc @@ -257,7 +257,7 @@ def change_access_request(server_url, api_token, pid, makeRestricted): server_url + '/api/access/:persistentId/allowAccessRequest?persistentId=' + pid , data=json.dumps(makeRestricted), headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) except requests.exceptions.RequestException as re: print("RequestException: ", re) raise @@ -273,7 +273,7 @@ def change_file_restrict(server_url, api_token, file_id, makeRestricted): server_url + '/api/files/{}/restrict'.format(file_id), data=json.dumps(makeRestricted), headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) except requests.exceptions.RequestException as re: print("RequestException: ", re) raise @@ -288,7 +288,7 @@ def replace_dataset_metadata(server_url, api_token, pid, json_data): server_url + '/api/datasets/:persistentId/metadata?persistentId=' + pid + '&replace=true', data=json_data, headers=headers) - dv_resp.raise_for_status() + raise_for_status(dv_resp) except requests.exceptions.RequestException as re: print("RequestException: ", re) raise