diff --git a/cax/api.py b/cax/api.py index ed39fc4..ce802b0 100644 --- a/cax/api.py +++ b/cax/api.py @@ -7,6 +7,7 @@ class api(): def __init__(self): + logging.getLogger('requests').setLevel(logging.ERROR) if ( config.API_URL is None or config.api_user() is None or config.api_key() is None or config.DETECTOR is None ): @@ -73,10 +74,35 @@ def add_location(self, uuid, parameters): # BSON/JSON confusion. Get rid of date field. if 'creation_time' in parameters: parameters.pop('creation_time') - parameters=dumps(parameters) - ret = requests.put(url, data=parameters, + pars=dumps(parameters) + ret = requests.put(url, data=pars, headers=self.data_set_headers) - + + # This checks to make sure the location was added/removed/updated + # GET request + params = self.get_params + doc = json_util.loads(requests.get(self.api_url+str(uuid), + params=params).text)['doc'] + + # We removed the location + if parameters['status'] == 'remove': + + if 'data' not in doc: + return True + for entry in doc['data']: + + if self.verify_site(entry, parameters): + print(entry) + print(parameters) + raise RuntimeError("Failed to update run doc") + else: + if 'data' not in doc: + raise RuntimeError("Failed to update run doc") + for entry in doc['data']: + if self.verify_site(entry, parameters): + return True + raise RuntimeError("Failed to update run doc") + def remove_location(self, uuid, parameters): # Removes a data location from the list parameters['status'] = "remove" @@ -86,3 +112,10 @@ def update_location(self, uuid, remove_parameters, add_parameters): # Removes location from the list then adds a new one self.remove_location(uuid, remove_parameters) self.add_location(uuid, add_parameters) + + def verify_site(self, sitea, siteb): + # We assume two data entries are identical if the host, type, + # and path are the same + return ( (sitea['host'] == siteb['host']) and + (sitea['type'] == siteb['type']) and + (sitea['location'] == siteb['location'])) diff --git a/cax/tasks/clear.py b/cax/tasks/clear.py index 3571345..49b1873 100644 --- a/cax/tasks/clear.py +++ b/cax/tasks/clear.py @@ -44,7 +44,7 @@ def each_location(self, data_doc): except FileNotFoundError: time_modified = 0 time_modified = datetime.datetime.fromtimestamp(time_modified) - time_made = datetime.datetime.strptime(data_doc['creation_time'][:-7], + time_made = datetime.datetime.strptime(data_doc['creation_time'][0][:-7], "%Y-%m-%dT%H:%M:%S") difference = datetime.datetime.utcnow() - max(time_modified, time_made) diff --git a/cax/tasks/data_mover.py b/cax/tasks/data_mover.py index 028d1a6..ea08c5c 100644 --- a/cax/tasks/data_mover.py +++ b/cax/tasks/data_mover.py @@ -163,6 +163,7 @@ def copyGFAL(self, datum_original, datum_destination, server, option_type, nstre def copySCP(self, datum_original, datum_destination, server, username, option_type): """Copy data via SCP function """ + util.log_to_file('ssh.log') ssh = SSHClient() ssh.load_system_host_keys()