Fix some issues. Now modifications of data fields do some basic verif…

…ications to ensure they were successful and throw if they weren't

cax/api.py

@@ -7,6 +7,7 @@
 class api():
     def __init__(self):
 
+        logging.getLogger('requests').setLevel(logging.ERROR)
 
         if ( config.API_URL is None or config.api_user() is None
              or config.api_key() is None or config.DETECTOR is None ):
@@ -73,10 +74,35 @@ def add_location(self, uuid, parameters):
         # BSON/JSON confusion. Get rid of date field.
         if 'creation_time' in parameters:
             parameters.pop('creation_time')
-        parameters=dumps(parameters)
-        ret = requests.put(url, data=parameters,
+        pars=dumps(parameters)
+        ret = requests.put(url, data=pars,
                            headers=self.data_set_headers)
-
+
+        # This checks to make sure the location was added/removed/updated
+        # GET request
+        params = self.get_params
+        doc = json_util.loads(requests.get(self.api_url+str(uuid),
+                                           params=params).text)['doc']
+
+        # We removed the location
+        if parameters['status'] == 'remove':
+
+            if 'data' not in doc:
+                return True
+            for entry in doc['data']:
+
+                if self.verify_site(entry, parameters):
+                    print(entry)
+                    print(parameters)
+                    raise RuntimeError("Failed to update run doc")
+        else:
+            if 'data' not in doc:
+                raise RuntimeError("Failed to update run doc")
+            for entry in doc['data']:
+                if self.verify_site(entry, parameters):
+                    return True
+            raise RuntimeError("Failed to update run doc")
+
     def remove_location(self, uuid, parameters):    
         # Removes a data location from the list        
         parameters['status'] = "remove"
@@ -86,3 +112,10 @@ def update_location(self, uuid, remove_parameters, add_parameters):
         # Removes location from the list then adds a new one
         self.remove_location(uuid, remove_parameters)
         self.add_location(uuid, add_parameters)
+
+    def verify_site(self, sitea, siteb):
+        # We assume two data entries are identical if the host, type,
+        # and path are the same
+        return ( (sitea['host'] == siteb['host']) and
+                 (sitea['type'] == siteb['type']) and
+                 (sitea['location'] == siteb['location']))

cax/tasks/clear.py

@@ -44,7 +44,7 @@ def each_location(self, data_doc):
         except FileNotFoundError:
             time_modified = 0
         time_modified = datetime.datetime.fromtimestamp(time_modified)
-        time_made = datetime.datetime.strptime(data_doc['creation_time'][:-7],
+        time_made = datetime.datetime.strptime(data_doc['creation_time'][0][:-7],
                                                "%Y-%m-%dT%H:%M:%S")
         difference = datetime.datetime.utcnow() - max(time_modified,
                                                       time_made)

cax/tasks/data_mover.py

@@ -163,6 +163,7 @@ def copyGFAL(self, datum_original, datum_destination, server, option_type, nstre
     def copySCP(self, datum_original, datum_destination, server, username, option_type):
         """Copy data via SCP function
         """
+
         util.log_to_file('ssh.log')
         ssh = SSHClient()
         ssh.load_system_host_keys()