diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 4320e58ab..7b1b164ea 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -507,7 +507,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): """ user_projects = dict() - user_info = dict() + user_info = defaultdict(dict) # parse dbGaP sftp server information dbgap_key = dbgap_config.get("decrypt_key", None) @@ -542,6 +542,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): ] # when converting the YAML from fence-config, python reads it as Python string literal. So "\" turns into "\\" which messes with the regex match project_id_patterns += patterns + self.logger.info(f"Using these file paths: {file_dict.items()}") for filepath, privileges in file_dict.items(): self.logger.info("Reading file {}".format(filepath)) if os.stat(filepath).st_size == 0: @@ -657,9 +658,9 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): tags["pi"] = row["downloader for names"] user_info[username] = { - "email": row.get("email") or "", + "email": row.get("email") or user_info[username].get('email') or "", "display_name": display_name, - "phone_number": row.get("phone") or "", + "phone_number": row.get("phone") or user_info[username].get('phone_number') or "", "tags": tags, } @@ -1570,6 +1571,8 @@ def _sync(self, sess): local_csv_file_list = glob.glob( os.path.join(self.sync_from_local_csv_dir, "*") ) + # Sort the list so the order of of files is consistent across platforms + local_csv_file_list.sort() user_projects_csv, user_info_csv = self._merge_multiple_local_csv_files( local_csv_file_list,