redis key store path for object_id

diycrate/cache_utils.py

@@ -17,7 +17,7 @@
 crate_logger = logging.getLogger(__name__)
 
 
-def redis_key(key):
+def redis_key(key: str) -> str:
     """
 
     :param key:
@@ -26,6 +26,11 @@ def redis_key(key):
     return "diy_crate.version.{}".format(key)
 
 
+def redis_path_for_object_id_key(key: PathLike):
+    key_str = key.as_posix() if isinstance(key, Path) else key
+    return f"diy_crate.path_for_object_id:{key_str}"
+
+
 def redis_set(
     cache_client: redis.Redis,
     cloud_item: Union[File, Folder],
@@ -81,12 +86,16 @@ def redis_set(
     cache_client.set(key, json.dumps(item_info))
     last_save_time_stamp = int(time.time())
     cache_client.set("diy_crate.last_save_time_stamp", last_save_time_stamp)
+    cache_client.set(
+        redis_path_for_object_id_key((path / cloud_item["name"])), cloud_item.object_id
+    )
+
     crate_logger.debug(f"Storing/updating info to redis:  {key=} {item_info=}")
     # assert redis_get(obj)
     return {key: item_info, "diy_crate.last_save_time_stamp": last_save_time_stamp}
 
 
-def redis_get(cache_client, obj):
+def redis_get(cache_client, obj: Union[File, Folder]):
     """
 
     :param cache_client:
@@ -104,6 +113,9 @@ def id_for_file_path(cache_client, file_path):
     :param file_path:
     :return:
     """
+    maybe_obj_id = cache_client.get(redis_path_for_object_id_key(file_path))
+    if maybe_obj_id:
+        return str(maybe_obj_id)
     for key in cache_client.keys("diy_crate.version.*"):
         value = json.loads(
             str(cache_client.get(key), encoding="utf-8", errors="strict")

diycrate/file_operations.py

@@ -30,6 +30,7 @@
     r_c,
     local_or_box_file_m_time_key_func,
     redis_set,
+    redis_path_for_object_id_key,
 )
 from .item_queue_io import (
     UploadQueueItem,
@@ -111,7 +112,7 @@ def operation_coalesce(self):
             ]  # keep a local copy for this loop-run
             # operations list could have changed since the previous two instructions
             # pycharm complained that I was re-assigning the instance
-            # variable outside of the __init__.
+            # variable outside the __init__.
             self.operations.clear()
             self.operations.extend(
                 self.operations[cur_num_operations:]
@@ -659,7 +660,7 @@ def process_move_event(self, event: pyinotify.Event):
                 is_dir and entry["name"] == src_event.name and entry["type"] == "folder"
             )
             if did_find_src_file:
-                src_file = client.file(file_id=entry["id"]).get()
+                src_file: File = client.file(file_id=entry["id"]).get()
                 if is_rename:
                     last_modified_time = (
                         datetime.fromtimestamp(
@@ -682,25 +683,40 @@ def process_move_event(self, event: pyinotify.Event):
                         .timestamp(),
                     )
                     src_file.rename(dest_event.name)
-                    file_obj = client.file(file_id=src_file.object_id).get()
+                    file_obj: File = client.file(file_id=src_file.object_id).get()
                     version_info["file_path"] = dest_event.pathname
                     version_info["etag"] = file_obj["etag"]
+                    r_c.set(
+                        redis_path_for_object_id_key(dest_event.pathname),
+                        src_file.object_id,
+                    )
+                    r_c.delete(
+                        redis_path_for_object_id_key(
+                            Path(src_event.path) / src_file.name
+                        )
+                    )
                     r_c.set(redis_key(src_file.object_id), json.dumps(version_info))
                     r_c.set("diy_crate.last_save_time_stamp", int(time.time()))
                     path_builder = BOX_DIR
                     oauth = setup_remote_oauth(r_c, conf=conf_obj)
                     client = Client(oauth)
-                    for updated_entry in (
+                    parent_folders: list[Folder] = (
                         client.file(file_obj.object_id)
                         .get(fields=["path_collection"])
                         .path_collection["entries"]
-                    ):
+                    )
+                    for updated_entry in parent_folders:
                         if updated_entry.id == "0":
                             continue
                         path_builder /= updated_entry.name
-                        folder_entry = client.folder(updated_entry.id).get(
+                        folder_entry: Folder = client.folder(updated_entry.id).get(
                             fields=["modified_at"]
                         )
+                        if not r_c.exists(redis_path_for_object_id_key(path_builder)):
+                            r_c.set(
+                                redis_path_for_object_id_key(path_builder),
+                                updated_entry.object_id,
+                            )
                         r_c.set(
                             local_or_box_file_m_time_key_func(path_builder, True),
                             parse(folder_entry.modified_at)
@@ -713,9 +729,10 @@ def process_move_event(self, event: pyinotify.Event):
                         dest_event.pathname
                     )  # should check box instead
                     cur_offset = 0
-                    for cur_entry in cur_box_folder.get_items(
-                        offset=cur_offset, limit=limit
-                    ):
+                    cur_box_folder_items: list[
+                        Union[File, Folder]
+                    ] = cur_box_folder.get_items(offset=cur_offset, limit=limit)
+                    for cur_entry in cur_box_folder_items:
                         matching_name = cur_entry["name"] == os.path.basename(
                             dest_event.pathname
                         )
@@ -1020,7 +1037,7 @@ def process_IN_MOVED_TO(self, event: pyinotify.Event):
             self.incoming_operations.put(
                 [event, "modify"]
             )  # "close"/"modify" seems appropriate
-            # allow moving from a ~.lock file...i guess that may be okay
+            # allow moving from a ~.lock file...I guess that may be okay
             crate_logger.debug("Moved to: {}".format(event.pathname))
 
     # noinspection PyPep8Naming,PyMethodMayBeStatic

diycrate/item_queue_io.py

@@ -27,6 +27,7 @@
     redis_get,
     r_c,
     local_or_box_file_m_time_key_func,
+    redis_path_for_object_id_key,
 )
 from .gui import notify_user_with_gui
 from .log_utils import setup_logger
@@ -111,23 +112,22 @@ def perform_upload(
                 item = ret_val  # is the new/updated item
                 if isinstance(item, File):
                     client = Client(oauth)
-                    file_obj: File = client.file(file_id=item.object_id).get()
+                    file_obj: File = client.file(file_id=item.object_id).get(
+                        fields=["path_collection", "name", "etag"]
+                    )
                     redis_set(r_c, file_obj, last_modified_time, box_dir_path=BOX_DIR)
                     r_c.set(
                         local_or_box_file_m_time_key_func(path_name, False),
                         datetime.fromtimestamp(Path(path_name).stat().st_mtime)
                         .astimezone(dateutil.tz.tzutc())
                         .timestamp(),
                     )
+                    r_c.set(redis_path_for_object_id_key(path_name), item.object_id)
 
                     path_builder = BOX_DIR
-                    oauth = setup_remote_oauth(r_c, conf=conf_obj)
-                    client = Client(oauth)
-                    for entry in (
-                        client.file(item.object_id)
-                        .get(fields=["path_collection"])
-                        .path_collection["entries"]
-                    ):
+                    # oauth = setup_remote_oauth(r_c, conf=conf_obj)
+                    # client = Client(oauth)
+                    for entry in file_obj.path_collection["entries"]:
                         if entry.id == "0":
                             continue
                         path_builder /= entry.name
@@ -140,6 +140,10 @@ def perform_upload(
                             .astimezone(dateutil.tz.tzutc())
                             .timestamp(),
                         )
+                        r_c.set(
+                            redis_path_for_object_id_key(path_builder),
+                            folder_entry.object_id,
+                        )
 
             break
         except BoxAPIException as e:
@@ -296,6 +300,7 @@ def perform_download(item: File, path: Union[str, Path], retry_limit=15):
                 local_or_box_file_m_time_key_func(path / item.name, True),
                 parse(item.modified_at).astimezone(dateutil.tz.tzutc()).timestamp(),
             )
+            r_c.set(redis_path_for_object_id_key(path / item.name), item.object_id)
             path_builder = BOX_DIR
             oauth = setup_remote_oauth(r_c, conf=conf_obj)
             client = Client(oauth)
@@ -312,12 +317,25 @@ def perform_download(item: File, path: Union[str, Path], retry_limit=15):
                         .astimezone(dateutil.tz.tzutc())
                         .timestamp(),
                     )
+                    r_c.set(
+                        redis_path_for_object_id_key(path_builder),
+                        folder_entry.object_id,
+                    )
             except boxsdk.exception.BoxAPIException as box_exc:
                 if box_exc.status == 404 and box_exc.code == "trashed":
                     crate_logger.debug(
                         f"Object {item.object_id=} was previously deleted from Box. "
                         f"{(path / item.name)=}"
                     )
+
+                    object_id_lookup = r_c.get(
+                        redis_path_for_object_id_key(path / item.name)
+                    )
+                    object_id_lookup = (
+                        str(object_id_lookup) if object_id_lookup else object_id_lookup
+                    )
+                    if object_id_lookup == item.object_id:
+                        r_c.delete(redis_path_for_object_id_key(path / item.name))
                     r_c.delete(redis_key(item.object_id))
                     r_c.set("diy_crate.last_save_time_stamp", int(time.time()))
                 else:

diycrate/long_poll_processing.py

@@ -29,6 +29,7 @@
     redis_key,
     redis_get,
     local_or_box_file_m_time_key_func,
+    redis_path_for_object_id_key,
 )
 from diycrate.gui import notify_user_with_gui
 from diycrate.item_queue_io import (
@@ -119,6 +120,9 @@ def process_item_create_long_poll(client: Client, event: Union[Event, Mapping]):
                 local_or_box_file_m_time_key_func(path / box_item.name, True),
                 parse(box_item.modified_at).astimezone(tzutc()).timestamp(),
             )
+            r_c.set(
+                redis_path_for_object_id_key(path / box_item.name), box_item.object_id
+            )
             r_c.setex(
                 f"diy_crate:event_ids:{event.event_id}",
                 timedelta(days=32),
@@ -218,6 +222,9 @@ def process_item_copy_long_poll(client: Client, event: Union[Event, Mapping]):
                 local_or_box_file_m_time_key_func(path / box_item.name, True),
                 parse(box_item.modified_at).astimezone(tzutc()).timestamp(),
             )
+            r_c.set(
+                redis_path_for_object_id_key(path / box_item.name), box_item.object_id
+            )
             r_c.setex(
                 f"diy_crate:event_ids:{event.event_id}",
                 timedelta(days=32),
@@ -304,7 +311,7 @@ def process_item_trash_file(event: Union[Event, Mapping], obj_id):
     if file_path.exists():
         send2trash(file_path.as_posix())
     if r_c.exists(redis_key(obj_id)):
-        r_c.delete(redis_key(obj_id))
+        r_c.delete(redis_key(obj_id), redis_path_for_object_id_key(file_path))
         r_c.set("diy_crate.last_save_time_stamp", int(time.time()))
     notify_user_with_gui(
         "Box message: Deleted:",
@@ -337,14 +344,20 @@ def process_item_trash_folder(event: Union[Event, Mapping], obj_id):
         file_path = path / item_info["file_path"]
     if file_path.is_dir():
         # still need to get the parent_id
-        for box_id in get_sub_ids(obj_id):
-            r_c.delete(redis_key(box_id))
+        for sub_box_id in get_sub_ids(obj_id):
+            cur_sub_box_redis_data = r_c.get(redis_key(sub_box_id))
+            if cur_sub_box_redis_data:
+                cur_sub_box_item_info = json.loads(
+                    str(cur_sub_box_redis_data, encoding="utf-8", errors="strict")
+                )
+                r_c.delete(cur_sub_box_item_info["file_name"])
+            r_c.delete(redis_key(sub_box_id))
         r_c.delete(redis_key(obj_id))
         shutil.rmtree(file_path)
-        obj_cache_data = r_c.get(redis_key(obj_id))
+        obj_cache_data = json.loads(str(r_c.get(redis_key(obj_id))))
         parent_id = obj_cache_data.get("parent_id") if obj_cache_data else None
         if parent_id:
-            parent_folder = r_c.get(redis_key(parent_id))
+            parent_folder = json.loads(str(r_c.get(redis_key(parent_id))))
             sub_ids = parent_folder.get("sub_ids", [])
             if sub_ids:
                 sub_ids.remove(obj_id)
@@ -514,8 +527,15 @@ def process_item_rename_long_poll(client: Client, event: Union[Event, Mapping]):
                     local_or_box_file_m_time_key_func(path / file_obj.name, True),
                     parse(file_obj.modified_at).astimezone(tzutc()).timestamp(),
                 )
-                r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False))
-                r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True))
+                r_c.set(
+                    redis_path_for_object_id_key(path / file_obj.name),
+                    file_obj.object_id,
+                )
+                r_c.delete(
+                    redis_path_for_object_id_key(src_file_path),
+                    local_or_box_file_m_time_key_func(src_file_path, False),
+                    local_or_box_file_m_time_key_func(src_file_path, True),
+                )
                 r_c.setex(
                     f"diy_crate:event_ids:{event.event_id}",
                     timedelta(days=32),
@@ -603,6 +623,10 @@ def process_item_rename_long_poll(client: Client, event: Union[Event, Mapping]):
                     local_or_box_file_m_time_key_func(path / folder_obj.name, True),
                     parse(folder_obj.modified_at).astimezone(tzutc()).timestamp(),
                 )
+                r_c.set(
+                    redis_path_for_object_id_key(path / folder_obj.name),
+                    folder_obj.object_id,
+                )
                 r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False))
                 r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True))
                 r_c.setex(
@@ -677,6 +701,13 @@ def process_item_move_long_poll(event: Union[Event, Mapping]):
                                     json.dumps(sub_item_info),
                                 )
                             )
+                            to_set.append(
+                                partial(
+                                    r_c.set,
+                                    redis_path_for_object_id_key(new_sub_path),
+                                    sub_id,
+                                )
+                            )
                 shutil.move(src_file_path, file_path)
                 for item in to_set:
                     item()
@@ -698,6 +729,8 @@ def process_item_move_long_poll(event: Union[Event, Mapping]):
                     .astimezone(tzutc())
                     .timestamp(),
                 )
+                r_c.set(redis_path_for_object_id_key(file_path), obj_id)
+                r_c.delete(redis_path_for_object_id_key(src_file_path))
                 r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False))
                 r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True))
                 r_c.setex(

diycrate/path_utils.py

@@ -31,6 +31,7 @@
     redis_set,
     redis_get,
     local_or_box_file_m_time_key_func,
+    redis_path_for_object_id_key,
 )
 from .iter_utils import SafeIter
 from .log_utils import setup_logger
@@ -143,6 +144,7 @@ def walk_and_notify_and_download_tree(
             local_or_box_file_m_time_key_func(path / box_item.name, True),
             parse(box_item.modified_at).astimezone(tzutc()).timestamp(),
         )
+        r_c.set(redis_path_for_object_id_key(path / box_item.name), box_item.object_id)
     if not local_only:
         redis_set(
             cache_client=r_c,