Merge pull request #146 from tymmej/all_albums_configurable

Add option to preserve album structure

README.md

@@ -112,15 +112,18 @@ photos:
   destination: "photos"
   remove_obsolete: false
   sync_interval: 500
-  filters: # Optional, use it only if you want to download specific albums. Else, all photos are downloaded to `all` folder.
+  all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem
+  filters:
+    # if all_albums is false list of albums to download, if all_albums is true list of ignored albums
+    # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders
     albums:
       - "album 1"
       - "album2"
     file_sizes: # valid values are original, medium and/or thumb
       - "original"
       # - "medium"
       # - "thumb"
-    extensions: #Optional, media extensions to be included in syncing iCloud Photos content
+    extensions: # Optional, media extensions to be included in syncing iCloud Photos content
       # - jpg
       # - heic
       # - png

config.yaml

@@ -45,7 +45,10 @@ photos:
   destination: "photos"
   remove_obsolete: false
   sync_interval: 500
+  all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem
   filters:
+    # if all_albums is false list of albums to download, if all_albums is true list of ignored albums
+    # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders
     albums:
       - "album 1"
       - "album2"

src/config_parser.py

@@ -96,6 +96,16 @@ def get_photos_sync_interval(config):
     return sync_interval
 
 
+def get_photos_all_albums(config):
+    """Return flag to download all albums from config."""
+    download_all = False
+    config_path = ["photos", "all_albums"]
+    if traverse_config_path(config=config, config_path=config_path):
+        download_all = get_config_value(config=config, config_path=config_path)
+        LOGGER.info("Syncing all albums.")
+    return download_all
+
+
 def prepare_root_destination(config):
     """Prepare root destination."""
     LOGGER.debug("Checking root destination ...")

src/sync_photos.py

@@ -144,7 +144,17 @@ def sync_photos(config, photos):
     destination_path = config_parser.prepare_photos_destination(config=config)
     filters = config_parser.get_photos_filters(config=config)
     files = set()
-    if filters["albums"]:
+    download_all = config_parser.get_photos_all_albums(config=config)
+    if download_all:
+        for album in photos.albums.keys():
+            sync_album(
+                album=photos.albums[album],
+                destination_path=os.path.join(destination_path, album),
+                file_sizes=filters["file_sizes"],
+                extensions=filters["extensions"],
+                files=files,
+            )
+    elif filters["albums"]:
         for album in iter(filters["albums"]):
             sync_album(
                 album=photos.albums[album],

tests/data/__init__.py

@@ -3834,6 +3834,64 @@ def request(self, method, url, **kwargs):
                             "query?remapEnums=True&getCurrentSyncToken=True"
                         ][0]["response"]
                     )
+                if (
+                    data.get("query").get("recordType")
+                    == "CPLAssetAndMasterHiddenByAssetDate"
+                ):
+                    return ResponseMock(
+                        photos_data.DATA[
+                            "query?remapEnums=True&getCurrentSyncToken=True"
+                        ][5]["response"]
+                    )
+                if (
+                    data.get("query").get("recordType")
+                    == "CPLAssetAndMasterDeletedByExpungedDate"
+                ):
+                    return ResponseMock(
+                        photos_data.DATA[
+                            "query?remapEnums=True&getCurrentSyncToken=True"
+                        ][5]["response"]
+                    )
+                if (
+                    data.get("query").get("recordType")
+                    == "CPLBurstStackAssetAndMasterByAssetDate"
+                ):
+                    return ResponseMock(
+                        photos_data.DATA[
+                            "query?remapEnums=True&getCurrentSyncToken=True"
+                        ][5]["response"]
+                    )
+
+                if data.get("query").get("recordType") in (
+                    "CPLAssetAndMasterInSmartAlbumByAssetDate"
+                ):
+                    if "filterBy" in data["query"] and data.get("query").get(
+                        "filterBy"
+                    )[2]["fieldValue"]["value"] in (
+                        "TIMELAPSE",
+                        "LIVE",
+                        "VIDEO",
+                        "SLOMO",
+                        "FAVORITE",
+                        "SCREENSHOT",
+                        "BURST",
+                        "PANORAMA",
+                    ):
+                        return ResponseMock(
+                            photos_data.DATA[
+                                "query?remapEnums=True&getCurrentSyncToken=True"
+                            ][5]["response"]
+                        )
+                    if (
+                        "filterBy" in data["query"]
+                        and data.get("query").get("filterBy")[2]["fieldValue"]["value"]
+                        == "VIDEO"
+                    ):
+                        return ResponseMock(
+                            photos_data.DATA[
+                                "query?remapEnums=True&getCurrentSyncToken=True"
+                            ][5]["response"]
+                        )
                 if data.get("query").get("recordType") == "CPLAlbumByPositionLive":
                     if (
                         "filterBy" in data["query"]

tests/data/test_config.yaml

@@ -53,7 +53,10 @@ photos:
   destination: photos
   remove_obsolete: false
   sync_interval: -1
+  all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem
   filters:
+    # if all_albums is false list of albums to download, if all_albums is true list of ignored albums
+    # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders
     albums:
       - "album 2"
       - album-1

tests/test_config_parser.py

@@ -399,3 +399,21 @@ def test_get_region_valid(self):
         config["app"]["region"] = "china"
         actual = config_parser.get_region(config=config)
         self.assertEqual(actual, "china")
+
+    def test_get_all_albums_empty(self):
+        """Empty all_albums."""
+        config = read_config(config_path=tests.CONFIG_PATH)
+        self.assertFalse(config_parser.get_photos_all_albums(config=config))
+
+    def test_get_all_albums_true(self):
+        """True all_albums."""
+        config = read_config(config_path=tests.CONFIG_PATH)
+        config["photos"]["all_albums"] = True
+        self.assertTrue(config_parser.get_photos_all_albums(config=config))
+
+    def test_get_all_albums_false(self):
+        """False all_albums."""
+        config = read_config(config_path=tests.CONFIG_PATH)
+        config["photos"]["all_albums"] = False
+        self.assertFalse(config_parser.get_photos_all_albums(config=config))
+

tests/test_sync_photos.py

@@ -50,6 +50,34 @@ def test_sync_photos_original(
         self.assertIsNone(
             sync_photos.sync_photos(config=config, photos=mock_service.photos)
         )
+        album_2_path = os.path.join(self.destination_path, "album 2")
+        album_1_1_path = os.path.join(album_2_path, "album-1-1")
+        album_1_path = os.path.join(self.destination_path, "album-1")
+        self.assertTrue(os.path.isdir(album_2_path))
+        self.assertTrue(os.path.isdir(album_1_path))
+        self.assertTrue(os.path.isdir(album_1_1_path))
+        self.assertTrue(len(os.listdir(album_2_path)) > 1)
+        self.assertTrue(len(os.listdir(album_1_path)) > 0)
+
+    @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD)
+    @patch(
+        target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER
+    )
+    @patch("icloudpy.ICloudPyService")
+    @patch("src.read_config")
+    def test_sync_photos_all_albums(
+        self, mock_read_config, mock_service, mock_get_username, mock_get_password
+    ):
+        """Test for successful original photo size download."""
+        mock_service = self.service
+        config = self.config.copy()
+        config["photos"]["destination"] = self.destination_path
+        config["photos"]["all_albums"] = True
+        mock_read_config.return_value = config
+        # Sync original photos
+        self.assertIsNone(
+            sync_photos.sync_photos(config=config, photos=mock_service.photos)
+        )
         album_0_path = os.path.join(
             self.destination_path, config["photos"]["filters"]["albums"][0]
         )