Merge pull request #873 from sabeechen/dev

Release v0.111.1

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit =
+    hassio-google-drive-backup/backup/logger.py
+    hassio-google-drive-backup/backup/tracing_session.py
+    hassio-google-drive-backup/backup/ui/debug.py
+    hassio-google-drive-backup/backup/server/cloudlogger.py
+    hassio-google-drive-backup/backup/debug/*

.devcontainer/requirements-dev.txt

@@ -30,4 +30,5 @@ aiofile
 grpcio
 aioping
 pytz
-tzlocal
+tzlocal
+pytest-cov

hassio-google-drive-backup/CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.111.1 [2023-06-19]
+- Support for the new network storage features in Home Assistant.  The addon will now create backups in what Home Assistant has configured as its default backup location.  This can be overridden in the addon's settings.
+- Raised the addon's required permissions to "Admin" in order to access the supervisor's mount API.
+- Fixed a CSS error causing toast messages to render partially off screen on small displays.
+- Fixed misreporting of some error codes from Google Drive when a partial upload can't be resumed.
+
 ## v0.110.4 [2023-04-28]
 - Fix a whitespace error causing authorization to fail.
 
@@ -18,19 +24,3 @@
   - Caching data from Google Drive for short periods during periodic syncing.
   - Backing off for a longer time (2 hours) when the addon hits permanent errors.
 - Fixes CSS issues that made the logs page hard to use.
-
-## v0.109.2 [2022-11-15]
-* Fixed a bug where disabling deletion from Google Drive and enabling deltes after upload could cause backups in Google Drive to be deleted.
-
-## v0.109.1 [2022-11-07]
-* If configured from the browser, defaults to a "dark" theme if haven't already configured custom colors
-* Makes the interval at which the addon publishes sensors to Home Assistant configurable (see the "Uncommon Options" settings)
-* "Free space in Google Drive" is now published as an attribute of the "sensor.backup_state" sensor.
-* The "binary_sensor.backups_stale" sensor will now report a problem if creating a backup hangs for more than a day.
-* Fixes potential whitespace errors when copy-pasting Google Drive credentials.
-* Fixes an exception when using generational backup and no backups are present.
-
-## v0.108.4 [2022-08-22]
-* Fixed an error causing "Undefined" to show up for addon descriptions.
-* Fixed an error preventing addon thumbnails from showing up.
-* Fixed an error causing username/password authentication to fail.

hassio-google-drive-backup/DOCS.md

@@ -10,7 +10,8 @@ _Note_: The configuration can be changed easily by starting the add-on and click
 The UI explains what each setting is and you don't need to modify anything before clicking `Start`.
 If you would still prefer to modify the settings in yaml, the options are detailed below.
 
-Add-on configuration example. Don't use this directly, the addon has a lot of configuration options that most users don't need or want:
+### Add-on configuration example
+Don't use this directly, the addon has a lot of configuration options that most users don't need or want:
 
 ```yaml
 # Keep 10 backups in Home Assistant
@@ -19,6 +20,9 @@ max_backups_in_ha: 10
 # Keep 10 backups in Google Drive
 max_backups_in_google_drive: 10
 
+# Create backups in Home Assistant on network storage 
+backup_location: my_nfs_share
+
 # Ignore backups the add-on hasn't created
 ignore_other_backups: True
 
@@ -80,6 +84,9 @@ The number of backups the add-on will allow Home Assistant to store locally befo
 
 The number of backups the add-on will keep in Google Drive before old ones are deleted. Google Drive gives you 15GB of free storage (at the time of writing) so plan accordingly if you know how big your backups are.
 
+### Option: `backup_location` (default: None)
+The place where backups are created in Home Assistant before uploading to Google Drive.  Can be "local-disk" or the name of any backup network storage you've configured in Home Assistant.  Leave unspecified (the default) to have backups created in whatever Home Assistant uses as the default backup location. 
+
 ### Option: `ignore_other_backups` (default: False)
 Make the addon ignore any backups it didn't directly create.  Any backup already uploaded to Google Drive will not be ignored until you delete it from Google Drive.
 

hassio-google-drive-backup/backup/config/settings.py

@@ -31,6 +31,7 @@ class Setting(Enum):
     ENABLE_BACKUP_STALE_SENSOR = "enable_backup_stale_sensor"
     ENABLE_BACKUP_STATE_SENSOR = "enable_backup_state_sensor"
     BACKUP_PASSWORD = "backup_password"
+    BACKUP_STORAGE = "backup_storage"
     CALL_BACKUP_SNAPSHOT = "call_backup_snapshot"
 
     # Basic backup settings
@@ -177,6 +178,7 @@ def key(self):
     Setting.ENABLE_BACKUP_STALE_SENSOR: True,
     Setting.ENABLE_BACKUP_STATE_SENSOR: True,
     Setting.BACKUP_PASSWORD: "",
+    Setting.BACKUP_STORAGE: "",
     Setting.WATCH_BACKUP_DIRECTORY: True,
     Setting.TRACE_REQUESTS: False,
 
@@ -316,6 +318,7 @@ def key(self):
     Setting.ENABLE_BACKUP_STALE_SENSOR: "bool?",
     Setting.ENABLE_BACKUP_STATE_SENSOR: "bool?",
     Setting.BACKUP_PASSWORD: "str?",
+    Setting.BACKUP_STORAGE: "str?",
     Setting.WATCH_BACKUP_DIRECTORY: "bool?",
     Setting.TRACE_REQUESTS: "bool?",
 

hassio-google-drive-backup/backup/config/version.py

@@ -1,7 +1,6 @@
 STAGING_KEY = ".staging."
 EXPECTED_VERISON_CHARS = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.']
 
-
 class Version:
     def __init__(self, *args):
         self._identifiers = args
@@ -11,6 +10,10 @@ def __init__(self, *args):
     def default(cls):
         return Version(0)
 
+    @classmethod
+    def max(cls):
+        return Version(99999999)
+
     @classmethod
     def parse(cls, version: str):
         staging_version = None

hassio-google-drive-backup/backup/const.py

@@ -34,6 +34,10 @@
 LOG_IN_TO_DRIVE = "log_in_to_drive"
 SUPERVISOR_PERMISSION = "supervisor_permission"
 
+# Network storage errors
+UNKONWN_NETWORK_STORAGE = "unknown_network_storage"
+INACTIVE_NETWORK_STORAGE = "inactive_network_storage"
+
 # these keys are necessary because they use the name "snapshot" in non-user-visible
 # places persisted outside the codebase.  They can't be changed without an upgrade path.
 NECESSARY_OLD_BACKUP_NAME = "snapshot"

hassio-google-drive-backup/backup/drive/driverequests.py

@@ -246,6 +246,15 @@ async def create(self, stream, metadata, mime_type):
                     # Drive doesn't recognize the resume token, so we'll just have to start over.
                     logger.debug("Drive upload session wasn't recognized, restarting upload from the beginning.")
                     location = None
+                    self.last_attempt_location = None
+                    self.last_attempt_metadata = None
+                    raise GoogleUnexpectedError()
+                if e.status == 404:
+                    logger.error("Drive upload session wasn't recognized (http 404), restarting upload from the beginning.")
+                    location = None
+                    self.last_attempt_location = None
+                    self.last_attempt_metadata = None
+                    raise GoogleUnexpectedError()
                 else:
                     raise
 

hassio-google-drive-backup/backup/exceptions/__init__.py

@@ -1,2 +1,2 @@
 # flake8: noqa
-from .exceptions import GoogleCredGenerateError, SupervisorUnexpectedError, SupervisorTimeoutError, GoogleUnexpectedError, SupervisorFileSystemError, SupervisorPermissionError, LogInToGoogleDriveError, KnownTransient, GoogleInternalError, GoogleRateLimitError, CredRefreshGoogleError, CredRefreshMyError, BackupFolderInaccessible, BackupFolderMissingError, DeleteMutlipleBackupsError, DriveQuotaExceeded, ensureKey, ExistingBackupFolderError, UserCancelledError, UploadFailed, SupervisorConnectionError, BackupPasswordKeyInvalid, BackupInProgress, SimulatedError, ProtocolError, PleaseWait, NotUploadable, NoBackup, LowSpaceError, LogicError, KnownError, InvalidConfigurationValue, HomeAssistantDeleteError, GoogleTimeoutError, GoogleSessionError, GoogleInternalError, GoogleDrivePermissionDenied, GoogleDnsFailure, GoogleCredentialsExpired, GoogleCantConnect, ExistingBackupFolderError
+from .exceptions import UnknownNetworkStorageError, InactiveNetworkStorageError, GoogleCredGenerateError, SupervisorUnexpectedError, SupervisorTimeoutError, GoogleUnexpectedError, SupervisorFileSystemError, SupervisorPermissionError, LogInToGoogleDriveError, KnownTransient, GoogleInternalError, GoogleRateLimitError, CredRefreshGoogleError, CredRefreshMyError, BackupFolderInaccessible, BackupFolderMissingError, DeleteMutlipleBackupsError, DriveQuotaExceeded, ensureKey, ExistingBackupFolderError, UserCancelledError, UploadFailed, SupervisorConnectionError, BackupPasswordKeyInvalid, BackupInProgress, SimulatedError, ProtocolError, PleaseWait, NotUploadable, NoBackup, LowSpaceError, LogicError, KnownError, InvalidConfigurationValue, HomeAssistantDeleteError, GoogleTimeoutError, GoogleSessionError, GoogleInternalError, GoogleDrivePermissionDenied, GoogleDnsFailure, GoogleCredentialsExpired, GoogleCantConnect, ExistingBackupFolderError

hassio-google-drive-backup/backup/exceptions/exceptions.py

@@ -10,7 +10,8 @@
                      ERROR_LOW_SPACE, ERROR_MULTIPLE_DELETES, ERROR_NO_BACKUP,
                      ERROR_NOT_UPLOADABLE, ERROR_PLEASE_WAIT, ERROR_PROTOCOL,
                      ERROR_BACKUP_IN_PROGRESS, ERROR_UPLOAD_FAILED, LOG_IN_TO_DRIVE,
-                     SUPERVISOR_PERMISSION, ERROR_GOOGLE_UNEXPECTED, ERROR_SUPERVISOR_TIMEOUT, ERROR_SUPERVISOR_UNEXPECTED, ERROR_SUPERVISOR_FILE_SYSTEM)
+                     SUPERVISOR_PERMISSION, ERROR_GOOGLE_UNEXPECTED, ERROR_SUPERVISOR_TIMEOUT, ERROR_SUPERVISOR_UNEXPECTED, ERROR_SUPERVISOR_FILE_SYSTEM,
+                     UNKONWN_NETWORK_STORAGE, INACTIVE_NETWORK_STORAGE)
 
 
 def ensureKey(key, target, name):
@@ -444,3 +445,35 @@ def message(self):
 
     def code(self):
         return ERROR_GOOGLE_CRED_PROCESS
+
+
+class UnknownNetworkStorageError(KnownError):
+    def __init__(self, name: str="Unkown"):
+        self.name = name
+
+    def message(self):
+        return f"The network storage '{self.name}' isn't recognized.  Please visit the add-on web UI to select different storage or use the local disk."
+
+    def code(self):
+        return UNKONWN_NETWORK_STORAGE
+
+    def data(self):
+        return {
+            "storage_name": self.name
+        }
+
+
+class InactiveNetworkStorageError(KnownError):
+    def __init__(self, name: str="Unkown"):
+        self.name = name
+
+    def message(self):
+        return f"The network storage '{self.name}' isn't ready.  The network share must be available before it can be used for a backup."
+
+    def code(self):
+        return INACTIVE_NETWORK_STORAGE
+
+    def data(self):
+        return {
+            "storage_name": self.name
+        }

hassio-google-drive-backup/backup/ha/harequests.py

@@ -25,6 +25,7 @@
 EVENT_BACKUP_END = "backup_ended"
 
 VERSION_BACKUP_PATH = Version.parse("2021.8")
+VERSION_MOUNT_INFO = Version.parse("2023.6")
 
 
 def supervisor_call(func):
@@ -55,7 +56,7 @@ def __init__(self, config: Config, session: ClientSession, time: Time, data_cach
         self._data_cache = data_cache
 
         # default the supervisor versio to using the "most featured" when it can't be parsed.
-        self._super_version = VERSION_BACKUP_PATH
+        self._super_version = VERSION_MOUNT_INFO
 
     def getSupervisorURL(self) -> URL:
         if len(self.config.get(Setting.SUPERVISOR_URL)) > 0:
@@ -72,6 +73,10 @@ def _getBackupPath(self):
 
     def supportsBackupPaths(self):
         return not self._super_version or self._super_version >= VERSION_BACKUP_PATH
+
+    @property
+    def supportsMountInfo(self):
+        return not self._super_version or self._super_version >= VERSION_MOUNT_INFO
 
     @supervisor_call
     async def createBackup(self, info):
@@ -168,6 +173,18 @@ async def supervisorInfo(self):
             self._super_version = Version.parse(info['version'])
         return info
 
+    @supervisor_call
+    async def mountInfo(self):
+        if self.supportsMountInfo:
+            url = self.getSupervisorURL().with_path("mounts")
+            info = await self._getHassioData(url)
+            return info
+        else:
+            return {
+                "default_backup_mount": None,
+                "mounts": []
+            }
+
     @supervisor_call
     async def restore(self, slug: str, password: str = None) -> None:
         url = self.getSupervisorURL().with_path("{1}/{0}/restore/full".format(slug, self._getBackupPath()))

hassio-google-drive-backup/backup/ha/hasource.py

@@ -3,16 +3,16 @@
 from datetime import timedelta
 from io import IOBase
 from threading import Lock, Thread
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Any, Union
 
 from aiohttp.client_exceptions import ClientResponseError
 from injector import inject, singleton
 
 from backup.util import AsyncHttpGetter, GlobalInfo, Estimator, DataCache, KEY_NOTE, KEY_LAST_SEEN, KEY_PENDING, KEY_NAME, KEY_CREATED, KEY_I_MADE_THIS, KEY_IGNORE
-from ..config import Config, Setting, CreateOptions, Startable
+from ..config import Config, Setting, CreateOptions, Startable, Version
 from ..const import SOURCE_HA
 from ..model import BackupSource, AbstractBackup, HABackup, Backup
-from ..exceptions import (LogicError, BackupInProgress,
+from ..exceptions import (LogicError, BackupInProgress, UnknownNetworkStorageError, InactiveNetworkStorageError,
                           UploadFailed, ensureKey)
 from .harequests import HaRequests
 from .password import Password
@@ -24,7 +24,6 @@
 
 logger: StandardLogger = getLogger(__name__)
 
-
 class PendingBackup(AbstractBackup):
     def __init__(self, backupType, protected, options: CreateOptions, request_info, config, time):
         super().__init__(
@@ -134,6 +133,7 @@ def __init__(self, config: Config, time: Time, ha: HaRequests, info: GlobalInfo,
         self.host_info = None
         self.ha_info = None
         self.super_info = None
+        self.mount_info = {}
         self.lock: Lock = Lock()
         self.time = time
         self.harequests = ha
@@ -244,6 +244,16 @@ async def stop(self):
             self._pending_backup_task.cancel()
             await asyncio.wait([self._pending_backup_task])
 
+    @property
+    def needsSpaceCheck(self):
+        if not self.harequests.supportsMountInfo:
+            return True
+        if self.config.get(Setting.BACKUP_STORAGE) == 'local-disk':
+            return True
+        if self.mount_info.get("default_backup_mount") is None and len(self.config.get(Setting.BACKUP_STORAGE)) == 0:
+            return True
+        return False
+
     @property
     def query_had_changes(self):
         return self._changes_from_last_query
@@ -341,7 +351,7 @@ async def ignore(self, backup: Backup, ignore: bool):
         self._data_cache.backup(slug)[KEY_IGNORE] = ignore
         self._data_cache.makeDirty()
 
-    async def note(self, backup, note: str) -> None:
+    async def note(self, backup, note: Union[str, None]) -> None:
         if isinstance(backup, HABackup):
             validated = backup
         else:
@@ -401,6 +411,8 @@ async def _refreshInfo(self) -> None:
             self.host_info = await self.harequests.info()
             self.ha_info = await self.harequests.haInfo()
             self.super_info = await self.harequests.supervisorInfo()
+            self.mount_info = await self.harequests.mountInfo()
+
             addon_info = ensureKey("addons", await self.harequests.getAddons(), "Supervisor Metadata")
             self.config.update(
                 ensureKey("options", self.self_info, "addon metdata"))
@@ -508,7 +520,7 @@ def _buildBackupInfo(self, options: CreateOptions):
         addons: List[str] = []
         for addon in self.super_info.get('addons', {}):
             addons.append(addon['slug'])
-        request_info = {
+        request_info: Dict[str, Any] = {
             'addons': [],
             'folders': []
         }
@@ -534,4 +546,29 @@ def _buildBackupInfo(self, options: CreateOptions):
         name = BackupName().resolve(type_name, options.name_template,
                                     self.time.toLocal(options.when), self.host_info)
         request_info['name'] = name
+
+        if self.harequests.supportsMountInfo:
+            # Validate the mount location and set it if necessary
+            mount_name = self.config.get(Setting.BACKUP_STORAGE)
+
+            # Default is to use Home Assistant's default configured mount
+            if not mount_name or len(mount_name) == 0:
+                ha_default = self.mount_info.get("default_backup_mount", None)
+                if ha_default:
+                    mount_name = ha_default
+                else:
+                    mount_name = "local-disk"
+
+            if mount_name != "local-disk":
+                # check to make sure the mount location is valid
+                for mount in self.mount_info.get("mounts", []):
+                    if mount.get("name", None) == mount_name:
+                        if mount.get("state", False) != "active":
+                            raise InactiveNetworkStorageError(mount_name)
+                        request_info['location'] = mount_name
+                        break
+                if request_info.get('location', None) is None:
+                    raise UnknownNetworkStorageError(mount_name)
+            else:
+                request_info['location'] = None
         return request_info, type_name, protected

hassio-google-drive-backup/backup/model/backups.py

@@ -1,6 +1,6 @@
 
 from datetime import datetime, timedelta
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 from dateutil.tz import tzutc
 from ..util import Estimator
 
@@ -56,7 +56,7 @@ def slug(self) -> str:
     def size(self) -> int:
         return self._size
 
-    def note(self) -> str:
+    def note(self) -> Union[str, None]:
         return self._note
 
     def sizeInt(self) -> int:
@@ -98,7 +98,7 @@ def setUploadable(self, uploadable):
     def details(self):
         return self._details
 
-    def setNote(self, note: str):
+    def setNote(self, note: Union[str, None]):
         self._note = note
 
     def status(self):
@@ -207,7 +207,7 @@ def backupType(self) -> str:
             return backup.backupType()
         return "error"
 
-    def version(self) -> str:
+    def version(self) -> Union[str, None]:
         for backup in self.sources.values():
             if backup.version() is not None:
                 return backup.version()