From 7a470bb3ac9e37d801cb9adac2f28caba5ad603a Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 29 Jun 2017 12:03:00 +0200 Subject: [PATCH 1/7] Pump version 0.39 --- hassio/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hassio/const.py b/hassio/const.py index d1c9df809d2..384dc5d6a65 100644 --- a/hassio/const.py +++ b/hassio/const.py @@ -1,7 +1,7 @@ """Const file for HassIO.""" from pathlib import Path -HASSIO_VERSION = '0.38' +HASSIO_VERSION = '0.39' URL_HASSIO_VERSION = ('https://raw.githubusercontent.com/home-assistant/' 'hassio/master/version.json') From 3c4240a8a80199b44a8b4ae430048ea07f0b7bd2 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 29 Jun 2017 21:10:42 +0200 Subject: [PATCH 2/7] Update __init__.py --- hassio/dock/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hassio/dock/__init__.py b/hassio/dock/__init__.py index e53a361c449..e84dd4a5a74 100644 --- a/hassio/dock/__init__.py +++ b/hassio/dock/__init__.py @@ -199,7 +199,7 @@ def _stop(self): container.remove(force=True) async def remove(self): - """Remove docker container.""" + """Remove docker images.""" if self._lock.locked(): _LOGGER.error("Can't excute remove while a task is in progress") return False @@ -208,7 +208,7 @@ async def remove(self): return await self.loop.run_in_executor(None, self._remove) def _remove(self): - """remove docker container. + """remove docker images. Need run inside executor. """ From 872f1d0ae38275a113363e7563fb1c7008a98ad7 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 3 Jul 2017 07:03:03 +0200 Subject: [PATCH 3/7] update homeassistant 0.48 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 21b6da62bf9..ac4bed797ae 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "hassio": "0.38", - "homeassistant": "0.47.1", + "homeassistant": "0.48", "resinos": "0.8", "resinhup": "0.1", "generic": "0.3" From 2c68e5801fb2261c61d70d579ceda7e5fc29af15 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 5 Jul 2017 11:15:57 +0200 Subject: [PATCH 4/7] Update HomeAssistant to 0.48.1 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index ac4bed797ae..1e5b7b67edb 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "hassio": "0.38", - "homeassistant": "0.48", + "homeassistant": "0.48.1", "resinos": "0.8", "resinhup": "0.1", "generic": "0.3" From e2a29b7290a51bc7deac64c0d582accf83f46025 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 5 Jul 2017 18:16:14 +0200 Subject: [PATCH 5/7] Add snapshot feature (#88) * Add API layout for snapshot * Update api * Add support for export/import docker images * Move restore into addon * Add restore to addon * Fix lint * fix lint * cleanup * init object * fix executor * cleanup * Change flow of init * Revert "Change flow of init" This reverts commit 6b3215e44c990d16f728f766df52b20c091de807. * allow restore from none * forward working * add size * add context for snapshot * add init function to set meta data * update local addon on load * add more validate and optimaze code * Optimaze code for restore data * add validate layer * Add more function to snapshot / cleanup others * finish snapshot function * Cleanup config / optimaze code * Finish snapshot on core * Some improvments first object for api * finish * fix lint p1 * fix lint p2 * fix lint p3 * fix async with * fix lint p4 * fix lint p5 * fix p6 * make staticmethod * fix schema * fix parse system data * fix bugs * fix get function * extend snapshot/restore * add type * fix lint * move to gz / xz is to slow * move to gz / xz is to slow p2 * Fix config folder * small compresslevel for more speed * fix lint * fix load * fix tar stream * fix tar stream p2 * fix parse * fix partial * fix start hass * fix rep * fix set * fix real * fix generator * Cleanup old image * add log * fix lint * fix lint p2 * fix load from tar --- API.md | 64 ++++++++ hassio/__main__.py | 2 +- hassio/addons/addon.py | 172 +++++++++++++++---- hassio/addons/data.py | 77 +++++---- hassio/addons/util.py | 16 +- hassio/addons/validate.py | 31 +++- hassio/api/__init__.py | 26 ++- hassio/api/snapshots.py | 113 +++++++++++++ hassio/api/supervisor.py | 21 ++- hassio/api/util.py | 3 +- hassio/config.py | 44 ++--- hassio/const.py | 17 ++ hassio/core.py | 17 +- hassio/dock/addon.py | 75 +++++++-- hassio/scheduler.py | 13 +- hassio/snapshots/__init__.py | 309 +++++++++++++++++++++++++++++++++++ hassio/snapshots/snapshot.py | 255 +++++++++++++++++++++++++++++ hassio/snapshots/util.py | 21 +++ hassio/snapshots/validate.py | 27 +++ 19 files changed, 1178 insertions(+), 125 deletions(-) create mode 100644 hassio/api/snapshots.py create mode 100644 hassio/snapshots/__init__.py create mode 100644 hassio/snapshots/snapshot.py create mode 100644 hassio/snapshots/util.py create mode 100644 hassio/snapshots/validate.py diff --git a/API.md b/API.md index ddd0e95a483..ca66274ea77 100644 --- a/API.md +++ b/API.md @@ -51,6 +51,13 @@ The addons from `addons` are only installed one. ], "addons_repositories": [ "REPO_URL" + ], + "snapshots": [ + { + "slug": "SLUG", + "data": "ISO", + "name": "Custom name" + } ] } ``` @@ -148,7 +155,62 @@ Return QR-Code } ``` +### Backup/Snapshot + +- POST `/snapshots/new/full` +```json +{ + "name": "Optional" +} +``` + +- POST `/snapshots/new/partial` +```json +{ + "name": "Optional", + "addons": ["ADDON_SLUG"], + "folders": ["FOLDER_NAME"] +} +``` + +- POST `/snapshots/reload` + +- GET `/snapshots/{slug}/info` +```json +{ + "slug": "SNAPSHOT ID", + "type": "full|partial", + "name": "custom snapshot name", + "date": "ISO", + "size": "SIZE_IN_MB", + "homeassistant": "INSTALLED_HASS_VERSION", + "addons": [ + { + "slug": "ADDON_SLUG", + "name": "NAME", + "version": "INSTALLED_VERSION" + } + ], + "repositories": ["URL"], + "folders": ["NAME"] +} +``` + +- POST `/snapshots/{slug}/remove` + +- POST `/snapshots/{slug}/restore/full` + +- POST `/snapshots/{slug}/restore/partial` +```json +{ + "homeassistant": "bool", + "addons": ["ADDON_SLUG"], + "folders": ["FOLDER_NAME"] +} +``` + ### Host +- POST `/host/reload` - POST `/host/shutdown` @@ -231,6 +293,8 @@ Output the raw docker log ### REST API addons +- POST `/addons/reload` + - GET `/addons/{addon}/info` ```json { diff --git a/hassio/__main__.py b/hassio/__main__.py index d8ba5bd6c3b..44b9dcd7efa 100644 --- a/hassio/__main__.py +++ b/hassio/__main__.py @@ -30,7 +30,7 @@ _LOGGER.info("Run Hassio setup") loop.run_until_complete(hassio.setup()) - _LOGGER.info("Start Hassio task") + _LOGGER.info("Start Hassio") loop.call_soon_threadsafe(loop.create_task, hassio.start()) loop.call_soon_threadsafe(bootstrap.reg_signal, loop, hassio) diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index c43b01558a5..b5d10395a40 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -1,22 +1,29 @@ """Init file for HassIO addons.""" from copy import deepcopy import logging +import json from pathlib import Path, PurePath import re import shutil +import tarfile +from tempfile import TemporaryDirectory import voluptuous as vol from voluptuous.humanize import humanize_error -from .validate import validate_options, MAP_VOLUME +from .validate import ( + validate_options, SCHEMA_ADDON_USER, SCHEMA_ADDON_SYSTEM, + SCHEMA_ADDON_SNAPSHOT, MAP_VOLUME) from ..const import ( ATTR_NAME, ATTR_VERSION, ATTR_SLUG, ATTR_DESCRIPTON, ATTR_BOOT, ATTR_MAP, ATTR_OPTIONS, ATTR_PORTS, ATTR_SCHEMA, ATTR_IMAGE, ATTR_REPOSITORY, ATTR_URL, ATTR_ARCH, ATTR_LOCATON, ATTR_DEVICES, ATTR_ENVIRONMENT, ATTR_HOST_NETWORK, ATTR_TMPFS, ATTR_PRIVILEGED, ATTR_STARTUP, - STATE_STARTED, STATE_STOPPED, STATE_NONE) + STATE_STARTED, STATE_STOPPED, STATE_NONE, ATTR_USER, ATTR_SYSTEM, + ATTR_STATE) +from .util import check_installed from ..dock.addon import DockerAddon -from ..tools import write_json_file +from ..tools import write_json_file, read_json_file _LOGGER = logging.getLogger(__name__) @@ -26,22 +33,33 @@ class Addon(object): """Hold data for addon inside HassIO.""" - def __init__(self, config, loop, dock, data, addon_slug): + def __init__(self, config, loop, dock, data, slug): """Initialize data holder.""" + self.loop = loop self.config = config self.data = data - self._id = addon_slug - - if self._mesh is None: - raise RuntimeError("{} not a valid addon!".format(self._id)) + self._id = slug self.addon_docker = DockerAddon(config, loop, dock, self) async def load(self): """Async initialize of object.""" if self.is_installed: + self._validate_system_user() await self.addon_docker.attach() + def _validate_system_user(self): + """Validate internal data they read from file.""" + for data, schema in ((self.data.system, SCHEMA_ADDON_SYSTEM), + (self.data.user, SCHEMA_ADDON_USER)): + try: + data[self._id] = schema(data[self._id]) + except vol.Invalid as err: + _LOGGER.warning("Can't validate addon load %s -> %s", self._id, + humanize_error(data[self._id], err)) + except KeyError: + pass + @property def slug(self): """Return slug/id of addon.""" @@ -88,6 +106,12 @@ def _set_update(self, version): self.data.user[self._id][ATTR_VERSION] = version self.data.save() + def _restore_data(self, user, system): + """Restore data to addon.""" + self.data.user[self._id] = deepcopy(user) + self.data.system[self._id] = deepcopy(system) + self.data.save() + @property def options(self): """Return options with local changes.""" @@ -281,12 +305,9 @@ async def install(self, version=None): self._set_install(version) return True + @check_installed async def uninstall(self): """Remove a addon.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - if not await self.addon_docker.remove(): return False @@ -307,29 +328,21 @@ async def state(self): return STATE_STARTED return STATE_STOPPED + @check_installed async def start(self): """Set options and start addon.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - return await self.addon_docker.run() + @check_installed async def stop(self): """Stop addon.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - return await self.addon_docker.stop() + @check_installed async def update(self, version=None): """Update addon.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - version = version or self.last_version + if version == self.version_installed: _LOGGER.warning( "Addon %s is already installed in %s", self._id, version) @@ -341,18 +354,113 @@ async def update(self, version=None): self._set_update(version) return True + @check_installed async def restart(self): """Restart addon.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - return await self.addon_docker.restart() + @check_installed async def logs(self): """Return addons log output.""" - if not self.is_installed: - _LOGGER.error("Addon %s is not installed", self._id) - return False - return await self.addon_docker.logs() + + @check_installed + async def snapshot(self, tar_file): + """Snapshot a state of a addon.""" + with TemporaryDirectory(dir=str(self.config.path_tmp)) as temp: + # store local image + if self.need_build and not await \ + self.addon_docker.export_image(Path(temp, "image.tar")): + return False + + data = { + ATTR_USER: self.data.user.get(self._id, {}), + ATTR_SYSTEM: self.data.system.get(self._id, {}), + ATTR_VERSION: self.version_installed, + ATTR_STATE: await self.state(), + } + + # store local configs/state + if not write_json_file(Path(temp, "addon.json"), data): + _LOGGER.error("Can't write addon.json for %s", self._id) + return False + + # write into tarfile + def _create_tar(): + """Write tar inside loop.""" + with tarfile.open(tar_file, "w:gz", + compresslevel=1) as snapshot: + snapshot.add(temp, arcname=".") + snapshot.add(self.path_data, arcname="data") + + try: + await self.loop.run_in_executor(None, _create_tar) + except tarfile.TarError as err: + _LOGGER.error("Can't write tarfile %s -> %s", tar_file, err) + return False + + return True + + async def restore(self, tar_file): + """Restore a state of a addon.""" + with TemporaryDirectory(dir=str(self.config.path_tmp)) as temp: + # extract snapshot + def _extract_tar(): + """Extract tar snapshot.""" + with tarfile.open(tar_file, "r:gz") as snapshot: + snapshot.extractall(path=Path(temp)) + + try: + await self.loop.run_in_executor(None, _extract_tar) + except tarfile.TarError as err: + _LOGGER.error("Can't read tarfile %s -> %s", tar_file, err) + return False + + # read snapshot data + try: + data = read_json_file(Path(temp, "addon.json")) + except (OSError, json.JSONDecodeError) as err: + _LOGGER.error("Can't read addon.json -> %s", err) + + # validate + try: + data = SCHEMA_ADDON_SNAPSHOT(data) + except vol.Invalid as err: + _LOGGER.error("Can't validate %s, snapshot data -> %s", + self._id, humanize_error(data, err)) + return False + + # restore data / reload addon + self._restore_data(data[ATTR_USER], data[ATTR_SYSTEM]) + + # check version / restore image + if data[ATTR_VERSION] != self.addon_docker.version: + image_file = Path(temp, "image.tar") + if image_file.is_file(): + if not await self.addon_docker.import_image(image_file): + return False + else: + if not await self.addon_docker.install(data[ATTR_VERSION]): + return False + await self.addon_docker.cleanup() + else: + await self.addon_docker.stop() + + # restore data + def _restore_data(): + """Restore data.""" + if self.path_data.is_dir(): + shutil.rmtree(str(self.path_data), ignore_errors=True) + shutil.copytree(str(Path(temp, "data")), str(self.path_data)) + + try: + await self.loop.run_in_executor(None, _restore_data) + except shutil.Error as err: + _LOGGER.error("Can't restore origin data -> %s", err) + return False + + # run addon + if data[ATTR_STATE] == STATE_STARTED: + return await self.start() + + return True diff --git a/hassio/addons/data.py b/hassio/addons/data.py index ffb685dd929..c6fac33e1fc 100644 --- a/hassio/addons/data.py +++ b/hassio/addons/data.py @@ -10,65 +10,72 @@ from .util import extract_hash_from_path from .validate import ( - SCHEMA_ADDON_CONFIG, SCHEMA_REPOSITORY_CONFIG, MAP_VOLUME) + SCHEMA_ADDON, SCHEMA_REPOSITORY_CONFIG, MAP_VOLUME) from ..const import ( FILE_HASSIO_ADDONS, ATTR_VERSION, ATTR_SLUG, ATTR_REPOSITORY, ATTR_LOCATON, - REPOSITORY_CORE, REPOSITORY_LOCAL) -from ..config import Config -from ..tools import read_json_file + REPOSITORY_CORE, REPOSITORY_LOCAL, ATTR_USER, ATTR_SYSTEM) +from ..tools import read_json_file, write_json_file _LOGGER = logging.getLogger(__name__) -SYSTEM = 'system' -USER = 'user' - RE_VOLUME = re.compile(MAP_VOLUME) -class Data(Config): +class Data(object): """Hold data for addons inside HassIO.""" def __init__(self, config): """Initialize data holder.""" - super().__init__(FILE_HASSIO_ADDONS) + self._file = FILE_HASSIO_ADDONS + self._data = {} self.config = config - self._system_data = self._data.get(SYSTEM, {}) - self._user_data = self._data.get(USER, {}) - self._cache_data = {} - self._repositories_data = {} + self._cache = {} + self._repositories = {} + + # init or load data + if self._file.is_file(): + try: + self._data = read_json_file(self._file) + except (OSError, json.JSONDecodeError): + _LOGGER.warning("Can't read %s", self._file) + self._data = {} + + # init data + if not self._data: + self._data[ATTR_USER] = {} + self._data[ATTR_SYSTEM] = {} def save(self): """Store data to config file.""" - self._data = { - USER: self._user_data, - SYSTEM: self._system_data, - } - super().save() + if not write_json_file(self._file, self._data): + _LOGGER.error("Can't store config in %s", self._file) + return False + return True @property def user(self): """Return local addon user data.""" - return self._user_data + return self._data[ATTR_USER] @property def system(self): """Return local addon data.""" - return self._system_data + return self._data[ATTR_SYSTEM] @property def cache(self): """Return addon data from cache/repositories.""" - return self._cache_data + return self._cache @property def repositories(self): """Return addon data from repositories.""" - return self._repositories_data + return self._repositories def reload(self): """Read data from addons repository.""" - self._cache_data = {} - self._repositories_data = {} + self._cache = {} + self._repositories = {} # read core repository self._read_addons_folder( @@ -100,7 +107,7 @@ def _read_git_repository(self, path): read_json_file(repository_file) ) - except OSError: + except (OSError, json.JSONDecodeError): _LOGGER.warning("Can't read repository information from %s", repository_file) return @@ -110,7 +117,7 @@ def _read_git_repository(self, path): return # process data - self._repositories_data[slug] = repository_info + self._repositories[slug] = repository_info self._read_addons_folder(path, slug) def _read_addons_folder(self, path, repository): @@ -120,7 +127,7 @@ def _read_addons_folder(self, path, repository): addon_config = read_json_file(addon) # validate - addon_config = SCHEMA_ADDON_CONFIG(addon_config) + addon_config = SCHEMA_ADDON(addon_config) # Generate slug addon_slug = "{}_{}".format( @@ -129,7 +136,7 @@ def _read_addons_folder(self, path, repository): # store addon_config[ATTR_REPOSITORY] = repository addon_config[ATTR_LOCATON] = str(addon.parent) - self._cache_data[addon_slug] = addon_config + self._cache[addon_slug] = addon_config except OSError: _LOGGER.warning("Can't read %s", addon) @@ -148,11 +155,11 @@ def _set_builtin_repositories(self): return # core repository - self._repositories_data[REPOSITORY_CORE] = \ + self._repositories[REPOSITORY_CORE] = \ builtin_data[REPOSITORY_CORE] # local repository - self._repositories_data[REPOSITORY_LOCAL] = \ + self._repositories[REPOSITORY_LOCAL] = \ builtin_data[REPOSITORY_LOCAL] def _merge_config(self): @@ -162,16 +169,16 @@ def _merge_config(self): """ have_change = False - for addon in set(self._system_data): + for addon in set(self.system): # detached - if addon not in self._cache_data: + if addon not in self._cache: continue - cache = self._cache_data[addon] - data = self._system_data[addon] + cache = self._cache[addon] + data = self.system[addon] if data[ATTR_VERSION] == cache[ATTR_VERSION]: if data != cache: - self._system_data[addon] = copy.deepcopy(cache) + self.system[addon] = copy.deepcopy(cache) have_change = True if have_change: diff --git a/hassio/addons/util.py b/hassio/addons/util.py index 152c2886684..f7a2a9514d2 100644 --- a/hassio/addons/util.py +++ b/hassio/addons/util.py @@ -1,10 +1,12 @@ """Util addons functions.""" import hashlib +import logging import re -RE_SLUGIFY = re.compile(r'[^a-z0-9_]+') RE_SHA1 = re.compile(r"[a-f0-9]{8}") +_LOGGER = logging.getLogger(__name__) + def get_hash_from_repository(name): """Generate a hash from repository.""" @@ -19,3 +21,15 @@ def extract_hash_from_path(path): if not RE_SHA1.match(repo_dir): return get_hash_from_repository(repo_dir) return repo_dir + + +def check_installed(method): + """Wrap function with check if addon is installed.""" + async def wrap_check(addon, *args, **kwargs): + """Return False if not installed or the function.""" + if not addon.is_installed: + _LOGGER.error("Addon %s is not installed", addon.slug) + return False + return await method(addon, *args, **kwargs) + + return wrap_check diff --git a/hassio/addons/validate.py b/hassio/addons/validate.py index 95795c02e5f..b94ad4e71a5 100644 --- a/hassio/addons/validate.py +++ b/hassio/addons/validate.py @@ -7,7 +7,8 @@ STARTUP_BEFORE, STARTUP_INITIALIZE, BOOT_AUTO, BOOT_MANUAL, ATTR_SCHEMA, ATTR_IMAGE, ATTR_URL, ATTR_MAINTAINER, ATTR_ARCH, ATTR_DEVICES, ATTR_ENVIRONMENT, ATTR_HOST_NETWORK, ARCH_ARMHF, ARCH_AARCH64, ARCH_AMD64, - ARCH_I386, ATTR_TMPFS, ATTR_PRIVILEGED) + ARCH_I386, ATTR_TMPFS, ATTR_PRIVILEGED, ATTR_USER, ATTR_STATE, ATTR_SYSTEM, + STATE_STARTED, STATE_STOPPED, ATTR_LOCATON, ATTR_REPOSITORY) MAP_VOLUME = r"^(config|ssl|addons|backup|share)(?::(rw|:ro))?$" @@ -41,7 +42,7 @@ def check_network(data): # pylint: disable=no-value-for-parameter -SCHEMA_ADDON_CONFIG = vol.Schema(vol.All({ +SCHEMA_ADDON_CONFIG = vol.Schema({ vol.Required(ATTR_NAME): vol.Coerce(str), vol.Required(ATTR_VERSION): vol.Coerce(str), vol.Required(ATTR_SLUG): vol.Coerce(str), @@ -68,8 +69,8 @@ def check_network(data): ]) }, False), vol.Optional(ATTR_IMAGE): vol.Match(r"\w*/\w*"), -}, check_network), extra=vol.ALLOW_EXTRA) - +}, extra=vol.ALLOW_EXTRA) +SCHEMA_ADDON = vol.Schema(vol.All(SCHEMA_ADDON_CONFIG, check_network)) # pylint: disable=no-value-for-parameter SCHEMA_REPOSITORY_CONFIG = vol.Schema({ @@ -79,6 +80,28 @@ def check_network(data): }, extra=vol.ALLOW_EXTRA) +SCHEMA_ADDON_USER = vol.Schema({ + vol.Required(ATTR_VERSION): vol.Coerce(str), + vol.Required(ATTR_OPTIONS): dict, + vol.Optional(ATTR_BOOT): + vol.In([BOOT_AUTO, BOOT_MANUAL]), +}) + + +SCHEMA_ADDON_SYSTEM = SCHEMA_ADDON_CONFIG.extend({ + vol.Required(ATTR_LOCATON): vol.Coerce(str), + vol.Required(ATTR_REPOSITORY): vol.Coerce(str), +}) + + +SCHEMA_ADDON_SNAPSHOT = vol.Schema({ + vol.Required(ATTR_USER): SCHEMA_ADDON_USER, + vol.Required(ATTR_SYSTEM): SCHEMA_ADDON_SYSTEM, + vol.Required(ATTR_STATE): vol.In([STATE_STARTED, STATE_STOPPED]), + vol.Required(ATTR_VERSION): vol.Coerce(str), +}) + + def validate_options(raw_schema): """Validate schema.""" def validate(struct): diff --git a/hassio/api/__init__.py b/hassio/api/__init__.py index 856f8aabfce..3d3e954652d 100644 --- a/hassio/api/__init__.py +++ b/hassio/api/__init__.py @@ -10,6 +10,7 @@ from .network import APINetwork from .supervisor import APISupervisor from .security import APISecurity +from .snapshots import APISnapshots _LOGGER = logging.getLogger(__name__) @@ -43,12 +44,12 @@ def register_network(self, host_control): self.webapp.router.add_get('/network/info', api_net.info) self.webapp.router.add_post('/network/options', api_net.options) - def register_supervisor(self, supervisor, addons, host_control, + def register_supervisor(self, supervisor, snapshots, addons, host_control, websession): """Register supervisor function.""" api_supervisor = APISupervisor( - self.config, self.loop, supervisor, addons, host_control, - websession) + self.config, self.loop, supervisor, snapshots, addons, + host_control, websession) self.webapp.router.add_get('/supervisor/ping', api_supervisor.ping) self.webapp.router.add_get('/supervisor/info', api_supervisor.info) @@ -100,6 +101,25 @@ def register_security(self): self.webapp.router.add_post('/security/totp', api_security.totp) self.webapp.router.add_post('/security/session', api_security.session) + def register_snapshots(self, snapshots): + """Register snapshots function.""" + api_snapshots = APISnapshots(self.config, self.loop, snapshots) + + self.webapp.router.add_post( + '/snapshots/new/full', api_snapshots.snapshot_full) + self.webapp.router.add_post( + '/snapshots/new/partial', api_snapshots.snapshot_partial) + + self.webapp.router.add_get( + '/snapshots/{snapshot}/info', api_snapshots.info) + self.webapp.router.add_post( + '/snapshots/{snapshot}/remove', api_snapshots.remove) + self.webapp.router.add_post( + '/snapshots/{snapshot}/restore/full', api_snapshots.restore_full) + self.webapp.router.add_post( + '/snapshots/{snapshot}/restore/partial', + api_snapshots.restore_partial) + def register_panel(self): """Register panel for homeassistant.""" panel = Path(__file__).parents[1].joinpath('panel/hassio-main.html') diff --git a/hassio/api/snapshots.py b/hassio/api/snapshots.py new file mode 100644 index 00000000000..4ba7652da10 --- /dev/null +++ b/hassio/api/snapshots.py @@ -0,0 +1,113 @@ +"""Init file for HassIO snapshot rest api.""" +import asyncio +import logging + +import voluptuous as vol + +from .util import api_process, api_validate +from ..snapshots.validate import ALL_FOLDERS +from ..const import ( + ATTR_NAME, ATTR_SLUG, ATTR_DATE, ATTR_ADDONS, ATTR_REPOSITORIES, + ATTR_HOMEASSISTANT, ATTR_VERSION, ATTR_SIZE, ATTR_FOLDERS, ATTR_TYPE) + +_LOGGER = logging.getLogger(__name__) + + +# pylint: disable=no-value-for-parameter +SCHEMA_RESTORE_PARTIAL = vol.Schema({ + vol.Optional(ATTR_HOMEASSISTANT): vol.Boolean(), + vol.Optional(ATTR_ADDONS): [vol.Coerce(str)], + vol.Optional(ATTR_FOLDERS): [vol.In(ALL_FOLDERS)], +}) + +SCHEMA_SNAPSHOT_FULL = vol.Schema({ + vol.Optional(ATTR_NAME): vol.Coerce(str), +}) + +SCHEMA_SNAPSHOT_PARTIAL = SCHEMA_SNAPSHOT_FULL.extend({ + vol.Optional(ATTR_ADDONS): [vol.Coerce(str)], + vol.Optional(ATTR_FOLDERS): [vol.In(ALL_FOLDERS)], +}) + + +class APISnapshots(object): + """Handle rest api for snapshot functions.""" + + def __init__(self, config, loop, snapshots): + """Initialize network rest api part.""" + self.config = config + self.loop = loop + self.snapshots = snapshots + + def _extract_snapshot(self, request): + """Return addon and if not exists trow a exception.""" + snapshot = self.snapshots.get(request.match_info.get('snapshot')) + if not snapshot: + raise RuntimeError("Snapshot not exists") + return snapshot + + @staticmethod + def _addons_list(snapshot): + """Generate a list with addons data.""" + data = [] + for addon_data in snapshot.addons: + data.append({ + ATTR_SLUG: addon_data[ATTR_SLUG], + ATTR_NAME: addon_data[ATTR_NAME], + ATTR_VERSION: addon_data[ATTR_VERSION], + }) + return data + + @api_process + async def info(self, request): + """Return snapshot info.""" + snapshot = self._extract_snapshot(request) + + return { + ATTR_SLUG: snapshot.slug, + ATTR_TYPE: snapshot.sys_type, + ATTR_NAME: snapshot.name, + ATTR_DATE: snapshot.date, + ATTR_SIZE: snapshot.size, + ATTR_HOMEASSISTANT: snapshot.homeassistant, + ATTR_ADDONS: self._addons_list(snapshot), + ATTR_REPOSITORIES: snapshot.repositories, + ATTR_FOLDERS: snapshot.folders, + } + + @api_process + async def snapshot_full(self, request): + """Full-Snapshot a snapshot.""" + body = await api_validate(SCHEMA_SNAPSHOT_FULL, request) + return await asyncio.shield( + self.snapshots.do_snapshot_full(**body), loop=self.loop) + + @api_process + async def snapshot_partial(self, request): + """Partial-Snapshot a snapshot.""" + body = await api_validate(SCHEMA_SNAPSHOT_PARTIAL, request) + return await asyncio.shield( + self.snapshots.do_snapshot_partial(**body), loop=self.loop) + + @api_process + async def restore_full(self, request): + """Full-Restore a snapshot.""" + snapshot = self._extract_snapshot(request) + return await asyncio.shield( + self.snapshots.do_restore_full(snapshot), loop=self.loop) + + @api_process + async def restore_partial(self, request): + """Partial-Restore a snapshot.""" + snapshot = self._extract_snapshot(request) + body = await api_validate(SCHEMA_SNAPSHOT_PARTIAL, request) + + return await asyncio.shield( + self.snapshots.do_restore_partial(snapshot, **body), + loop=self.loop) + + @api_process + async def remove(self, request): + """Remove a snapshot.""" + snapshot = self._extract_snapshot(request) + return self.snapshots.remove(snapshot) diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index 8bddfd9b736..05d13f6cbbe 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -10,7 +10,7 @@ HASSIO_VERSION, ATTR_ADDONS_REPOSITORIES, ATTR_REPOSITORIES, ATTR_REPOSITORY, ATTR_DESCRIPTON, ATTR_NAME, ATTR_SLUG, ATTR_INSTALLED, ATTR_DETACHED, ATTR_SOURCE, ATTR_MAINTAINER, ATTR_URL, ATTR_ARCH, - ATTR_BUILD, ATTR_TIMEZONE) + ATTR_BUILD, ATTR_TIMEZONE, ATTR_DATE, ATTR_SNAPSHOTS) from ..tools import validate_timezone _LOGGER = logging.getLogger(__name__) @@ -30,13 +30,14 @@ class APISupervisor(object): """Handle rest api for supervisor functions.""" - def __init__(self, config, loop, supervisor, addons, host_control, - websession): + def __init__(self, config, loop, supervisor, snapshots, addons, + host_control, websession): """Initialize supervisor rest api part.""" self.config = config self.loop = loop self.supervisor = supervisor self.addons = addons + self.snapshots = snapshots self.host_control = host_control self.websession = websession @@ -76,6 +77,18 @@ def _repositories_list(self): return data + def _snapshots_list(self): + """Return a list of available snapshots.""" + data = [] + for snapshot in self.snapshots.list_snapshots: + data.append({ + ATTR_SLUG: snapshot.slug, + ATTR_NAME: snapshot.name, + ATTR_DATE: snapshot.date, + }) + + return data + @api_process async def ping(self, request): """Return ok for signal that the api is ready.""" @@ -92,6 +105,7 @@ async def info(self, request): ATTR_TIMEZONE: self.config.timezone, ATTR_ADDONS: self._addons_list(only_installed=True), ATTR_ADDONS_REPOSITORIES: self.config.addons_repositories, + ATTR_SNAPSHOTS: self._snapshots_list(), } @api_process @@ -136,6 +150,7 @@ async def reload(self, request): """Reload addons, config ect.""" tasks = [ self.addons.reload(), + self.snapshots.reload(), self.config.fetch_update_infos(self.websession), self.host_control.load() ] diff --git a/hassio/api/util.py b/hassio/api/util.py index 0c8583c4327..c1e52e249ce 100644 --- a/hassio/api/util.py +++ b/hassio/api/util.py @@ -81,7 +81,8 @@ async def wrap_api(api, *args, **kwargs): def api_return_error(message=None): """Return a API error message.""" - _LOGGER.error(message) + if message: + _LOGGER.error(message) return web.json_response({ JSON_RESULT: RESULT_ERROR, diff --git a/hassio/config.py b/hassio/config.py index 30ff88d08a2..33e61d1f0d8 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -60,12 +60,13 @@ }, extra=vol.REMOVE_EXTRA) -class Config(object): - """Hold all config data.""" +class CoreConfig(object): + """Hold all core config data.""" - def __init__(self, config_file): + def __init__(self): """Initialize config object.""" - self._file = config_file + self.arch = None + self._file = FILE_HASSIO_CONFIG self._data = {} # init or load data @@ -76,30 +77,31 @@ def __init__(self, config_file): _LOGGER.warning("Can't read %s", self._file) self._data = {} - def save(self): - """Store data to config file.""" - if not write_json_file(self._file, self._data): - _LOGGER.error("Can't store config in %s", self._file) - return False - return True - - -class CoreConfig(Config): - """Hold all core config data.""" - - def __init__(self): - """Initialize config object.""" - self.arch = None - - super().__init__(FILE_HASSIO_CONFIG) + # validate data + if not self._validate_config(): + self._data = SCHEMA_CONFIG({}) + def _validate_config(self): + """Validate config and return True or False.""" # validate data try: self._data = SCHEMA_CONFIG(self._data) - self.save() except vol.Invalid as ex: _LOGGER.warning( "Invalid config %s", humanize_error(self._data, ex)) + return False + + return True + + def save(self): + """Store data to config file.""" + if not self._validate_config(): + return False + + if not write_json_file(self._file, self._data): + _LOGGER.error("Can't store config in %s", self._file) + return False + return True async def fetch_update_infos(self, websession): """Read current versions from web.""" diff --git a/hassio/const.py b/hassio/const.py index 384dc5d6a65..1725f299a68 100644 --- a/hassio/const.py +++ b/hassio/const.py @@ -15,6 +15,7 @@ RUN_UPDATE_INFO_TASKS = 28800 RUN_UPDATE_SUPERVISOR_TASKS = 29100 RUN_RELOAD_ADDONS_TASKS = 28800 +RUN_RELOAD_SNAPSHOTS_TASKS = 72000 RUN_WATCHDOG_HOMEASSISTANT = 15 RUN_CLEANUP_API_SESSIONS = 900 @@ -41,6 +42,7 @@ RESULT_ERROR = 'error' RESULT_OK = 'ok' +ATTR_DATE = 'date' ATTR_ARCH = 'arch' ATTR_HOSTNAME = 'hostname' ATTR_TIMEZONE = 'timezone' @@ -81,6 +83,13 @@ ATTR_HOST_NETWORK = 'host_network' ATTR_TMPFS = 'tmpfs' ATTR_PRIVILEGED = 'privileged' +ATTR_USER = 'user' +ATTR_SYSTEM = 'system' +ATTR_SNAPSHOTS = 'snapshots' +ATTR_HOMEASSISTANT = 'homeassistant' +ATTR_FOLDERS = 'folders' +ATTR_SIZE = 'size' +ATTR_TYPE = 'type' STARTUP_INITIALIZE = 'initialize' STARTUP_BEFORE = 'before' @@ -107,3 +116,11 @@ REPOSITORY_CORE = 'core' REPOSITORY_LOCAL = 'local' + +FOLDER_HOMEASSISTANT = 'homeassistant' +FOLDER_SHARE = 'share' +FOLDER_ADDONS = 'addons/local' +FOLDER_SSL = 'ssl' + +SNAPSHOT_FULL = 'full' +SNAPSHOT_PARTIAL = 'partial' diff --git a/hassio/core.py b/hassio/core.py index ffaa717b665..c5f51e065fa 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -12,10 +12,11 @@ SOCKET_DOCKER, RUN_UPDATE_INFO_TASKS, RUN_RELOAD_ADDONS_TASKS, RUN_UPDATE_SUPERVISOR_TASKS, RUN_WATCHDOG_HOMEASSISTANT, RUN_CLEANUP_API_SESSIONS, STARTUP_AFTER, STARTUP_BEFORE, - STARTUP_INITIALIZE) + STARTUP_INITIALIZE, RUN_RELOAD_SNAPSHOTS_TASKS) from .scheduler import Scheduler from .dock.homeassistant import DockerHomeAssistant from .dock.supervisor import DockerSupervisor +from .snapshots import SnapshotsManager from .tasks import ( hassio_update, homeassistant_watchdog, homeassistant_setup, api_sessions_cleanup) @@ -48,6 +49,10 @@ def __init__(self, loop, config): # init addon system self.addons = AddonManager(config, loop, self.dock) + # init snapshot system + self.snapshots = SnapshotsManager( + config, loop, self.scheduler, self.addons, self.homeassistant) + async def setup(self): """Setup HassIO orchestration.""" # supervisor @@ -76,10 +81,12 @@ async def setup(self): self.api.register_host(self.host_control) self.api.register_network(self.host_control) self.api.register_supervisor( - self.supervisor, self.addons, self.host_control, self.websession) + self.supervisor, self.snapshots, self.addons, self.host_control, + self.websession) self.api.register_homeassistant(self.homeassistant) self.api.register_addons(self.addons) self.api.register_security() + self.api.register_snapshots(self.snapshots) self.api.register_panel() # schedule api session cleanup @@ -107,6 +114,10 @@ async def setup(self): hassio_update(self.config, self.supervisor, self.websession), RUN_UPDATE_SUPERVISOR_TASKS) + # schedule snapshot update tasks + self.scheduler.register_task( + self.snapshots.reload, RUN_RELOAD_SNAPSHOTS_TASKS, now=True) + # start addon mark as initialize await self.addons.auto_boot(STARTUP_INITIALIZE) @@ -147,7 +158,7 @@ async def start(self): async def stop(self, exit_code=0): """Stop a running orchestration.""" # don't process scheduler anymore - self.scheduler.stop() + self.scheduler.suspend = True # process stop tasks self.websession.close() diff --git a/hassio/dock/addon.py b/hassio/dock/addon.py index ffa0e9e21eb..77c87825154 100644 --- a/hassio/dock/addon.py +++ b/hassio/dock/addon.py @@ -135,15 +135,6 @@ def _install(self, tag): return super()._install(tag) - async def build(self, tag): - """Build a docker container.""" - if self._lock.locked(): - _LOGGER.error("Can't excute build while a task is in progress") - return False - - async with self._lock: - return await self.loop.run_in_executor(None, self._build, tag) - def _build(self, tag): """Build a docker container. @@ -157,7 +148,7 @@ def _build(self, tag): shutil.copytree(str(source), str(build_dir)) except shutil.Error as err: _LOGGER.error("Can't copy %s to temporary build folder -> %s", - source, build_dir) + source, err) return False # prepare Dockerfile @@ -189,6 +180,65 @@ def _build(self, tag): finally: shutil.rmtree(str(build_dir), ignore_errors=True) + async def export_image(self, path): + """Export current images into a tar file.""" + if self._lock.locked(): + _LOGGER.error("Can't excute export while a task is in progress") + return False + + async with self._lock: + return await self.loop.run_in_executor( + None, self._export_image, path) + + def _export_image(self, tar_file): + """Export current images into a tar file. + + Need run inside executor. + """ + try: + image = self.dock.api.get_image("{}:latest".format(self.image)) + except docker.errors.DockerException as err: + _LOGGER.error("Can't fetch image %s -> %s", self.image, err) + return False + + try: + with tar_file.open("wb") as write_tar: + for chunk in image.stream(): + write_tar.write(chunk) + except OSError() as err: + _LOGGER.error("Can't write tar file %s -> %s", tar_file, err) + return False + + return True + + async def import_image(self, path): + """Import a tar file as image.""" + if self._lock.locked(): + _LOGGER.error("Can't excute import while a task is in progress") + return False + + async with self._lock: + return await self.loop.run_in_executor( + None, self._import_image, path) + + def _import_image(self, tar_file): + """Import a tar file as image. + + Need run inside executor. + """ + try: + with tar_file.open("rb") as read_tar: + self.dock.api.load_image(read_tar) + + image = self.dock.images.get(self.image) + except (docker.errors.DockerException, OSError) as err: + _LOGGER.error("Can't import image %s -> %s", self.image, err) + return False + + self.process_metadata(image.attrs, force=True) + self._cleanup() + return True + def _restart(self): """Restart docker container. @@ -200,8 +250,11 @@ def _restart(self): except docker.errors.DockerException: return False - _LOGGER.info("Restart %s", self.image) + # for restart it need to run! + if container.status != 'running': + return False + _LOGGER.info("Restart %s", self.image) with suppress(docker.errors.DockerException): container.stop(timeout=15) diff --git a/hassio/scheduler.py b/hassio/scheduler.py index b09a212c6ee..4125a3662f5 100644 --- a/hassio/scheduler.py +++ b/hassio/scheduler.py @@ -16,11 +16,7 @@ def __init__(self, loop): """Initialize task schedule.""" self.loop = loop self._data = {} - self._stop = False - - def stop(self): - """Stop to execute tasks in scheduler.""" - self._stop = True + self.suspend = False def register_task(self, coro_callback, seconds, repeat=True, now=False): @@ -51,11 +47,8 @@ def _run_task(self, idx): """Run a scheduled task.""" data = self._data.pop(idx) - # stop execute tasks - if self._stop: - return - - self.loop.create_task(data[CALL]()) + if not self.suspend: + self.loop.create_task(data[CALL]()) if data[REPEAT]: task = self.loop.call_later(data[SEC], self._run_task, idx) diff --git a/hassio/snapshots/__init__.py b/hassio/snapshots/__init__.py new file mode 100644 index 00000000000..e757ac487d8 --- /dev/null +++ b/hassio/snapshots/__init__.py @@ -0,0 +1,309 @@ +"""Snapshot system control.""" +import asyncio +from datetime import datetime +import logging +from pathlib import Path +import tarfile + +from .snapshot import Snapshot +from .util import create_slug +from ..const import ( + ATTR_SLUG, FOLDER_HOMEASSISTANT, SNAPSHOT_FULL, SNAPSHOT_PARTIAL) + +_LOGGER = logging.getLogger(__name__) + + +class SnapshotsManager(object): + """Manage snapshots.""" + + def __init__(self, config, loop, sheduler, addons, homeassistant): + """Initialize a snapshot manager.""" + self.config = config + self.loop = loop + self.sheduler = sheduler + self.addons = addons + self.homeassistant = homeassistant + self.snapshots = {} + self._lock = asyncio.Lock(loop=loop) + + @property + def list_snapshots(self): + """Return a list of all snapshot object.""" + return set(self.snapshots.values()) + + def get(self, slug): + """Return snapshot object.""" + return self.snapshots.get(slug) + + def _create_snapshot(self, name, sys_type): + """Initialize a new snapshot object from name.""" + date_str = str(datetime.utcnow()) + slug = create_slug(name, date_str) + tar_file = Path(self.config.path_backup, "{}.tar".format(slug)) + + snapshot = Snapshot(self.config, self.loop, tar_file) + snapshot.create(slug, name, date_str, sys_type) + + return snapshot + + async def reload(self): + """Load exists backups.""" + self.snapshots = {} + + async def _load_snapshot(tar_file): + """Internal function to load snapshot.""" + snapshot = Snapshot(self.config, self.loop, tar_file) + if await snapshot.load(): + self.snapshots[snapshot.slug] = snapshot + + tasks = [_load_snapshot(tar_file) for tar_file in + self.config.path_backup.glob("*.tar")] + + _LOGGER.info("Found %d snapshot files", len(tasks)) + if tasks: + await asyncio.wait(tasks, loop=self.loop) + + def remove(self, snapshot): + """Remove a snapshot.""" + try: + snapshot.tar_file.unlink() + self.snapshots.pop(snapshot.slug, None) + except OSError as err: + _LOGGER.error("Can't remove snapshot %s -> %s", snapshot.slug, err) + return False + + return True + + async def do_snapshot_full(self, name=""): + """Create a full snapshot.""" + if self._lock.locked(): + _LOGGER.error("It is already a snapshot/restore process running") + return False + + snapshot = self._create_snapshot(name, SNAPSHOT_FULL) + _LOGGER.info("Full-Snapshot %s start", snapshot.slug) + try: + self.sheduler.suspend = True + await self._lock.acquire() + + async with snapshot: + snapshot.homeassistant = self.homeassistant.version + snapshot.repositories = self.config.addons_repositories + + # snapshot addons + tasks = [] + for addon in self.addons.list_addons: + if not addon.is_installed: + continue + tasks.append(snapshot.import_addon(addon)) + + if tasks: + _LOGGER.info("Full-Snapshot %s run %d addons", + snapshot.slug, len(tasks)) + await asyncio.wait(tasks, loop=self.loop) + + # snapshot folders + _LOGGER.info("Full-Snapshot %s store folders", snapshot.slug) + await snapshot.store_folders() + + _LOGGER.info("Full-Snapshot %s done", snapshot.slug) + self.snapshots[snapshot.slug] = snapshot + return True + + except (OSError, tarfile.TarError) as err: + _LOGGER.info("Full-Snapshot %s error -> %s", snapshot.slug, err) + return False + + finally: + self.sheduler.suspend = False + self._lock.release() + + async def do_snapshot_partial(self, name="", addons=None, folders=None): + """Create a partial snapshot.""" + if self._lock.locked(): + _LOGGER.error("It is already a snapshot/restore process running") + return False + + addons = addons or [] + folders = folders or [] + snapshot = self._create_snapshot(name, SNAPSHOT_PARTIAL) + + _LOGGER.info("Partial-Snapshot %s start", snapshot.slug) + try: + self.sheduler.suspend = True + await self._lock.acquire() + + async with snapshot: + snapshot.homeassistant = self.homeassistant.version + snapshot.repositories = self.config.addons_repositories + + # snapshot addons + tasks = [] + for slug in addons: + addon = self.addons.get(slug) + if addon.is_installed: + tasks.append(snapshot.import_addon(addon)) + + if tasks: + _LOGGER.info("Partial-Snapshot %s run %d addons", + snapshot.slug, len(tasks)) + await asyncio.wait(tasks, loop=self.loop) + + # snapshot folders + _LOGGER.info("Partial-Snapshot %s store folders %s", + snapshot.slug, folders) + await snapshot.store_folders(folders) + + _LOGGER.info("Partial-Snapshot %s done", snapshot.slug) + self.snapshots[snapshot.slug] = snapshot + return True + + except (OSError, tarfile.TarError) as err: + _LOGGER.info("Partial-Snapshot %s error -> %s", snapshot.slug, err) + return False + + finally: + self.sheduler.suspend = False + self._lock.release() + + async def do_restore_full(self, snapshot): + """Restore a snapshot.""" + if self._lock.locked(): + _LOGGER.error("It is already a snapshot/restore process running") + return False + + if snapshot.sys_type != SNAPSHOT_FULL: + _LOGGER.error( + "Full-Restore %s is only a partial snapshot!", snapshot.slug) + return False + + _LOGGER.info("Full-Restore %s start", snapshot.slug) + try: + self.sheduler.suspend = True + await self._lock.acquire() + + async with snapshot: + # stop system + tasks = [] + tasks.append(self.homeassistant.stop()) + + for addon in self.addons.list_addons: + if addon.is_installed: + tasks.append(addon.stop()) + + await asyncio.wait(tasks, loop=self.loop) + + # restore folders + _LOGGER.info("Full-Restore %s restore folders", snapshot.slug) + await snapshot.restore_folders() + + # start homeassistant restore + task_hass = self.loop.create_task( + self.homeassistant.update(snapshot.homeassistant)) + + # restore repositories + await self.addons.load_repositories(snapshot.repositories) + + # restore addons + tasks = [] + actual_addons = \ + set(addon.slug for addon in self.addons.list_addons + if addon.is_installed) + restore_addons = \ + set(data[ATTR_SLUG] for data in snapshot.addons) + remove_addons = actual_addons - restore_addons + + _LOGGER.info("Full-Restore %s restore addons %s, remove %s", + snapshot.slug, restore_addons, remove_addons) + + for slug in remove_addons: + addon = self.addons.get(slug) + if addon: + tasks.append(addon.uninstall()) + else: + _LOGGER.warning("Can't remove addon %s", slug) + + for slug in restore_addons: + addon = self.addons.get(slug) + if addon: + tasks.append(snapshot.export_addon(addon)) + else: + _LOGGER.warning("Can't restore addon %s", slug) + + if tasks: + _LOGGER.info("Full-Restore %s restore addons tasks %d", + snapshot.slug, len(tasks)) + await asyncio.wait(tasks, loop=self.loop) + + # finish homeassistant task + _LOGGER.info("Full-Restore %s wait until homeassistant ready", + snapshot.slug) + await task_hass + await self.homeassistant.run() + + _LOGGER.info("Full-Restore %s done", snapshot.slug) + return True + + except (OSError, tarfile.TarError) as err: + _LOGGER.info("Full-Restore %s error -> %s", slug, err) + return False + + finally: + self.sheduler.suspend = False + self._lock.release() + + async def do_restore_partial(self, snapshot, homeassistant=False, + addons=None, folders=None): + """Restore a snapshot.""" + if self._lock.locked(): + _LOGGER.error("It is already a snapshot/restore process running") + return False + + addons = addons or [] + folders = folders or [] + + _LOGGER.info("Partial-Restore %s start", snapshot.slug) + try: + self.sheduler.suspend = True + await self._lock.acquire() + + async with snapshot: + tasks = [] + + if FOLDER_HOMEASSISTANT in folders: + await self.homeassistant.stop() + + if folders: + _LOGGER.info("Partial-Restore %s restore folders %s", + snapshot.slug, folders) + await snapshot.restore_folders(folders) + + if homeassistant: + tasks.append(self.homeassistant.update( + snapshot.homeassistant)) + + for slug in addons: + addon = self.addons.get(slug) + if addon: + tasks.append(snapshot.export_addon(addon)) + else: + _LOGGER.warning("Can't restore addon %s", slug) + + if tasks: + _LOGGER.info("Partial-Restore %s run %d tasks", + snapshot.slug, len(tasks)) + await asyncio.wait(tasks, loop=self.loop) + + # make sure homeassistant run agen + await self.homeassistant.run() + + _LOGGER.info("Partial-Restore %s done", snapshot.slug) + return True + + except (OSError, tarfile.TarError) as err: + _LOGGER.info("Partial-Restore %s error -> %s", slug, err) + return False + + finally: + self.sheduler.suspend = False + self._lock.release() diff --git a/hassio/snapshots/snapshot.py b/hassio/snapshots/snapshot.py new file mode 100644 index 00000000000..729f722e847 --- /dev/null +++ b/hassio/snapshots/snapshot.py @@ -0,0 +1,255 @@ +"""Represent a snapshot file.""" +import asyncio +import json +import logging +from pathlib import Path +import tarfile +from tempfile import TemporaryDirectory + +import voluptuous as vol +from voluptuous.humanize import humanize_error + +from .validate import SCHEMA_SNAPSHOT, ALL_FOLDERS +from .util import remove_folder +from ..const import ( + ATTR_SLUG, ATTR_NAME, ATTR_DATE, ATTR_ADDONS, ATTR_REPOSITORIES, + ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_VERSION, ATTR_TYPE) +from ..tools import write_json_file + +_LOGGER = logging.getLogger(__name__) + + +class Snapshot(object): + """A signle hassio snapshot.""" + + def __init__(self, config, loop, tar_file): + """Initialize a snapshot.""" + self.loop = loop + self.config = config + self.tar_file = tar_file + self._data = {} + self._tmp = None + + @property + def slug(self): + """Return snapshot slug.""" + return self._data.get(ATTR_SLUG) + + @property + def sys_type(self): + """Return snapshot type.""" + return self._data.get(ATTR_TYPE) + + @property + def name(self): + """Return snapshot name.""" + return self._data.get(ATTR_NAME) + + @property + def date(self): + """Return snapshot date.""" + return self._data.get(ATTR_DATE) + + @property + def addons(self): + """Return snapshot date.""" + return self._data.get(ATTR_ADDONS, []) + + @property + def folders(self): + """Return list of saved folders.""" + return self._data.get(ATTR_FOLDERS, []) + + @property + def repositories(self): + """Return snapshot date.""" + return self._data.get(ATTR_REPOSITORIES, []) + + @repositories.setter + def repositories(self, value): + """Set snapshot date.""" + self._data[ATTR_REPOSITORIES] = value + + @property + def homeassistant(self): + """Return snapshot homeassistant version.""" + return self._data.get(ATTR_HOMEASSISTANT) + + @homeassistant.setter + def homeassistant(self, value): + """Set snapshot homeassistant version.""" + self._data[ATTR_HOMEASSISTANT] = value + + @property + def size(self): + """Return snapshot size.""" + if not self.tar_file.is_file(): + return 0 + return self.tar_file.stat().st_size / 1048576 # calc mbyte + + def create(self, slug, name, date, sys_type): + """Initialize a new snapshot.""" + # init metadata + self._data[ATTR_SLUG] = slug + self._data[ATTR_NAME] = name + self._data[ATTR_DATE] = date + self._data[ATTR_TYPE] = sys_type + + # init other constructs + self._data[ATTR_ADDONS] = [] + self._data[ATTR_REPOSITORIES] = [] + self._data[ATTR_FOLDERS] = [] + + async def load(self): + """Read snapshot.json from tar file.""" + if not self.tar_file.is_file(): + _LOGGER.error("No tarfile %s", self.tar_file) + return False + + def _load_file(): + """Read snapshot.json.""" + with tarfile.open(self.tar_file, "r:") as snapshot: + json_file = snapshot.extractfile("./snapshot.json") + return json_file.read() + + # read snapshot.json + try: + raw = await self.loop.run_in_executor(None, _load_file) + except (tarfile.TarError, KeyError) as err: + _LOGGER.error( + "Can't read snapshot tarfile %s -> %s", self.tar_file, err) + return False + + # parse data + try: + raw_dict = json.loads(raw) + except json.JSONDecodeError as err: + _LOGGER.error("Can't read data for %s -> %s", self.tar_file, err) + return False + + # validate + try: + self._data = SCHEMA_SNAPSHOT(raw_dict) + except vol.Invalid as err: + _LOGGER.error("Can't validate data for %s -> %s", self.tar_file, + humanize_error(raw_dict, err)) + return False + + return True + + async def __aenter__(self): + """Async context to open a snapshot.""" + self._tmp = TemporaryDirectory(dir=str(self.config.path_tmp)) + + # create a snapshot + if not self.tar_file.is_file(): + return self + + # extract a exists snapshot + def _extract_snapshot(): + """Extract a snapshot.""" + with tarfile.open(self.tar_file, "r:") as tar: + tar.extractall(path=self._tmp.name) + + await self.loop.run_in_executor(None, _extract_snapshot) + + async def __aexit__(self, exception_type, exception_value, traceback): + """Async context to close a snapshot.""" + # exists snapshot or exception on build + if self.tar_file.is_file() or exception_type is not None: + return self._tmp.cleanup() + + # new snapshot, build it + def _create_snapshot(): + """Create a new snapshot.""" + with tarfile.open(self.tar_file, "w:") as tar: + tar.add(self._tmp.name, arcname=".") + + if write_json_file(Path(self._tmp.name, "snapshot.json"), self._data): + try: + await self.loop.run_in_executor(None, _create_snapshot) + except tarfile.TarError as err: + _LOGGER.error("Can't create tar %s", err) + else: + _LOGGER.error("Can't write snapshot.json") + + self._tmp.cleanup() + self._tmp = None + + async def import_addon(self, addon): + """Add a addon into snapshot.""" + snapshot_file = Path(self._tmp.name, "{}.tar.gz".format(addon.slug)) + + if not await addon.snapshot(snapshot_file): + _LOGGER.error("Can't make snapshot from %s", addon.slug) + return False + + # store to config + self._data[ATTR_ADDONS].append({ + ATTR_SLUG: addon.slug, + ATTR_NAME: addon.name, + ATTR_VERSION: addon.version_installed, + }) + + return True + + async def export_addon(self, addon): + """Restore a addon from snapshot.""" + snapshot_file = Path(self._tmp.name, "{}.tar.gz".format(addon.slug)) + + if not await addon.restore(snapshot_file): + _LOGGER.error("Can't restore snapshot for %s", addon.slug) + return False + + return True + + async def store_folders(self, folder_list=None): + """Backup hassio data into snapshot.""" + folder_list = folder_list or ALL_FOLDERS + + def _folder_save(name): + """Intenal function to snapshot a folder.""" + slug_name = name.replace("/", "_") + snapshot_tar = Path(self._tmp.name, "{}.tar.gz".format(slug_name)) + origin_dir = Path(self.config.path_hassio, name) + + try: + with tarfile.open(snapshot_tar, "w:gz", + compresslevel=1) as tar_file: + tar_file.add(origin_dir, arcname=".") + + self._data[ATTR_FOLDERS].append(name) + except tarfile.TarError as err: + _LOGGER.warning("Can't snapshot folder %s -> %s", name, err) + + # run tasks + tasks = [self.loop.run_in_executor(None, _folder_save, folder) + for folder in folder_list] + if tasks: + await asyncio.wait(tasks, loop=self.loop) + + async def restore_folders(self, folder_list=None): + """Backup hassio data into snapshot.""" + folder_list = folder_list or ALL_FOLDERS + + def _folder_restore(name): + """Intenal function to restore a folder.""" + slug_name = name.replace("/", "_") + snapshot_tar = Path(self._tmp.name, "{}.tar.gz".format(slug_name)) + origin_dir = Path(self.config.path_hassio, name) + + # clean old stuff + if origin_dir.is_dir(): + remove_folder(origin_dir) + + try: + with tarfile.open(snapshot_tar, "r:gz") as tar_file: + tar_file.extractall(path=origin_dir) + except tarfile.TarError as err: + _LOGGER.warning("Can't restore folder %s -> %s", name, err) + + # run tasks + tasks = [self.loop.run_in_executor(None, _folder_restore, folder) + for folder in folder_list] + if tasks: + await asyncio.wait(tasks, loop=self.loop) diff --git a/hassio/snapshots/util.py b/hassio/snapshots/util.py new file mode 100644 index 00000000000..af75b3ffafc --- /dev/null +++ b/hassio/snapshots/util.py @@ -0,0 +1,21 @@ +"""Util addons functions.""" +import hashlib +import shutil + + +def create_slug(name, date_str): + """Generate a hash from repository.""" + key = "{} - {}".format(date_str, name).lower().encode() + return hashlib.sha1(key).hexdigest()[:8] + + +def remove_folder(folder): + """Remove folder data but not the folder itself.""" + for obj in folder.iterdir(): + try: + if obj.is_dir(): + shutil.rmtree(str(obj), ignore_errors=True) + else: + obj.unlink() + except (OSError, shutil.Error): + pass diff --git a/hassio/snapshots/validate.py b/hassio/snapshots/validate.py new file mode 100644 index 00000000000..aabea42e392 --- /dev/null +++ b/hassio/snapshots/validate.py @@ -0,0 +1,27 @@ +"""Validate some things around restore.""" + +import voluptuous as vol + +from ..const import ( + ATTR_REPOSITORIES, ATTR_ADDONS, ATTR_NAME, ATTR_SLUG, ATTR_DATE, + ATTR_VERSION, ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_TYPE, FOLDER_SHARE, + FOLDER_HOMEASSISTANT, FOLDER_ADDONS, FOLDER_SSL, SNAPSHOT_FULL, + SNAPSHOT_PARTIAL) + +ALL_FOLDERS = [FOLDER_HOMEASSISTANT, FOLDER_SHARE, FOLDER_ADDONS, FOLDER_SSL] + +# pylint: disable=no-value-for-parameter +SCHEMA_SNAPSHOT = vol.Schema({ + vol.Required(ATTR_SLUG): vol.Coerce(str), + vol.Required(ATTR_TYPE): vol.In([SNAPSHOT_FULL, SNAPSHOT_PARTIAL]), + vol.Required(ATTR_NAME): vol.Coerce(str), + vol.Required(ATTR_DATE): vol.Coerce(str), + vol.Required(ATTR_HOMEASSISTANT): vol.Coerce(str), + vol.Required(ATTR_FOLDERS): [vol.In(ALL_FOLDERS)], + vol.Required(ATTR_ADDONS): [vol.Schema({ + vol.Required(ATTR_SLUG): vol.Coerce(str), + vol.Required(ATTR_NAME): vol.Coerce(str), + vol.Required(ATTR_VERSION): vol.Coerce(str), + })], + vol.Required(ATTR_REPOSITORIES): [vol.Url()], +}, extra=vol.ALLOW_EXTRA) From 0ac96c207e6d5ad1dc487320dc0dc2e305fa6b24 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 6 Jul 2017 01:29:09 +0200 Subject: [PATCH 6/7] Convert homeassistant into a dict (snapshot) (#90) * Convert homeassistant into a dict * fix lint * fix bugs * cleanup code * fix cleanup --- API.md | 7 ++++-- hassio/addons/addon.py | 11 ++++----- hassio/api/snapshots.py | 8 +++++-- hassio/dock/__init__.py | 3 +++ hassio/dock/addon.py | 14 +++++++---- hassio/snapshots/__init__.py | 28 ++++++++++++---------- hassio/snapshots/snapshot.py | 46 ++++++++++++++++++++++++------------ hassio/snapshots/validate.py | 17 +++++++------ 8 files changed, 85 insertions(+), 49 deletions(-) diff --git a/API.md b/API.md index ca66274ea77..899082608d6 100644 --- a/API.md +++ b/API.md @@ -180,10 +180,13 @@ Return QR-Code { "slug": "SNAPSHOT ID", "type": "full|partial", - "name": "custom snapshot name", + "name": "custom snapshot name / description", "date": "ISO", "size": "SIZE_IN_MB", - "homeassistant": "INSTALLED_HASS_VERSION", + "homeassistant": { + "version": "INSTALLED_HASS_VERSION", + "devices": [] + }, "addons": [ { "slug": "ADDON_SLUG", diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index b5d10395a40..633ac674c9b 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -434,15 +434,14 @@ def _extract_tar(): self._restore_data(data[ATTR_USER], data[ATTR_SYSTEM]) # check version / restore image - if data[ATTR_VERSION] != self.addon_docker.version: + version = data[ATTR_VERSION] + if version != self.addon_docker.version: image_file = Path(temp, "image.tar") if image_file.is_file(): - if not await self.addon_docker.import_image(image_file): - return False + await self.addon_docker.import_image(image_file, version) else: - if not await self.addon_docker.install(data[ATTR_VERSION]): - return False - await self.addon_docker.cleanup() + if await self.addon_docker.install(version): + await self.addon_docker.cleanup() else: await self.addon_docker.stop() diff --git a/hassio/api/snapshots.py b/hassio/api/snapshots.py index 4ba7652da10..0fdab2fec2b 100644 --- a/hassio/api/snapshots.py +++ b/hassio/api/snapshots.py @@ -8,7 +8,8 @@ from ..snapshots.validate import ALL_FOLDERS from ..const import ( ATTR_NAME, ATTR_SLUG, ATTR_DATE, ATTR_ADDONS, ATTR_REPOSITORIES, - ATTR_HOMEASSISTANT, ATTR_VERSION, ATTR_SIZE, ATTR_FOLDERS, ATTR_TYPE) + ATTR_HOMEASSISTANT, ATTR_VERSION, ATTR_SIZE, ATTR_FOLDERS, ATTR_TYPE, + ATTR_DEVICES) _LOGGER = logging.getLogger(__name__) @@ -69,7 +70,10 @@ async def info(self, request): ATTR_NAME: snapshot.name, ATTR_DATE: snapshot.date, ATTR_SIZE: snapshot.size, - ATTR_HOMEASSISTANT: snapshot.homeassistant, + ATTR_HOMEASSISTANT: { + ATTR_VERSION: snapshot.homeassistant_version, + ATTR_DEVICES: snapshot.homeassistant_devices, + }, ATTR_ADDONS: self._addons_list(snapshot), ATTR_REPOSITORIES: snapshot.repositories, ATTR_FOLDERS: snapshot.folders, diff --git a/hassio/dock/__init__.py b/hassio/dock/__init__.py index e84dd4a5a74..6cfd9d678d4 100644 --- a/hassio/dock/__init__.py +++ b/hassio/dock/__init__.py @@ -231,6 +231,9 @@ def _remove(self): _LOGGER.warning("Can't remove image %s -> %s", self.image, err) return False + # clean metadata + self.version = None + self.arch = None return True async def update(self, tag): diff --git a/hassio/dock/addon.py b/hassio/dock/addon.py index 77c87825154..3db9f87b1e6 100644 --- a/hassio/dock/addon.py +++ b/hassio/dock/addon.py @@ -5,6 +5,7 @@ import shutil import docker +import requests from . import DockerBase from .util import dockerfile_template @@ -196,7 +197,7 @@ def _export_image(self, tar_file): Need run inside executor. """ try: - image = self.dock.api.get_image("{}:latest".format(self.image)) + image = self.dock.api.get_image(self.image) except docker.errors.DockerException as err: _LOGGER.error("Can't fetch image %s -> %s", self.image, err) return False @@ -205,13 +206,14 @@ def _export_image(self, tar_file): with tar_file.open("wb") as write_tar: for chunk in image.stream(): write_tar.write(chunk) - except OSError() as err: + except (OSError, requests.exceptions.ReadTimeout) as err: _LOGGER.error("Can't write tar file %s -> %s", tar_file, err) return False + _LOGGER.info("Export image %s to %s", self.image, tar_file) return True - async def import_image(self, path): + async def import_image(self, path, tag): """Import a tar file as image.""" if self._lock.locked(): _LOGGER.error("Can't excute import while a task is in progress") @@ -219,9 +221,9 @@ async def import_image(self, path): async with self._lock: return await self.loop.run_in_executor( - None, self._import_image, path) + None, self._import_image, path, tag) - def _import_image(self, tar_file): + def _import_image(self, tar_file, tag): """Import a tar file as image. Need run inside executor. @@ -231,10 +233,12 @@ def _import_image(self, tar_file): self.dock.api.load_image(read_tar) image = self.dock.images.get(self.image) + image.tag(self.image, tag=tag) except (docker.errors.DockerException, OSError) as err: _LOGGER.error("Can't import image %s -> %s", self.image, err) return False + _LOGGER.info("Import image %s and tag %s", tar_file, tag) self.process_metadata(image.attrs, force=True) self._cleanup() return True diff --git a/hassio/snapshots/__init__.py b/hassio/snapshots/__init__.py index e757ac487d8..29f3b277a54 100644 --- a/hassio/snapshots/__init__.py +++ b/hassio/snapshots/__init__.py @@ -41,9 +41,15 @@ def _create_snapshot(self, name, sys_type): slug = create_slug(name, date_str) tar_file = Path(self.config.path_backup, "{}.tar".format(slug)) + # init object snapshot = Snapshot(self.config, self.loop, tar_file) snapshot.create(slug, name, date_str, sys_type) + # set general data + snapshot.homeassistant_version = self.homeassistant.version + snapshot.homeassistant_devices = self.config.homeassistant_devices + snapshot.repositories = self.config.addons_repositories + return snapshot async def reload(self): @@ -87,9 +93,6 @@ async def do_snapshot_full(self, name=""): await self._lock.acquire() async with snapshot: - snapshot.homeassistant = self.homeassistant.version - snapshot.repositories = self.config.addons_repositories - # snapshot addons tasks = [] for addon in self.addons.list_addons: @@ -110,7 +113,7 @@ async def do_snapshot_full(self, name=""): self.snapshots[snapshot.slug] = snapshot return True - except (OSError, tarfile.TarError) as err: + except (OSError, ValueError, tarfile.TarError) as err: _LOGGER.info("Full-Snapshot %s error -> %s", snapshot.slug, err) return False @@ -134,9 +137,6 @@ async def do_snapshot_partial(self, name="", addons=None, folders=None): await self._lock.acquire() async with snapshot: - snapshot.homeassistant = self.homeassistant.version - snapshot.repositories = self.config.addons_repositories - # snapshot addons tasks = [] for slug in addons: @@ -158,7 +158,7 @@ async def do_snapshot_partial(self, name="", addons=None, folders=None): self.snapshots[snapshot.slug] = snapshot return True - except (OSError, tarfile.TarError) as err: + except (OSError, ValueError, tarfile.TarError) as err: _LOGGER.info("Partial-Snapshot %s error -> %s", snapshot.slug, err) return False @@ -198,8 +198,10 @@ async def do_restore_full(self, snapshot): await snapshot.restore_folders() # start homeassistant restore + self.config.homeassistant_devices = \ + snapshot.homeassistant_devices task_hass = self.loop.create_task( - self.homeassistant.update(snapshot.homeassistant)) + self.homeassistant.update(snapshot.homeassistant_version)) # restore repositories await self.addons.load_repositories(snapshot.repositories) @@ -244,7 +246,7 @@ async def do_restore_full(self, snapshot): _LOGGER.info("Full-Restore %s done", snapshot.slug) return True - except (OSError, tarfile.TarError) as err: + except (OSError, ValueError, tarfile.TarError) as err: _LOGGER.info("Full-Restore %s error -> %s", slug, err) return False @@ -279,8 +281,10 @@ async def do_restore_partial(self, snapshot, homeassistant=False, await snapshot.restore_folders(folders) if homeassistant: + self.config.homeassistant_devices = \ + snapshot.homeassistant_devices tasks.append(self.homeassistant.update( - snapshot.homeassistant)) + snapshot.homeassistant_version)) for slug in addons: addon = self.addons.get(slug) @@ -300,7 +304,7 @@ async def do_restore_partial(self, snapshot, homeassistant=False, _LOGGER.info("Partial-Restore %s done", snapshot.slug) return True - except (OSError, tarfile.TarError) as err: + except (OSError, ValueError, tarfile.TarError) as err: _LOGGER.info("Partial-Restore %s error -> %s", slug, err) return False diff --git a/hassio/snapshots/snapshot.py b/hassio/snapshots/snapshot.py index 729f722e847..1882a5f0747 100644 --- a/hassio/snapshots/snapshot.py +++ b/hassio/snapshots/snapshot.py @@ -13,7 +13,7 @@ from .util import remove_folder from ..const import ( ATTR_SLUG, ATTR_NAME, ATTR_DATE, ATTR_ADDONS, ATTR_REPOSITORIES, - ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_VERSION, ATTR_TYPE) + ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_VERSION, ATTR_TYPE, ATTR_DEVICES) from ..tools import write_json_file _LOGGER = logging.getLogger(__name__) @@ -43,27 +43,27 @@ def sys_type(self): @property def name(self): """Return snapshot name.""" - return self._data.get(ATTR_NAME) + return self._data[ATTR_NAME] @property def date(self): """Return snapshot date.""" - return self._data.get(ATTR_DATE) + return self._data[ATTR_DATE] @property def addons(self): """Return snapshot date.""" - return self._data.get(ATTR_ADDONS, []) + return self._data[ATTR_ADDONS] @property def folders(self): """Return list of saved folders.""" - return self._data.get(ATTR_FOLDERS, []) + return self._data[ATTR_FOLDERS] @property def repositories(self): """Return snapshot date.""" - return self._data.get(ATTR_REPOSITORIES, []) + return self._data[ATTR_REPOSITORIES] @repositories.setter def repositories(self, value): @@ -71,14 +71,24 @@ def repositories(self, value): self._data[ATTR_REPOSITORIES] = value @property - def homeassistant(self): + def homeassistant_version(self): """Return snapshot homeassistant version.""" - return self._data.get(ATTR_HOMEASSISTANT) + return self._data[ATTR_HOMEASSISTANT].get(ATTR_VERSION) - @homeassistant.setter - def homeassistant(self, value): + @homeassistant_version.setter + def homeassistant_version(self, value): """Set snapshot homeassistant version.""" - self._data[ATTR_HOMEASSISTANT] = value + self._data[ATTR_HOMEASSISTANT][ATTR_VERSION] = value + + @property + def homeassistant_devices(self): + """Return snapshot homeassistant devices.""" + return self._data[ATTR_HOMEASSISTANT].get(ATTR_DEVICES) + + @homeassistant_devices.setter + def homeassistant_devices(self, value): + """Set snapshot homeassistant devices.""" + self._data[ATTR_HOMEASSISTANT][ATTR_DEVICES] = value @property def size(self): @@ -96,6 +106,7 @@ def create(self, slug, name, date, sys_type): self._data[ATTR_TYPE] = sys_type # init other constructs + self._data[ATTR_HOMEASSISTANT] = {} self._data[ATTR_ADDONS] = [] self._data[ATTR_REPOSITORIES] = [] self._data[ATTR_FOLDERS] = [] @@ -159,6 +170,14 @@ async def __aexit__(self, exception_type, exception_value, traceback): if self.tar_file.is_file() or exception_type is not None: return self._tmp.cleanup() + # validate data + try: + self._data = SCHEMA_SNAPSHOT(self._data) + except vol.Invalid as err: + _LOGGER.error("Invalid data for %s -> %s", self.tar_file, + humanize_error(self._data, err)) + raise ValueError("Invalid config") from None + # new snapshot, build it def _create_snapshot(): """Create a new snapshot.""" @@ -166,10 +185,7 @@ def _create_snapshot(): tar.add(self._tmp.name, arcname=".") if write_json_file(Path(self._tmp.name, "snapshot.json"), self._data): - try: - await self.loop.run_in_executor(None, _create_snapshot) - except tarfile.TarError as err: - _LOGGER.error("Can't create tar %s", err) + await self.loop.run_in_executor(None, _create_snapshot) else: _LOGGER.error("Can't write snapshot.json") diff --git a/hassio/snapshots/validate.py b/hassio/snapshots/validate.py index aabea42e392..0efe2ee288d 100644 --- a/hassio/snapshots/validate.py +++ b/hassio/snapshots/validate.py @@ -4,9 +4,9 @@ from ..const import ( ATTR_REPOSITORIES, ATTR_ADDONS, ATTR_NAME, ATTR_SLUG, ATTR_DATE, - ATTR_VERSION, ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_TYPE, FOLDER_SHARE, - FOLDER_HOMEASSISTANT, FOLDER_ADDONS, FOLDER_SSL, SNAPSHOT_FULL, - SNAPSHOT_PARTIAL) + ATTR_VERSION, ATTR_HOMEASSISTANT, ATTR_FOLDERS, ATTR_TYPE, ATTR_DEVICES, + FOLDER_SHARE, FOLDER_HOMEASSISTANT, FOLDER_ADDONS, FOLDER_SSL, + SNAPSHOT_FULL, SNAPSHOT_PARTIAL) ALL_FOLDERS = [FOLDER_HOMEASSISTANT, FOLDER_SHARE, FOLDER_ADDONS, FOLDER_SSL] @@ -16,12 +16,15 @@ vol.Required(ATTR_TYPE): vol.In([SNAPSHOT_FULL, SNAPSHOT_PARTIAL]), vol.Required(ATTR_NAME): vol.Coerce(str), vol.Required(ATTR_DATE): vol.Coerce(str), - vol.Required(ATTR_HOMEASSISTANT): vol.Coerce(str), - vol.Required(ATTR_FOLDERS): [vol.In(ALL_FOLDERS)], - vol.Required(ATTR_ADDONS): [vol.Schema({ + vol.Required(ATTR_HOMEASSISTANT): vol.Schema({ + vol.Required(ATTR_VERSION): vol.Coerce(str), + vol.Optional(ATTR_DEVICES, default=[]): [vol.Match(r"^[^/]*$")], + }), + vol.Optional(ATTR_FOLDERS, default=[]): [vol.In(ALL_FOLDERS)], + vol.Optional(ATTR_ADDONS, default=[]): [vol.Schema({ vol.Required(ATTR_SLUG): vol.Coerce(str), vol.Required(ATTR_NAME): vol.Coerce(str), vol.Required(ATTR_VERSION): vol.Coerce(str), })], - vol.Required(ATTR_REPOSITORIES): [vol.Url()], + vol.Optional(ATTR_REPOSITORIES, default=[]): [vol.Url()], }, extra=vol.ALLOW_EXTRA) From f731c630a6f65db85ff15978a3560be24bb82efe Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 6 Jul 2017 01:34:37 +0200 Subject: [PATCH 7/7] Update version.json --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 1e5b7b67edb..e8a9d4c41ef 100644 --- a/version.json +++ b/version.json @@ -1,5 +1,5 @@ { - "hassio": "0.38", + "hassio": "0.39", "homeassistant": "0.48.1", "resinos": "0.8", "resinhup": "0.1",