From 722735e356384ccd6b7c191db6a5b42f294387c7 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sun, 14 Jul 2019 15:34:42 +0300 Subject: [PATCH] Initial version This version is based on vdsm tests/storage/userstorage.py, which is based on the older sanlock tests/storage.py tool. Improvements compared with vdsm version: - Create a package - Separate configuration from the tool, so the same tool can be used by other projects, or different configuration can be used in different environments. - Make it easier to run using python -m userstorage - Refine storage directory layout. - Remove oVirt CI policy from the tool; policy should be implemented in configuration files, not in the tool. - Add required option, for marking some configuration as optional - Improve error handling so setup code can handled both unsupported configuration and failed setup in a generic way. --- README.md | 125 +++++++++++++++++++++++++++++++++- exampleconf.py | 78 ++++++++++++++++++++++ test/__init__.py | 0 test/userstorage_test.py | 140 +++++++++++++++++++++++++++++++++++++++ tox.ini | 26 ++++++++ userstorage/__init__.py | 12 ++++ userstorage/__main__.py | 50 ++++++++++++++ userstorage/backend.py | 66 ++++++++++++++++++ userstorage/config.py | 36 ++++++++++ userstorage/file.py | 58 ++++++++++++++++ userstorage/loop.py | 100 ++++++++++++++++++++++++++++ userstorage/mount.py | 82 +++++++++++++++++++++++ userstorage/osutil.py | 35 ++++++++++ 13 files changed, 807 insertions(+), 1 deletion(-) create mode 100644 exampleconf.py create mode 100644 test/__init__.py create mode 100644 test/userstorage_test.py create mode 100644 tox.ini create mode 100644 userstorage/__init__.py create mode 100644 userstorage/__main__.py create mode 100644 userstorage/backend.py create mode 100644 userstorage/config.py create mode 100644 userstorage/file.py create mode 100644 userstorage/loop.py create mode 100644 userstorage/mount.py create mode 100644 userstorage/osutil.py diff --git a/README.md b/README.md index 7f6c2e6..d514cdb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,125 @@ # userstorage -Helper for setting up storage for tests + +Helper for setting up storage for tests. + + +## Overview + +Some tests need more than a temporary directory on the local file +system. One example is testing block device with 4k sector size, or +testing a filesystem on top of such a block device. + +You can create storage using loop devices and mounts in test fixtures, +but creating devices and mounts requires root. Do you really want to run +all your tests as root, when the code under test should not run as root? + +The userstorage tool solves this problem by creating storage for tests +before running the tests, and making the storage available to the +current user. Once you created the storage, you can run the tests +quickly as yourself directly from your editor. + + +## Installing + +Currently the only way to install this is cloning the repo and +installing the package manually. We will make this available via pip +soon. + + +## Creating configuration file + +The userstorage tool creates storage based on configuration file that +you must provide. + +The configuration module is use both by the userstorage tool to +provision the storage, and by the tests consuming the storage. + +The configuration module typically starts by importing the backends you +want to provision: + + from userstorage import File + +The configuration module must define these names: + + # Where storage is provisioned. + BASE_DIR = "/path/to/my/storage" + + # Storage configurations needed by the tests. + BACKENDS = {} + +See exampleconf.py for example configuration used by the tests for this +project. + + +## Setting up user storage + +To setup storage for these tests, run: + + python -m userstorage setup exampleconf.py + +This can be run once when setting up development environment, and must +be run again after rebooting the host. + +If you want to tear down the user storage, run: + + python -m userstorage teardown exampleconf.py + +There is no need to tear down the storage normally. The loop devices are +backed up by sparse files and do not consume much resources. + + +## Consuming the storage in your tests + +See test/userstorage_test.py for example test module consuming storage +set up by userstorage tool, and the exampleconf.py module. + +Note that some storage may not be available on some systems. Your tests +can check if a storage is available and skip or mark the test as xfail +if needed. + + +## How it works? + +The userstorage tool creates this directory layout in the BASE_DIR +defined in the configuration module: + +$ tree /var/tmp/example-storage/ +/var/tmp/example-storage/ +├── block-4k-backing +├── block-4k-loop -> /dev/loop2 +├── block-512-backing +├── block-512-loop -> /dev/loop3 +├── file-4k-backing +├── file-4k-loop -> /dev/loop4 +├── file-4k-mount +│   ├── file +│   └── lost+found [error opening dir] +├── file-512-backing +├── file-512-loop -> /dev/loop5 +└── file-512-mount + ├── file + └── lost+found [error opening dir] + +The symbolic links (e.g. file-4k-loop) link to the loop devices created +by the tool (/dev/loop4), and used to tear down the storage. + +The actual file used for the tests are created inside the mounted +filesystem (/var/tmp/example-storage/file-4k-mount/file). + + +## Projects using userstorage + +- sanlock - using very early version of this tool +- vdsm - using more recent version of this tool + +(Please add your project here) + + +## Contributing + +If you found a bug, please open an issue. + +If you have an idea for improving this tool, please open an issue to +discuss the idea. + +For trivial changes please send a pull request. diff --git a/exampleconf.py b/exampleconf.py new file mode 100644 index 0000000..d1101b2 --- /dev/null +++ b/exampleconf.py @@ -0,0 +1,78 @@ +""" +Example userstorage configuration module. +""" + +from userstorage import LoopDevice, Mount, File + +GiB = 1024**3 + +# This is the directory where backing files, symlinks to loop devices, and +# mount directories are created. + +BASE_DIR = "/var/tmp/example-storage" + + +# Dictionary of backends. Here is an example configuration providing file and +# block storage with 512 and 4k sector size. Note that 4k storage is defined as +# optional since creating loop device with 4k storgae is not supported on all +# environments and may be flaky in some supported environments. + +BACKENDS = { + + "block-512": LoopDevice( + base_dir=BASE_DIR, + name="block-512", + size=GiB, + sector_size=512, + ), + + "block-4k": LoopDevice( + base_dir=BASE_DIR, + name="block-4k", + size=GiB, + sector_size=4096, + required=False, + ), + + "mount-512": Mount( + LoopDevice( + base_dir=BASE_DIR, + name="mount-512", + size=GiB, + sector_size=512, + ) + ), + + "mount-4k": Mount( + LoopDevice( + base_dir=BASE_DIR, + name="mount-4k", + size=GiB, + sector_size=4096, + required=False, + ) + ), + + "file-512": File( + Mount( + LoopDevice( + base_dir=BASE_DIR, + name="file-512", + size=GiB, + sector_size=512, + ) + ) + ), + + "file-4k": File( + Mount( + LoopDevice( + base_dir=BASE_DIR, + name="file-4k", + size=GiB, + sector_size=4096, + required=False, + ) + ) + ), +} diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/userstorage_test.py b/test/userstorage_test.py new file mode 100644 index 0000000..9108f46 --- /dev/null +++ b/test/userstorage_test.py @@ -0,0 +1,140 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +from __future__ import absolute_import +from __future__ import division + +import errno +import glob +import io +import mmap +import os +import shutil +import stat +import subprocess + +from contextlib import closing + +import pytest + +import userstorage + +BACKENDS = userstorage.load_config("exampleconf.py").BACKENDS + + +@pytest.fixture( + params=[ + BACKENDS["block-512"], + BACKENDS["block-4k"], + ], + ids=str, +) +def user_loop(request): + backend = validate_backend(request.param) + yield backend + + # Discard loop device to ensure next test is not affected. + # TODO: Should be implemented by backend. + + subprocess.check_output(["blkdiscard", backend.path]) + + +@pytest.fixture( + params=[ + BACKENDS["mount-512"], + BACKENDS["mount-4k"] + ], + ids=str, +) +def user_mount(request): + backend = validate_backend(request.param) + yield backend + + # Remove files and directories created by the current tests to ensure that + # the next test is not affected. + # TODO: Should be implemented by backend. + + for path in glob.glob(os.path.join(backend.path, "*")): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + +@pytest.fixture( + params=[ + BACKENDS["file-512"], + BACKENDS["file-4k"], + ], + ids=str, +) +def user_file(request): + backend = validate_backend(request.param) + yield backend + + # Truncate file to ensure that next test is not affected. + # TODO: Should be implemented by backend. + + with open(backend.path, "w") as f: + f.truncate(0) + + +def validate_backend(backend): + if not backend.exists(): + pytest.xfail("backend {} not available".format(backend)) + return backend + + +def test_loop_device(user_loop): + assert is_block_device(user_loop.path) + assert logical_block_size(user_loop.path) == user_loop.sector_size + + +def test_mount(user_mount): + assert os.path.isdir(user_mount.path) + + filename = os.path.join(user_mount.path, "file") + with open(filename, "w") as f: + f.truncate(4096) + + assert detect_block_size(filename) == user_mount.sector_size + + +def test_file(user_file): + assert os.path.isfile(user_file.path) + assert detect_block_size(user_file.path) == user_file.sector_size + + +def is_block_device(path): + mode = os.stat(path).st_mode + return stat.S_ISBLK(mode) + + +def logical_block_size(path): + realpath = os.path.realpath(path) + dev = os.path.split(realpath)[1] + lbs = "/sys/block/{}/queue/logical_block_size".format(dev) + with open(lbs) as f: + return int(f.readline()) + + +def detect_block_size(path): + """ + Detect the minimal block size for direct I/O. This is typically the sector + size of the underlying storage. + + Copied from ovirt-imageio. + """ + fd = os.open(path, os.O_RDONLY | os.O_DIRECT) + with io.FileIO(fd, "r") as f: + for block_size in (512, 4096): + buf = mmap.mmap(-1, block_size) + with closing(buf): + try: + f.readinto(buf) + except EnvironmentError as e: + if e.errno != errno.EINVAL: + raise + else: + return block_size + raise RuntimeError("Cannot detect block size") diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..42892aa --- /dev/null +++ b/tox.ini @@ -0,0 +1,26 @@ +[tox] +envlist = py27,py36,py37,flake8,pylint +skip_missing_interpreters = True +skipsdist = True +usedevelop = True + +[testenv] +deps = + pytest +commands = + py.test {posargs} + +[testenv:flake8] +deps = + flake8 +commands = + flake8 + +[testenv:pylint] +deps = + pylint +commands = + pylint -E userstorage + +[pytest] +addopts = -v -rxXs --durations=10 diff --git a/userstorage/__init__.py b/userstorage/__init__.py new file mode 100644 index 0000000..24ad3b2 --- /dev/null +++ b/userstorage/__init__.py @@ -0,0 +1,12 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +# flake8: noqa + +# Backends. +from userstorage.loop import LoopDevice +from userstorage.mount import Mount +from userstorage.file import File + +# Helpers. +from userstorage.config import load_config diff --git a/userstorage/__main__.py b/userstorage/__main__.py new file mode 100644 index 0000000..2ef72fc --- /dev/null +++ b/userstorage/__main__.py @@ -0,0 +1,50 @@ +import argparse +import logging + +from . import backend +from . import config +from . import osutil + +log = logging.getLogger("userstorage") + + +def main(): + parser = argparse.ArgumentParser( + description='Set up storage tests') + parser.add_argument("command", choices=["setup", "teardown"]) + parser.add_argument("config_file", help="Configuration file") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="[%(name)s] %(levelname)-7s %(message)s") + + cfg = config.load_config(args.config_file) + + if args.command == "setup": + setup(cfg) + elif args.command == "teardown": + teardown(cfg) + + +def setup(cfg): + osutil.create_dir(cfg.BASE_DIR) + + for b in cfg.BACKENDS.values(): + try: + b.setup() + except backend.Error as e: + if b.required: + raise + log.warning("Skipping %s storage: %s", b.name, e) + + +def teardown(cfg): + for b in cfg.BACKENDS.values(): + b.teardown() + + osutil.remove_dir(cfg.BASE_DIR) + + +if __name__ == "__main__": + main() diff --git a/userstorage/backend.py b/userstorage/backend.py new file mode 100644 index 0000000..62720d2 --- /dev/null +++ b/userstorage/backend.py @@ -0,0 +1,66 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +from __future__ import absolute_import +from __future__ import division + + +class Error(Exception): + """ + Base class for backend errors. + """ + + +class Unsupported(Error): + """ + May be raised in setup() if backend is not supported on the current system. + """ + + +class SetupFailed(Error): + """ + May be raised in setup() if backend should be supported but setup has + failed. + """ + + +class Base(object): + """ + Base class for backend objects. + """ + + # Name used by the tests to locate this backend. Storage backends must be + # configured with a unique name. + name = None + + # Storage logical block size. + sector_size = 512 + + # Path to backend. This can be a regular file, a directory, a block device, + # or a symlink, depending on the backend. + path = None + + # Set to False to ignore setup errors. Tests using this storage will handle + # the missing storage. + required = True + + def setup(self): + """ + Set up backend for testing. + """ + raise NotImplementedError + + def teardown(self): + """ + Clean up backend when it not needed any more. + """ + raise NotImplementedError + + def exists(self): + """ + Return True if backend is set up and can be used. + """ + raise NotImplementedError + + def __str__(self): + return self.name diff --git a/userstorage/config.py b/userstorage/config.py new file mode 100644 index 0000000..819c19b --- /dev/null +++ b/userstorage/config.py @@ -0,0 +1,36 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +""" +Storage configuration loader. + +The configuration file is loaded by userstorage tool providing the storage and +by test module consuming the storage. + +The configuration file is a python module, providing these names: + + # Directory keeping storage files. + BASE_DIR = "/var/tmp/my-project-storage" + + # Dictionary of backends. + BACKENDS = {} + +See testconf.py example for more info. +""" + +import imp +import os + + +def load_config(filename): + """ + Load user configuration module. + """ + basepath = os.path.splitext(filename)[0] + module_dir, module_name = os.path.split(basepath) + fp, pathname, description = imp.find_module(module_name, [module_dir]) + try: + return imp.load_module(module_name, fp, pathname, description) + finally: + if fp: + fp.close() diff --git a/userstorage/file.py b/userstorage/file.py new file mode 100644 index 0000000..fa656a0 --- /dev/null +++ b/userstorage/file.py @@ -0,0 +1,58 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +from __future__ import absolute_import +from __future__ import division + +import os +import logging + +from . import backend +from . import osutil + +log = logging.getLogger("userstorage") + + +class File(backend.Base): + """ + A single file on a mounted file system. + """ + + def __init__(self, mount): + """ + Create file based storage. + """ + self._mount = mount + self.path = os.path.join(mount.path, "file") + + # Backend interface. + + @property + def name(self): + return self._mount.name + + @property + def sector_size(self): + return self._mount.sector_size + + @property + def required(self): + return self._mount.required + + def setup(self): + if self.exists(): + log.debug("Reusing file %s", self.path) + return + + self._mount.setup() + + log.info("Creating file %s", self.path) + open(self.path, "w").close() + + def teardown(self): + log.info("Removing file %s", self.path) + osutil.remove_file(self.path) + self._mount.teardown() + + def exists(self): + return os.path.exists(self.path) diff --git a/userstorage/loop.py b/userstorage/loop.py new file mode 100644 index 0000000..55f46d1 --- /dev/null +++ b/userstorage/loop.py @@ -0,0 +1,100 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +from __future__ import absolute_import +from __future__ import division + +import os +import subprocess +import logging + +from . import backend +from . import osutil + +log = logging.getLogger("userstorage") + + +class LoopDevice(backend.Base): + """ + A loop device with optionally specific sector size. + """ + + _have_sector_size = None + + def __init__(self, base_dir, name, size, sector_size=512, required=True): + self.base_dir = base_dir + self.name = name + self.size = size + self.sector_size = sector_size + self.required = required + self.path = os.path.join(base_dir, name + "-loop") + self._backing = os.path.join(base_dir, name + "-backing") + + # Backend interface + + def setup(self): + if self.sector_size == 4096 and not self.have_sector_size(): + raise backend.Unsupported( + "Sector size {} not supported" .format(self.sector_size)) + + if self.exists(): + log.debug("Reusing loop device %s", self.path) + return + + log.info("Creating backing file %s", self._backing) + with open(self._backing, "w") as f: + f.truncate(self.size) + + log.info("Creating loop device %s", self.path) + try: + device = self._create_loop_device() + except subprocess.CalledProcessError as e: + # Creating loop devices using --sector-size is flaky on some setups + # like oVirt CI, running in under mock. + raise backend.SetupFailed( + "Error creating loop device: {}".format(e)) + + # Remove stale symlink. + if os.path.islink(self.path): + os.unlink(self.path) + + os.symlink(device, self.path) + + if os.geteuid() != 0: + osutil.chown(self.path) + + def teardown(self): + log.info("Removing loop device %s", self.path) + if self.exists(): + self._remove_loop_device() + osutil.remove_file(self.path) + + log.info("Removing backing file %s", self._backing) + osutil.remove_file(self._backing) + + def exists(self): + return os.path.exists(self.path) + + # LoopDevice interface. + + @classmethod + def have_sector_size(cls): + if cls._have_sector_size is None: + out = subprocess.check_output(["losetup", "-h"]).decode() + cls._have_sector_size = "--sector-size " in out + return cls._have_sector_size + + # Helpers + + def _create_loop_device(self): + cmd = ["sudo", "losetup", "--find", self._backing, "--show"] + + if self.sector_size != 512: + cmd.append("--sector-size") + cmd.append(str(self.sector_size)) + + out = subprocess.check_output(cmd) + return out.decode("utf-8").strip() + + def _remove_loop_device(self): + subprocess.check_call(["sudo", "losetup", "-d", self.path]) diff --git a/userstorage/mount.py b/userstorage/mount.py new file mode 100644 index 0000000..4a648d6 --- /dev/null +++ b/userstorage/mount.py @@ -0,0 +1,82 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +from __future__ import absolute_import +from __future__ import division + +import os +import subprocess +import logging + +from . import backend +from . import osutil + +log = logging.getLogger("userstorage") + + +class Mount(backend.Base): + """ + A mounted filesystem on top of a loop device. + """ + + def __init__(self, loop): + self._loop = loop + self.path = os.path.join(loop.base_dir, loop.name + "-mount") + + # Backend interface + + @property + def name(self): + return self._loop.name + + @property + def sector_size(self): + return self._loop.sector_size + + @property + def required(self): + return self._loop.required + + def setup(self): + if self.exists(): + log.debug("Reusing mount %s", self.path) + return + + self._loop.setup() + + log.info("Creating filesystem %s", self.path) + self._create_filesystem() + osutil.create_dir(self.path) + self._mount_loop() + + if os.geteuid() != 0: + osutil.chown(self.path) + + def teardown(self): + log.info("Unmounting filesystem %s", self.path) + + if self.exists(): + self._unmount_loop() + + osutil.remove_dir(self.path) + + self._loop.teardown() + + def exists(self): + with open("/proc/self/mounts") as f: + for line in f: + if self.path in line: + return True + return False + + # Helpers + + def _create_filesystem(self): + # TODO: Use -t xfs (requires xfsprogs package). + subprocess.check_call(["sudo", "mkfs", "-q", self._loop.path]) + + def _mount_loop(self): + subprocess.check_call(["sudo", "mount", self._loop.path, self.path]) + + def _unmount_loop(self): + subprocess.check_call(["sudo", "umount", self.path]) diff --git a/userstorage/osutil.py b/userstorage/osutil.py new file mode 100644 index 0000000..b1086d7 --- /dev/null +++ b/userstorage/osutil.py @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Nir Soffer +# This program is free software; see LICENSE for more info. + +import errno +import os +import subprocess + + +def chown(path): + user_group = "%(USER)s:%(USER)s" % os.environ + subprocess.check_call(["sudo", "chown", "-R", user_group, path]) + + +def create_dir(path): + try: + os.makedirs(path) + except EnvironmentError as e: + if e.errno != errno.EEXIST: + raise + + +def remove_file(path): + try: + os.remove(path) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + + +def remove_dir(path): + try: + os.rmdir(path) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise