From dd8e687e1fb41f2cf372d4eb57888e556c299a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Mon, 11 Jun 2018 14:31:18 +0200 Subject: [PATCH] Define metadata tree root [fix #26] Add suppport for ".fmf" directory which defines root of the metadata tree. This special directory contains a "version" file holding format version. Documentation, examples and tests adjusted accordingly. --- docs/concept.rst | 16 ++++++++-- examples/child/.fmf/version | 1 + examples/deep/.fmf/version | 1 + examples/merge/.fmf/version | 1 + examples/scatter/.fmf/version | 1 + examples/touch/.fmf/version | 1 + examples/wget/.fmf/version | 1 + fmf/base.py | 58 +++++++++++++++++++++++++++-------- fmf/utils.py | 3 ++ tests/test_base.py | 10 +++--- tests/test_cli.py | 22 ++++++++----- 11 files changed, 86 insertions(+), 29 deletions(-) create mode 100644 examples/child/.fmf/version create mode 100644 examples/deep/.fmf/version create mode 100644 examples/merge/.fmf/version create mode 100644 examples/scatter/.fmf/version create mode 100644 examples/touch/.fmf/version create mode 100644 examples/wget/.fmf/version diff --git a/docs/concept.rst b/docs/concept.rst index 7aa2fd21..64dce553 100644 --- a/docs/concept.rst +++ b/docs/concept.rst @@ -80,6 +80,16 @@ collisions between similar attributes. For example: * test_description, requirement_description +Tree +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Metadata form a tree where inheritance is applied. The tree root +is defined by an ``.fmf`` directory (similarly as ``.git`` +identifies top of the git repository). The ``.fmf`` directory +contains at least a ``version`` file with a single integer number +defining version of the format. + + Objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,9 +99,9 @@ file name ``main.fmf`` works similarly as ``index.html``: +-------------------------------+-----------------------+ | Location | Identifier | +===============================+=======================+ - | wget/main.fmf | wget | + | wget/main.fmf | / | +-------------------------------+-----------------------+ - | wget/download/main.fmf | wget/download | + | wget/download/main.fmf | /download | +-------------------------------+-----------------------+ - | wget/download/smoke.fmf | wget/download/smoke | + | wget/download/smoke.fmf | /download/smoke | +-------------------------------+-----------------------+ diff --git a/examples/child/.fmf/version b/examples/child/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/child/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/examples/deep/.fmf/version b/examples/deep/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/deep/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/examples/merge/.fmf/version b/examples/merge/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/merge/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/examples/scatter/.fmf/version b/examples/scatter/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/scatter/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/examples/touch/.fmf/version b/examples/touch/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/touch/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/examples/wget/.fmf/version b/examples/wget/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/wget/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/fmf/base.py b/fmf/base.py index c2721e99..334694b8 100644 --- a/fmf/base.py +++ b/fmf/base.py @@ -19,6 +19,7 @@ SUFFIX = ".fmf" MAIN = "main" + SUFFIX +VERSION = 1 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # YAML @@ -43,35 +44,66 @@ class Tree(object): """ Metadata Tree """ def __init__(self, data, name=None, parent=None): """ - Initialize data dictionary, optionally update data + Initialize metadata tree from directory path or data dictionary - Data can be either string with directory path to be explored or - a dictionary with the values already prepared. + Data parameter can be either a string with directory path to be + explored or a dictionary with the values already prepared. """ - # Family relations and name (identifier) + # Initialize family relations, object data and source files self.parent = parent self.children = dict() self.data = dict() self.sources = list() - if name is None: - self.name = os.path.basename(os.path.realpath(data)) - self.root = os.path.dirname(os.path.realpath(data)) + self.root = None + self.version = VERSION + + # Special handling for top parent + if self.parent is None: + self.name = "/" + if not isinstance(data, dict): + self._initialize(path=data) + data = self.root + # Handle child node creation else: - self.name = "/".join([self.parent.name, name]) self.root = self.parent.root - log.debug("New tree '{0}' created.".format(self)) - - # Update data from dictionary or explore directory + self.name = os.path.join(self.parent.name, name) + # Initialize data if isinstance(data, dict): self.update(data) else: self.grow(data) + log.debug("New tree '{0}' created.".format(self)) def __unicode__(self): """ Use tree name as identifier """ return self.name + def _initialize(self, path): + """ Find metadata tree root, detect format version """ + # Find the tree root + root = os.path.abspath(path) + try: + while ".fmf" not in next(os.walk(root))[1]: + if root == "/": + raise utils.FileError( + "Unable to find root directory for '{0}'".format(path)) + root = os.path.abspath(os.path.join(root, os.pardir)) + except StopIteration: + raise utils.FileError("Invalid directory path: {0}".format(root)) + log.info("Root directory found: {0}".format(root)) + self.root = root + # Detect format version + try: + with open(os.path.join(self.root, ".fmf", "version")) as version: + self.version = int(version.read()) + log.info("Format version detected: {0}".format(self.version)) + except IOError as error: + raise utils.FormatError( + "Unable to detect format version: {0}".format(error)) + except ValueError: + raise utils.FormatError("Invalid version format") + def inherit(self): """ Apply inheritance and attribute merging """ if self.parent is not None: @@ -156,7 +188,7 @@ def grow(self, path): return path = path.rstrip("/") log.info("Walking through directory {0}".format( - os.path.realpath(path))) + os.path.abspath(path))) try: dirpath, dirnames, filenames = list(os.walk(path))[0] except IndexError: @@ -173,7 +205,7 @@ def grow(self, path): for filename in filenames: if filename.startswith("."): continue - fullpath = os.path.realpath(os.path.join(dirpath, filename)) + fullpath = os.path.abspath(os.path.join(dirpath, filename)) log.info("Checking file {0}".format(fullpath)) with open(fullpath) as datafile: data = yaml.load(datafile) diff --git a/fmf/utils.py b/fmf/utils.py index 1d88d8ec..f5955b21 100644 --- a/fmf/utils.py +++ b/fmf/utils.py @@ -37,6 +37,9 @@ class GeneralError(Exception): """ General error """ +class FormatError(GeneralError): + """ Metadata format error """ + class FileError(GeneralError): """ File reading error """ diff --git a/tests/test_base.py b/tests/test_base.py index 2db537ee..1ff37c38 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -40,14 +40,14 @@ def test_hidden(self): def test_inheritance(self): """ Inheritance and data types """ - deep = self.wget.find('wget/recursion/deep') + deep = self.wget.find('/recursion/deep') assert(deep.data['depth'] == 1000) assert(deep.data['description'] == 'Check recursive download options') assert(deep.data['tags'] == ['Tier2']) def test_scatter(self): """ Scattered files """ - scatter = Tree(EXAMPLES + "scatter").find("scatter/object") + scatter = Tree(EXAMPLES + "scatter").find("/object") assert(len(list(scatter.climb())) == 1) assert(scatter.data['one'] == 1) assert(scatter.data['two'] == 2) @@ -55,7 +55,7 @@ def test_scatter(self): def test_scattered_inheritance(self): """ Inheritance of scattered files """ - grandson = Tree(EXAMPLES + "child").find("child/son/grandson") + grandson = Tree(EXAMPLES + "child").find("/son/grandson") assert(grandson.data['name'] == 'Hugo') assert(grandson.data['eyes'] == 'blue') assert(grandson.data['nose'] == 'long') @@ -68,7 +68,7 @@ def test_deep_hierarchy(self): def test_merge(self): """ Attribute merging """ - child = self.merge.find('merge/parent/child') + child = self.merge.find('/parent/child') assert('General' in child.data['description']) assert('Specific' in child.data['description']) assert(child.data['tags'] == ['Tier1', 'Tier2']) @@ -90,7 +90,7 @@ def test_show(self): assert(self.wget.show(brief=True).endswith("\n")) assert(isinstance(self.wget.show(), type(""))) assert(self.wget.show().endswith("\n")) - assert('wget' in self.wget.show()) + assert('tester' in self.wget.show()) def test_update(self): """ Update data """ diff --git a/tests/test_cli.py b/tests/test_cli.py index 435b9adb..f14a1eb9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,9 @@ import os import sys +import pytest import fmf.cli +import fmf.utils as utils # Prepare path to examples PATH = os.path.dirname(os.path.realpath(__file__)) @@ -16,11 +18,15 @@ class TestCommandLine(object): def test_smoke(self): """ Smoke test """ - fmf.cli.main("") fmf.cli.main(WGET) fmf.cli.main(WGET + " --debug") fmf.cli.main(WGET + " --verbose") + def test_missing_root(self): + """ Missing root """ + with pytest.raises(utils.FileError): + fmf.cli.main("") + def test_output(self): """ There is some output """ output = fmf.cli.main(WGET) @@ -53,20 +59,20 @@ def test_filtering(self): """ Filtering """ output = fmf.cli.main(WGET + " --filter tags:Tier1 --filter tags:TierSecurity") - assert "wget/download/test" in output + assert "/download/test" in output output = fmf.cli.main(WGET + " --filter tags:Tier1 --filter tags:Wrong") assert "wget" not in output output = fmf.cli.main(WGET + " --filter 'tags: Tier[A-Z].*'") - assert "wget/download/test" in output - assert "wget/recursion" not in output + assert "/download/test" in output + assert "/recursion" not in output def test_key_content(self): """ Key content """ output = fmf.cli.main(WGET + " --key depth") - assert "wget/recursion/deep" in output - assert "wget/download/test" not in output + assert "/recursion/deep" in output + assert "/download/test" not in output def test_format_basic(self): """ Custom format (basic) """ @@ -77,11 +83,11 @@ def test_format_basic(self): def test_format_key(self): """ Custom format (find by key, check the name) """ output = fmf.cli.main(WGET + " --key depth --format {0} --value name") - assert "wget/recursion/deep" in output + assert "/recursion/deep" in output def test_format_functions(self): """ Custom format (using python functions) """ output = fmf.cli.main( WGET + " --key depth --format {0} --value os.path.basename(name)") assert "deep" in output - assert "wget/recursion" not in output + assert "/recursion" not in output