Merge pull request #45 from ameily/issue-43

Issue #43 - Environment Variables; fixes #43

CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v0.7.0] - 2020-02-09
+### Added
+- Support for the `~` home directory symbol. All filenames are passed through the
+  `os.path.expanduser` function.
+- `IPv4NetworkField` now support setting a minimum and maximum prefix length, in bits.
+- `Field.help` attribute to document the field, similar to a docstring. The `help` and
+  autogenerated `short_help` attribute, can be used in UI to display information and
+  documentation about the field.
+- `BaseSchema.generate_argparse_parser` method to autogenerate an `argparse.ArgumentParser`
+  object to parse command line arguments.
+- `Field.env` and `Schema.env` to control automatically loading configuration values from
+  environment variables.
+
+### Changed
+- Improved error reporting with the `ValidationError` exception that contains the offending
+  field's full name and path.
+
+
 ## [v0.6.0] - 2020-11-05
 ### Added
 - `Field.sensitive` property to mark a value as sensitive.

cincoconfig/abc.py

@@ -111,13 +111,52 @@ def validate(self, cfg, value):
 
     Each Field subclass can define a class or instance level ``storage_type`` which holds the
     annotation of the value being stored in memory.
+
+    .. _field-env-variables:
+
+    **Environment Variables**
+
+    Fields can load their default value from an environment variable. The Schema and Field accept
+    an ``env`` argument in the constructor that controls whether and how environment variables are
+    loaded. The default behavior is to not load any environment variables and to honor the
+    :attr:`Field.default` value.
+
+    There are two ways to load a field's default value from an environment variable.
+
+    - ``Schema.env``: Provide ``True`` or a string.
+    - ``Field.env``: Provide ``True`` or a string.
+
+    When ``Schema.env`` or ``Field.env`` is ``None`` (the default), the environment variable
+    configuration is inherited from the parent schema. A value of ``True`` will load the the
+    field's default value from an autogenerated environment variable name, based on the field's
+    full path. For example:
+
+    .. code-block:: python
+
+        schema = Schema(env=True)
+        schema.mode = ApplicationModeField(env="APP_MODE")
+        schema.port = PortField(env=False)
+
+        schema.db.host = HostnameField()
+
+        schema.auth = Schema(env="SECRET")
+        schema.auth.username = StringField()
+
+    - The top-level schema is configured to autogenerate and load environment variables for all
+      fields.
+    - ``mode`` is loaded from the ``APP_MODE`` environment variable.
+    - ``port`` is not loaded from any the environment variabale.
+    - ``db.host`` is loaded from the ``DB_HOST`` environment variable.
+    - The ``auth`` schema has a environment variable prefix of ``SECRET``. All children and nested
+      fields/schemas will start with ``SECRET_``.
+    - The ``auth.username`` field is loaded from the ``SECRET_USERNAME`` environment variable.
     '''
     storage_type = Any
 
     def __init__(self, *, name: str = None, key: str = None, required: bool = False,
                  default: Union[Callable, Any] = None,
                  validator: Callable[['BaseConfig', Any], Any] = None, sensitive: bool = False,
-                 description: str = None, help: str = None):
+                 description: str = None, help: str = None, env: Union[bool, str] = None):
         '''
         All builtin Fields accept the following keyword parameters.
 
@@ -140,6 +179,7 @@ def __init__(self, *, name: str = None, key: str = None, required: bool = False,
         self.sensitive = sensitive
         self.description = description
         self.help = help.strip() if help else None
+        self.env = env
 
     @property
     def short_help(self) -> Optional[str]:
@@ -236,14 +276,43 @@ def __setkey__(self, schema: 'BaseSchema', key: str):
         '''
         self.key = key
 
-    def __setdefault__(self, cfg: 'BaseConfig'):
+        if self.env is False:
+            return
+
+        if self.env is True or (self.env is None and isinstance(schema._env_prefix, str)):
+            # Set our environment variable name based on the schema's prefix and our key
+            if isinstance(schema._env_prefix, str) and schema._env_prefix:
+                prefix = schema._env_prefix + '_'
+            else:
+                prefix = ''
+
+            self.env = prefix + self.key.upper()
+
+    def __setdefault__(self, cfg: 'BaseConfig') -> None:
         '''
         Set the default value of the field in the config. This is called when the config is first
         created.
 
         :param cfg: current config
         '''
-        cfg._data[self.key] = self.default
+        value = None
+
+        if isinstance(self.env, str) and self.env:
+            env_value = os.environ.get(self.env)
+            if env_value:
+                try:
+                    env_value = self.validate(cfg, env_value)
+                except ValidationError:
+                    raise
+                except Exception as exc:
+                    raise ValidationError(cfg, self, exc) from exc
+                else:
+                    value = env_value
+
+        if value is None:
+            value = self.default
+
+        cfg._data[self.key] = value
 
     def to_python(self, cfg: 'BaseConfig', value: Any) -> Any:
         '''
@@ -345,15 +414,19 @@ class BaseSchema:
     '''
     storage_type = 'BaseSchema'
 
-    def __init__(self, key: str = None, dynamic: bool = False):
+    def __init__(self, key: str = None, dynamic: bool = False, env: Union[str, bool] = None):
         '''
         :param key: the schema key, only used for sub-schemas, and stored in the instance as
             *_key*
         :param dynamic: the schema is dynamic and can contain fields not originally specified in
             the schema and stored in the instance as *_dynamic*
+        :param env: the environment variable prefix for this schema and all children schemas, for
+            information, see :ref:`Field Environment Variables <field-env-variables>`
         '''
         self._key = key
         self._dynamic = dynamic
+        self._env_prefix = '' if env is True else env
+
         self._fields = OrderedDict()  # type: Dict[str, SchemaField]
         self.__post_init__()
 
@@ -370,6 +443,14 @@ def __setkey__(self, parent: 'BaseSchema', key: str) -> None:
         '''
         self._key = key
 
+        if self._env_prefix is False:
+            return
+
+        if self._env_prefix is None and isinstance(parent._env_prefix, str):
+            # Set our environment variable prefix to be "{parent}_{key}"
+            prefix = (parent._env_prefix + '_') if parent._env_prefix else ''
+            self._env_prefix = prefix + self._key.upper()
+
     def _add_field(self, key: str, field: SchemaField) -> SchemaField:
         '''
         Add a field to the schema. This method will call ``field.__setkey__(self, key)``.
@@ -379,6 +460,7 @@ def _add_field(self, key: str, field: SchemaField) -> SchemaField:
         self._fields[key] = field
         if isinstance(field, (Field, BaseSchema)):
             field.__setkey__(self, key)
+
         return field
 
     def _get_field(self, key: str) -> Optional[SchemaField]:

cincoconfig/config.py

@@ -127,7 +127,7 @@ def __getattr__(self, name: str) -> SchemaField:
         '''
         field = self._fields.get(name)
         if field is None:
-            field = self._fields[name] = Schema(name)
+            field = self._add_field(name, Schema())
         return field
 
     def __iter__(self) -> Iterator[Tuple[str, SchemaField]]:
@@ -662,6 +662,9 @@ def load_tree(self, tree: dict) -> None:
         for key, value in tree.items():
             field = self._get_field(key)
             if isinstance(field, Field):
+                if isinstance(field.env, str) and field.env and os.environ.get(field.env):
+                    continue
+
                 value = field.to_python(self, value)
 
             self.__setattr__(key, value)

cincoconfig/version.py

@@ -1 +1 @@
-__version__ = '0.6.0'
+__version__ = '0.7.0'

tests/test_baseschema.py

@@ -6,7 +6,7 @@ class TestBaseSchema:
 
     def test_setkey(self):
         schema = BaseSchema()
-        schema.__setkey__(None, 'hello')
+        schema.__setkey__(BaseSchema(), 'hello')
         assert schema._key == 'hello'
 
     def test_add_field_field(self):
@@ -35,3 +35,25 @@ def test_get_field_exists(self):
     def test_get_field_no_exists(self):
         schema = BaseSchema()
         assert schema._get_field('hello') is None
+
+    def test_env_true(self):
+        schema = BaseSchema(env=True)
+        assert schema._env_prefix == '' and isinstance(schema._env_prefix, str)
+
+    def test_setkey_inherit_env(self):
+        schema = BaseSchema(env=True)
+        child = BaseSchema()
+        child.__setkey__(schema, 'child')
+        assert child._env_prefix == 'CHILD'
+
+    def test_setkey_inherit_env_append(self):
+        schema = BaseSchema(env='ASDF')
+        child = BaseSchema()
+        child.__setkey__(schema, 'child')
+        assert child._env_prefix == 'ASDF_CHILD'
+
+    def test_setkey_env_false(self):
+        schema = BaseSchema(env='ASDF')
+        child = BaseSchema(env=False)
+        child.__setkey__(schema, 'child')
+        assert child._env_prefix is False

tests/test_config.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 from unittest.mock import MagicMock, patch, mock_open
 
 import pytest
@@ -325,6 +326,17 @@ def test_load_tree_validate(self):
         config.load_tree({'x': 1})
         mock_validate.assert_called_once_with()
 
+    @patch('cincoconfig.config.os')
+    def test_load_tree_ignore_env(self, mock_os):
+        env = mock_os.environ.get.return_value = object()
+        schema = Schema()
+        schema.x = Field(env='ASDF')
+        cfg = schema()
+        cfg._data = {'x': 'qwer'}
+        cfg.load_tree({'x': 'asdf'})
+        assert cfg._data == {'x': 'qwer'}
+        mock_os.environ.get.assert_called_once_with('ASDF')
+
     def test_validator(self):
         validator = MagicMock()
         schema = Schema()

tests/test_fields/test_field.py

@@ -4,10 +4,11 @@
 # This file is subject to the terms and conditions defined in the file 'LICENSE', which is part of
 # this source code package.
 #
+import os
 from unittest.mock import patch, MagicMock
 import pytest
 
-from cincoconfig.abc import Field
+from cincoconfig.abc import Field, BaseSchema, BaseConfig, ValidationError
 
 
 class MockConfig:
@@ -16,6 +17,7 @@ def __init__(self, data=None, parent=None, key=None):
         self._data = data or {}
         self._parent = parent
         self._key = key
+        self._schema = BaseSchema()
 
     def _full_path(self):
         return ''
@@ -54,7 +56,7 @@ def test_getval(self):
 
     def test_setkey(self):
         field = Field()
-        field.__setkey__(self.cfg, 'key')
+        field.__setkey__(self.cfg._schema, 'key')
         assert field.key == 'key'
 
     def test_setdefault(self):
@@ -152,3 +154,85 @@ def test_short_help_paragraph(self):
         field = Field(help='\n\nfirst\nsecond\nthird.\n\nmore\n\n')
         assert field.short_help == 'first\nsecond\nthird.'
         assert field.help == 'first\nsecond\nthird.\n\nmore'
+
+    def test_env_true(self):
+        schema = BaseSchema()
+        field = Field(env=True)
+        field.__setkey__(schema, 'field')
+        assert field.env == 'FIELD'
+
+    def test_setkey_inherit_env(self):
+        schema = BaseSchema(env=True)
+        field = Field()
+        field.__setkey__(schema, 'field')
+        assert field.env == 'FIELD'
+
+    def test_setkey_inherit_env_append(self):
+        schema = BaseSchema(env='APP')
+        field = Field()
+        field.__setkey__(schema, 'field')
+        assert field.env == 'APP_FIELD'
+
+    def test_setkey_env_false(self):
+        schema = BaseSchema(env=True)
+        field = Field(env=False)
+        field.__setkey__(schema, 'field')
+        assert field.env is False
+
+    @patch.object(os.environ, 'get')
+    def test_setdefault_env_exists(self, mock_environ_get):
+        retval = mock_environ_get.return_value = object()
+        cfg = BaseConfig(schema=BaseSchema())
+        field = Field(env='ASDF', key='field')
+        field.__setdefault__(cfg)
+        assert cfg._data == {'field': retval}
+        mock_environ_get.assert_called_once_with('ASDF')
+
+    @patch.object(os.environ, 'get')
+    def test_setdefault_env_exists_valid(self, mock_environ_get):
+        env = mock_environ_get.return_value = object()
+        retval = object()
+        cfg = BaseConfig(schema=BaseSchema())
+        field = Field(env='ASDF', key='field')
+        field.validate = MagicMock(return_value=retval)
+        field.__setdefault__(cfg)
+        field.validate.assert_called_once_with(cfg, env)
+        assert cfg._data == {'field': retval}
+
+    @patch.object(os.environ, 'get')
+    def test_setdefault_env_exists_invalid(self, mock_environ_get):
+        env = mock_environ_get.return_value = object()
+        retval = object()
+        cfg = BaseConfig(schema=BaseSchema())
+        field = Field(env='ASDF', key='field')
+        field.validate = MagicMock(side_effect=ValueError())
+        field._default = retval
+        with pytest.raises(ValidationError):
+            field.__setdefault__(cfg)
+
+        field.validate.assert_called_once_with(cfg, env)
+
+    @patch.object(os.environ, 'get')
+    def test_setdefault_env_exists_invalid_validationerror(self, mock_environ_get):
+        env = mock_environ_get.return_value = object()
+        retval = object()
+        cfg = BaseConfig(schema=BaseSchema())
+        field = Field(env='ASDF', key='field')
+        err = ValidationError(cfg, field, ValueError('asdf'))
+        field.validate = MagicMock(side_effect=err)
+        field._default = retval
+        with pytest.raises(ValidationError) as exc:
+            field.__setdefault__(cfg)
+
+        assert exc.value is err
+
+    @patch.object(os.environ, 'get')
+    def test_setdefault_env_not_exists(self, mock_environ_get):
+        mock_environ_get.return_value = None
+        retval = object()
+        cfg = BaseConfig(schema=BaseSchema())
+        field = Field(env='ASDF', key='field')
+        field._default = retval
+        field.__setdefault__(cfg)
+        assert cfg._data == {'field': retval}
+        mock_environ_get.assert_called_once_with('ASDF')

tests/test_fields/test_string.py

@@ -24,6 +24,7 @@ class MockSchema:
 
     def __init__(self):
         self._fields = {}
+        self._env_prefix = False
 
     def _add_field(self, name, field):
         self._fields[name] = field

tests/test_schema.py

@@ -116,6 +116,13 @@ def test_getitem_keyerror_not_schema(self):
         with pytest.raises(KeyError):
             y = schema['x.y']
 
+    def test_getattr_add_field(self):
+        schema = Schema()
+        mock_add_field = MagicMock(return_value=Schema())
+        object.__setattr__(schema, '_add_field', mock_add_field)
+        schema.x.y = 2
+        mock_add_field.assert_called_once()
+
     @patch('cincoconfig.config.ArgumentParser')
     def test_generate_argparse_parser(self, mock_argparse):
         parser = MagicMock()