mypy, split contruct_object

fixes issue #306


*When this change indeed resolves your problem, please **Close** this issue*.
*(You can do so using the WorkFlow pull-down (close to the top right of this page))*

CHANGES

@@ -1,3 +1,7 @@
+[0, 16, 3]: 2019-08-15
+  - move setting of version based on YAML directive to scanner, allowing to
+    check for file version during TAG directive scanning
+
 [0, 16, 2]: 2019-08-15
   - preserve YAML and TAG directives on roundtrip, correctly output #
     in URL for YAML 1.2 (both reported by `Thomas Smith

README.rst

@@ -4,7 +4,7 @@ ruamel.yaml
 
 ``ruamel.yaml`` is a YAML 1.2 loader/dumper package for Python.
 
-:version:       0.16.2
+:version:       0.16.3
 :updated:       2019-08-15
 :documentation: http://yaml.readthedocs.io
 :repository:    https://bitbucket.org/ruamel/yaml
@@ -54,6 +54,12 @@ ChangeLog
 
 .. should insert NEXT: at the beginning of line for next key (with empty line)
 
+0.16.3 (2019-08-15):
+  - split construct_object
+  - change stuff back to keep mypy happy
+  - move setting of version based on YAML directive to scanner, allowing to
+    check for file version during TAG directive scanning
+
 0.16.2 (2019-08-15):
   - preserve YAML and TAG directives on roundtrip, correctly output #
     in URL for YAML 1.2 (both reported by `Thomas Smith

__init__.py

@@ -7,8 +7,8 @@
 
 _package_data = dict(
     full_package_name='ruamel.yaml',
-    version_info=(0, 16, 2),
-    __version__='0.16.2',
+    version_info=(0, 16, 3),
+    __version__='0.16.3',
     author='Anthon van der Neut',
     author_email='[email protected]',
     description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order',  # NOQA

comments.py

@@ -12,7 +12,8 @@
 import copy
 
 
-from ruamel.yaml.compat import ordereddict, PY2, string_types, MutableSliceableSequence
+from ruamel.yaml.compat import ordereddict  # type: ignore
+from ruamel.yaml.compat import PY2, string_types, MutableSliceableSequence
 from ruamel.yaml.scalarstring import ScalarString
 from ruamel.yaml.anchor import Anchor
 
@@ -159,7 +160,7 @@ def item(self, idx):
     def add_idx_line_col(self, key, data):
         # type: (Any, Any) -> None
         if self.data is None:
-            self.data = {}  # type: Dict[Any, Any]
+            self.data = {}
         self.data[key] = data
 
 
@@ -351,7 +352,7 @@ def yaml_set_tag(self, value):
         self.tag.value = value
 
     def copy_attributes(self, t, memo=None):
-        # type: (Any, bool) -> None
+        # type: (Any, Any) -> None
         # fmt: off
         for a in [Comment.attrib, Format.attrib, LineCol.attrib, Anchor.attrib,
                   Tag.attrib, merge_attrib]:
@@ -629,7 +630,7 @@ def __iter__(self):
             yield self._mapping[key]
 
 
-class CommentedMap(ordereddict, CommentedBase):
+class CommentedMap(ordereddict, CommentedBase):  # type: ignore
     __slots__ = (Comment.attrib, '_ok', '_ref')
 
     def __init__(self, *args, **kw):
@@ -695,7 +696,7 @@ def _yaml_get_pre_comment(self):
             self.ca.comment[1] = pre_comments
         return pre_comments
 
-    def update(self, vals):  # type: ignore
+    def update(self, vals):
         # type: (Any) -> None
         try:
             ordereddict.update(self, vals)
@@ -828,7 +829,7 @@ def _keys(self):
 
     def __len__(self):
         # type: () -> int
-        return ordereddict.__len__(self)
+        return int(ordereddict.__len__(self))
 
     def __eq__(self, other):
         # type: (Any) -> bool

compat.py

@@ -13,7 +13,8 @@
 
 # fmt: off
 if False:  # MYPY
-    from typing import Any, Dict, Optional, List, Union, BinaryIO, IO, Text, Tuple, Optional  # NOQA
+    from typing import Any, Dict, Optional, List, Union, BinaryIO, IO, Text, Tuple  # NOQA
+    from typing import Optional  # NOQA
 # fmt: on
 
 _DEFAULT_YAML_VERSION = (1, 2)
@@ -106,7 +107,7 @@ def to_unicode(s):
     binary_type = str
 
     # to allow importing
-    unichr = unichr  # type: ignore
+    unichr = unichr
     from StringIO import StringIO as _StringIO
 
     StringIO = _StringIO
@@ -122,7 +123,7 @@ def to_unicode(s):
     # StreamType = Union[BinaryIO, IO[str], StringIO]  # type: ignore
     StreamType = Any
 
-    StreamTextType = Union[Text, StreamType]
+    StreamTextType = StreamType  # Union[Text, StreamType]
     VersionType = Union[List[int], str, Tuple[int, int]]
 
 if PY3:

constructor.py

@@ -16,8 +16,10 @@
 from ruamel.yaml.nodes import *                               # NOQA
 from ruamel.yaml.nodes import (SequenceNode, MappingNode, ScalarNode)
 from ruamel.yaml.compat import (utf8, builtins_module, to_str, PY2, PY3,  # NOQA
-                                ordereddict, text_type, nprint, nprintf, version_tnf,
-                                Hashable, MutableSequence, MutableMapping)
+                                text_type, nprint, nprintf, version_tnf)
+from ruamel.yaml.compat import ordereddict, Hashable, MutableSequence  # type: ignore
+from ruamel.yaml.compat import MutableMapping  # type: ignore
+
 from ruamel.yaml.comments import *                               # NOQA
 from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet,
                                   CommentedKeySeq, CommentedSeq, TaggedScalar,
@@ -141,19 +143,31 @@ def construct_object(self, node, deep=False):
             #     None, None, 'found unconstructable recursive node', node.start_mark
             # )
         self.recursive_objects[node] = None
+        data = self.construct_non_recursive_object(node)
+
+        self.constructed_objects[node] = data
+        del self.recursive_objects[node]
+        if deep:
+            self.deep_construct = old_deep
+        return data
+
+    def construct_non_recursive_object(self, node, tag=None):
+        # type: (Any, Optional[str]) -> Any
         constructor = None  # type: Any
         tag_suffix = None
-        if node.tag in self.yaml_constructors:
-            constructor = self.yaml_constructors[node.tag]
+        if tag is None:
+            tag = node.tag
+        if tag in self.yaml_constructors:
+            constructor = self.yaml_constructors[tag]
         else:
             for tag_prefix in self.yaml_multi_constructors:
-                if node.tag.startswith(tag_prefix):
-                    tag_suffix = node.tag[len(tag_prefix) :]
+                if tag.startswith(tag_prefix):
+                    tag_suffix = tag[len(tag_prefix) :]
                     constructor = self.yaml_multi_constructors[tag_prefix]
                     break
             else:
                 if None in self.yaml_multi_constructors:
-                    tag_suffix = node.tag
+                    tag_suffix = tag
                     constructor = self.yaml_multi_constructors[None]
                 elif None in self.yaml_constructors:
                     constructor = self.yaml_constructors[None]
@@ -175,10 +189,6 @@ def construct_object(self, node, deep=False):
                     pass
             else:
                 self.state_generators.append(generator)
-        self.constructed_objects[node] = data
-        del self.recursive_objects[node]
-        if deep:
-            self.deep_construct = old_deep
         return data
 
     def construct_scalar(self, node):

emitter.py

@@ -241,7 +241,7 @@ def flow_level(self):
     def dispose(self):
         # type: () -> None
         # Reset the state attributes (to clear self-references)
-        self.states = []  # type: List[Any]
+        self.states = []
         self.state = None
 
     def emit(self, event):
@@ -1591,7 +1591,7 @@ def write_plain(self, text, split=True):
                         self.stream.write(data)
                     start = end
             elif breaks:
-                if ch not in u'\n\x85\u2028\u2029':
+                if ch not in u'\n\x85\u2028\u2029':  # type: ignore
                     if text[start] == u'\n':
                         self.write_line_break()
                     for br in text[start:end]:

error.py

@@ -40,13 +40,15 @@ def __str__(self):
         return where
 
     def __eq__(self, other):
+        # type: (Any) -> bool
         if self.line != other.line or self.column != other.column:
             return False
         if self.name != other.name or self.index != other.index:
             return False
         return True
 
     def __ne__(self, other):
+        # type: (Any) -> bool
         return not self.__eq__(other)
 
 

main.py

@@ -35,7 +35,7 @@
 from ruamel.yaml.loader import Loader as UnsafeLoader
 
 if False:  # MYPY
-    from typing import List, Set, Dict, Union, Any, Callable  # NOQA
+    from typing import List, Set, Dict, Union, Any, Callable, Optional, Text  # NOQA
     from ruamel.yaml.compat import StreamType, StreamTextType, VersionType  # NOQA
 
     if PY3:
@@ -99,9 +99,9 @@ def __init__(
             self.default_flow_style = False
             # no optimized rt-dumper yet
             self.Emitter = ruamel.yaml.emitter.Emitter  # type: Any
-            self.Serializer = ruamel.yaml.serializer.Serializer  # type: Any
+            self.Serializer = ruamel.yaml.serializer.Serializer
             self.Representer = ruamel.yaml.representer.RoundTripRepresenter  # type: Any
-            self.Scanner = ruamel.yaml.scanner.RoundTripScanner  # type: Any
+            self.Scanner = ruamel.yaml.scanner.RoundTripScanner
             # no optimized rt-parser yet
             self.Parser = ruamel.yaml.parser.RoundTripParser  # type: Any
             self.Composer = ruamel.yaml.composer.Composer  # type: Any
@@ -326,7 +326,7 @@ def load(self, stream):
         """
         if not hasattr(stream, 'read') and hasattr(stream, 'open'):
             # pathlib.Path() instance
-            with stream.open('rb') as fp:  # type: ignore
+            with stream.open('rb') as fp:
                 return self.load(fp)
         constructor, parser = self.get_constructor_parser(stream)
         try:
@@ -351,7 +351,7 @@ def load_all(self, stream, _kw=enforce):  # , skip=None):
             )
         if not hasattr(stream, 'read') and hasattr(stream, 'open'):
             # pathlib.Path() instance
-            with stream.open('r') as fp:  # type: ignore
+            with stream.open('r') as fp:
                 for d in self.load_all(fp, _kw=enforce):
                     yield d
                 return

representer.py

@@ -5,7 +5,8 @@
 
 from ruamel.yaml.error import *  # NOQA
 from ruamel.yaml.nodes import *  # NOQA
-from ruamel.yaml.compat import text_type, binary_type, to_unicode, PY2, PY3, ordereddict
+from ruamel.yaml.compat import text_type, binary_type, to_unicode, PY2, PY3
+from ruamel.yaml.compat import ordereddict  # type: ignore
 from ruamel.yaml.compat import nprint, nprintf  # NOQA
 from ruamel.yaml.scalarstring import (
     LiteralScalarString,
@@ -83,8 +84,8 @@ def represent(self, data):
         # type: (Any) -> None
         node = self.represent_data(data)
         self.serializer.serialize(node)
-        self.represented_objects = {}  # type: Dict[Any, Any]
-        self.object_keeper = []  # type: List[Any]
+        self.represented_objects = {}
+        self.object_keeper = []
         self.alias_key = None
 
     def represent_data(self, data):

scalarstring.py

@@ -129,7 +129,8 @@ def walk_tree(base, map=None):
         map[':'] = SingleQuotedScalarString
         walk_tree(data, map=map)
     """
-    from ruamel.yaml.compat import string_types, MutableMapping, MutableSequence
+    from ruamel.yaml.compat import string_types
+    from ruamel.yaml.compat import MutableMapping, MutableSequence  # type: ignore
 
     if map is None:
         map = {'\n': preserve_literal}

scanner.py

@@ -81,7 +81,7 @@ def __init__(self, loader=None):
             self.loader._scanner = self
         self.reset_scanner()
         self.first_time = False
-        self.yaml_version = None
+        self.yaml_version = None  # type: Any
 
     @property
     def flow_level(self):
@@ -156,10 +156,10 @@ def reader(self):
 
     @property
     def scanner_processing_version(self):  # prefix until un-composited
-        # type: () -> VersionType
+        # type: () -> Any
         if hasattr(self.loader, 'typ'):
-            return self.loader.resolver.processing_version  # type: ignore
-        return self.loader.processing_version  # type: ignore
+            return self.loader.resolver.processing_version
+        return self.loader.processing_version
 
     # Public methods.
 
@@ -454,7 +454,7 @@ def fetch_stream_end(self):
         # Reset simple keys.
         self.remove_possible_simple_key()
         self.allow_simple_key = False
-        self.possible_simple_keys = {}  # type: Dict[Any, Any]
+        self.possible_simple_keys = {}
         # Read the token.
         mark = self.reader.get_mark()
         # Add STREAM-END.

setup.py

@@ -1,6 +1,6 @@
 # # header
 # coding: utf-8
-# dd: 20190807
+# dd: 20190815
 
 from __future__ import print_function, absolute_import, division, unicode_literals
 
@@ -20,8 +20,12 @@
 from setuptools.command import install_lib  # NOQA
 from setuptools.command.sdist import sdist as _sdist  # NOQA
 
-from setuptools.namespaces import Installer as NameSpaceInstaller # NOQA
-
+try:
+    from setuptools.namespaces import Installer as NameSpaceInstaller # NOQA
+except ImportError:
+    print("You should use the latest setuptools. The namespaces.py file that this setup.py"
+          " uses was added in setuptools 28.7.0 (Oct 2016)")
+    sys.exit()
 
 if __name__ != '__main__':
     raise NotImplementedError('should never include setup.py')
@@ -30,9 +34,6 @@
 
 full_package_name = None
 
-if __name__ != '__main__':
-    raise NotImplementedError('should never include setup.py')
-
 if sys.version_info < (3,):
     string_type = basestring
 else:

tokens.py

@@ -272,6 +272,7 @@ def __repr__(self):
         return 'CommentToken({})'.format(v)
 
     def __eq__(self, other):
+        # type: (Any) -> bool
         if self.start_mark != other.start_mark:
             return False
         if self.end_mark != other.end_mark:
@@ -281,4 +282,5 @@ def __eq__(self, other):
         return True
 
     def __ne__(self, other):
+        # type: (Any) -> bool
         return not self.__eq__(other)

util.py

@@ -77,9 +77,10 @@ def leading_spaces(l):
         return idx
 
     if isinstance(stream, text_type):
-        yaml_str = stream
+        yaml_str = stream  # type: Any
     elif isinstance(stream, binary_type):
-        yaml_str = stream.decode('utf-8')  # most likely, but the Reader checks BOM for this
+        # most likely, but the Reader checks BOM for this
+        yaml_str = stream.decode('utf-8')
     else:
         yaml_str = stream.read()
     map_indent = None