Make the ssg module to pass mypy

pyproject.toml

@@ -42,3 +42,7 @@ git_describe_command = ["sh", "-c", "tag=$(git tag | grep -v '-' | sort | tail -
 
 [tool.setuptools.packages.find]
 include = ["ssg*"]
+
+[[tool.mypy.overrides]]
+module = "pkg_resources"
+ignore_missing_imports = true

ssg/build_cpe.py

@@ -6,8 +6,8 @@
 from __future__ import print_function
 import os
 import sys
-import ssg.id_translate
 
+import ssg.id_translate
 from .constants import oval_namespace
 from .constants import PREFIX_TO_NS
 from .utils import required_key, apply_formatting_on_dict_values
@@ -207,9 +207,9 @@ class CPEItem(XCCDFEntity, Templatable):
     )
     KEYS.update(**Templatable.KEYS)
 
-    MANDATORY_KEYS = [
+    MANDATORY_KEYS = {
         "name",
-    ]
+    }
 
     prefix = "cpe-dict"
     ns = PREFIX_TO_NS[prefix]

ssg/build_remediations.py

@@ -37,7 +37,7 @@
                            'strategy']
 REMEDIATION_ELM_KEYS = ['complexity', 'disruption', 'reboot', 'strategy']
 
-RemediationObject = namedtuple('remediation', ['contents', 'config'])
+RemediationObject = namedtuple('RemediationObject', ['contents', 'config'])
 
 
 def is_supported_filename(remediation_type, filename):

ssg/build_renumber.py

@@ -2,7 +2,7 @@
 from __future__ import print_function
 import sys
 import os
-
+from typing import Optional
 
 from .constants import (
     OSCAP_RULE, OSCAP_VALUE, oval_namespace, XCCDF12_NS, cce_uri, ocil_cs,
@@ -26,8 +26,8 @@ class FileLinker(object):
     Bass class which represents the linking of checks to their identifiers.
     """
 
-    CHECK_SYSTEM = None
-    CHECK_NAMESPACE = None
+    CHECK_SYSTEM: Optional[str] = None
+    CHECK_NAMESPACE: Optional[str] = None
 
     def __init__(self, translator, xccdftree, checks, output_file_name):
         self.translator = translator

ssg/build_yaml.py

@@ -3426,13 +3426,13 @@ class Platform(XCCDFEntity):
         ** XCCDFEntity.KEYS
     )
 
-    MANDATORY_KEYS = [
+    MANDATORY_KEYS = {
         "name",
         "xml_content",
         "original_expression",
         "bash_conditional",
         "ansible_conditional"
-    ]
+    }
 
     prefix = "cpe-lang"
     ns = PREFIX_TO_NS[prefix]

ssg/cce.py

@@ -5,9 +5,7 @@
 import re
 import random
 import os
-
-
-CCE_POOLS = dict()
+from typing import Dict, Type
 
 
 class CCEFile:
@@ -136,6 +134,7 @@ def absolute_path(self):
         return os.path.join(self.project_root, "shared", "references", "cce-sle15-avail.txt")
 
 
+CCE_POOLS: Dict[str, Type[CCEFile]] = {}
 CCE_POOLS["redhat"] = RedhatCCEFile
 CCE_POOLS["sle12"] = SLE12CCEFile
 CCE_POOLS["sle15"] = SLE15CCEFile

ssg/controls.py

@@ -7,7 +7,7 @@
 import copy
 import sys
 from glob import glob
-from typing import List, Dict
+from typing import Dict, List, Set
 
 import ssg.entities.common
 import ssg.yaml
@@ -144,7 +144,7 @@ class Control(ssg.entities.common.SelectionHandler, ssg.entities.common.XCCDFEnt
         description=str,
         rationale=str,
         automated=str,
-        status=None,
+        status=lambda: None,
         mitigation=str,
         artifact_description=str,
         status_justification=str,
@@ -812,7 +812,7 @@ def __init__(self, controls_dirs: List[str], env_yaml=None, existing_rules=None)
         self.controls_dirs = [os.path.abspath(controls_dir) for controls_dir in controls_dirs]
         self.env_yaml = env_yaml
         self.existing_rules = existing_rules
-        self.policies = {}
+        self.policies: Dict = {}
 
     def _load(self, format):
         for controls_dir in self.controls_dirs:

ssg/entities/common.py

@@ -6,12 +6,12 @@
 import yaml
 from collections import defaultdict
 from copy import deepcopy
+from typing import Set, Dict, Callable, Any, Optional
 
 from ssg.yaml import yaml_Dumper
 
 from ..xml import ElementTree as ET, add_xhtml_namespace
 from ..yaml import DocumentationNotComplete, open_and_expand
-from ..shims import unicode_func
 
 from ..constants import (
     xhtml_namespace,
@@ -112,7 +112,7 @@ def add_sub_element(parent, tag, ns, data):
     # and therefore it does not add child elements
     # we need to do a hack instead
     # TODO: Remove this function after we move to Markdown everywhere in SSG
-    ustr = unicode_func('<{0} xmlns="{3}" xmlns:xhtml="{2}">{1}</{0}>').format(
+    ustr = str('<{0} xmlns="{3}" xmlns:xhtml="{2}">{1}</{0}>').format(
         tag, namespaced_data, xhtml_namespace, ns)
 
     try:
@@ -156,15 +156,15 @@ class XCCDFEntity(object):
     when entities are defined in the benchmark tree,
     and they are compiled into flat YAMLs to the build directory.
     """
-    KEYS = dict(
+    KEYS: Dict[str, Callable[[], Optional[Any]]] = dict(
             id_=lambda: "",
             title=lambda: "",
             definition_location=lambda: "",
     )
 
-    MANDATORY_KEYS = set()
+    MANDATORY_KEYS: Set[str] = set()
 
-    ALTERNATIVE_KEYS = dict()
+    ALTERNATIVE_KEYS: Dict[str, str] = {}
 
     GENERIC_FILENAME = ""
     ID_LABEL = "id"

ssg/ext/boolean/boolean.py

@@ -29,19 +29,8 @@
 from operator import and_ as and_operator
 from operator import or_ as or_operator
 
-# Python 2 and 3
-try:
-    basestring  # NOQA
-except NameError:
-    basestring = str  # NOQA
-
-# Python 2 and 3
-try:
-    # Python 2
-    reduce  # NOQA
-except NameError:
-    # Python 3
-    from functools import reduce  # NOQA
+from functools import reduce
+from typing import Optional, Any
 
 # Set to True to enable tracing for parsing
 TRACE_PARSE = False
@@ -208,7 +197,7 @@ def parse(self, expr, simplify=False):
 
         precedence = {self.NOT: 5, self.AND: 10, self.OR: 15, TOKEN_LPAR: 20}
 
-        if isinstance(expr, basestring):
+        if isinstance(expr, str):
             tokenized = self.tokenize(expr)
         else:
             tokenized = iter(expr)
@@ -445,7 +434,7 @@ def tokenize(self, expr):
             - True symbols: 1 and True
             - False symbols: 0, False and None
         """
-        if not isinstance(expr, basestring):
+        if not isinstance(expr, str):
             raise TypeError('expr must be string but it is %s.' % type(expr))
 
         # mapping of lowercase token strings to a token type id for the standard
@@ -556,17 +545,17 @@ class Expression(object):
     variable symbols.
     """
     # Defines sort and comparison order between expressions arguments
-    sort_order = None
+    sort_order: Optional[int] = None
 
     # Store arguments aka. subterms of this expressions.
     # subterms are either literals or expressions.
-    args = tuple()
+    args: Any = ()
 
     # True is this is a literal expression such as a Symbol, TRUE or FALSE
-    isliteral = False
+    isliteral: bool = False
 
     # True if this expression has been simplified to in canonical form.
-    iscanonical = False
+    iscanonical: bool = False
 
     # these class attributes are configured when a new BooleanAlgebra is created
     TRUE = None
@@ -809,7 +798,7 @@ def __lt__(self, other):
             return self == self.FALSE
         return NotImplemented
 
-    __nonzero__ = __bool__ = lambda s: None
+    __nonzero__ = __bool__ = lambda s: None # type: ignore
 
     def pretty(self, indent=0, debug=False):
         """
@@ -843,7 +832,7 @@ def __repr__(self):
     def __call__(self):
         return self
 
-    __nonzero__ = __bool__ = lambda s: True
+    __nonzero__ = __bool__ = lambda s: True # type: ignore
 
 
 class _FALSE(BaseElement):
@@ -871,7 +860,7 @@ def __repr__(self):
     def __call__(self):
         return self
 
-    __nonzero__ = __bool__ = lambda s: False
+    __nonzero__ = __bool__ = lambda s: False # type: ignore
 
 
 class Symbol(Expression):
@@ -920,7 +909,7 @@ def __str__(self):
         return str(self.obj)
 
     def __repr__(self):
-        obj = "'%s'" % self.obj if isinstance(self.obj, basestring) else repr(self.obj)
+        obj = "'%s'" % self.obj if isinstance(self.obj, str) else repr(self.obj)
         return '%s(%s)' % (self.__class__.__name__, obj)
 
     def pretty(self, indent=0, debug=False):
@@ -931,7 +920,7 @@ def pretty(self, indent=0, debug=False):
         if debug:
             debug_details += '<isliteral=%r, iscanonical=%r>' % (self.isliteral, self.iscanonical)
 
-        obj = "'%s'" % self.obj if isinstance(self.obj, basestring) else repr(self.obj)
+        obj = "'%s'" % self.obj if isinstance(self.obj, str) else repr(self.obj)
         return (' ' * indent) + ('%s(%s%s)' % (self.__class__.__name__, debug_details, obj))
 
 
@@ -1470,7 +1459,7 @@ class AND(DualBase):
     """
 
     sort_order = 10
-    _pyoperator = and_operator
+    _pyoperator = and_operator # type: ignore
 
     def __init__(self, arg1, arg2, *args):
         super(AND, self).__init__(arg1, arg2, *args)
@@ -1496,7 +1485,7 @@ class OR(DualBase):
     """
 
     sort_order = 25
-    _pyoperator = or_operator
+    _pyoperator = or_operator # type: ignore
 
     def __init__(self, arg1, arg2, *args):
         super(OR, self).__init__(arg1, arg2, *args)

ssg/jinja.py

@@ -9,15 +9,8 @@
 import sys
 import jinja2
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
-try:
-    from shlex import quote as shell_quote
-except ImportError:
-    from pipes import quote as shell_quote
+from urllib.parse import quote
+from shlex import quote as shell_quote
 
 from .constants import JINJA_MACROS_DIRECTORY
 from .utils import (required_key,
@@ -126,6 +119,10 @@ def __init__(self, bytecode_cache=None):
         )
 
 
+# Module-level cached environment for jinja environment
+_jinja_env = None
+
+
 def _get_jinja_environment(substitutions_dict):
     """
     Initializes and returns a Jinja2 Environment with custom settings and filters.
@@ -145,30 +142,28 @@ def _get_jinja_environment(substitutions_dict):
     Returns:
         jinja2.Environment: The configured Jinja2 Environment instance.
     """
-    if _get_jinja_environment.env is None:
+    global _jinja_env
+    if _jinja_env is None:
         bytecode_cache = None
         if substitutions_dict.get("jinja2_cache_enabled") == "true":
             bytecode_cache = jinja2.FileSystemBytecodeCache(
                 required_key(substitutions_dict, "jinja2_cache_dir")
             )
 
         # TODO: Choose better syntax?
-        _get_jinja_environment.env = JinjaEnvironment(bytecode_cache=bytecode_cache)
+        _jinja_env = JinjaEnvironment(bytecode_cache=bytecode_cache)
         add_python_functions(substitutions_dict)
-        _get_jinja_environment.env.filters['banner_anchor_wrap'] = banner_anchor_wrap
-        _get_jinja_environment.env.filters['banner_regexify'] = banner_regexify
-        _get_jinja_environment.env.filters['escape_id'] = escape_id
-        _get_jinja_environment.env.filters['escape_regex'] = escape_regex
-        _get_jinja_environment.env.filters['escape_yaml_key'] = escape_yaml_key
-        _get_jinja_environment.env.filters['quote'] = shell_quote
-        _get_jinja_environment.env.filters['sha256'] = sha256
-        _get_jinja_environment.env.globals.update(substitutions_dict)
-        preload_macros(_get_jinja_environment.env)
-
-    return _get_jinja_environment.env
-
-
-_get_jinja_environment.env = None
+        _jinja_env.filters['banner_anchor_wrap'] = banner_anchor_wrap
+        _jinja_env.filters['banner_regexify'] = banner_regexify
+        _jinja_env.filters['escape_id'] = escape_id
+        _jinja_env.filters['escape_regex'] = escape_regex
+        _jinja_env.filters['escape_yaml_key'] = escape_yaml_key
+        _jinja_env.filters['quote'] = shell_quote
+        _jinja_env.filters['sha256'] = sha256
+        _jinja_env.globals.update(substitutions_dict)
+        preload_macros(_jinja_env)
+
+    return _jinja_env
 
 
 def initialize(substitutions_dict):

ssg/oval.py

@@ -19,13 +19,8 @@
 
 ASSUMED_OVAL_VERSION_STRING = "5.11"
 # globals, to make recursion easier in case we encounter extend_definition
-try:
-    ET.register_namespace("oval", ovalns)
-except AttributeError:
-    # Legacy Python 2.6 fix, see e.g.
-    # https://www.programcreek.com/python/example/57552/xml.etree.ElementTree._namespace_map
-    from xml.etree import ElementTree as ET
-    ET._namespace_map[ovalns] = "oval"
+ET.register_namespace("oval", ovalns)
+
 
 
 def applicable_platforms(oval_file, oval_version_string=None):

ssg/products.py

@@ -353,7 +353,7 @@ def read_properties_from_directory(self, path):
             self.expand_by_acquired_data(new_symbols)
 
 
-def load_product_yaml(product_yaml_path):
+def load_product_yaml(product_yaml_path: str) -> Product:
     """
     Reads a product data from disk and returns it.