Enable file type plugins if corresponding flag is set

src/cfgnet/launcher.py

@@ -38,6 +38,15 @@
     "`--disable-linker foo --disable-linker bar`.",
 )
 
+add_enable_file_type_plugins = click.option(
+    "--enable_file_type_plugins",
+    type=click.BOOL,
+    is_flag=True,
+    default=False,
+    help="Enable file type plugins."
+    "Besides concept plugins also file type plugins will be used to construct the network.",
+)
+
 
 @click.group()
 @click.option(
@@ -57,6 +66,7 @@ def main(verbose: bool):
 @add_project_root_argument
 @add_enable_linker_option
 @add_disable_linker_option
+@add_enable_file_type_plugins
 def init(
     enable_static_blacklist: bool,
     enable_internal_links: bool,
@@ -65,6 +75,7 @@ def init(
     project_root: str,
     enable_linker: List[str],
     disable_linker: List[str],
+    enable_file_type_plugins: bool,
     config_files: List,
 ):
     """Initialize configuration network."""
@@ -78,6 +89,7 @@ def init(
         enable_internal_links=enable_internal_links,
         enabled_linkers=list(set(enable_linker) - set(disable_linker)),
         enable_all_conflicts=enable_all_conflicts,
+        enable_file_type_plugins=enable_file_type_plugins,
         system_level=system_level,
     )
     LinkerManager.set_enabled_linkers(network_configuration.enabled_linkers)
@@ -142,6 +154,7 @@ def validate(project_root: str):
 @add_project_root_argument
 @add_enable_linker_option
 @add_disable_linker_option
+@add_enable_file_type_plugins
 def analyze(
     enable_static_blacklist: bool,
     enable_internal_links: bool,
@@ -151,6 +164,7 @@ def analyze(
     disable_linker: List[str],
     config_files: List,
     system_level: bool,
+    enable_file_type_plugins: bool,
 ):
     """Run self-evaluating analysis of commit history."""
     project_name = os.path.basename(project_root)
@@ -165,6 +179,7 @@ def analyze(
         enabled_linkers=list(set(enable_linker) - set(disable_linker)),
         enable_all_conflicts=enable_all_conflicts,
         system_level=system_level,
+        enable_file_type_plugins=enable_file_type_plugins,
     )
     LinkerManager.set_enabled_linkers(network_configuration.enabled_linkers)
     logger.configure_repo_logger(network_configuration.logfile_path())
@@ -231,8 +246,13 @@ def export(
 @click.option("-o", "--output", required=True)
 @click.option("-f", "--system_level", is_flag=False)
 @add_project_root_argument
+@add_enable_file_type_plugins
 def extract(
-    project_root: str, config_files: List, output: str, system_level: bool
+    project_root: str,
+    config_files: List,
+    output: str,
+    enable_file_type_plugins: bool,
+    system_level: bool,
 ):
     """Extract key-value pairs."""
     project_name = os.path.basename(project_root)
@@ -244,6 +264,7 @@ def extract(
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=enable_file_type_plugins,
         system_level=system_level,
     )
 

src/cfgnet/network/network.py

@@ -37,7 +37,7 @@
 )
 from cfgnet.network.network_configuration import NetworkConfiguration
 from cfgnet.exporter.exporter import DotExporter, JSONExporter
-from cfgnet.utility.util import is_test_file, get_system_files
+from cfgnet.utility.util import get_system_files
 
 
 class Network:
@@ -300,31 +300,52 @@ def init_network(cfg: NetworkConfiguration) -> Network:
         network = Network(project_name=project_name, root=root, cfg=cfg)
 
         tracked_files = IgnoreFile.filter(tracked_files)
-        plugins = PluginManager.get_plugins()
+        concept_plugins = PluginManager.get_concept_plugins()
+        file_type_plugins = PluginManager.get_file_type_plugins()
 
         for file in sorted(tracked_files):
             abs_file_path = os.path.join(cfg.project_root_abs, file)
 
-            if is_test_file(abs_file_path=abs_file_path):
-                continue
-
-            plugin = PluginManager.get_responsible_plugin(
-                plugins, abs_file_path
+            concept_plugin = PluginManager.get_responsible_plugin(
+                concept_plugins, abs_file_path
             )
-            if plugin:
+
+            if concept_plugin:
                 try:
-                    plugin.parse_file(
+                    concept_plugin.parse_file(
                         abs_file_path=abs_file_path,
                         rel_file_path=file,
                         root=root,
                     )
                 except UnicodeDecodeError as error:
                     logging.warning(
                         "%s: %s (%s)",
-                        plugin.__class__.__name__,
+                        concept_plugin.__class__.__name__,
                         error.reason,
                         file,
                     )
+                continue
+
+            if cfg.enable_file_type_plugins:
+                file_type_plugin = PluginManager.get_responsible_plugin(
+                    file_type_plugins, abs_file_path
+                )
+
+                if file_type_plugin:
+                    try:
+                        file_type_plugin.parse_file(
+                            abs_file_path=abs_file_path,
+                            rel_file_path=file,
+                            root=root,
+                        )
+                    except UnicodeDecodeError as error:
+                        logging.warning(
+                            "%s: %s (%s)",
+                            file_type_plugin.__class__.__name__,
+                            error.reason,
+                            file,
+                        )
+                    continue
 
         LinkerManager.apply_linkers(network)
 

src/cfgnet/network/network_configuration.py

@@ -11,6 +11,7 @@ class NetworkConfiguration:
     enable_static_blacklist: bool
     enable_internal_links: bool
     enable_all_conflicts: bool
+    enable_file_type_plugins: bool
     system_level: bool
     # Path to CfgNet data directory relative to project_root
     cfgnet_path_rel: str = ".cfgnet"

src/cfgnet/plugins/plugin_manager.py

@@ -31,7 +31,7 @@
 from cfgnet.plugins.file_type.configparser_plugin import ConfigParserPlugin
 from cfgnet.plugins.file_type.yaml_plugin import YAMLPlugin
 from cfgnet.plugins.file_type.toml_plugin import TomlPlugin
-from cfgnet.plugins.file_type.hadoop_plugin import HadoopPlugin
+from cfgnet.plugins.file_type.json_plugin import JsonPlugin
 from cfgnet.plugins.concept.mysql_plugin import MysqlPlugin
 from cfgnet.plugins.concept.ansible_plugin import AnsiblePlugin
 from cfgnet.plugins.concept.ansible_playbook_plugin import (
@@ -101,14 +101,19 @@ class PluginManager:
         ConfigParserPlugin(),
         YAMLPlugin(),
         TomlPlugin(),
-        HadoopPlugin(),
+        JsonPlugin(),
     ]
 
     @staticmethod
-    def get_plugins() -> List:
-        """Return all plugins except vcs plugins."""
+    def get_concept_plugins() -> List:
+        """Return all concept plugins."""
         return PluginManager.concept_plugins
 
+    @staticmethod
+    def get_file_type_plugins() -> List:
+        """Return all file type plugins."""
+        return PluginManager.file_type_plugins
+
     @staticmethod
     def get_responsible_plugin(
         plugins: List[Plugin], artifact_path: str

tests/cfgnet/analyze/test_analyzer.py

@@ -37,6 +37,7 @@ def get_config_(get_repo):
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
 
@@ -50,6 +51,7 @@ def get_config_all_conflicts_(get_repo):
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=True,
+        enable_file_type_plugins=False,
         system_level=False
     )
 

tests/cfgnet/conflicts/test_conflict_detector.py

@@ -38,6 +38,7 @@ def get_maven_docker_networks():
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
     ref_network = Network.init_network(cfg=network_configuration)
@@ -61,6 +62,7 @@ def get_docker_networks():
         enable_static_blacklist=False,
         enable_internal_links=True,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
     ref_network = Network.init_network(cfg=network_configuration)
@@ -84,6 +86,7 @@ def get_port_db_networks():
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
     ref_network = Network.init_network(cfg=network_configuration)
@@ -107,6 +110,7 @@ def get_nodejs_networks():
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=True,
+        enable_file_type_plugins=False,
         system_level=False
     )
     ref_network = Network.init_network(cfg=network_configuration)
@@ -130,6 +134,7 @@ def get_networks_equally_changed():
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
     ref_network = Network.init_network(cfg=network_configuration)

tests/cfgnet/exporter/test_exporter.py

@@ -42,6 +42,7 @@ def get_config_(get_repo):
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
 

tests/cfgnet/network/test_ignorefile.py

@@ -36,6 +36,7 @@ def test_ignorefile(repo):
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
 

tests/cfgnet/network/test_network.py

@@ -37,13 +37,27 @@ def get_repo_():
     return repo
 
 
+@pytest.fixture(name="get_file_type_repo")
+def get_file_type_repo_():
+    repo = TemporaryRepository(
+        "tests/test_repos/file_type_repo/0001-Initial-commit.patch"
+    )
+
+    repo.apply_patch(
+        "tests/test_repos/file_type_repo/0002-Add-config-file.patch"
+    )
+
+    return repo
+
+
 @pytest.fixture(name="get_config")
 def get_config_(get_repo):
     network_configuration = NetworkConfiguration(
         project_root_abs=os.path.abspath(get_repo.root),
         enable_static_blacklist=False,
         enable_internal_links=False,
         enable_all_conflicts=False,
+        enable_file_type_plugins=False,
         system_level=False
     )
 
@@ -61,6 +75,21 @@ def test_init_network(get_repo, get_config):
     assert os.path.isdir(network.cfg.data_dir_path())
 
 
+def test_init_network_with_file_type_plugins(get_file_type_repo, get_config):
+    config = get_config
+    config.project_root_abs = os.path.abspath(get_file_type_repo.root)
+    network = Network.init_network(cfg=config)
+
+    assert network
+    assert len(network.get_nodes(node_type=ArtifactNode)) == 2
+
+    config.enable_file_type_plugins = True
+    network = Network.init_network(cfg=config)
+
+    assert network
+    assert len(network.get_nodes(node_type=ArtifactNode)) == 3
+
+
 def test_links(get_config):
     network = Network.init_network(cfg=get_config)
     expected_links = {

tests/cfgnet/plugins/test_plugin_manager.py

@@ -16,14 +16,14 @@
 from cfgnet.plugins.plugin_manager import PluginManager
 
 
-def test_get_all_plugins():
-    all_plugins = PluginManager.get_plugins()
+def test_get_all_concept_plugins():
+    all_concept_plugins = PluginManager.get_concept_plugins()
 
-    assert len(all_plugins) == 32
+    assert len(all_concept_plugins) == 32
 
 
-def test_get_responsible_plugin():
-    plugins = PluginManager.get_plugins()
+def test_get_responsible_concept_plugin():
+    plugins = PluginManager.get_concept_plugins()
 
     docker_plugin = PluginManager.get_responsible_plugin(plugins, "path/to/Dockerfile")
     maven_plugin = PluginManager.get_responsible_plugin(plugins, "path/to/pom.xml")