Merge pull request #140 from Honny1/PerformanceImprovementOfBuildHtml

Improves performance when creating HTML

benchmark.py

@@ -2,7 +2,7 @@
 import time
 
 rule = 'xccdf_org.ssgproject.content_rule_enable_fips_mode'
-src = 'tests/test_data/results-2019-08-27_03-15-00-x86_64@kvm.xml'
+src = 'tests/test_data/ssg-fedora-ds-arf.xml'
 
 print("Benchmark xml to oval_tree")
 print("Start process rule: ", rule)

benchmark_full_command.py

@@ -0,0 +1,23 @@
+import time
+import os
+import subprocess
+import tempfile
+import uuid
+
+print("Start process all rules")
+src = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+
+start_time_all_rules = time.time()
+
+subprocess.check_call(['python3',
+                       '-m',
+                       'oval_graph.command_line',
+                       'arf-to-graph',
+                       '--all',
+                       '-o',
+                       src,
+                       'tests/test_data/ssg-fedora-ds-arf.xml',
+                       '.'
+                       ])
+
+print("--- %s seconds ---" % (time.time() - start_time_all_rules))

oval_graph/_builder_html_graph.py

@@ -2,29 +2,32 @@
 import json
 import re
 import sys
+import lxml.html
 from lxml import etree
 from lxml.builder import ElementMaker, E
-import lxml.html
+from io import BytesIO
 
 
 class BuilderHtmlGraph():
     def __init__(self, parts, verbose):
         self.parts = parts
         self.verbose = verbose
+        self.html_head = self._get_html_head()
+        self.script = self._get_part('script.js')
 
     def save_html(self, dict_oval_trees, src, rules):
         self.save_html_report(dict_oval_trees, src)
         self.print_output_message(src, self._format_rules_output(rules))
 
     def save_html_report(self, dict_of_rules, src):
-        with open(src, "w+") as data_file:
+        with open(src, "wb+") as data_file:
             data_file.writelines(self._get_html(dict_of_rules))
 
     def _get_html(self, dict_of_rules):
         maker = ElementMaker(namespace=None,
                              nsmap={None: "http://www.w3.org/1999/xhtml"})
         html = maker.html(
-            self._get_html_head(),
+            self.html_head,
             self._get_html_body(dict_of_rules))
         result = etree.tostring(
             html,
@@ -36,7 +39,7 @@ def _get_html(self, dict_of_rules):
             with_tail=False,
             method='html',
             pretty_print=True)
-        return result.decode('UTF-8')
+        return BytesIO(result)
 
     def _get_html_head(self):
         return E.head(
@@ -60,7 +63,7 @@ def _get_html_body(self, dict_of_rules):
                         E.div({'id': 'content'}),
                         )
                   ),
-            E.script(self._get_part('script.js')),
+            E.script(self.script),
         )
 
     def _get_script_graph_data(self, dict_of_rules):

oval_graph/arf_to_html.py

@@ -1,16 +1,18 @@
 from .client import Client
 from .converter import Converter
+from ._builder_html_graph import BuilderHtmlGraph
+from .xml_parser import XmlParser
 
 
 class ArfToHtml(Client):
-    def __init__(self, args):
-        super().__init__(args)
+    def _set_attributes(self):
+        self.all_in_one = self.arg.all_in_one
+        self.all_rules = True if self.all_in_one else self.arg.all
         self.display_html = True if self.out is None else self.arg.display
         self.show_failed_rules = self.arg.show_failed_rules
         self.show_not_selected_rules = self.arg.show_not_selected_rules
-        self.all_in_one = self.arg.all_in_one
-        if self.all_in_one:
-            self.all_rules = True
+        self.html_builder = BuilderHtmlGraph(self.parts, self.verbose)
+        self.xml_parser = XmlParser(self.source_filename)
 
     def _get_message(self):
         MESSAGES = {

oval_graph/arf_to_json.py

@@ -1,20 +1,16 @@
-import webbrowser
 import json
 import os
-import shutil
-import sys
-import uuid
 
-from .converter import Converter
 from .client import Client
 from .exceptions import NotChecked
+from .xml_parser import XmlParser
 
 
 class ArfToJson(Client):
-    def __init__(self, args):
-        super().__init__(args)
+    def _set_attributes(self):
         self.show_failed_rules = self.arg.show_failed_rules
         self.show_not_selected_rules = self.arg.show_not_selected_rules
+        self.xml_parser = XmlParser(self.source_filename)
 
     def _get_message(self):
         MESSAGES = {

oval_graph/client.py

@@ -3,14 +3,11 @@
 import tempfile
 import os
 import webbrowser
-import json
-import shutil
 from datetime import datetime
 import sys
 
 from .xml_parser import XmlParser
 from .exceptions import NotChecked
-from ._builder_html_graph import BuilderHtmlGraph
 from .__init__ import __version__
 
 
@@ -23,18 +20,28 @@ def __init__(self, args):
         self.source_filename = self.arg.source_filename
         self.rule_name = self.arg.rule_id
         self.out = self.arg.output
+        self.verbose = self.arg.verbose
+
+        self.parts = self.get_src('parts')
+        self.START_OF_FILE_NAME = 'graph-of-'
+        self.date = str(datetime.now().strftime("-%d_%m_%Y-%H_%M_%S"))
+        self.isatty = sys.stdout.isatty()
+
         self.all_rules = self.arg.all
         self.all_in_one = None
         self.display_html = None
-        self.isatty = sys.stdout.isatty()
+
         self.show_failed_rules = False
         self.show_not_selected_rules = False
-        self.xml_parser = XmlParser(
-            self.source_filename)
-        self.parts = self.get_src('parts')
-        self.START_OF_FILE_NAME = 'graph-of-'
-        self.date = str(datetime.now().strftime("-%d_%m_%Y-%H_%M_%S"))
-        self.verbose = self.arg.verbose
+
+        self.xml_parser = None
+
+        self.html_builder = None
+
+        self._set_attributes()
+
+    def _set_attributes(self):
+        self.xml_parser = self.xml_parser = XmlParser(self.source_filename)
 
     def _get_message(self):
         MESSAGES = {
@@ -166,9 +173,7 @@ def get_src(self, src):
         return str(os.path.join(_dir, src))
 
     def _build_and_save_html(self, dict_oval_trees, src, rules, out_src):
-        builder = BuilderHtmlGraph(
-            self.parts, self.verbose)
-        builder.save_html(dict_oval_trees, src, rules)
+        self.html_builder.save_html(dict_oval_trees, src, rules)
         out_src.append(src)
 
     def open_html(self, out):

oval_graph/json_to_html.py

@@ -1,39 +1,23 @@
-import webbrowser
 import json
-import os
-import argparse
-from datetime import datetime
-import shutil
-import sys
-import re
 
 from .client import Client
 from .oval_node import restore_dict_to_tree
 from .converter import Converter
 from .exceptions import NotChecked
+from ._builder_html_graph import BuilderHtmlGraph
 
 
 class JsonToHtml(Client):
     def __init__(self, args):
-        self.parser = None
-        self.MESSAGES = self._get_message()
-        self.arg = self.parse_arguments(args)
-        self.hide_passing_tests = self.arg.hide_passing_tests
-        self.source_filename = self.arg.source_filename
-        self.rule_name = self.arg.rule_id
-        self.out = self.arg.output
+        super().__init__(args)
+        self.json_data_file = self.get_json_data_file()
+        self.oval_tree = None
+
+    def _set_attributes(self):
         self.all_in_one = self.arg.all_in_one
         self.all_rules = True if self.all_in_one else self.arg.all
-        self.isatty = sys.stdout.isatty()
-        self.show_failed_rules = False
-        self.show_not_selected_rules = False
-        self.oval_tree = None
         self.display_html = True if self.out is None else self.arg.display
-        self.json_data_file = self.get_json_data_file()
-        self.parts = self.get_src('parts')
-        self.START_OF_FILE_NAME = 'graph-of-'
-        self.date = datetime.now().strftime("-%d_%m_%Y-%H_%M_%S")
-        self.verbose = self.arg.verbose
+        self.html_builder = BuilderHtmlGraph(self.parts, self.verbose)
 
     def _get_message(self):
         MESSAGES = {

tests/test_arf_to_json.py

@@ -69,14 +69,13 @@ def test_prepare_json(capsys):
     rule = 'xccdf_org.ssgproject.content_rule_package_abrt_removed'
     client = get_client_arf_to_json(src, rule)
     rules = {'rules': [rule]}
-    date = str(datetime.now().strftime("-%d_%m_%Y-%H_%M_%S"))
     results_src = client.prepare_data(rules)
     assert not results_src
     captured = capsys.readouterr()
     assert captured.out == (
         '{\n'
         '    "graph-of-xccdf_org.ssgproject.content_rule_package_abrt_removed' +
-        date +
+        client.date +
         '": {\n'
         '        "node_id": "xccdf_org.ssgproject.content_rule_package_abrt_removed",\n'
         '        "type": "operator",\n'