[testing] Add most testing via tox.ini

Consolidate testing, so that things can be run from CLI, CI and
various other means in a consistent way by using tox.

* Updated all the flake8 tests in the tests folder.
* Added pylint as a neutral test so that we can work on this in stages
  and have some collaboration on what we test and don't
* The tox tests for unit, stageone, stagetwo testing makes is easier
  for users to know how to run tests, and not have to do things
  manually
* Using tox for CI doesn't make sense, as that will create virtual
  envs and will disregard system/snap based python modules so may not
  work

Signed-off-by: Arif Ali <[email protected]>

.cirrus.yml

@@ -60,7 +60,21 @@ flake8_task:
     name: "Flake8 linting test"
     container:
         image: alpine/flake8:latest
-    flake_script: flake8 sos
+    setup_script: |
+        apk update
+        apk add --upgrade py3-tox
+    flake_script: tox -e flake8
+
+pylint_task:
+    alias: "pylint_test"
+    name: "pylint linting test"
+    allow_failures: true
+    container:
+        image: "python:latest"
+    setup_script: |
+        apt update
+        apt -y install tox
+    pylint_script: tox -e pylint
 
 # Run a check on newer upstream python versions to check for possible
 # breaks/changes in common modules. This is not meant to check any of the actual

.gitignore

@@ -28,3 +28,6 @@ debian/sosreport*
 debian/files
 debian/.debhelper
 debian/debhelper-build-stamp
+
+# tox
+.tox

plugins_overview.py

@@ -44,9 +44,9 @@ def add_valid_item(dest, item):
 
 # method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..)
 # split by comma; add each valid item to the `dest` list
-def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
+def add_all_items(method, dest, wrapopen=r'\(', wrapclose=r'\)'):
     regexp = "%s%s(.*?)%s" % (method, wrapopen, wrapclose)
-    for match in re.findall(regexp, plugcontent, flags=re.MULTILINE|re.DOTALL):
+    for match in re.findall(regexp, plugcontent, flags=re.MULTILINE | re.DOTALL):
         # tuple of distros ended by either (class|from|import)
         if isinstance(match, tuple):
             for item in list(match):
@@ -89,7 +89,8 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
             'journals': [],
             'env': [],
     }
-    plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n', '')
+    plugcontent = open(
+        os.path.join(PLUGDIR, plugfile)).read().replace('\n', '')
     add_all_items(
         "from sos.report.plugins import ",
         plugs_data[plugname]['distros'],
@@ -112,7 +113,8 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
           "service_status;journals;env_vars")
     for plugname in plugs_data.keys():
         plugin = plugs_data[plugname]
-        # determine max number of lines - usually "max(len(copyspec),len(commands))"
+        # determine max number of lines - usually
+        # "max(len(copyspec),len(commands))"
         # ignore 'sourcecode' key as it
         maxline = 1
         plugkeys = list(plugin.keys())

tests/cleaner_tests/basic_function_tests/binary_test.py

@@ -16,6 +16,5 @@ class BinaryPlugin(Plugin, IndependentPlugin):
     plugin_name = 'binary_test'
     short_desc = 'test plugin for removing binaries with --clean'
 
-
     def setup(self):
         self.add_copy_spec('/var/log/binary_test.tar.xz')

tests/cleaner_tests/basic_function_tests/report_with_mask.py

@@ -25,7 +25,8 @@ def pre_sos_setup(self):
         # obfuscate a random word from /etc/hosts and ensure the updated
         # sanitised file has same permissions (a+r)
         try:
-            self.hosts_obfuscated = open('/etc/hosts').read().strip('#\n').split()[-1]
+            self.hosts_obfuscated = open(
+                '/etc/hosts').read().strip('#\n').split()[-1]
         except (FileNotFoundError, IndexError) as e:
             self.warning(f"Unable to process /etc/hosts: {e}")
         if self.hosts_obfuscated:
@@ -36,8 +37,10 @@ def test_mask_was_run(self):
         self.assertOutputContains('Obfuscation completed')
 
     def test_private_map_was_generated(self):
-        self.assertOutputContains('A mapping of obfuscated elements is available at')
-        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        self.assertOutputContains(
+            'A mapping of obfuscated elements is available at')
+        map_file = re.findall(
+            '/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
         self.assertFileExists(map_file)
 
     def test_tarball_named_obfuscated(self):
@@ -53,40 +56,53 @@ def test_ip_address_was_obfuscated(self):
         # Note: do not test for starting with the 100.* block here, as test
         # machines may have /32 addresses. Instead, test that the actual
         # IP address is not present
-        self.assertFileNotHasContent('ip_addr', self.sysinfo['pre']['networking']['ip_addr'])
+        self.assertFileNotHasContent(
+            'ip_addr',
+            self.sysinfo['pre']['networking']['ip_addr']
+        )
 
     def test_loopback_was_not_obfuscated(self):
         self.assertFileHasContent('ip_addr', '127.0.0.1/8')
 
     def test_mac_addrs_were_obfuscated(self):
-        content = self.get_file_content('sos_commands/networking/ip_maddr_show')
+        content = self.get_file_content(
+            'sos_commands/networking/ip_maddr_show'
+        )
         for line in content.splitlines():
             if line.strip().startswith('link'):
                 mac = line.strip().split()[1]
-                assert mac.startswith('53:4f:53'), "Found unobfuscated mac addr %s" % mac
+                assert \
+                    mac.startswith('53:4f:53'), \
+                    f"Found unobfuscated mac addr {mac}"
 
     def test_perms_unchanged_on_modified_file(self):
         if self.hosts_obfuscated:
             imode_orig = stat('/etc/hosts').st_mode
-            imode_obfuscated = stat(self.get_name_in_archive('etc/hosts')).st_mode
+            imode_obfuscated = stat(
+                self.get_name_in_archive('etc/hosts')).st_mode
             self.assertEqual(imode_orig, imode_obfuscated)
 
 
 class ReportWithUserCustomisations(StageOneReportTest):
-    """Testing for 1) obfuscated keywords provided by the user (--keywords option),
-    and 2) skipping to clean specific files (--skip-cleaning-files option)
+    """Testing for 1) obfuscated keywords provided by the user (--keywords
+    option), and 2) skipping to clean specific files (--skip-cleaning-files
+    option)
 
     :avocado: tags=stageone
     """
 
-    sos_cmd = '--clean -o filesys,kernel --keywords=fstab,Linux,tmp,BOOT_IMAGE,fs.dentry-state \
-        --skip-cleaning-files proc/cmdline,sos_commands/*/sysctl* --no-update'
+    sos_cmd = ('--clean -o filesys,kernel --keywords=fstab,Linux,tmp,'
+               'BOOT_IMAGE,fs.dentry-state --skip-cleaning-files '
+               'proc/cmdline,sos_commands/*/sysctl* --no-update')
 
-    # Will the 'tmp' be properly treated in path to working dir without raising an error?
-    # To make this test effective, we assume the test runs on a system / with Policy
-    # returning '/var/tmp' as temp.dir
+    # Will the 'tmp' be properly treated in path to working dir without
+    # raising an error?
+    # To make this test effective, we assume the test runs on a system / with
+    # Policy returning '/var/tmp' as temp.dir
     def test_keyword_in_tempdir_path(self):
-        self.assertOutputContains('Your sosreport has been generated and saved in:')
+        self.assertOutputContains(
+            'Your sosreport has been generated and saved in:'
+        )
         self.assertTrue('tmp/' in self.archive)
 
     # Ok, sort of cheesy here but this does actually test filename changes on
@@ -102,7 +118,10 @@ def test_skip_cleaning_single_file(self):
         self.assertFileHasContent('proc/cmdline', 'BOOT_IMAGE')
 
     def test_skip_cleaning_glob_file(self):
-        self.assertFileHasContent('sos_commands/kernel/sysctl_-a', 'fs.dentry-state')
+        self.assertFileHasContent(
+            'sos_commands/kernel/sysctl_-a',
+            'fs.dentry-state'
+        )
 
 
 class DefaultRemoveBinaryFilesTest(StageTwoReportTest):
@@ -134,8 +153,8 @@ class KeepBinaryFilesTest(StageTwoReportTest):
 
     def test_warning_message_shown(self):
         self.assertOutputContains(
-            'WARNING: binary files that potentially contain sensitive information '
-            'will NOT be removed from the final archive'
+            'WARNING: binary files that potentially contain sensitive '
+            'information will NOT be removed from the final archive'
         )
 
     def test_binary_is_in_archive(self):

tests/cleaner_tests/existing_archive.py

@@ -13,6 +13,7 @@
 
 ARCHIVE = 'sosreport-cleanertest-2021-08-03-qpkxdid'
 
+
 class ExistingArchiveCleanTest(StageTwoReportTest):
     """Ensure that we can extract an already created archive and clean it the
     same as we would an in-line run of `report --clean`.
@@ -26,34 +27,52 @@ class ExistingArchiveCleanTest(StageTwoReportTest):
     sos_component = 'clean'
 
     def test_obfuscation_log_created(self):
-        self.assertFileExists(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE))
+        self.assertFileExists(
+            os.path.join(self.tmpdir, f'{ARCHIVE}-obfuscation.log')
+        )
 
     def test_archive_type_correct(self):
-        with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log:
+        with open(os.path.join(
+                self.tmpdir,
+                f'{ARCHIVE}-obfuscation.log'), 'r') as log:
             for line in log:
                 if "Loaded %s" % ARCHIVE in line:
-                    assert 'as type sos report archive' in line, "Incorrect archive type detected: %s" % line
+                    assert \
+                        'as type sos report archive' in line, \
+                        f"Incorrect archive type detected: {line}"
                     break
 
     def test_from_cmdline_logged(self):
-        with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log:
+        with open(os.path.join(
+                self.tmpdir,
+                f'{ARCHIVE}-obfuscation.log'), 'r') as log:
             for line in log:
                 if 'From cmdline' in line:
-                    assert 'From cmdline: True' in line, "Did not properly log cmdline run"
+                    assert \
+                        'From cmdline: True' in line, \
+                        "Did not properly log cmdline run"
                     break
 
     def test_extraction_completed_successfully(self):
-        with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log:
+        with open(os.path.join(
+                self.tmpdir,
+                f'{ARCHIVE}-obfuscation.log'), 'r') as log:
             for line in log:
                 if 'Extracted path is' in line:
                     path = line.split('Extracted path is')[-1].strip()
-                    assert path.startswith(self.tmpdir), "Extracted path appears wrong: %s (tmpdir: %s)" % (path, self.tmpdir)
+                    assert \
+                        path.startswith(self.tmpdir), \
+                        (f"Extracted path appears wrong: {path} "
+                         f"(tmpdir: {self.tmpdir})")
                     return
             self.fail("Extracted path not logged")
 
     def test_private_map_was_generated(self):
-        self.assertOutputContains('A mapping of obfuscated elements is available at')
-        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        self.assertOutputContains(
+            'A mapping of obfuscated elements is available at'
+        )
+        map_file = re.findall(
+            '/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
         self.assertFileExists(map_file)
 
     def test_tarball_named_obfuscated(self):
@@ -70,7 +89,9 @@ def test_hostname_not_in_any_file(self):
 
     def test_no_empty_obfuscations(self):
         # get the private map file name
-        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        map_file = re.findall(
+            '/.*sosreport-.*-private_map', self.cmd_output.stdout
+        )[-1]
         with open(map_file, 'r') as mf:
             map_json = json.load(mf)
         for mapping in map_json:
@@ -83,12 +104,17 @@ def test_ip_not_in_any_file(self):
         if not content:
             assert True
         else:
-            self.fail("IP appears in files: %s" % "\n".join(f for f in content))
+            new_content = "\n".join(f for f in content)
+            self.fail(f'IP appears in files: {new_content}')
 
     def test_user_is_obfuscated(self):
         """Ensure that the 'testuser1' user created at install is obfuscated
         """
-        self.assertFileNotHasContent('var/log/anaconda/journal.log', 'testuser1')
+        self.assertFileNotHasContent(
+            'var/log/anaconda/journal.log',
+            'testuser1'
+        )
+
 
 class ExistingArchiveCleanTmpTest(StageTwoReportTest):
     """Continuation of above tests which requires cleaning var / tmp keywords
@@ -98,13 +124,14 @@ class ExistingArchiveCleanTmpTest(StageTwoReportTest):
     :avocado: tags=stagetwo
     """
 
-    sos_cmd = '-v --keywords var,tmp,avocado --disable-parsers ip,ipv6,mac,username \
-        --no-update tests/test_data/%s.tar.xz' % ARCHIVE
+    sos_cmd = f'-v --keywords var,tmp,avocado --disable-parsers \
+        ip,ipv6,mac,username --no-update tests/test_data/{ARCHIVE}.tar.xz'
     sos_component = 'clean'
 
     def test_sys_tmp_not_obfuscated(self):
-        """ Ensure that keywords var, tmp and avocado remains in the final archive
-        path despite they are parts of the --tmp-dir
+        """ Ensure that keywords var, tmp and avocado remains in the final
+        archive path despite they are parts of the --tmp-dir
         """
-        self.assertTrue(self.archive.startswith(os.getenv('AVOCADO_TESTS_COMMON_TMPDIR')))
-
+        self.assertTrue(
+            self.archive.startswith(os.getenv('AVOCADO_TESTS_COMMON_TMPDIR'))
+        )

tests/cleaner_tests/full_report/full_report_run.py

@@ -9,7 +9,6 @@
 import json
 import re
 
-from avocado.utils import process
 from sos_tests import StageTwoReportTest
 
 
@@ -46,8 +45,13 @@ def pre_sos_setup(self):
         )
 
     def test_private_map_was_generated(self):
-        self.assertOutputContains('A mapping of obfuscated elements is available at')
-        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        self.assertOutputContains(
+            'A mapping of obfuscated elements is available at'
+        )
+        map_file = re.findall(
+            '/.*sosreport-.*-private_map',
+            self.cmd_output.stdout
+        )[-1]
         self.assertFileExists(map_file)
 
     def test_tarball_named_obfuscated(self):
@@ -69,7 +73,10 @@ def test_hostname_not_in_any_file(self):
 
     def test_no_empty_obfuscations(self):
         # get the private map file name
-        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        map_file = re.findall(
+            '/.*sosreport-.*-private_map',
+            self.cmd_output.stdout
+        )[-1]
         with open(map_file, 'r') as mf:
             map_json = json.load(mf)
         for mapping in map_json:
@@ -83,4 +90,5 @@ def test_ip_not_in_any_file(self):
         if not content:
             assert True
         else:
-            self.fail("IP appears in files: %s" % "\n".join(f for f in content))
+            new_content = "\n".join(f for f in content)
+            self.fail(f'IP appears in files: {new_content}')

tests/cleaner_tests/help_output_tests.py

@@ -21,7 +21,8 @@ class CleanHelpTest(StageOneOutputTest):
     def test_all_help_sections_present(self):
         self.assertOutputContains('Global Options:')
         self.assertOutputContains('Cleaner/Masking Options:')
-        self.assertOutputContains('TARGET                The directory or archive to obfuscate')
+        self.assertOutputContains('TARGET                The directory or '
+                                  'archive to obfuscate')
 
 
 class MaskHelpTest(CleanHelpTest):

tests/cleaner_tests/ipv6_test/ipv6_test.py

@@ -10,6 +10,7 @@
 
 MOCK_FILE = '/tmp/sos-test-ipv6.txt'
 
+
 class IPv6Test(StageTwoReportTest):
     """Place artificial plugin collecting crafted text file with ipv6 adresses
     to make sure ipv6 obfuscation works when calling 'sos clean' like a user
@@ -31,8 +32,14 @@ class IPv6Test(StageTwoReportTest):
     def test_valid_ipv6(self):
         self.assertFileCollected(MOCK_FILE)
         self.assertFileHasContent(MOCK_FILE, 'GOOD_IP=')
-        self.assertFileNotHasContent(MOCK_FILE, 'GOOD_IP=3000:505f:505f:505f:505f:505f:505f:505f')
+        self.assertFileNotHasContent(
+            MOCK_FILE,
+            'GOOD_IP=3000:505f:505f:505f:505f:505f:505f:505f'
+        )
 
     def test_bad_ipv6(self):
         self.assertFileHasContent(MOCK_FILE, 'BAD_IP=')
-        self.assertFileNotHasContent(MOCK_FILE, 'BAD_IP=505f:505f:505f:505f:505f:505f:505f:505f')
+        self.assertFileNotHasContent(
+            MOCK_FILE,
+            'BAD_IP=505f:505f:505f:505f:505f:505f:505f:505f'
+        )