[archive, component] Obfuscate upload password in sos.log and manifes…

…t.json

sos_logs/sos.log and sos_reports/manifest.json tracks command line where
we must obfuscate upload passwords like:

--upload-pass=PASSWORD
--upload-url https://user:[email protected]

So let move the do_file_sub functionality into archive class and call
that from report before finalizing the archive.

Resolves: sosreport#3463
Closes: sosreport#3462

Signed-off-by: Pavel Moravec <[email protected]>

man/en/sos-report.1

@@ -439,7 +439,7 @@ If this option is omitted and upload is requested, you will be prompted for one.
 If --batch is used, this prompt will not occur, so any uploads are likely to fail unless
 this option is used.
 
-Note that this will result in the plaintext string appearing in `ps` output that may
+Note that this may result in the plaintext string appearing in `ps` output that may
 be collected by sos and be in the archive. If a password must be provided by you
 for uploading, it is strongly recommended to not use --batch and enter the password
 when prompted rather than using this option.

sos/archive.py

@@ -559,6 +559,43 @@ def rename_archive_root(self, cleaner):
         self._archive_root = _new_root
         self._archive_name = os.path.join(self._tmp_dir, self.name())
 
+    def do_file_sub(self, path, regexp, subst):
+        """Apply a regexp substitution to a file in the archive.
+
+        :param path: Path in the archive where the file can be found
+        :type path: ``str``
+
+        :param regexp:  A regex to match the contents of the file
+        :type regexp: ``str`` or compiled ``re`` object
+
+        :param subst: The substitution string to be used to replace matches
+                      within the file
+        :type subst: ``str``
+
+        :returns: Number of replacements made
+        :rtype: ``int``
+        """
+        common_flags = re.IGNORECASE | re.MULTILINE
+        if hasattr(regexp, "pattern"):
+            pattern = regexp.pattern
+            flags = regexp.flags | common_flags
+        else:
+            pattern = regexp
+            flags = common_flags
+
+        content = ""
+        with self.open_file(path) as readable:
+            content = readable.read()
+        if not isinstance(content, str):
+            content = content.decode('utf8', 'ignore')
+        result, replacements = re.subn(pattern, subst, content,
+                                       flags=flags)
+        if replacements:
+            self.add_string(result, path)
+        else:
+            replacements = 0
+        return replacements
+
     def finalize(self, method):
         self.log_info("finalizing archive '%s' using method '%s'"
                       % (self._archive_root, method))

sos/collector/__init__.py

@@ -1360,6 +1360,7 @@ def create_cluster_archive(self):
                 self.archive.add_final_manifest_data(
                     self.opts.compression_type
                 )
+            self._obfuscate_upload_passwords()
             if do_clean:
                 _dir = os.path.join(self.tmpdir, self.archive._name)
                 cleaner.obfuscate_file(

sos/component.py

@@ -69,6 +69,15 @@ class SoSComponent():
         "verbosity": 0
     }
 
+    # files in collected archive that might contain upload password
+    files_with_upload_passwd = [
+        "sos_logs/sos.log",
+        "sos_reports/manifest.json",
+        "sos_commands/process/ps_*",
+        "sos_commands/selinux/ps_*",
+        "sos_commands/systemd/systemctl_status_--all",
+    ]
+
     def __init__(self, parser, parsed_args, cmdline_args):
         self.parser = parser
         self.args = parsed_args
@@ -367,6 +376,22 @@ def setup_archive(self, name=''):
 
         self.archive.set_debug(self.opts.verbosity > 2)
 
+    def _obfuscate_upload_passwords(self):
+        # obfuscate strings like:
+        # --upload-pass=PASSWORD
+        # --upload-pass PASSWORD
+        # --upload-url https://user:[email protected]
+        # in both sos_logs/sos.log and in sos_reports/manifest.json
+        # and several sos_commands/* places from plugins's collected data
+        _arc_path = self.archive.get_archive_path()
+        for path in self.files_with_upload_passwd:
+            for f in Path(_arc_path).glob(path):
+                # get just the relative path that archive works with
+                f = os.path.relpath(f, _arc_path)
+                for re in [r"(--upload-pass[\s=]+)\S+",
+                           r"(--upload-url[\s=]+\S+://.*:)([^@]*)"]:
+                    self.archive.do_file_sub(f, re, r"\1********")
+
     def add_ui_log_to_stdout(self):
         ui_console = logging.StreamHandler(sys.stdout)
         ui_console.setFormatter(logging.Formatter('%(message)s'))

sos/report/__init__.py

@@ -1515,6 +1515,8 @@ def final_work(self):
         self._add_sos_logs()
         if self.manifest is not None:
             self.archive.add_final_manifest_data(self.opts.compression_type)
+        # Hide upload passwords in the log files
+        self._obfuscate_upload_passwords()
         # Now, separately clean the log files that cleaner also wrote to
         if do_clean:
             _dir = os.path.join(self.tmpdir, self.archive._name)

sos/report/plugins/__init__.py

@@ -1279,28 +1279,15 @@ def do_file_sub(self, srcpath, regexp, subst):
         """
         try:
             path = self._get_dest_for_srcpath(srcpath)
-            common_flags = re.IGNORECASE | re.MULTILINE
-            if hasattr(regexp, "pattern"):
-                pattern = regexp.pattern
-                flags = regexp.flags | common_flags
-            else:
-                pattern = regexp
-                flags = common_flags
             self._log_debug("substituting scrpath '%s'" % srcpath)
             self._log_debug("substituting '%s' for '%s' in '%s'"
-                            % (subst, pattern, path))
+                            % (subst,
+                               regexp.pattern if hasattr(regexp, "pattern")
+                               else regexp,
+                               path))
             if not path:
                 return 0
-            readable = self.archive.open_file(path)
-            content = readable.read()
-            if not isinstance(content, str):
-                content = content.decode('utf8', 'ignore')
-            result, replacements = re.subn(pattern, subst, content,
-                                           flags=flags)
-            if replacements:
-                self.archive.add_string(result, self.strip_sysroot(srcpath))
-            else:
-                replacements = 0
+            replacements = self.archive.do_file_sub(path, regexp, subst)
         except (OSError, IOError) as e:
             # if trying to regexp a nonexisting file, dont log it as an
             # error to stdout