From f99726b1d1b5c5db78bd6b102228e774afe6844d Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Thu, 8 Aug 2024 21:10:02 +0100 Subject: [PATCH] [sos] Add 'upload' component to upload existing reports and files This commit marks the beginning of the addition of a new 'upload' component for sos, which can be used to upload already created sos reports, collects, or other files like logs or vmcores to a policy defined location. The user needs to specify a file location, and can make use of any of the options that exist nowadays for the --upload option. This first commit includes: - The initial framework for the 'upload' component. - The new man page for 'sos upload'. - The code in the component 'help' to show information about the component. - The code in sos/__init__.py to deal with the component. - The code for uploads to Red Hat and Ubuntu systems. - The code to allow uploads specifying remote destination, called targets in this implementation. For example, you could generate a sos report in a CentOS system and specify a target defined as 'redhat' or 'RedHatUpload' to upload to the Red Hat Customer Portal. - And modifications to setup.py to build the man pages. Related: RHEL-23032, SUPDEV-138, CLIOT-481 Co-authored-by: Jose Castillo Co-authored-by: Pavel Moravec Co-authored-by: Trevor Benson Signed-off-by: Jose Castillo --- man/en/sos-upload.1 | 151 ++++++ setup.py | 3 +- sos/__init__.py | 4 +- sos/collector/__init__.py | 21 +- sos/help/__init__.py | 6 +- sos/policies/distros/__init__.py | 685 +-------------------------- sos/policies/distros/redhat.py | 204 +------- sos/policies/distros/ubuntu.py | 18 - sos/report/__init__.py | 17 +- sos/upload/__init__.py | 323 +++++++++++++ sos/upload/targets/__init__.py | 778 +++++++++++++++++++++++++++++++ sos/upload/targets/redhat.py | 251 ++++++++++ sos/upload/targets/ubuntu.py | 46 ++ 13 files changed, 1594 insertions(+), 913 deletions(-) create mode 100644 man/en/sos-upload.1 create mode 100644 sos/upload/__init__.py create mode 100644 sos/upload/targets/__init__.py create mode 100644 sos/upload/targets/redhat.py create mode 100644 sos/upload/targets/ubuntu.py diff --git a/man/en/sos-upload.1 b/man/en/sos-upload.1 new file mode 100644 index 0000000000..7acbad355a --- /dev/null +++ b/man/en/sos-upload.1 @@ -0,0 +1,151 @@ +.TH UPLOAD 1 "December 2024" + +.SH NAME +sos_upload \- Upload files like previously generated sos reports or logs to a policy specific location +.SH SYNOPSIS +.B sos upload FILE [options] + [--case-id id]\fR + [--upload-url url]\fR + [--upload-user user]\fR + [--upload-pass pass]\fR + [--upload-directory dir]\fR + [--upload-method]\fR + [--upload-no-ssl-verify]\fR + [--upload-protocol protocol]\fR + [--upload-target target]\fR + [--upload-s3-endpoint endpoint]\fR + [--upload-s3-region region]\fR + [--upload-s3-bucket bucket]\fR + [--upload-s3-access-key key]\fR + [--upload-s3-secret-key key]\fR + [--upload-s3-object-prefix prefix]\fR + +.PP +.SH DESCRIPTION +upload is an sos subcommand to upload sos reports, +logs, vmcores, or other files to a policy defined +remote location, or a user defined one. +.SH REQUIRED ARGUMENTS +.B FILE +.TP +The path to the archive that is to be uploaded. +.SH OPTIONS +.TP +.B \--case-id NUMBER +Specify a case identifier to associate with the archive. +Identifiers may include alphanumeric characters, commas and periods ('.'). +.TP +.B \--upload-url URL +If a vendor does not provide a default upload location, or if you would like to upload +the archive to a different location, specify the address here. + +An upload protocol MUST be specified in this URL. Currently uploading is supported +for HTTPS, SFTP, and FTP protocols. + +If your destination server listens on a non-standard port, specify the listening +port in the URL. +.TP +.B \-\-upload-user USER +If a vendor does not provide a default user for uploading, specify the username here. + +If --batch is used and this option is omitted, no username will +be collected and thus uploads will fail if no vendor default is set. + +You also have the option of providing this value via the SOSUPLOADUSER environment +variable. If this variable is set, then no username prompt will occur and --batch +may be used provided all other required values (case number, upload password) +are provided. +.TP +.B \-\-upload-pass PASS +Specify the password to use for authentication with the destination server. + +If this option is omitted and upload is requested, you will be prompted for one. + +If --batch is used, this prompt will not occur, so any uploads are likely to fail unless +this option is used. + +Note that this will result in the plaintext string appearing in `ps` output that may +be collected by sos and be in the archive. If a password must be provided by you +for uploading, it is strongly recommended to not use --batch and enter the password +when prompted rather than using this option. + +You also have the option of providing this value via the SOSUPLOADPASSWORD environment +variable. If this variable is set, then no password prompt will occur and --batch may +be used provided all other required values (case number, upload user) are provided. +.TP +.B \--upload-directory DIR +Specify a directory to upload to, if one is not specified by a vendor default location +or if your destination server does not allow writes to '/'. +.TP +.B \--upload-method METHOD +Specify the HTTP method to use for uploading to the provided --upload-url. Valid +values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to +the method required by the policy-default upload location, if one exists. + +This option has no effect on upload protocols other than HTTPS. +.TP +.B \--upload-no-ssl-verify +Disable SSL verification for HTTPS uploads. This may be used to allow uploading +to locations that have self-signed certificates, or certificates that are otherwise +untrusted by the local system. + +Default behavior is to perform SSL verification against all upload locations. +.TP +.B \--upload-protocol PROTO +Manually specify the protocol to use for uploading to the target \fBupload-url\fR. + +Normally this is determined via the upload address, assuming that the protocol is part +of the address provided, e.g. 'https://example.com'. By using this option, sos will skip +the protocol check and use the method defined for the specified PROTO. + +For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to +upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server, +which is typically used as a fallback target. + +Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp', 's3'. + +.TP +.B \--upload-target TARGET +Manually specify the upload target to use for uploading. Possible targets +are: RHELUpload, UbuntuUpload, local. +sos to upload a file to a vendor-specific location explicitly. +If this option is not called, sos will try to determine the local target +and use it for uploads. + +.TP +.B \---upload-s3-endpoint ENDPOINT +Endpoint to upload to for S3 bucket + +.TP +.B \---upload-s3-region REGION +Region to upload to for S3 bucket + +.TP +.B \---upload-s3-bucket BUCKET +Name of the S3 bucket to upload to + +.TP +.B \---upload-s3-access-key KEY +Access key for the S3 bucket + +.TP +.B \- --upload-s3-secret-key KEY +Secret key for the S3 bucket + +.TP +.B \- --upload-s3-object-prefix PREFIX +Prefix for the S3 object/key + +.SH SEE ALSO +.BR sos (1) +.BR sos-report (1) +.BR sos-clean (1) +.BR sos.conf (5) +.BR sos-collect (1) + +.SH MAINTAINER +.nf +Maintained on GitHub at https://github.com/sosreport/sos +.fi +.SH AUTHORS & CONTRIBUTORS +See \fBAUTHORS\fR file in the package documentation. diff --git a/setup.py b/setup.py index 9ad4bb75f4..3942a26d98 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,8 @@ ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1', 'man/en/sos.1', 'man/en/sos-collect.1', 'man/en/sos-collector.1', 'man/en/sos-clean.1', - 'man/en/sos-mask.1', 'man/en/sos-help.1']), + 'man/en/sos-mask.1', 'man/en/sos-help.1', + 'man/en/sos-upload.1']), ('share/man/man5', ['man/en/sos.conf.5']), ('share/licenses/sos', ['LICENSE']), ('share/doc/sos', ['AUTHORS', 'README.md']), diff --git a/sos/__init__.py b/sos/__init__.py index c0baa3b4a6..c11b88fa7e 100644 --- a/sos/__init__.py +++ b/sos/__init__.py @@ -54,10 +54,12 @@ def __init__(self, args): import sos.report import sos.cleaner import sos.help + import sos.upload self._components = { 'report': (sos.report.SoSReport, ['rep']), 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']), - 'help': (sos.help.SoSHelper, []) + 'help': (sos.help.SoSHelper, []), + 'upload': (sos.upload.SoSUpload, []) } # some distros do not want pexpect as a default dep, so try to load # collector here, and if it fails add an entry that implies it is at diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py index 24b2da9754..9e978c2f92 100644 --- a/sos/collector/__init__.py +++ b/sos/collector/__init__.py @@ -33,6 +33,7 @@ from sos.component import SoSComponent from sos.utilities import bold from sos import __version__ +from sos.upload import SoSUpload COLLECTOR_CONFIG_DIR = '/etc/sos/groups.d' @@ -145,7 +146,8 @@ class SoSCollector(SoSComponent): 'upload_s3_bucket': None, 'upload_s3_access_key': None, 'upload_s3_secret_key': None, - 'upload_s3_object_prefix': None + 'upload_s3_object_prefix': None, + 'upload_target': None } def __init__(self, parser, parsed_args, cmdline_args): @@ -1304,10 +1306,23 @@ def collect(self): msg = 'No sos reports were collected, nothing to archive...' self.exit(msg, 1) - if (self.opts.upload and self.policy.get_upload_url()) or \ + if self.opts.upload or \ self.opts.upload_s3_endpoint: try: - self.policy.upload_archive(self.arc_name) + hook_commons = { + 'policy': self.policy, + 'tmpdir': self.tmpdir, + 'sys_tmp': self.sys_tmp, + 'options': self.opts, + 'manifest': self.manifest + } + uploader = SoSUpload(parser=self.parser, + args=self.args, + cmdline=self.cmdline, + in_place=True, + hook_commons=hook_commons, + archive=self.arc_name) + uploader.execute() self.ui_log.info("Uploaded archive successfully") except Exception as err: self.ui_log.error(f"Upload attempt failed: {err}") diff --git a/sos/help/__init__.py b/sos/help/__init__.py index e5162d98e7..b057d17430 100644 --- a/sos/help/__init__.py +++ b/sos/help/__init__.py @@ -100,7 +100,8 @@ def get_obj_for_topic(self): 'collector': 'SoSCollector', 'collector.transports': 'RemoteTransport', 'collector.clusters': 'Cluster', - 'policies': 'Policy' + 'policies': 'Policy', + 'upload': 'SoSUpload' } cls = None @@ -206,7 +207,8 @@ def display_self_help(self): 'report.plugins.$plugin': 'Information on a specific $plugin', 'clean': 'Detailed help on the clean command', 'collect': 'Detailed help on the collect command', - 'policies': 'How sos operates on different distributions' + 'upload': 'Detailed help on the upload command', + 'policies': 'How sos operates on different distributions', } for sect, value in sections.items(): diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py index a2a07a91f8..06327a6b05 100644 --- a/sos/policies/distros/__init__.py +++ b/sos/policies/distros/__init__.py @@ -13,9 +13,6 @@ import os import re -from getpass import getpass - -from sos import _sos as _ from sos.policies import Policy from sos.policies.init_systems import InitSystem from sos.policies.init_systems.systemd import SystemdInit @@ -25,21 +22,8 @@ from sos.policies.runtimes.lxd import LxdContainerRuntime from sos.utilities import (shell_out, is_executable, bold, - sos_get_command_output, TIMEOUT_DEFAULT) - - -try: - import requests - REQUESTS_LOADED = True -except ImportError: - REQUESTS_LOADED = False + sos_get_command_output) -try: - import boto3 - from botocore.config import Config as BotocoreConfig - BOTO3_LOADED = True -except ImportError: - BOTO3_LOADED = False OS_RELEASE = "/etc/os-release" # Container environment variables for detecting if we're in a container @@ -58,34 +42,9 @@ class LinuxPolicy(Policy): os_release_file = '' os_release_name = '' os_release_id = '' - # _ prefixed class attrs are used for storing any vendor-defined defaults - # the non-prefixed attrs are used by the upload methods, and will be set - # to the cmdline/config file values, if provided. If not provided, then - # those attrs will be set to the _ prefixed values as a fallback. - # TL;DR Use _upload_* for policy default values, use upload_* when wanting - # to actual use the value in a method/override _upload_url = None - _upload_directory = '/' - _upload_user = None - _upload_password = None - _upload_method = None - _upload_s3_endpoint = 'https://s3.amazonaws.com' - _upload_s3_bucket = None - _upload_s3_access_key = None - _upload_s3_secret_key = None - _upload_s3_region = None - _upload_s3_object_prefix = '' default_container_runtime = 'docker' _preferred_hash_name = None - upload_url = None - upload_user = None - upload_password = None - upload_s3_endpoint = None - upload_s3_bucket = None - upload_s3_access_key = None - upload_s3_secret_key = None - upload_s3_region = None - upload_s3_object_prefix = None # collector-focused class attrs containerized = False container_image = None @@ -337,56 +296,13 @@ def pre_work(self): # this method will be called before the gathering begins cmdline_opts = self.commons['cmdlineopts'] - caseid = cmdline_opts.case_id if cmdline_opts.case_id else "" if cmdline_opts.low_priority: self._configure_low_priority() - # Set the cmdline settings to the class attrs that are referenced later - # The policy default '_' prefixed versions of these are untouched to - # allow fallback - self.upload_url = cmdline_opts.upload_url - self.upload_user = cmdline_opts.upload_user - self.upload_directory = cmdline_opts.upload_directory - self.upload_password = cmdline_opts.upload_pass - self.upload_archive_name = '' - - self.upload_s3_endpoint = cmdline_opts.upload_s3_endpoint - self.upload_s3_region = cmdline_opts.upload_s3_region - self.upload_s3_access_key = cmdline_opts.upload_s3_access_key - self.upload_s3_bucket = cmdline_opts.upload_s3_bucket - self.upload_s3_object_prefix = cmdline_opts.upload_s3_object_prefix - self.upload_s3_secret_key = cmdline_opts.upload_s3_secret_key - - # set or query for case id - if not cmdline_opts.batch and not \ - cmdline_opts.quiet: - if caseid: - self.commons['cmdlineopts'].case_id = caseid - else: - self.commons['cmdlineopts'].case_id = input( - _("Optionally, please enter the case id that you are " - "generating this report for [%s]: ") % caseid - ) if cmdline_opts.case_id: self.case_id = cmdline_opts.case_id - # set or query for upload credentials; this needs to be done after - # setting case id, as below methods might rely on detection of it - if not cmdline_opts.batch and not \ - cmdline_opts.quiet: - # Policies will need to handle the prompts for user information - if cmdline_opts.upload and self.get_upload_url() and \ - not cmdline_opts.upload_protocol == 's3': - self.prompt_for_upload_user() - self.prompt_for_upload_password() - elif cmdline_opts.upload_protocol == 's3': - self.prompt_for_upload_s3_bucket() - self.prompt_for_upload_s3_endpoint() - self.prompt_for_upload_s3_access_key() - self.prompt_for_upload_s3_secret_key() - self.ui_log.info('') - def _configure_low_priority(self): """Used to constrain sos to a 'low priority' execution, potentially letting individual policies set their own definition of what that is. @@ -418,605 +334,6 @@ def _configure_low_priority(self): except Exception as err: self.soslog.error(f"Error setting report niceness to 19: {err}") - def prompt_for_upload_s3_access_key(self): - """Should be overridden by policies to determine if an access key needs - to be provided for upload or not - """ - if not self.get_upload_s3_access_key(): - - msg = ( - "Please provide the upload access key for bucket" - f" {self.get_upload_s3_bucket()} via endpoint" - f" {self.get_upload_s3_endpoint()}: " - ) - self.upload_s3_access_key = input(_(msg)) - - def prompt_for_upload_s3_secret_key(self): - """Should be overridden by policies to determine if a secret key needs - to be provided for upload or not - """ - if not self.get_upload_s3_secret_key(): - msg = ( - "Please provide the upload secret key for bucket" - f" {self.get_upload_s3_bucket()} via endpoint" - f" {self.get_upload_s3_endpoint()}: " - ) - self.upload_s3_secret_key = getpass(msg) - - def prompt_for_upload_s3_bucket(self): - """Should be overridden by policies to determine if a bucket needs to - be provided for upload or not - """ - if not self.upload_s3_bucket: - if self.upload_url and self.upload_url.startswith('s3://'): - self.upload_s3_bucket = self.upload_url[5:] - else: - user_input = input(_("Please provide the upload bucket: ")) - self.upload_s3_bucket = user_input.strip('/') - return self.upload_s3_bucket - - def prompt_for_upload_s3_endpoint(self): - """Should be overridden by policies to determine if an endpoint needs - to be provided for upload or not - """ - default_endpoint = self._upload_s3_endpoint - if not self.upload_s3_endpoint: - msg = ( - "Please provide the upload endpoint for bucket" - f" {self.get_upload_s3_bucket()}" - f" (default: {default_endpoint}): " - ) - user_input = input(_(msg)) - self.upload_s3_endpoint = user_input or default_endpoint - return self.upload_s3_endpoint - - def prompt_for_upload_user(self): - """Should be overridden by policies to determine if a user needs to - be provided or not - """ - if not self.get_upload_user(): - msg = f"Please provide upload user for {self.get_upload_url()}: " - self.upload_user = input(_(msg)) - - def prompt_for_upload_password(self): - """Should be overridden by policies to determine if a password needs to - be provided for upload or not - """ - if not self.get_upload_password() and (self.get_upload_user() != - self._upload_user): - msg = ("Please provide the upload password for " - f"{self.get_upload_user()}: ") - self.upload_password = getpass(msg) - - def upload_archive(self, archive): - """ - Entry point for sos attempts to upload the generated archive to a - policy or user specified location. - - Currently there is support for HTTPS, SFTP, and FTP. HTTPS uploads are - preferred for policy-defined defaults. - - Policies that need to override uploading methods should override the - respective upload_https(), upload_sftp(), and/or upload_ftp() methods - and should NOT override this method. - - :param archive: The archive filepath to use for upload - :type archive: ``str`` - - In order to enable this for a policy, that policy needs to implement - the following: - - Required Class Attrs - - :_upload_url: The default location to use. Note these MUST include - protocol header - :_upload_user: Default username, if any else None - :_upload_password: Default password, if any else None - - The following Class Attrs may optionally be overidden by the Policy - - :_upload_directory: Default FTP server directory, if any - - - The following methods may be overridden by ``Policy`` as needed - - `prompt_for_upload_user()` - Determines if sos should prompt for a username or not. - - `get_upload_user()` - Determines if the default or a different username should be used - - `get_upload_https_auth()` - Format authentication data for HTTPS uploads - - `get_upload_url_string()` - Print a more human-friendly string than vendor URLs - """ - self.upload_archive_name = archive - if not self.upload_url: - self.upload_url = self.get_upload_url() - if not self.upload_url: - raise Exception("No upload destination provided by policy or by " - "--upload-url") - upload_func = self._determine_upload_type() - self.ui_log.info( - _(f"Attempting upload to {self.get_upload_url_string()}") - ) - return upload_func() - - def _determine_upload_type(self): - """Based on the url provided, determine what type of upload to attempt. - - Note that this requires users to provide a FQDN address, such as - https://myvendor.com/api or ftp://myvendor.com instead of - myvendor.com/api or myvendor.com - """ - prots = { - 'ftp': self.upload_ftp, - 'sftp': self.upload_sftp, - 'https': self.upload_https, - 's3': self.upload_s3 - } - if self.commons['cmdlineopts'].upload_protocol in prots: - return prots[self.commons['cmdlineopts'].upload_protocol] - if '://' not in self.upload_url: - raise Exception("Must provide protocol in upload URL") - prot, _ = self.upload_url.split('://') - if prot not in prots: - raise Exception(f"Unsupported or unrecognized protocol: {prot}") - return prots[prot] - - def get_upload_https_auth(self, user=None, password=None): - """Formats the user/password credentials using basic auth - - :param user: The username for upload - :type user: ``str`` - - :param password: Password for `user` to use for upload - :type password: ``str`` - - :returns: The user/password auth suitable for use in requests calls - :rtype: ``requests.auth.HTTPBasicAuth()`` - """ - if not user: - user = self.get_upload_user() - if not password: - password = self.get_upload_password() - - return requests.auth.HTTPBasicAuth(user, password) - - def get_upload_s3_access_key(self): - """Helper function to determine if we should use the policy default - upload access key or one provided by the user - - :returns: The access_key to use for upload - :rtype: ``str`` - """ - return (os.getenv('SOSUPLOADS3ACCESSKEY', None) or - self.upload_s3_access_key or - self._upload_s3_access_key) - - def get_upload_s3_endpoint(self): - """Helper function to determine if we should use the policy default - upload endpoint or one provided by the user - - :returns: The S3 Endpoint to use for upload - :rtype: ``str`` - """ - if not self.upload_s3_endpoint: - self.prompt_for_upload_s3_endpoint() - return self.upload_s3_endpoint - - def get_upload_s3_region(self): - """Helper function to determine if we should use the policy default - upload region or one provided by the user - - :returns: The S3 region to use for upload - :rtype: ``str`` - """ - return self.upload_s3_region or self._upload_s3_region - - def get_upload_s3_bucket(self): - """Helper function to determine if we should use the policy default - upload bucket or one provided by the user - - :returns: The S3 bucket to use for upload - :rtype: ``str`` - """ - if self.upload_url and self.upload_url.startswith('s3://'): - bucket_and_prefix = self.upload_url[5:].split('/', 1) - self.upload_s3_bucket = bucket_and_prefix[0] - if len(bucket_and_prefix) > 1: - self.upload_s3_object_prefix = bucket_and_prefix[1] - if not self.upload_s3_bucket: - self.prompt_for_upload_s3_bucket() - return self.upload_s3_bucket or self._upload_s3_bucket - - def get_upload_s3_object_prefix(self): - """Helper function to determine if we should use the policy default - upload object prefix or one provided by the user - - :returns: The S3 object prefix to use for upload - :rtype: ``str`` - """ - return self.upload_s3_object_prefix or self._upload_s3_object_prefix - - def get_upload_s3_secret_key(self): - """Helper function to determine if we should use the policy default - upload secret key or one provided by the user - - :returns: The S3 secret key to use for upload - :rtype: ``str`` - """ - return (os.getenv('SOSUPLOADS3SECRETKEY', None) or - self.upload_s3_secret_key or - self._upload_s3_secret_key) - - def get_upload_url(self): - """Helper function to determine if we should use the policy default - upload url or one provided by the user - - :returns: The URL to use for upload - :rtype: ``str`` - """ - if not self.upload_url and ( - self.upload_s3_bucket and - self.upload_s3_access_key and - self.upload_s3_secret_key - ): - bucket = self.get_upload_s3_bucket() - prefix = self.get_upload_s3_object_prefix() - self._upload_url = f"s3://{bucket}/{prefix}" - return self.upload_url or self._upload_url - - def _get_obfuscated_upload_url(self, url): - pattern = r"([^:]+://[^:]+:)([^@]+)(@.+)" - obfuscated_url = re.sub(pattern, r'\1********\3', url) - return obfuscated_url - - def get_upload_url_string(self): - """Used by distro policies to potentially change the string used to - report upload location from the URL to a more human-friendly string - """ - return self._get_obfuscated_upload_url(self.get_upload_url()) - - def get_upload_user(self): - """Helper function to determine if we should use the policy default - upload user or one provided by the user - - :returns: The username to use for upload - :rtype: ``str`` - """ - return (os.getenv('SOSUPLOADUSER', None) or - self.upload_user or - self._upload_user) - - def get_upload_password(self): - """Helper function to determine if we should use the policy default - upload password or one provided by the user - - A user provided password, either via option or the 'SOSUPLOADPASSWORD' - environment variable will have precendent over any policy value - - :returns: The password to use for upload - :rtype: ``str`` - """ - return (os.getenv('SOSUPLOADPASSWORD', None) or - self.upload_password or - self._upload_password) - - def upload_sftp(self, user=None, password=None): - """Attempts to upload the archive to an SFTP location. - - Due to the lack of well maintained, secure, and generally widespread - python libraries for SFTP, sos will shell-out to the system's local ssh - installation in order to handle these uploads. - - Do not override this method with one that uses python-paramiko, as the - upstream sos team will reject any PR that includes that dependency. - """ - # if we somehow don't have sftp available locally, fail early - if not is_executable('sftp'): - raise Exception('SFTP is not locally supported') - - # soft dependency on python3-pexpect, which we need to use to control - # sftp login since as of this writing we don't have a viable solution - # via ssh python bindings commonly available among downstreams - try: - import pexpect - except ImportError as err: - raise Exception('SFTP upload requires python3-pexpect, which is ' - 'not currently installed') from err - - sftp_connected = False - - if not user: - user = self.get_upload_user() - if not password: - password = self.get_upload_password() - - # need to strip the protocol prefix here - sftp_url = self.get_upload_url().replace('sftp://', '') - sftp_cmd = f"sftp -oStrictHostKeyChecking=no {user}@{sftp_url}" - ret = pexpect.spawn(sftp_cmd, encoding='utf-8') - - sftp_expects = [ - 'sftp>', - 'password:', - 'Connection refused', - pexpect.TIMEOUT, - pexpect.EOF - ] - - idx = ret.expect(sftp_expects, timeout=15) - - if idx == 0: - sftp_connected = True - elif idx == 1: - ret.sendline(password) - pass_expects = [ - 'sftp>', - 'Permission denied', - pexpect.TIMEOUT, - pexpect.EOF - ] - sftp_connected = ret.expect(pass_expects, timeout=10) == 0 - if not sftp_connected: - ret.close() - raise Exception("Incorrect username or password for " - f"{self.get_upload_url_string()}") - elif idx == 2: - raise Exception("Connection refused by " - f"{self.get_upload_url_string()}. Incorrect port?") - elif idx == 3: - raise Exception("Timeout hit trying to connect to " - f"{self.get_upload_url_string()}") - elif idx == 4: - raise Exception("Unexpected error trying to connect to sftp: " - f"{ret.before}") - - if not sftp_connected: - ret.close() - raise Exception("Unable to connect via SFTP to " - f"{self.get_upload_url_string()}") - - put_cmd = (f'put {self.upload_archive_name} ' - f'{self._get_sftp_upload_name()}') - ret.sendline(put_cmd) - - put_expects = [ - '100%', - pexpect.TIMEOUT, - pexpect.EOF, - 'No such file or directory' - ] - - put_success = ret.expect(put_expects, timeout=180) - - if put_success == 0: - ret.sendline('bye') - return True - if put_success == 1: - raise Exception("Timeout expired while uploading") - if put_success == 2: - raise Exception(f"Unknown error during upload: {ret.before}") - if put_success == 3: - raise Exception("Unable to write archive to destination") - raise Exception(f"Unexpected response from server: {ret.before}") - - def _get_sftp_upload_name(self): - """If a specific file name pattern is required by the SFTP server, - override this method in the relevant Policy. Otherwise the archive's - name on disk will be used - - :returns: Filename as it will exist on the SFTP server - :rtype: ``str`` - """ - fname = self.upload_archive_name.split('/')[-1] - if self.upload_directory: - fname = os.path.join(self.upload_directory, fname) - return fname - - def _upload_https_put(self, archive, verify=True): - """If upload_https() needs to use requests.put(), use this method. - - Policies should override this method instead of the base upload_https() - - :param archive: The open archive file object - """ - return requests.put(self.get_upload_url(), data=archive, - auth=self.get_upload_https_auth(), - verify=verify, timeout=TIMEOUT_DEFAULT) - - def _get_upload_headers(self): - """Define any needed headers to be passed with the POST request here - """ - return {} - - def _upload_https_post(self, archive, verify=True): - """If upload_https() needs to use requests.post(), use this method. - - Policies should override this method instead of the base upload_https() - - :param archive: The open archive file object - """ - files = { - 'file': (archive.name.split('/')[-1], archive, - self._get_upload_headers()) - } - return requests.post(self.get_upload_url(), files=files, - auth=self.get_upload_https_auth(), - verify=verify, timeout=TIMEOUT_DEFAULT) - - def upload_https(self): - """Attempts to upload the archive to an HTTPS location. - - :returns: ``True`` if upload is successful - :rtype: ``bool`` - - :raises: ``Exception`` if upload was unsuccessful - """ - if not REQUESTS_LOADED: - raise Exception("Unable to upload due to missing python requests " - "library") - - with open(self.upload_archive_name, 'rb') as arc: - if self.commons['cmdlineopts'].upload_method == 'auto': - method = self._upload_method - else: - method = self.commons['cmdlineopts'].upload_method - verify = self.commons['cmdlineopts'].upload_no_ssl_verify is False - if method == 'put': - r = self._upload_https_put(arc, verify) - else: - r = self._upload_https_post(arc, verify) - if r.status_code not in (200, 201): - if r.status_code == 401: - raise Exception( - "Authentication failed: invalid user credentials" - ) - raise Exception(f"POST request returned {r.status_code}: " - f"{r.reason}") - return True - - def upload_ftp(self, url=None, directory=None, user=None, password=None): - """Attempts to upload the archive to either the policy defined or user - provided FTP location. - - :param url: The URL to upload to - :type url: ``str`` - - :param directory: The directory on the FTP server to write to - :type directory: ``str`` or ``None`` - - :param user: The user to authenticate with - :type user: ``str`` - - :param password: The password to use for `user` - :type password: ``str`` - - :returns: ``True`` if upload is successful - :rtype: ``bool`` - - :raises: ``Exception`` if upload in unsuccessful - """ - import ftplib - import socket - - if not url: - url = self.get_upload_url() - if url is None: - raise Exception("no FTP server specified by policy, use --upload-" - "url to specify a location") - - url = url.replace('ftp://', '') - - if not user: - user = self.get_upload_user() - - if not password: - password = self.get_upload_password() - - if not directory: - directory = self.upload_directory or self._upload_directory - - try: - session = ftplib.FTP(url, user, password, timeout=15) - if not session: - raise Exception("connection failed, did you set a user and " - "password?") - session.cwd(directory) - except socket.timeout as err: - raise Exception(f"timeout hit while connecting to {url}") from err - except socket.gaierror as err: - raise Exception(f"unable to connect to {url}") from err - except ftplib.error_perm as err: - errno = str(err).split()[0] - if errno == '503': - raise Exception(f"could not login as '{user}'") from err - if errno == '530': - raise Exception(f"invalid password for user '{user}'") from err - if errno == '550': - raise Exception("could not set upload directory to " - f"{directory}") from err - raise Exception(f"error trying to establish session: {str(err)}") \ - from err - - with open(self.upload_archive_name, 'rb') as _arcfile: - session.storbinary( - f"STOR {self.upload_archive_name.split('/')[-1]}", _arcfile - ) - session.quit() - return True - - def upload_s3(self, endpoint=None, region=None, bucket=None, prefix=None, - access_key=None, secret_key=None): - """Attempts to upload the archive to an S3 bucket. - - :param endpoint: The S3 endpoint to upload to - :type endpoint: str - - :param region: The S3 region to upload to - :type region: str - - :param bucket: The name of the S3 bucket to upload to - :type bucket: str - - :param prefix: The prefix for the S3 object/key - :type prefix: str - - :param access_key: The access key for the S3 bucket - :type access_key: str - - :param secret_key: The secret key for the S3 bucket - :type secret_key: str - - :returns: True if upload is successful - :rtype: bool - - :raises: Exception if upload is unsuccessful - """ - if not BOTO3_LOADED: - raise Exception("Unable to upload due to missing python boto3 " - "library") - - if not endpoint: - endpoint = self.get_upload_s3_endpoint() - if not region: - region = self.get_upload_s3_region() - - if not bucket: - bucket = self.get_upload_s3_bucket().strip('/') - - if not prefix: - prefix = self.get_upload_s3_object_prefix() - if prefix != '' and prefix.startswith('/'): - prefix = prefix[1:] - if prefix != '' and not prefix.endswith('/'): - prefix = f'{prefix}/' if prefix else '' - - if not access_key: - access_key = self.get_upload_s3_access_key() - - if not secret_key: - secret_key = self.get_upload_s3_secret_key() - - boto3_config = BotocoreConfig(user_agent_extra='app/sos') - - s3_client = boto3.client('s3', endpoint_url=endpoint, - region_name=region, - aws_access_key_id=access_key, - aws_secret_access_key=secret_key, - config=boto3_config) - - try: - key = prefix + self.upload_archive_name.split('/')[-1] - s3_client.upload_file(self.upload_archive_name, - bucket, key) - return True - except Exception as e: - raise Exception(f"Failed to upload to S3: {str(e)}") from e - def set_sos_prefix(self): """If sos report commands need to always be prefixed with something, for example running in a specific container image, then it should be diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py index abe5cc1d90..3278d74f6a 100644 --- a/sos/policies/distros/redhat.py +++ b/sos/policies/distros/redhat.py @@ -8,11 +8,9 @@ # # See the LICENSE file in the source distribution for further information. -import json import os import sys import re -from sos.policies.auth import DeviceAuthorizationClass from sos.report.plugins import RedHatPlugin from sos.presets.redhat import (RHEL_PRESETS, RHV, RHEL, CB, RHOSP, @@ -22,14 +20,9 @@ from sos.policies.package_managers.rpm import RpmPackageManager from sos.policies.package_managers.flatpak import FlatpakPackageManager from sos.policies.package_managers import MultiPackageManager -from sos.utilities import bold, convert_bytes, TIMEOUT_DEFAULT +from sos.utilities import bold from sos import _sos as _ -try: - import requests - REQUESTS_LOADED = True -except ImportError: - REQUESTS_LOADED = False RHEL_RELEASE_STR = "Red Hat Enterprise Linux" @@ -228,201 +221,6 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, remote_exec=remote_exec) self.register_presets(RHEL_PRESETS) - def prompt_for_upload_user(self): - if self.commons['cmdlineopts'].upload_user: - self.ui_log.info( - _("The option --upload-user has been deprecated in favour" - " of device authorization in RHEL") - ) - if not self.case_id: - # no case id provided => failover to SFTP - self.upload_url = RH_SFTP_HOST - self.ui_log.info("No case id provided, uploading to SFTP") - - def prompt_for_upload_password(self): - # With OIDC we don't ask for user/pass anymore - if self.commons['cmdlineopts'].upload_pass: - self.ui_log.info( - _("The option --upload-pass has been deprecated in favour" - " of device authorization in RHEL") - ) - - def get_upload_url(self): - if self.upload_url: - return self.upload_url - if self.commons['cmdlineopts'].upload_url: - return self.commons['cmdlineopts'].upload_url - if self.commons['cmdlineopts'].upload_protocol == 'sftp': - return RH_SFTP_HOST - if not self.commons['cmdlineopts'].case_id: - self.ui_log.info("No case id provided, uploading to SFTP") - return RH_SFTP_HOST - rh_case_api = "/support/v1/cases/%s/attachments" - return RH_API_HOST + rh_case_api % self.case_id - - def _get_upload_https_auth(self): - str_auth = f"Bearer {self._device_token}" - return {'Authorization': str_auth} - - def _upload_https_post(self, archive, verify=True): - """If upload_https() needs to use requests.post(), use this method. - - Policies should override this method instead of the base upload_https() - - :param archive: The open archive file object - """ - files = { - 'file': (archive.name.split('/')[-1], archive, - self._get_upload_headers()) - } - # Get the access token at this point. With this, - # we cover the cases where report generation takes - # longer than the token timeout - RHELAuth = DeviceAuthorizationClass( - self.client_identifier_url, - self.token_endpoint - ) - self._device_token = RHELAuth.get_access_token() - self.ui_log.info("Device authorized correctly. Uploading file to " - f"{self.get_upload_url_string()}") - return requests.post(self.get_upload_url(), files=files, - headers=self._get_upload_https_auth(), - verify=verify, timeout=TIMEOUT_DEFAULT) - - def _get_upload_headers(self): - if self.get_upload_url().startswith(RH_API_HOST): - return {'isPrivate': 'false', 'cache-control': 'no-cache'} - return {} - - def get_upload_url_string(self): - if self.get_upload_url().startswith(RH_API_HOST): - return "Red Hat Customer Portal" - if self.get_upload_url().startswith(RH_SFTP_HOST): - return "Red Hat Secure FTP" - return self._get_obfuscated_upload_url(self.upload_url) - - def _get_sftp_upload_name(self): - """The RH SFTP server will only automatically connect file uploads to - cases if the filename _starts_ with the case number - """ - fname = self.upload_archive_name.split('/')[-1] - if self.case_id: - fname = f"{self.case_id}_{fname}" - if self.upload_directory: - fname = os.path.join(self.upload_directory, fname) - return fname - - # pylint: disable=too-many-branches - def upload_sftp(self, user=None, password=None): - """Override the base upload_sftp to allow for setting an on-demand - generated anonymous login for the RH SFTP server if a username and - password are not given - """ - if RH_SFTP_HOST.split('//')[1] not in self.get_upload_url(): - return super().upload_sftp() - - if not REQUESTS_LOADED: - raise Exception("python3-requests is not installed and is required" - " for obtaining SFTP auth token.") - _token = None - _user = None - - # We may have a device token already if we attempted - # to upload via http but the upload failed. So - # lets check first if there isn't one. - if not self._device_token: - try: - RHELAuth = DeviceAuthorizationClass( - self.client_identifier_url, - self.token_endpoint - ) - except Exception as e: - # We end up here if the user cancels the device - # authentication in the web interface - if "end user denied" in str(e): - self.ui_log.info( - "Device token authorization " - "has been cancelled by the user." - ) - else: - self._device_token = RHELAuth.get_access_token() - if self._device_token: - self.ui_log.info("Device authorized correctly. Uploading file to" - f" {self.get_upload_url_string()}") - - url = RH_API_HOST + '/support/v2/sftp/token' - ret = None - if self._device_token: - headers = self._get_upload_https_auth() - ret = requests.post(url, headers=headers, timeout=10) - if ret.status_code == 200: - # credentials are valid - _user = json.loads(ret.text)['username'] - _token = json.loads(ret.text)['token'] - else: - self.ui_log.debug( - f"DEBUG: auth attempt failed (status: {ret.status_code}): " - f"{ret.json()}" - ) - self.ui_log.error( - "Unable to retrieve Red Hat auth token using provided " - "credentials. Will try anonymous." - ) - else: - adata = {"isAnonymous": True} - anon = requests.post(url, data=json.dumps(adata), timeout=10) - if anon.status_code == 200: - resp = json.loads(anon.text) - _user = resp['username'] - _token = resp['token'] - self.ui_log.info( - _(f"User {_user} used for anonymous upload. Please inform " - f"your support engineer so they may retrieve the data.") - ) - else: - self.ui_log.debug( - f"DEBUG: anonymous request failed (status: " - f"{anon.status_code}): {anon.json()}" - ) - if _user and _token: - return super().upload_sftp(user=_user, password=_token) - raise Exception("Could not retrieve valid or anonymous credentials") - - def check_file_too_big(self, archive): - size = os.path.getsize(archive) - # Lets check if the size is bigger than the limit. - # There's really no need to transform the size to Gb, - # so we don't need to call any size converter implemented - # in tools.py - if size >= self._max_size_request: - self.ui_log.warning( - _("Size of archive is bigger than Red Hat Customer Portal " - "limit for uploads of " - f"{convert_bytes(self._max_size_request)} " - " via sos http upload. \n") - ) - self.upload_url = RH_SFTP_HOST - - def upload_archive(self, archive): - """Override the base upload_archive to provide for automatic failover - from RHCP failures to the public RH dropbox - """ - try: - if self.get_upload_url().startswith(RH_API_HOST): - self.check_file_too_big(archive) - uploaded = super().upload_archive(archive) - except Exception as e: - uploaded = False - if not self.upload_url.startswith(RH_API_HOST): - raise - self.ui_log.error( - _(f"Upload to Red Hat Customer Portal failed due to " - f"{e}. Trying {RH_SFTP_HOST}") - ) - self.upload_url = RH_SFTP_HOST - uploaded = super().upload_archive(archive) - return uploaded - def dist_version(self): try: rr = self.package_manager.all_pkgs_by_name_regex("redhat-release*") diff --git a/sos/policies/distros/ubuntu.py b/sos/policies/distros/ubuntu.py index f241f4f90f..8bc0962faa 100644 --- a/sos/policies/distros/ubuntu.py +++ b/sos/policies/distros/ubuntu.py @@ -6,7 +6,6 @@ # # See the LICENSE file in the source distribution for further information. -import os from sos.report.plugins import UbuntuPlugin from sos.policies.distros.debian import DebianPolicy @@ -66,22 +65,5 @@ def dist_version(self): except (IOError, ValueError): return False - def get_upload_https_auth(self, user=None, password=None): - if self.upload_url.startswith(self._upload_url): - return (self._upload_user, self._upload_password) - return super().get_upload_https_auth() - - def get_upload_url_string(self): - if self.upload_url.startswith(self._upload_url): - return "Canonical Support File Server" - return self._get_obfuscated_upload_url(self.get_upload_url()) - - def get_upload_url(self): - if not self.upload_url or self.upload_url.startswith(self._upload_url): - if not self.upload_archive_name: - return self._upload_url - fname = os.path.basename(self.upload_archive_name) - return self._upload_url + fname - return super().get_upload_url() # vim: set et ts=4 sw=4 : diff --git a/sos/report/__init__.py b/sos/report/__init__.py index 73f068ff47..bfed296ccd 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -37,6 +37,7 @@ CreatedFile, Alert, Note, PlainTextReport, JSONReport, HTMLReport) from sos.cleaner import SoSCleaner +from sos.upload import SoSUpload # file system errors that should terminate a run fatal_fs_errors = (errno.ENOSPC, errno.EROFS) @@ -140,6 +141,7 @@ class SoSReport(SoSComponent): 'upload_s3_access_key': None, 'upload_s3_secret_key': None, 'upload_s3_object_prefix': None, + 'upload_target': None, 'add_preset': '', 'del_preset': '' } @@ -1717,7 +1719,20 @@ def final_work(self): or self.opts.upload_s3_endpoint): if not self.opts.build: try: - self.policy.upload_archive(archive) + hook_commons = { + 'policy': self.policy, + 'tmpdir': self.tmpdir, + 'sys_tmp': self.sys_tmp, + 'options': self.opts, + 'manifest': self.manifest + } + uploader = SoSUpload(parser=self.parser, + args=self.args, + cmdline=self.cmdline, + in_place=True, + hook_commons=hook_commons, + archive=archive) + uploader.execute() self.ui_log.info(_("Uploaded archive successfully")) except Exception as err: self.ui_log.error(f"Upload attempt failed: {err}") diff --git a/sos/upload/__init__.py b/sos/upload/__init__.py new file mode 100644 index 0000000000..8f96e3dac8 --- /dev/null +++ b/sos/upload/__init__.py @@ -0,0 +1,323 @@ +# Copyright 2024 Red Hat, Inc. Jose Castillo + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import sys +import logging +import inspect + +from textwrap import fill +from sos.component import SoSComponent +from sos import _sos as _ +from sos import __version__ + + +class SoSUpload(SoSComponent): + """ + This class is designed to upload files to a distribution + defined location. These files can be either sos reports, + sos collections, or other kind of files like: vmcores, + application cores, logs, etc. + + """ + + desc = """Upload a file to a user or policy defined remote location""" + + arg_defaults = { + 'upload_file': '', + 'case_id': '', + 'low_priority': False, + 'upload_url': None, + 'upload_directory': None, + 'upload_user': None, + 'upload_pass': None, + 'upload_method': 'auto', + 'upload_no_ssl_verify': False, + 'upload_protocol': 'auto', + 'upload_s3_endpoint': 'https://s3.amazonaws.com', + 'upload_s3_region': None, + 'upload_s3_bucket': None, + 'upload_s3_access_key': None, + 'upload_s3_secret_key': None, + 'upload_s3_object_prefix': None, + 'upload_target': None + } + + def __init__(self, parser=None, args=None, cmdline=None, in_place=False, + hook_commons=None, archive=None): + if not in_place: + # we are running `sos upload` directly + super().__init__(parser, args, cmdline) + self.from_cmdline = True + else: + # we are being hooked by either SoSReport or SoSCollector, don't + # re-init everything as that will cause issues, but instead load + # the needed bits from the calling component + self.opts = hook_commons['options'] + self.policy = hook_commons['policy'] + self.manifest = hook_commons['manifest'] + self.parser = parser + self.cmdline = cmdline + self.args = args + self._upload_file = archive + + self.ui_log = logging.getLogger('sos_ui') + self.from_cmdline = False + self.archive = archive + self.upload_targets = self.load_upload_targets() + self.upload_target = None + + @classmethod + def add_parser_options(cls, parser): + parser.usage = 'sos upload FILE [options]' + upload_grp = parser.add_argument_group( + 'Upload Options', + 'These options control how upload manages files' + ) + upload_grp.add_argument("upload_file", metavar="FILE", + help="The file or archive to upload") + upload_grp.add_argument("--case-id", action="store", dest="case_id", + help="specify case identifier") + upload_grp.add_argument("--upload-url", default=None, + help="Upload the archive to specified server") + upload_grp.add_argument("--upload-user", default=None, + help="Username to authenticate with") + upload_grp.add_argument("--upload-pass", default=None, + help="Password to authenticate with") + upload_grp.add_argument("--upload-directory", action="store", + dest="upload_directory", + help="Specify upload directory for archive") + upload_grp.add_argument("--upload-s3-endpoint", default=None, + help="Endpoint to upload to for S3 bucket") + upload_grp.add_argument("--upload-s3-region", default=None, + help="Region to upload to for S3 bucket") + upload_grp.add_argument("--upload-s3-bucket", default=None, + help="Name of the S3 bucket to upload to") + upload_grp.add_argument("--upload-s3-access-key", default=None, + help="Access key for the S3 bucket") + upload_grp.add_argument("--upload-s3-secret-key", default=None, + help="Secret key for the S3 bucket") + upload_grp.add_argument("--upload-s3-object-prefix", default=None, + help="Prefix for the S3 object/key") + upload_grp.add_argument("--upload-method", default='auto', + choices=['auto', 'put', 'post'], + help="HTTP method to use for uploading") + upload_grp.add_argument("--upload-protocol", default='auto', + choices=['auto', 'https', 'ftp', 'sftp', 's3'], + help="Manually specify the upload protocol") + upload_grp.add_argument("--upload-no-ssl-verify", default=False, + action='store_true', + help="Disable SSL verification for upload url") + upload_grp.add_argument("--upload-target", default='local', + choices=[ + 'RHELUpload', + 'UbuntuUpload', + 'local'], + help=("Manually specify vendor-specific " + "target for uploads. Supported " + "options are:\n" + "RHELUpload, UbuntuUpload, local")) + + @classmethod + def display_help(cls, section): + section.set_title('SoS Upload Detailed Help') + + section.add_text( + 'The upload command is designed to upload already existing ' + 'sos reports, as well as other files like logs and vmcores ' + 'to a distribution specific location.' + ) + + def _fmt_msg(self, msg): + width = 80 + _fmt = '' + for line in msg.splitlines(): + _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' + return _fmt + + def intro(self): + """Print the intro message and prompts for a case ID if one is not + provided on the command line + """ + disclaimer = """\ +This utility is used to upload files to a target location \ +based either on a command line option or detecting the local \ +distribution. + +The archive to be uploaded may contain data considered sensitive \ +and its content should be reviewed by the originating \ +organization before being passed to any third party. + +No configuration changes will be made to the system running \ +this utility. +""" + self.ui_log.info(f"\nsos upload (version {__version__})") + intro_msg = self._fmt_msg(disclaimer) + self.ui_log.info(intro_msg) + + prompt = "\nPress ENTER to continue, or CTRL-C to quit\n" + if not self.opts.batch: + try: + input(prompt) + self.ui_log.info("") + except KeyboardInterrupt: + self._exit("Exiting on user cancel", 130) + except Exception as e: + self._exit(e, 1) + + def get_commons(self): + return { + 'cmdlineopts': self.opts, + 'policy': self.policy, + 'case_id': self.opts.case_id, + 'upload_directory': self.opts.upload_directory + } + + def set_commons(self, commons): + """Set common host data for the Upload targets + to reference + """ + self.commons = commons + + def determine_upload_target(self): + """This sets the upload target and loads that target's options. + + If no upload target is matched and no target is provided by + the user, then we abort. + + If an upload target is provided in the command line, + this will not run. + """ + checks = list(self.upload_targets.values()) + for upload_target in self.upload_targets.values(): + checks.remove(upload_target) + if upload_target.check_distribution(): + cname = upload_target.__class__.__name__ + self.ui_log.debug(f"Installation matches {cname}, checking for" + " upload targets") + self.upload_target = upload_target + self.upload_name = upload_target.name() + self.ui_log.info( + f'Upload target set to {self.upload_name}') + break + + def load_upload_targets(self): + """Loads all upload targets supported by the local installation + """ + import sos.upload.targets + supported_upload_targets = {} + for upload_target in self._load_modules(sos.upload.targets, 'targets'): + target_class = upload_target[1]( + parser=self.parser, + args=self.args, + cmdline=self.cmdline + ) + target_class.set_commons(self.get_commons()) + supported_upload_targets[upload_target[0]] = target_class + return supported_upload_targets + + @classmethod + def _load_modules(cls, package, submod): + """Helper to import upload targets""" + modules = [] + for path in package.__path__: + if os.path.isdir(path): + modules.extend(cls._find_modules_in_path(path, submod)) + return modules + + @classmethod + def _find_modules_in_path(cls, path, modulename): + """Given a path and a module name, find everything that can be imported + and then import it + + path - the filesystem path of the package + modulename - the name of the module in the package + + E.G. a path of 'targets', and a modulename of 'redhat' equates to + importing sos.upload.targets.redhat + """ + modules = [] + if os.path.exists(path): + for pyfile in sorted(os.listdir(path)): + if not pyfile.endswith('.py'): + continue + if '__' in pyfile: + continue + fname, _ = os.path.splitext(pyfile) + modname = f'sos.upload.{modulename}.{fname}' + modules.extend(cls._import_modules(modname)) + return modules + + @classmethod + def _import_modules(cls, modname): + """Import and return all found classes in a module""" + mod_short_name = modname.split('.')[2] + try: + module = __import__(modname, globals(), locals(), [mod_short_name]) + except ImportError as e: + raise e + modules = inspect.getmembers(module, inspect.isclass) + for mod in modules.copy(): + if mod[0] in ( + 'DeviceAuthorizationClass', + 'Upload', + 'RHELPolicy', + 'UbuntuPolicy'): + modules.remove(mod) + return modules + + def pre_work(self): + # This method will be called before upload begins + self.commons = self.get_commons() + cmdline_opts = self.commons['cmdlineopts'] + + if cmdline_opts.low_priority: + self.policy._configure_low_priority() + + def execute(self): + self.pre_work() + if self.from_cmdline: + self.intro() + self.archive = self.opts.upload_file + cmdline_target = self.opts.upload_target + if cmdline_target and cmdline_target != 'local': + self.upload_target = self.upload_targets[cmdline_target] + else: + self.determine_upload_target() + if not self.upload_target: + # There was no upload target set, so we'll throw + # an error here and exit + self.ui_log.error( + "No upload target set via command line options or" + " autodetected.\n" + "Please specify one using the option --upload-target.\n" + "Exiting." + ) + sys.exit(1) + self.upload_target.pre_work(self.get_commons()) + try: + if os.stat(self.archive).st_size > 0: + if os.path.isfile(self.archive): + try: + self.upload_target.upload_archive(self.archive) + self.ui_log.info( + _(f"File {self.archive} uploaded successfully") + ) + except Exception as err: + self.ui_log.error(_(f"Upload attempt failed: {err}")) + sys.exit(1) + else: + self.ui_log.error(_(f"{self.archive} is not a file.")) + else: + self.ui_log.error(_(f"File {self.archive} is empty.")) + except Exception as e: + self.ui_log.error(_(f"Cannot upload {self.archive}: {e} ")) + +# vim: set et ts=4 sw=4 : diff --git a/sos/upload/targets/__init__.py b/sos/upload/targets/__init__.py new file mode 100644 index 0000000000..074c0ce1bb --- /dev/null +++ b/sos/upload/targets/__init__.py @@ -0,0 +1,778 @@ +# Copyright 2024 Red Hat, Inc. Jose Castillo + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import re +import logging + +from getpass import getpass +from sos import _sos as _ +from sos.utilities import is_executable, TIMEOUT_DEFAULT + +try: + import requests + REQUESTS_LOADED = True +except ImportError: + REQUESTS_LOADED = False + +try: + import boto3 + BOTO3_LOADED = True +except ImportError: + BOTO3_LOADED = False + + +class Upload(): + """ + This class is designed to upload files to a distribution + defined location. These files can be either sos reports, + sos collections, or other kind of files like: vmcores, + application cores, logs, etc. + + """ + + desc = """ + Upload a file (can be a sos report, a must-gather, or others) to + a distribution defined remote location + """ + # _ prefixed class attrs are used for storing any vendor-defined defaults + # the non-prefixed attrs are used by the upload methods, and will be set + # to the cmdline/config file values, if provided. If not provided, then + # those attrs will be set to the _ prefixed values as a fallback. + # TL;DR Use _upload_* for target default values, use upload_* when wanting + # to actual use the value in a method/override + upload_target_name = None + _upload_file = None + _upload_url = None + _upload_directory = '/' + _upload_user = None + _upload_password = None + _upload_method = None + _upload_s3_endpoint = 'https://s3.amazonaws.com' + _upload_s3_bucket = None + _upload_s3_access_key = None + _upload_s3_secret_key = None + _upload_s3_region = None + _upload_s3_object_prefix = '' + upload_url = None + upload_user = None + upload_password = None + upload_s3_endpoint = None + upload_s3_bucket = None + upload_s3_access_key = None + upload_s3_secret_key = None + upload_s3_region = None + upload_s3_object_prefix = None + upload_target = None + + arg_defaults = { + 'upload_file': '', + 'case_id': '', + 'low_priority': False, + 'profiles': [], + 'upload_url': None, + 'upload_directory': None, + 'upload_user': None, + 'upload_pass': None, + 'upload_method': 'auto', + 'upload_no_ssl_verify': False, + 'upload_protocol': 'auto', + 'upload_s3_endpoint': None, + 'upload_s3_region': None, + 'upload_s3_bucket': None, + 'upload_s3_access_key': None, + 'upload_s3_secret_key': None, + 'upload_s3_object_prefix': None, + 'upload_target': None, + 'skip_plugins': [], + } + + def __init__(self, parser=None, args=None, cmdline=None): + + self.ui_log = logging.getLogger('sos_ui') + self.parser = parser + self.cmdline = cmdline + self.args = args + + @classmethod + def name(cls): + """Returns the upload target's name as a string.""" + if cls.upload_target_name: + return cls.upload_target_name + return cls.__name__.lower() + + def get_commons(self): + return { + 'cmdlineopts': self.hook_commons['cmdlineopts'], + 'policy': self.hook_commons['policy'], + 'case_id': self.hook_commons['cmdlineopts'].case_id, + 'upload_directory': self.hook_commons['cmdlineopts'] + .upload_directory + } + + def set_commons(self, commons): + """Set common host data for the Upload targets + to reference + """ + self.commons = commons + + def pre_work(self, hook_commons): + + self.hook_commons = hook_commons + self.commons = self.get_commons() + cmdline_opts = self.commons['cmdlineopts'] + policy = self.commons['policy'] + caseid = cmdline_opts.case_id if cmdline_opts.case_id else "" + + if cmdline_opts.low_priority: + policy._configure_low_priority() + + # Set the cmdline settings to the class attrs that are referenced later + # The target default '_' prefixed versions of these are untouched to + # allow fallback + self.upload_url = cmdline_opts.upload_url + self.upload_user = cmdline_opts.upload_user + self.upload_directory = cmdline_opts.upload_directory + self.upload_password = cmdline_opts.upload_pass + self.upload_archive_name = '' + + self.upload_s3_endpoint = cmdline_opts.upload_s3_endpoint + self.upload_s3_region = cmdline_opts.upload_s3_region + self.upload_s3_access_key = cmdline_opts.upload_s3_access_key + self.upload_s3_bucket = cmdline_opts.upload_s3_bucket + self.upload_s3_object_prefix = cmdline_opts.upload_s3_object_prefix + self.upload_s3_secret_key = cmdline_opts.upload_s3_secret_key + + # set or query for case id + if not cmdline_opts.batch and not \ + cmdline_opts.quiet: + if caseid: + self.commons['cmdlineopts'].case_id = caseid + else: + self.commons['cmdlineopts'].case_id = input( + _("Optionally, please enter the case id that you are " + "generating this report for [%s]: ") % caseid + ) + if cmdline_opts.case_id: + self.case_id = cmdline_opts.case_id + + # set or query for upload credentials; this needs to be done after + # setting case id, as below methods might rely on detection of it + if not cmdline_opts.batch and not \ + cmdline_opts.quiet: + # Targets will need to handle the prompts for user information + if self.get_upload_url() and \ + not cmdline_opts.upload_protocol == 's3': + self.prompt_for_upload_user() + self.prompt_for_upload_password() + elif cmdline_opts.upload_protocol == 's3': + self.prompt_for_upload_s3_bucket() + self.prompt_for_upload_s3_endpoint() + self.prompt_for_upload_s3_access_key() + self.prompt_for_upload_s3_secret_key() + self.ui_log.info('') + + def prompt_for_upload_s3_access_key(self): + """Should be overridden by targets to determine if an access key needs + to be provided for upload or not + """ + if not self.get_upload_s3_access_key(): + + msg = ( + "Please provide the upload access key for bucket" + f" {self.get_upload_s3_bucket()} via endpoint" + f" {self.get_upload_s3_endpoint()}: " + ) + self.upload_s3_access_key = input(_(msg)) + + def prompt_for_upload_s3_secret_key(self): + """Should be overridden by targets to determine if a secret key needs + to be provided for upload or not + """ + if not self.get_upload_s3_secret_key(): + msg = ( + "Please provide the upload secret key for bucket" + f" {self.get_upload_s3_bucket()} via endpoint" + f" {self.get_upload_s3_endpoint()}: " + ) + self.upload_s3_secret_key = getpass(msg) + + def prompt_for_upload_s3_bucket(self): + """Should be overridden by targets to determine if a bucket needs to + be provided for upload or not + """ + if not self.upload_s3_bucket: + if self.upload_url and self.upload_url.startswith('s3://'): + self.upload_s3_bucket = self.upload_url[5:] + else: + user_input = input(_("Please provide the upload bucket: ")) + self.upload_s3_bucket = user_input.strip('/') + return self.upload_s3_bucket + + def prompt_for_upload_s3_endpoint(self): + """Should be overridden by targets to determine if an endpoint needs + to be provided for upload or not + """ + default_endpoint = self._upload_s3_endpoint + if not self.upload_s3_endpoint: + msg = ( + "Please provide the upload endpoint for bucket" + f" {self.get_upload_s3_bucket()}" + f" (default: {default_endpoint}): " + ) + user_input = input(_(msg)) + self.upload_s3_endpoint = user_input or default_endpoint + return self.upload_s3_endpoint + + def prompt_for_upload_user(self): + """Should be overridden by targets to determine if a user needs to + be provided or not + """ + if not self.get_upload_user(): + msg = f"Please provide upload user for {self.get_upload_url()}: " + self.upload_user = input(_(msg)) + + def prompt_for_upload_password(self): + """Should be overridden by targets to determine if a password needs to + be provided for upload or not + """ + if not self.get_upload_password() and (self.get_upload_user() != + self._upload_user): + msg = ("Please provide the upload password for " + f"{self.get_upload_user()}: ") + self.upload_password = getpass(msg) + + def upload_archive(self, archive): + """ + Entry point for sos attempts to upload the generated archive to a + target or user specified location. + + Currently there is support for HTTPS, SFTP, and FTP. HTTPS uploads are + preferred for target-defined defaults. + + Targets that need to override uploading methods should override the + respective upload_https(), upload_sftp(), and/or upload_ftp() methods + and should NOT override this method. + + :param archive: The archive filepath to use for upload + :type archive: ``str`` + + In order to enable this for a target, that target needs to implement + the following: + + Required Class Attrs + + :_upload_url: The default location to use. Note these MUST include + protocol header + :_upload_user: Default username, if any else None + :_upload_password: Default password, if any else None + + The following Class Attrs may optionally be overidden by the Target + + :_upload_directory: Default FTP server directory, if any + + + The following methods may be overridden by ``Target`` as needed + + `prompt_for_upload_user()` + Determines if sos should prompt for a username or not. + + `get_upload_user()` + Determines if the default or a different username should be used + + `get_upload_https_auth()` + Format authentication data for HTTPS uploads + + `get_upload_url_string()` + Print a more human-friendly string than vendor URLs + """ + self.upload_archive_name = archive + if not self.upload_url: + self.upload_url = self.get_upload_url() + if not self.upload_url: + raise Exception("No upload destination provided by upload target" + " or by --upload-url") + upload_func = self._determine_upload_type() + self.ui_log.info( + _(f"Attempting upload to {self.get_upload_url_string()}") + ) + return upload_func() + + def _determine_upload_type(self): + """Based on the url provided, determine what type of upload to attempt. + + Note that this requires users to provide a FQDN address, such as + https://myvendor.com/api or ftp://myvendor.com instead of + myvendor.com/api or myvendor.com + """ + prots = { + 'ftp': self.upload_ftp, + 'sftp': self.upload_sftp, + 'https': self.upload_https, + 's3': self.upload_s3 + } + if self.commons['cmdlineopts'].upload_protocol in prots: + return prots[self.commons['cmdlineopts'].upload_protocol] + if '://' not in self.upload_url: + raise Exception("Must provide protocol in upload URL") + prot, _ = self.upload_url.split('://') + if prot not in prots: + raise Exception(f"Unsupported or unrecognized protocol: {prot}") + return prots[prot] + + def get_upload_https_auth(self, user=None, password=None): + """Formats the user/password credentials using basic auth + + :param user: The username for upload + :type user: ``str`` + + :param password: Password for `user` to use for upload + :type password: ``str`` + + :returns: The user/password auth suitable for use in requests calls + :rtype: ``requests.auth.HTTPBasicAuth()`` + """ + if not user: + user = self.get_upload_user() + if not password: + password = self.get_upload_password() + + return requests.auth.HTTPBasicAuth(user, password) + + def get_upload_s3_access_key(self): + """Helper function to determine if we should use the target default + upload access key or one provided by the user + + :returns: The access_key to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADS3ACCESSKEY', None) or + self.upload_s3_access_key or + self._upload_s3_access_key) + + def get_upload_s3_endpoint(self): + """Helper function to determine if we should use the target default + upload endpoint or one provided by the user + + :returns: The S3 Endpoint to use for upload + :rtype: ``str`` + """ + if not self.upload_s3_endpoint: + self.prompt_for_upload_s3_endpoint() + return self.upload_s3_endpoint + + def get_upload_s3_region(self): + """Helper function to determine if we should use the target default + upload region or one provided by the user + + :returns: The S3 region to use for upload + :rtype: ``str`` + """ + return self.upload_s3_region or self._upload_s3_region + + def get_upload_s3_bucket(self): + """Helper function to determine if we should use the target default + upload bucket or one provided by the user + + :returns: The S3 bucket to use for upload + :rtype: ``str`` + """ + if self.upload_url and self.upload_url.startswith('s3://'): + bucket_and_prefix = self.upload_url[5:].split('/', 1) + self.upload_s3_bucket = bucket_and_prefix[0] + if len(bucket_and_prefix) > 1: + self.upload_s3_object_prefix = bucket_and_prefix[1] + if not self.upload_s3_bucket: + self.prompt_for_upload_s3_bucket() + return self.upload_s3_bucket or self._upload_s3_bucket + + def get_upload_s3_object_prefix(self): + """Helper function to determine if we should use the target default + upload object prefix or one provided by the user + + :returns: The S3 object prefix to use for upload + :rtype: ``str`` + """ + return self.upload_s3_object_prefix or self._upload_s3_object_prefix + + def get_upload_s3_secret_key(self): + """Helper function to determine if we should use the target default + upload secret key or one provided by the user + + :returns: The S3 secret key to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADS3SECRETKEY', None) or + self.upload_s3_secret_key or + self._upload_s3_secret_key) + + def get_upload_url(self): + """Helper function to determine if we should use the target default + upload url or one provided by the user + + :returns: The URL to use for upload + :rtype: ``str`` + """ + if not self.upload_url and ( + self.upload_s3_bucket and + self.upload_s3_access_key and + self.upload_s3_secret_key + ): + bucket = self.get_upload_s3_bucket() + prefix = self.get_upload_s3_object_prefix() + self._upload_url = f"s3://{bucket}/{prefix}" + return self.upload_url or self._upload_url + + def _get_obfuscated_upload_url(self, url): + pattern = r"([^:]+://[^:]+:)([^@]+)(@.+)" + obfuscated_url = re.sub(pattern, r'\1********\3', url) + return obfuscated_url + + def get_upload_url_string(self): + """Used by upload targets to potentially change the string used to + report upload location from the URL to a more human-friendly string + """ + return self._get_obfuscated_upload_url(self.get_upload_url()) + + def get_upload_user(self): + """Helper function to determine if we should use the target default + upload user or one provided by the user + + :returns: The username to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADUSER', None) or + self.upload_user or + self._upload_user) + + def get_upload_password(self): + """Helper function to determine if we should use the target default + upload password or one provided by the user + + A user provided password, either via option or the 'SOSUPLOADPASSWORD' + environment variable will have precendent over any target value + + :returns: The password to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADPASSWORD', None) or + self.upload_password or + self._upload_password) + + def upload_sftp(self, user=None, password=None): + """Attempts to upload the archive to an SFTP location. + + Due to the lack of well maintained, secure, and generally widespread + python libraries for SFTP, sos will shell-out to the system's local ssh + installation in order to handle these uploads. + + Do not override this method with one that uses python-paramiko, as the + upstream sos team will reject any PR that includes that dependency. + """ + # if we somehow don't have sftp available locally, fail early + if not is_executable('sftp'): + raise Exception('SFTP is not locally supported') + + # soft dependency on python3-pexpect, which we need to use to control + # sftp login since as of this writing we don't have a viable solution + # via ssh python bindings commonly available among downstreams + try: + import pexpect + except ImportError as err: + raise Exception('SFTP upload requires python3-pexpect, which is ' + 'not currently installed') from err + + sftp_connected = False + + if not user: + user = self.get_upload_user() + if not password: + password = self.get_upload_password() + + # need to strip the protocol prefix here + sftp_url = self.get_upload_url().replace('sftp://', '') + sftp_cmd = f"sftp -oStrictHostKeyChecking=no {user}@{sftp_url}" + ret = pexpect.spawn(sftp_cmd, encoding='utf-8') + + sftp_expects = [ + 'sftp>', + 'password:', + 'Connection refused', + pexpect.TIMEOUT, + pexpect.EOF + ] + + idx = ret.expect(sftp_expects, timeout=15) + + if idx == 0: + sftp_connected = True + elif idx == 1: + ret.sendline(password) + pass_expects = [ + 'sftp>', + 'Permission denied', + pexpect.TIMEOUT, + pexpect.EOF + ] + sftp_connected = ret.expect(pass_expects, timeout=10) == 0 + if not sftp_connected: + ret.close() + raise Exception("Incorrect username or password for " + f"{self.get_upload_url_string()}") + elif idx == 2: + raise Exception("Connection refused by " + f"{self.get_upload_url_string()}. Incorrect port?") + elif idx == 3: + raise Exception("Timeout hit trying to connect to " + f"{self.get_upload_url_string()}") + elif idx == 4: + raise Exception("Unexpected error trying to connect to sftp: " + f"{ret.before}") + + if not sftp_connected: + ret.close() + raise Exception("Unable to connect via SFTP to " + f"{self.get_upload_url_string()}") + + put_cmd = (f'put {self.upload_archive_name} ' + f'{self._get_sftp_upload_name()}') + ret.sendline(put_cmd) + + put_expects = [ + '100%', + pexpect.TIMEOUT, + pexpect.EOF, + 'No such file or directory' + ] + + put_success = ret.expect(put_expects, timeout=180) + + if put_success == 0: + ret.sendline('bye') + return True + if put_success == 1: + raise Exception("Timeout expired while uploading") + if put_success == 2: + raise Exception(f"Unknown error during upload: {ret.before}") + if put_success == 3: + raise Exception("Unable to write archive to destination") + raise Exception(f"Unexpected response from server: {ret.before}") + + def _get_sftp_upload_name(self): + """If a specific file name pattern is required by the SFTP server, + override this method in the relevant Upload Target. Otherwise the + archive's name on disk will be used + + :returns: Filename as it will exist on the SFTP server + :rtype: ``str`` + """ + fname = self.upload_archive_name.split('/')[-1] + if self.upload_directory: + fname = os.path.join(self.upload_directory, fname) + return fname + + def _upload_https_put(self, archive, verify=True): + """If upload_https() needs to use requests.put(), use this method. + + Targets should override this method instead of the base upload_https() + + :param archive: The open archive file object + """ + return requests.put(self.get_upload_url(), data=archive, + auth=self.get_upload_https_auth(), + verify=verify, timeout=TIMEOUT_DEFAULT) + + def _get_upload_headers(self): + """Define any needed headers to be passed with the POST request here + """ + return {} + + def _upload_https_post(self, archive, verify=True): + """If upload_https() needs to use requests.post(), use this method. + + Targets should override this method instead of the base upload_https() + + :param archive: The open archive file object + """ + files = { + 'file': (archive.name.split('/')[-1], archive, + self._get_upload_headers()) + } + return requests.post(self.get_upload_url(), files=files, + auth=self.get_upload_https_auth(), + verify=verify, timeout=TIMEOUT_DEFAULT) + + def upload_https(self): + """Attempts to upload the archive to an HTTPS location. + + :returns: ``True`` if upload is successful + :rtype: ``bool`` + + :raises: ``Exception`` if upload was unsuccessful + """ + if not REQUESTS_LOADED: + raise Exception("Unable to upload due to missing python requests " + "library") + + with open(self.upload_archive_name, 'rb') as arc: + if self.commons['cmdlineopts'].upload_method == 'auto': + method = self._upload_method + else: + method = self.commons['cmdlineopts'].upload_method + verify = self.commons['cmdlineopts'].upload_no_ssl_verify is False + if method == 'put': + r = self._upload_https_put(arc, verify) + else: + r = self._upload_https_post(arc, verify) + if r.status_code not in (200, 201): + if r.status_code == 401: + raise Exception( + "Authentication failed: invalid user credentials" + ) + raise Exception(f"POST request returned {r.status_code}: " + f"{r.reason}") + return True + + def upload_ftp(self, url=None, directory=None, user=None, password=None): + """Attempts to upload the archive to either the target defined or user + provided FTP location. + + :param url: The URL to upload to + :type url: ``str`` + + :param directory: The directory on the FTP server to write to + :type directory: ``str`` or ``None`` + + :param user: The user to authenticate with + :type user: ``str`` + + :param password: The password to use for `user` + :type password: ``str`` + + :returns: ``True`` if upload is successful + :rtype: ``bool`` + + :raises: ``Exception`` if upload in unsuccessful + """ + import ftplib + import socket + + if not url: + url = self.get_upload_url() + if url is None: + raise Exception("no FTP server specified by upload target, " + "use --upload-url to specify a location") + + url = url.replace('ftp://', '') + + if not user: + user = self.get_upload_user() + + if not password: + password = self.get_upload_password() + + if not directory: + directory = self.upload_directory or self._upload_directory + + try: + session = ftplib.FTP(url, user, password, timeout=15) + if not session: + raise Exception("connection failed, did you set a user and " + "password?") + session.cwd(directory) + except socket.timeout as err: + raise Exception(f"timeout hit while connecting to {url}") from err + except socket.gaierror as err: + raise Exception(f"unable to connect to {url}") from err + except ftplib.error_perm as err: + errno = str(err).split()[0] + if errno == '503': + raise Exception(f"could not login as '{user}'") from err + if errno == '530': + raise Exception(f"invalid password for user '{user}'") from err + if errno == '550': + raise Exception("could not set upload directory to " + f"{directory}") from err + raise Exception(f"error trying to establish session: {str(err)}") \ + from err + + with open(self.upload_archive_name, 'rb') as _arcfile: + session.storbinary( + f"STOR {self.upload_archive_name.split('/')[-1]}", _arcfile + ) + session.quit() + return True + + def upload_s3(self, endpoint=None, region=None, bucket=None, prefix=None, + access_key=None, secret_key=None): + """Attempts to upload the archive to an S3 bucket. + + :param endpoint: The S3 endpoint to upload to + :type endpoint: str + + :param region: The S3 region to upload to + :type region: str + + :param bucket: The name of the S3 bucket to upload to + :type bucket: str + + :param prefix: The prefix for the S3 object/key + :type prefix: str + + :param access_key: The access key for the S3 bucket + :type access_key: str + + :param secret_key: The secret key for the S3 bucket + :type secret_key: str + + :returns: True if upload is successful + :rtype: bool + + :raises: Exception if upload is unsuccessful + """ + if not BOTO3_LOADED: + raise Exception("Unable to upload due to missing python boto3 " + "library") + + if not endpoint: + endpoint = self.get_upload_s3_endpoint() + if not region: + region = self.get_upload_s3_region() + + if not bucket: + bucket = self.get_upload_s3_bucket().strip('/') + + if not prefix: + prefix = self.get_upload_s3_object_prefix() + if prefix != '' and prefix.startswith('/'): + prefix = prefix[1:] + if prefix != '' and not prefix.endswith('/'): + prefix = f'{prefix}/' if prefix else '' + + if not access_key: + access_key = self.get_upload_s3_access_key() + + if not secret_key: + secret_key = self.get_upload_s3_secret_key() + + s3_client = boto3.client('s3', endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + + try: + key = prefix + self.upload_archive_name.split('/')[-1] + s3_client.upload_file(self.upload_archive_name, + bucket, key) + return True + except Exception as e: + raise Exception(f"Failed to upload to S3: {str(e)}") from e + +# vim: set et ts=4 sw=4 : diff --git a/sos/upload/targets/redhat.py b/sos/upload/targets/redhat.py new file mode 100644 index 0000000000..bbbd3cc155 --- /dev/null +++ b/sos/upload/targets/redhat.py @@ -0,0 +1,251 @@ +# Copyright 2024 Red Hat, Inc. Jose Castillo +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. +import os +import json +from sos.upload.targets import Upload +from sos.utilities import convert_bytes, TIMEOUT_DEFAULT +from sos.policies.auth import DeviceAuthorizationClass +from sos.policies.distros.redhat import RHELPolicy +from sos import _sos as _ + +try: + import requests + REQUESTS_LOADED = True +except ImportError: + REQUESTS_LOADED = False + + +class RHELUpload(Upload): + + client_identifier_url = "https://sso.redhat.com/auth/"\ + "realms/redhat-external/protocol/openid-connect/auth/device" + token_endpoint = "https://sso.redhat.com/auth/realms/"\ + "redhat-external/protocol/openid-connect/token" + upload_target_name = 'Red Hat Upload Profile' + + def __init__(self, parser=None, args=None, cmdline=None): + + super().__init__(parser=parser, args=args, cmdline=cmdline) + + RH_API_HOST = "https://api.access.redhat.com" + RH_SFTP_HOST = "sftp://sftp.access.redhat.com" + _upload_url = RH_SFTP_HOST + _upload_method = 'post' + _device_token = None + # Max size for an http single request is 1Gb + _max_size_request = 1073741824 + + def check_distribution(self): + """Return true if we are running in a RHEL system""" + return isinstance(self.commons['policy'], RHELPolicy) + + def prompt_for_upload_user(self): + if self.commons['cmdlineopts'].upload_user: + self.ui_log.info( + _("The option --upload-user has been deprecated in favour" + " of device authorization in RHEL") + ) + if not self.commons['cmdlineopts'].case_id: + # no case id provided => failover to SFTP + self.upload_url = self.RH_SFTP_HOST + self.ui_log.info("No case id provided, uploading to SFTP") + + def prompt_for_upload_password(self): + # With OIDC we don't ask for user/pass anymore + if self.commons['cmdlineopts'].upload_pass: + self.ui_log.info( + _("The option --upload-pass has been deprecated in favour" + " of device authorization in RHEL") + ) + + def get_upload_url(self): + rh_case_api = "/support/v1/cases/%s/attachments" + try: + if self.upload_url: + return self.upload_url + if self.commons['cmdlineopts'].upload_url: + return self.commons['cmdlineopts'].upload_url + if self.commons['cmdlineopts'].upload_protocol == 'sftp': + return self.RH_SFTP_HOST + if not self.commons['cmdlineopts'].case_id: + return self.RH_SFTP_HOST + + except Exception as e: + self.ui_log.info( + "There was a problem while setting the " + f"remote upload target: {e}" + ) + return self.RH_API_HOST + rh_case_api\ + % self.commons['cmdlineopts'].case_id + + def _get_upload_https_auth(self): + str_auth = f"Bearer {self._device_token}" + return {'Authorization': str_auth} + + def _upload_https_post(self, archive, verify=True): + """If upload_https() needs to use requests.post(), use this method. + + Policies should override this method instead of the base upload_https() + + :param archive: The open archive file object + """ + files = { + 'file': (archive.name.split('/')[-1], archive, + self._get_upload_headers()) + } + # Get the access token at this point. With this, + # we cover the cases where report generation takes + # longer than the token timeout + RHELAuth = DeviceAuthorizationClass( + self.client_identifier_url, + self.token_endpoint + ) + self._device_token = RHELAuth.get_access_token() + self.ui_log.info("Device authorized correctly. Uploading file to " + f"{self.get_upload_url_string()}") + return requests.post(self.get_upload_url(), files=files, + headers=self._get_upload_https_auth(), + verify=verify, timeout=TIMEOUT_DEFAULT) + + def _get_upload_headers(self): + if self.get_upload_url().startswith(self.RH_API_HOST): + return {'isPrivate': 'false', 'cache-control': 'no-cache'} + return {} + + def get_upload_url_string(self): + if self.get_upload_url().startswith(self.RH_API_HOST): + return "Red Hat Customer Portal" + if self.get_upload_url().startswith(self.RH_SFTP_HOST): + return "Red Hat Secure FTP" + return self._get_obfuscated_upload_url(self.upload_url) + + def _get_sftp_upload_name(self): + """The RH SFTP server will only automatically connect file uploads to + cases if the filename _starts_ with the case number + """ + fname = self.upload_archive_name.split('/')[-1] + + if self.commons['cmdlineopts'].case_id: + fname = f"{self.commons['cmdlineopts'].case_id}_{fname}" + if self.upload_directory: + fname = os.path.join(self.upload_directory, fname) + return fname + + # pylint: disable=too-many-branches + def upload_sftp(self, user=None, password=None): + """Override the base upload_sftp to allow for setting an on-demand + generated anonymous login for the RH SFTP server if a username and + password are not given + """ + if self.RH_SFTP_HOST.split('//')[1] not in self.get_upload_url(): + return super().upload_sftp() + + if not REQUESTS_LOADED: + raise Exception("python3-requests is not installed and is required" + " for obtaining SFTP auth token.") + _token = None + _user = None + + # We may have a device token already if we attempted + # to upload via http but the upload failed. So + # lets check first if there isn't one. + if not self._device_token: + try: + RHELAuth = DeviceAuthorizationClass( + self.client_identifier_url, + self.token_endpoint + ) + except Exception as e: + # We end up here if the user cancels the device + # authentication in the web interface + if "end user denied" in str(e): + self.ui_log.info( + "Device token authorization " + "has been cancelled by the user." + ) + else: + self._device_token = RHELAuth.get_access_token() + if self._device_token: + self.ui_log.info("Device authorized correctly. Uploading file to" + f" {self.get_upload_url_string()}") + + url = self.RH_API_HOST + '/support/v2/sftp/token' + ret = None + if self._device_token: + headers = self._get_upload_https_auth() + ret = requests.post(url, headers=headers, timeout=10) + if ret.status_code == 200: + # credentials are valid + _user = json.loads(ret.text)['username'] + _token = json.loads(ret.text)['token'] + else: + self.ui_log.debug( + f"DEBUG: auth attempt failed (status: {ret.status_code}): " + f"{ret.json()}" + ) + self.ui_log.error( + "Unable to retrieve Red Hat auth token using provided " + "credentials. Will try anonymous." + ) + else: + adata = {"isAnonymous": True} + anon = requests.post(url, data=json.dumps(adata), timeout=10) + if anon.status_code == 200: + resp = json.loads(anon.text) + _user = resp['username'] + _token = resp['token'] + self.ui_log.info( + _(f"User {_user} used for anonymous upload. Please inform " + f"your support engineer so they may retrieve the data.") + ) + else: + self.ui_log.debug( + f"DEBUG: anonymous request failed (status: " + f"{anon.status_code}): {anon.json()}" + ) + if _user and _token: + return super().upload_sftp(user=_user, password=_token) + raise Exception("Could not retrieve valid or anonymous credentials") + + def check_file_too_big(self, archive): + size = os.path.getsize(archive) + # Lets check if the size is bigger than the limit. + # There's really no need to transform the size to Gb, + # so we don't need to call any size converter implemented + # in tools.py + if size >= self._max_size_request: + self.ui_log.warning( + _("Size of archive is bigger than Red Hat Customer Portal " + "limit for uploads of " + f"{convert_bytes(self._max_size_request)} " + " via sos http upload. \n") + ) + self.upload_url = self.RH_SFTP_HOST + + def upload_archive(self, archive): + """Override the base upload_archive to provide for automatic failover + from RHCP failures to the public RH dropbox + """ + try: + if self.get_upload_url().startswith(self.RH_API_HOST): + self.check_file_too_big(archive) + uploaded = super().upload_archive(archive) + except Exception as e: + uploaded = False + if not self.upload_url.startswith(self.RH_API_HOST): + raise + self.ui_log.error( + _(f"Upload to Red Hat Customer Portal failed due to " + f"{e}. Trying {self.RH_SFTP_HOST}") + ) + self.upload_url = self.RH_SFTP_HOST + uploaded = super().upload_archive(archive) + return uploaded + +# vim: set et ts=4 sw=4 : diff --git a/sos/upload/targets/ubuntu.py b/sos/upload/targets/ubuntu.py new file mode 100644 index 0000000000..83bdd8a4d1 --- /dev/null +++ b/sos/upload/targets/ubuntu.py @@ -0,0 +1,46 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. +import os +from sos.upload.targets import Upload +from sos.policies.distros.ubuntu import UbuntuPolicy + + +class UbuntuUpload(Upload): + + upload_target_name = 'Ubuntu Upload Profile' + _upload_url = "https://files.support.canonical.com/uploads/" + _upload_user = "ubuntu" + _upload_password = "ubuntu" + _upload_method = "put" + + def __init__(self, parser=None, args=None, cmdline=None): + super().__init__(parser=parser, args=args, cmdline=cmdline) + + def check_distribution(self): + """ Return true if we are running in a Ubuntu system""" + return isinstance(self.commons['policy'], UbuntuPolicy) + + def get_upload_https_auth(self, user=None, password=None): + if self.upload_url.startswith(self._upload_url): + return (self._upload_user, self._upload_password) + return super().get_upload_https_auth() + + def get_upload_url_string(self): + if self.upload_url.startswith(self._upload_url): + return "Canonical Support File Server" + return self._get_obfuscated_upload_url(self.get_upload_url()) + + def get_upload_url(self): + if not self.upload_url or self.upload_url.startswith(self._upload_url): + if not self.upload_archive_name: + return self._upload_url + fname = os.path.basename(self.upload_archive_name) + return self._upload_url + fname + return super().get_upload_url() + +# vim: set et ts=4 sw=4 :