load data files: update hints in error msgs and ignore the download by default

repos/system_upgrade/common/actors/checkconsumedassets/libraries/check_consumed_assets.py

@@ -4,10 +4,27 @@
 from leapp import reporting
 from leapp.libraries.common.config import get_consumed_data_stream_id
 from leapp.libraries.common.fetch import ASSET_PROVIDED_DATA_STREAMS_FIELD
+from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents
 from leapp.libraries.stdlib import api
 from leapp.models import ConsumedDataAsset
 
 
+def _get_hint():
+    hint = (
+        'All official assets (data files) are part of the installed rpms these days.'
+        ' This issue is usually encountered when the data files are incorrectly'
+        ' customized, replaced, or removed. '
+        ' In case you want to recover the original files, remove them (if they still exist)'
+        ' and reinstall the following rpms: {rpms}.\n'
+        'The listed assets (data files) are usually inside the /etc/leapp/files/'
+        ' directory.'
+        .format(
+            rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY))
+        )
+    )
+    return hint
+
+
 def compose_summary_for_incompatible_assets(assets, incompatibility_reason):
     if not assets:
         return []
@@ -69,13 +86,16 @@ def report_incompatible_assets(assets):
         summary_lines += compose_summary_for_incompatible_assets(incompatible_assets, reason)
 
         for asset in incompatible_assets:
-            doc_url_to_title[asset.docs_url].append(asset.docs_title)
+            if asset.docs_url:
+                # Add URLs only when they are specified. docs_url could be empty string
+                doc_url_to_title[asset.docs_url].append(asset.docs_title)
 
     report_parts = [
         reporting.Title(title),
         reporting.Summary('\n'.join(summary_lines)),
         reporting.Severity(reporting.Severity.HIGH),
-        reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.REPOSITORY]),
+        reporting.Remediation(hint=_get_hint()),
+        reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.SANITY]),
     ]
 
     report_parts += make_report_entries_with_unique_urls(docs_url_to_title_map)
@@ -101,13 +121,16 @@ def report_malformed_assets(malformed_assets):
             details = ' - The asset file {filename} contains invalid value in its "{data_streams_field}"'
             details = details.format(filename=asset.filename, data_streams_field=ASSET_PROVIDED_DATA_STREAMS_FIELD)
         summary_lines.append(details)
-        docs_url_to_title_map[asset.docs_url].append(asset.docs_title)
+        if asset.docs_url:
+            # Add URLs only when they are specified. docs_url could be empty string
+            docs_url_to_title_map[asset.docs_url].append(asset.docs_title)
 
     report_parts = [
         reporting.Title(title),
         reporting.Summary('\n'.join(summary_lines)),
+        reporting.Remediation(hint=_get_hint()),
         reporting.Severity(reporting.Severity.HIGH),
-        reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.REPOSITORY]),
+        reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.SANITY]),
     ]
 
     report_parts += make_report_entries_with_unique_urls(docs_url_to_title_map)

repos/system_upgrade/common/actors/loaddevicedriverdeprecationdata/libraries/deviceanddriverdeprecationdataload.py

@@ -13,6 +13,9 @@ def process():
     supported_device_types = set(DeviceDriverDeprecationEntry.device_type.serialize()['choices'])
 
     data_file_name = 'device_driver_deprecation_data.json'
+    # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see
+    # the code for more info. Keeping the handling on the framework in such
+    # a case as we have no work to do in such a case here.
     deprecation_data = fetch.load_data_asset(api.current_actor(),
                                              data_file_name,
                                              asset_fulltext_name='Device driver deprecation data',

repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_event_parsing.py

@@ -8,7 +8,7 @@
 from leapp.exceptions import StopActorExecution
 from leapp.libraries.common import fetch
 from leapp.libraries.common.config import architecture
-from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version
+from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents
 from leapp.libraries.stdlib import api
 
 # NOTE(mhecko): The modulestream field contains a set of modulestreams until the very end when we generate a Package
@@ -67,6 +67,9 @@ def get_pes_events(pes_json_directory, pes_json_filename):
     :return: List of Event tuples, where each event contains event type and input/output pkgs
     """
     try:
+        # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see
+        # the code for more info. Keeping the handling on the framework in such
+        # a case as we have no work to do in such a case here.
         events_data = fetch.load_data_asset(api.current_actor(),
                                             pes_json_filename,
                                             asset_fulltext_name='PES events file',
@@ -83,22 +86,27 @@ def get_pes_events(pes_json_directory, pes_json_filename):
         events_matching_arch = [e for e in all_events if not e.architectures or arch in e.architectures]
         return events_matching_arch
     except (ValueError, KeyError):
-        rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version())
-        title = 'Missing/Invalid PES data file ({}/{})'.format(pes_json_directory, pes_json_filename)
+        local_path = os.path.join(pes_json_directory, pes_json_filename)
+        title = 'Missing/Invalid PES data file ({})'.format(local_path)
         summary = (
             'All official data files are nowadays part of the installed rpms.'
             ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed'
             ' (e.g. by custom scripts).'
-            ' In case you want to recover the original file, remove it (if still exists)'
-            ' and reinstall the {} rpm.'
-            .format(rpmname)
+        )
+        hint = (
+            ' In case you want to recover the original {lp} file, remove it (if it still exists)'
+            ' and reinstall the following rpms: {rpms}.'
+            .format(
+                lp=local_path,
+                rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY))
+            )
         )
         reporting.create_report([
             reporting.Title(title),
             reporting.Summary(summary),
+            reporting.Remediation(hint=hint),
             reporting.Severity(reporting.Severity.HIGH),
-            reporting.Groups([reporting.Groups.SANITY]),
-            reporting.Groups([reporting.Groups.INHIBITOR]),
+            reporting.Groups([reporting.Groups.SANITY, reporting.Groups.INHIBITOR]),
             reporting.RelatedResource('file', os.path.join(pes_json_directory, pes_json_filename))
         ])
         raise StopActorExecution()

repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py

@@ -4,6 +4,7 @@
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version
 from leapp.libraries.common.fetch import load_data_asset
+from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents
 from leapp.libraries.stdlib import api
 from leapp.models import PESIDRepositoryEntry, RepoMapEntry, RepositoriesMapping
 from leapp.models.fields import ModelViolationError
@@ -130,29 +131,31 @@ def load_from_dict(data):
 
 
 def _inhibit_upgrade(msg):
-    rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version())
+    local_path = os.path.join('/etc/leapp/file', REPOMAP_FILE)
     hint = (
         'All official data files are nowadays part of the installed rpms.'
         ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed'
         ' (e.g. by custom scripts).'
-        ' In case you want to recover the original file, remove it (if still exists)'
-        ' and reinstall the {} rpm.'
-        .format(rpmname)
+        ' In case you want to recover the original {lp} file, remove the current one (if it still exists)'
+        ' and reinstall the following packages: {rpms}.'
+        .format(
+            lp=local_path,
+            rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY))
+        )
     )
     raise StopActorExecutionError(msg, details={'hint': hint})
 
 
 def _read_repofile(repofile):
-    # NOTE: what about catch StopActorExecution error when the file cannot be
-    # obtained -> then check whether old_repomap file exists and in such a case
-    # inform user they have to provide the new repomap.json file (we have the
-    # warning now only which could be potentially overlooked)
+    # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see
+    # the code for more info. Keeping the handling on the framework in such
+    # a case as we have no work to do in such a case here.
     repofile_data = load_data_asset(api.current_actor(),
                                     repofile,
                                     asset_fulltext_name='Repositories mapping',
                                     docs_url='',
                                     docs_title='')
-    return repofile_data  # If the file does not contain a valid json then load_asset will do a stop actor execution
+    return repofile_data
 
 
 def scan_repositories(read_repofile_func=_read_repofile):

repos/system_upgrade/common/libraries/fetch.py

@@ -7,7 +7,7 @@
 from leapp import models
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.common.config import get_consumed_data_stream_id, get_env
-from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version
+from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents
 from leapp.libraries.stdlib import api
 
 SERVICE_HOST_DEFAULT = "https://cert.cloud.redhat.com"
@@ -16,15 +16,18 @@
 ASSET_PROVIDED_DATA_STREAMS_FIELD = 'provided_data_streams'
 
 
-def _get_hint():
-    rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version())
+def _get_hint(local_path):
     hint = (
-        'All official data files are nowadays part of the installed rpms.'
+        'All official data files are part of the installed rpms these days.'
+        ' The rpm is the only official source of the official data files for in-place upgrades.'
         ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed'
         ' (e.g. by custom scripts).'
-        ' In case you want to recover the original file, remove it (if still exists)'
-        ' and reinstall the {} rpm.'
-        .format(rpmname)
+        ' In case you want to recover the original {lp} file, remove the current one (if it still exists)'
+        ' and reinstall the following packages: {rpms}.'
+        .format(
+            lp=local_path,
+            rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY))
+        )
     )
     return hint
 
@@ -34,7 +37,8 @@ def _raise_error(local_path, details):
     If the file acquisition fails in any way, throw an informative error to stop the actor.
     """
     summary = 'Data file {lp} is missing or invalid.'.format(lp=local_path)
-    raise StopActorExecutionError(summary, details={'details': details, 'hint': _get_hint()})
+
+    raise StopActorExecutionError(summary, details={'details': details, 'hint': _get_hint(local_path)})
 
 
 def _request_data(service_path, cert, proxies, timeout=REQUEST_TIMEOUT):
@@ -145,6 +149,7 @@ def load_data_asset(actor_requesting_asset,
                     docs_title):
     """
     Load the content of the data asset with given asset_filename
+    and produce :class:`leapp.model.ConsumedDataAsset` message.
 
     :param Actor actor_requesting_asset: The actor instance requesting the asset file. It is necessary for the actor
                                          to be able to produce ConsumedDataAsset message in order for leapp to be able
@@ -154,6 +159,10 @@ def load_data_asset(actor_requesting_asset,
     :param str docs_url: Docs url to provide if an asset is malformed or outdated.
     :param str docs_title: Title of the documentation to where `docs_url` points to.
     :returns: A dict with asset contents (a parsed JSON), or None if the asset was outdated.
+    :raises StopActorExecutionError: In following cases:
+        * ConsumedDataAsset is not specified in the produces tuple of the actor_requesting_asset actor
+        * The content of the required data file is not valid JSON format
+        * The required data cannot be obtained (e.g. due to missing file)
     """
 
     # Check that the actor that is attempting to obtain the asset meets the contract to call this function
@@ -164,7 +173,7 @@ def load_data_asset(actor_requesting_asset,
         error_hint = {'hint': ('Read documentation at the following link for more information about how to retrieve '
                                'the valid file: {0}'.format(docs_url))}
     else:
-        error_hint = {'hint': _get_hint()}
+        error_hint = {'hint': _get_hint(os.path.join('/etc/leapp/files', asset_filename))}
 
     data_stream_id = get_consumed_data_stream_id()
     data_stream_major = data_stream_id.split('.', 1)[0]
@@ -174,7 +183,7 @@ def load_data_asset(actor_requesting_asset,
 
     try:
         # The asset family ID has the form (major, minor), include only `major` in the URL
-        raw_asset_contents = read_or_fetch(asset_filename, data_stream=data_stream_major)
+        raw_asset_contents = read_or_fetch(asset_filename, data_stream=data_stream_major, allow_download=False)
         asset_contents = json.loads(raw_asset_contents)
     except ValueError:
         msg = 'The {0} file (at {1}) does not contain a valid JSON object.'.format(asset_fulltext_name, asset_filename)