Merge branch 'master' into uptime_tracking

.rubocop.yml

@@ -50,3 +50,11 @@ Style/OpenStructUse:
 
 RSpec/MessageChain:
   Enabled: false
+
+# We disable this check since we are stubing class methods
+# that are already tested in their respective tests.
+# This way we circumvent tight coupling between tests and
+# implementation.
+# Count: 107 offenses
+RSpec/SubjectStub:
+  Enabled: false

.rubocop_todo.yml

@@ -16,7 +16,6 @@ Layout/ArgumentAlignment:
     - 'app/models/product.rb'
     - 'engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb'
     - 'spec/factories/products.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/models/migration_engine_spec.rb'
 
 # Offense count: 3
@@ -40,7 +39,6 @@ Layout/EmptyLineAfterGuardClause:
     - 'lib/rmt/cli/repos_custom.rb'
     - 'lib/rmt/cli/smt_importer.rb'
     - 'lib/rmt/fiber_request.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -102,7 +100,6 @@ Layout/LineEndStringConcatenationIndentation:
 # Cop supports --auto-correct.
 Lint/AmbiguousOperatorPrecedence:
   Exclude:
-    - 'lib/rmt/cli/mirror.rb'
     - 'lib/rmt/cli/products.rb'
     - 'lib/rmt/cli/repos.rb'
     - 'lib/rmt/cli/repos_custom.rb'
@@ -241,15 +238,13 @@ Performance/RegexpMatch:
     - 'engines/registration_sharing/lib/registration_sharing.rb'
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
     - 'lib/rmt/cli/base.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect.
 Performance/StringInclude:
   Exclude:
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 167
 # Configuration parameters: Prefixes.
@@ -279,7 +274,6 @@ RSpec/EmptyLineAfterHook:
     - 'engines/strict_authentication/spec/requests/services_controller_spec.rb'
     - 'engines/strict_authentication/spec/requests/strict_authentication/authentication_controller_spec.rb'
     - 'spec/lib/rmt/config_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/models/product_spec.rb'
     - 'spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb'
     - 'spec/requests/api/connect/v3/systems/activations_controller_spec.rb'
@@ -321,7 +315,6 @@ RSpec/ExpectInHook:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/repos_custom_spec.rb'
     - 'spec/lib/rmt/cli/repos_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
 
 # Offense count: 3
@@ -358,11 +351,6 @@ RSpec/ImplicitSubject:
     - 'spec/models/migration_engine_spec.rb'
     - 'spec/models/product_spec.rb'
 
-# Offense count: 34
-# Configuration parameters: AssignmentOnly.
-RSpec/InstanceVariable:
-  Exclude:
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 1
 RSpec/IteratedExpectation:
@@ -430,7 +418,6 @@ RSpec/Rails/HttpStatus:
 RSpec/ReceiveCounts:
   Exclude:
     - 'spec/lib/rmt/lockfile_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/suse/connect/api_spec.rb'
 
 # Offense count: 3
@@ -454,7 +441,6 @@ RSpec/StubbedMock:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/smt_importer_spec.rb'
     - 'spec/lib/rmt/cli/systems_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
     - 'spec/support/shared_examples/cli.rb'
     - 'spec/suse/connect/api_spec.rb'

Gemfile

@@ -17,7 +17,7 @@ gem 'actionview', '~> 6.1.7'
 gem 'activemodel', '~> 6.1.7'
 gem 'activerecord', '~> 6.1.7'
 gem 'railties', '~> 6.1.7'
-gem 'repomd_parser', '~> 0.1.6'
+gem 'repomd_parser', '~> 1.1.0'
 
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'

Gemfile.lock

@@ -1,15 +1,15 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.1.7.6)
-      actionview (= 6.1.7.6)
-      activesupport (= 6.1.7.6)
+    actionpack (6.1.7.7)
+      actionview (= 6.1.7.7)
+      activesupport (= 6.1.7.7)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.1.7.6)
-      activesupport (= 6.1.7.6)
+    actionview (6.1.7.7)
+      activesupport (= 6.1.7.7)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -19,12 +19,12 @@ GEM
       activemodel (>= 4.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activemodel (6.1.7.6)
-      activesupport (= 6.1.7.6)
-    activerecord (6.1.7.6)
-      activemodel (= 6.1.7.6)
-      activesupport (= 6.1.7.6)
-    activesupport (6.1.7.6)
+    activemodel (6.1.7.7)
+      activesupport (= 6.1.7.7)
+    activerecord (6.1.7.7)
+      activemodel (= 6.1.7.7)
+      activesupport (= 6.1.7.7)
+    activesupport (6.1.7.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -38,10 +38,12 @@ GEM
     bigdecimal (3.1.6)
     builder (3.2.4)
     byebug (11.1.3)
+    bzip2-ffi (1.1.1)
+      ffi (~> 1.0)
     case_transform (0.2)
       activesupport
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     config (3.0.0)
       deep_merge (~> 1.2, >= 1.2.1)
       dry-validation (~> 1.0, >= 1.0.0)
@@ -144,7 +146,7 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     locale (2.1.3)
-    loofah (2.21.4)
+    loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     lumberjack (1.2.8)
@@ -175,7 +177,7 @@ GEM
     public_suffix (4.0.7)
     puma (5.6.8)
       nio4r (~> 2.0)
-    racc (1.7.1)
+    racc (1.7.3)
     rack (2.2.8.1)
     rack-test (2.1.0)
       rack (>= 1.3)
@@ -185,20 +187,21 @@ GEM
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.5.0)
       loofah (~> 2.19, >= 2.19.1)
-    railties (6.1.7.6)
-      actionpack (= 6.1.7.6)
-      activesupport (= 6.1.7.6)
+    railties (6.1.7.7)
+      actionpack (= 6.1.7.7)
+      activesupport (= 6.1.7.7)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rb-fsevent (0.11.0)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rdiscount (2.2.0.2)
     regexp_parser (2.6.0)
-    repomd_parser (0.1.6)
+    repomd_parser (1.1.0)
+      bzip2-ffi (~> 1.1.1)
       nokogiri (~> 1.8)
       zstd-ruby (~> 1.3, >= 1.3.5.0)
     responders (3.1.1)
@@ -314,9 +317,9 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    yard (0.9.25)
-    zeitwerk (2.6.12)
-    zstd-ruby (1.5.5.0)
+    zeitwerk (2.6.13)
+    yard (0.9.35)
+    zstd-ruby (1.5.6.1)
 
 PLATFORMS
   ruby
@@ -350,7 +353,7 @@ DEPENDENCIES
   public_suffix (< 5)
   puma (~> 5.6.2)
   railties (~> 6.1.7)
-  repomd_parser (~> 0.1.6)
+  repomd_parser (~> 1.1.0)
   responders (~> 3.1.1)
   ronn
   rspec-command (= 1.0.3)

app/controllers/api/connect/v3/systems/products_controller.rb

@@ -92,9 +92,13 @@ def check_product_service_and_repositories
 
     mandatory_repos = @product.repositories.only_enabled
     mirrored_repos = @product.repositories.only_enabled.only_fully_mirrored
+    missing_repo_ids = (mandatory_repos - mirrored_repos).pluck(:id).join(', ')
 
     unless mandatory_repos.size == mirrored_repos.size
-      fail ActionController::TranslatedError.new(N_('Not all mandatory repositories are mirrored for product %s'), @product.friendly_name)
+      # rubocop:disable Layout/LineLength
+      fail ActionController::TranslatedError.new(N_('Not all mandatory repositories are mirrored for product %s. Missing Repositories (by ids): %s. On the RMT server, the missing repositories can get enabled with: rmt-cli repos enable %s;  rmt-cli mirror'),
+                                                 @product.friendly_name, missing_repo_ids, missing_repo_ids)
+      # rubocop:enable Layout/LineLength
     end
   end
 

config/rmt.yml

@@ -27,7 +27,6 @@ scc:
 
 mirroring:
   mirror_src: false
-  verify_rpm_checksums: false
   dedup_method: hardlink
 
 http_client:

docs/smt_and_rmt.md

@@ -3,7 +3,7 @@
 RMT replaces some functionality of [SMT](https://github.com/SUSE/smt).
 The following table outlines the differences and similarities between the three tools.
 The last SUSE Linux Enterprise Server version where SMT is available is 12 SP5.
-From SUSE Linux Enterprise Server 15 onward, only RMT or SUMA is available.
+From SUSE Linux Enterprise Server 15 onward, only RMT and SUMA are available.
 
 > Note: RMT is fully maintained and receives new features, bug fixes, and perfomance improvements.
 > SMT no longer receives new features, only critical security and bug fixes.
@@ -23,8 +23,8 @@ From SUSE Linux Enterprise Server 15 onward, only RMT or SUMA is available.
 |Registration of SLE 11 systems|:heavy_check_mark:|:x:|:heavy_check_mark:|
 |Registration of openSUSE Leap 15 systems|:x:|:x:|:heavy_check_mark:|
 |Registration of non-SUSE products (RHEL, Ubuntu, etc)|:x:|:x:|:heavy_check_mark:|
-|Red Hat 7 and earlier support ([Expanded Support](https://www.suse.com/products/expandedsupport/))|:heavy_check_mark:|:x:|:heavy_check_mark:|
-|Red Hat 8 support ([Expanded Support](https://www.suse.com/products/expandedsupport/))|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
+|Red Hat 6 and earlier support (via [SUSE Liberty](https://www.suse.com/products/suse-liberty-linux/))|:heavy_check_mark:|:x:|:heavy_check_mark:|
+|Red Hat 7+ support (via [SUSE Liberty](https://www.suse.com/products/suse-liberty-linux/))|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |Support for migrating SLE 12 systems to 15|:warning:<sup>[1](#partial-migration)</sup>|:heavy_check_mark:|:heavy_check_mark:|
 |Support for migrating SLE 15 SPx systems to 15 SPx+1|:warning:<sup>[1](#partial-migration)</sup>|:heavy_check_mark:|:heavy_check_mark:|
 |Staging repositories|:heavy_check_mark:|:x:|:heavy_check_mark:|
@@ -41,7 +41,7 @@ From SUSE Linux Enterprise Server 15 onward, only RMT or SUMA is available.
 |Clean up data from repositories that are not used any longer|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |Bash completion|:x:|:heavy_check_mark:|:x:|
 |Available on [openSUSE Leap 15](https://github.com/SUSE/rmt/blob/master/docs/installation.md#installation-on-opensuse-leap-15)|:x:|:heavy_check_mark:<sup>[2](#self-support)</sup>|:heavy_check_mark: (Uyuni, #self-support)|
-|Option to [run as container](https://github.com/SUSE/rmt/blob/master/README.md#development-setup---docker-compose)|:x:|:heavy_check_mark:<sup>[2](#self-support)</sup>|:x:|
+|Option to [deploy on Kubernetes](https://documentation.suse.com/sles/html/SLES-all/cha-rmt-installation.html#sec-rmt-deploy-kubernetes)|:x:|:heavy_check_mark:|:x:|
 |Easy development setup |:x:|:heavy_check_mark:|:heavy_check_mark:|
 |100% test [coverage](https://coveralls.io/github/SUSE/rmt?branch=master)|:x:|:heavy_check_mark:|:x:|
 |[Plugin functionality](https://github.com/SUSE/rmt/blob/master/docs/PLUGINS.md)|:x:|:heavy_check_mark:|:heavy_check_mark:|

engines/scc_proxy/lib/scc_proxy/engine.rb

@@ -352,17 +352,14 @@ def authenticate_system(skip_on_duplicated: false)
         def get_system(systems)
           return nil if systems.blank?
 
-          matched_systems = systems.select { |system| system.proxy_byos && system.system_token }
-          if matched_systems.empty?
-            logger.info _('BYOS system with login \"%{login}\" authenticated without system token') %
-              { login: systems.first.login }
-            return systems.first
-          end
+          byos_systems_with_token = systems.select { |system| system.proxy_byos && system.system_token }
+
+          return systems.first if byos_systems_with_token.empty?
 
-          system = matched_systems.first
-          if matched_systems.length > 1
+          system = byos_systems_with_token.first
+          if byos_systems_with_token.length > 1
             # check for possible duplicated system_tokens
-            duplicated_system_tokens = matched_systems.group_by { |sys| sys[:system_token] }.keys
+            duplicated_system_tokens = byos_systems_with_token.group_by { |sys| sys[:system_token] }.keys
 
             if duplicated_system_tokens.length > 1
               logger.info _('BYOS system with login \"%{login}\" authenticated and duplicated due to token (system tokens %{system_tokens}) mismatch') %

lib/rmt/cli/base.rb

@@ -117,8 +117,6 @@ def logger
     @logger
   end
 
-  private
-
   def needs_path(path, writable: false)
     # expand the path to make it easier to work with
     path = File.expand_path(path)
@@ -137,6 +135,8 @@ def needs_path(path, writable: false)
     path
   end
 
+  private
+
   # Allows to have any type of multi input that you want:
   #
   # 1575 (alone)

lib/rmt/cli/clean.rb

@@ -143,7 +143,7 @@ def dangling_packages_list(repo_base_dir, repomd_file)
   end
 
   def parse_packages_data(repomd_file, repo_base_dir)
-    metadata_files = RepomdParser::RepomdXmlParser.new(repomd_file).parse
+    metadata_files = RepomdParser::RepomdXmlParser.new.parse_file(repomd_file)
 
     xml_parsers = { deltainfo: RepomdParser::DeltainfoXmlParser,
                     primary: RepomdParser::PrimaryXmlParser }
@@ -152,7 +152,7 @@ def parse_packages_data(repomd_file, repo_base_dir)
       next acc unless xml_parsers.key?(metadata.type)
 
       metadata_path = File.join(repo_base_dir, metadata.location)
-      acc << xml_parsers[metadata.type].new(metadata_path).parse
+      acc << xml_parsers[metadata.type].new.parse_file(metadata_path)
     end.flatten
   end
 

lib/rmt/cli/export.rb

@@ -29,26 +29,30 @@ def settings(path)
   REPOS
   def repos(path)
     path = needs_path(path, writable: true)
-
-    mirror = RMT::Mirror.new(mirroring_base_dir: path, logger: logger, airgap_mode: true)
+    suma_product_tree = RMT::Mirror::SumaProductTree.new(mirroring_base_dir: path, logger: logger)
 
     begin
-      mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')
+      suma_product_tree.mirror
     rescue RMT::Mirror::Exception => e
-      logger.warn(e.message)
+      logger.warn(_('Exporting SUSE Manager product tree failed: %{error_message}') % { error_message: e.message })
     end
 
     repos_file = File.join(path, 'repos.json')
     raise RMT::CLI::Error.new(_('%{file} does not exist.') % { file: repos_file }) unless File.exist?(repos_file)
 
     repos = JSON.parse(File.read(repos_file))
-    repos.each do |repo|
+    repos.each do |repo_json|
+      repo = Repository.find_by(external_url: repo_json['url'])
+
       begin
-        mirror.mirror(
-          repository_url: repo['url'],
-          local_path: Repository.make_local_path(repo['url']),
-          auth_token: repo['auth_token']
-        )
+        configuration = {
+          repository: repo,
+          logger: logger,
+          mirroring_base_dir: path,
+          mirror_sources: RMT::Config.mirror_src_files?,
+          is_airgapped: true
+        }
+        RMT::Mirror.new(**configuration).mirror_now
       rescue RMT::Mirror::Exception => e
         warn e.to_s
       end