diff --git a/Dockerfile b/Dockerfile index 95ace119..25fb816b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN ARCH=$(uname -m) && \ http://mirror.stream.centos.org/9-stream/BaseOS/${ARCH}/os/Packages/centos-gpg-keys-9.0-24.el9.noarch.rpm && \ dnf -y install \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ - https://rpm.manageiq.org/release/18-radjabov/el9/noarch/manageiq-release-18.0-1.el9.noarch.rpm && \ + https://rpm.manageiq.org/release/19-spassky/el9/noarch/manageiq-release-19.0-1.el9.noarch.rpm && \ dnf -y --disablerepo=ubi-9-baseos-rpms swap openssl-fips-provider openssl-libs && \ dnf -y update && \ dnf -y module enable ruby:3.1 && \ diff --git a/bin/parse_requirements.rb b/bin/parse_requirements.rb new file mode 100755 index 00000000..7a47537a --- /dev/null +++ b/bin/parse_requirements.rb @@ -0,0 +1,246 @@ +#!/usr/bin/env ruby + +# This script takes the existing requirements.txt file +# and updates it with the version for our supported packages +# +# USAGE: +# +# 1. Setup environment +# +# upload bin/parse_requirements.rb and config/requirements.txt to /tmp +# source /var/lib/manageiq/venv/bin/activate +# chmod 755 parse_requirements.rb +# +# 2. Get all module requirements +# +# ./parse_requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt +# +# 3. Resolve conflicts and determine if new one is correct +# double check that the legacy ones are still needed +# +# diff {,new_}requirements.txt +# # cp new_requirements.txt requirements.txt +# +# 4. Update dev files +# +# download /tmp/requirements.txt to local machine +# create a PR with updates +# +class ParseRequirements + # this is the list of supported collections + PACKAGES = %w[ + amazon/aws/requirements.txt + ansible/netcommon/requirements.txt + ansible/utils/requirements.txt + awx/awx/requirements.txt + azure/azcollection/requirements-azure.txt + cisco/intersight/requirements.txt + community/aws/requirements.txt + community/okd/requirements.txt + community/vmware/requirements.txt + google/cloud/requirements.txt + kubernetes/core/requirements.txt + openstack/cloud/requirements.txt + ovirt/ovirt/requirements.txt + theforeman/foreman/requirements.txt + ].freeze + attr_reader :filenames, :non_modules, :final, :verbose + + # These packages are installed via rpm + def os_packages + # Leaving this as pure bash so we can run from the command line to fix issues. + @os_packages ||= + `rpm -ql $(rpm -qa | grep python3- | sort) | awk -F/ '/site-packages.*-info$/ { print $6 }' | sed 's/-[0-9].*//' | tr '_A-Z' '-a-z' | sort -u`.chomp.split + end + + def os_package_regex + @os_package_regex ||= Regexp.union(os_packages) + end + + # for test + def os_packages=(values) + @os_packages = values + @os_package_regex = nil + end + + def initialize + @filenames = [] + @non_modules = [] + + @final = {} + @verbose = false + end + + def verbose! + @verbose = true + end + + def add_target(filename) + if Dir.exist?(filename) + add_dir(filename) + elsif File.exist?(filename) + add_file(filename) + else + warn("File not found: #{filename}") + end + end + + def add_file(filename) + @filenames << filename + @non_modules << filename unless filename.include?("ansible_collections") + + self + end + + def add_dir(dirname) + dirname = dirname[0..-2] if dirname.end_with?("/") + PACKAGES.each do |package| + filename = "#{dirname}/#{package}" + if File.exist?(filename) + @filenames << filename + else + warn("NOTICE: missing #{filename}") + end + end + + self + end + + def add_line(line, mod) + lib, ver = parse_line(line) + return unless lib + + final[lib] ||= {} + (final[lib][ver] ||= []) << mod + end + + def parse + filenames.each do |filename| + mod = module_name_from_filename(filename) + File.foreach(filename, :chomp => true).each do |line| + add_line(line, mod) + end + end + + self + end + + def output + result = final.flat_map do |lib, vers| + ver, modules = consolidate_vers(vers, :lib => lib) + + "#{lib}#{ver} # #{modules.join(", ")}" + end.sort.join("\n") + + puts result + end + + private + + def module_name_from_filename(filename) + if non_modules.include?(filename) + "legacy" + else + filename.gsub(%r{.*ansible_collections/}, "") + .gsub(%r{/requirements.*}, "") + end + end + + def parse_line(line) + line.downcase! + # TODO: do we want to keep legacy comments? Only useful for our requirements.txt file + line.gsub!(/#.*/, "") + line.strip! + return if line.empty? + + # Some libraries list "python" instead of "python_version" + # Dropping since just listing the python version isn't useful + return if line.match?(/^python([ <=>]|_version)/) + + # Some libraries list version "5+" instead of ">=5" + line.gsub!(/\([0-9.]*\)\+/, '>=\1') + line.gsub!("= ", "=") + # Ignore package requirements for older version of pythons (assumption here) + return if line.match?(/python_version ?[=<]/) + + lib, ver = split_lib_ver(line) + + # NOTE: Already normalized for lowercase + # Normalize library name with dash. All these characters are treated the same. + lib.gsub!(/[-_.]+/, "-") + ver ||= "" + + # TODO: split off ;python_version in split_lib_version - evaluate it properly + return if ver.match?(/python_version *[=<]/) + + # Skip git libraries. The 'git>=.*' line from vsphere gave us problems. + return if lib.start_with?("git") + + # Defer to version requirements provided by rpm system packages. + ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/) + + [lib, ver] + end + + # ipaddress>=1.0,<=2.0;python_version<3.0 + # currently returning "ipaddress", ">=1.0,<=2.0;python_version<3.0" + # @return lib, version + def split_lib_ver(line) + # split on first space (or =) + # version can have multiple spaces + lib, ver = line.match(/([^ >=]*) ?(.*)/).captures + # azure uses ==, we are instead using >= + ver.gsub!("==", ">=") + + [lib, ver] + end + + # @return [Numeric, Numeric] + # highest, lowest for version comparison + # boolean is true if there is a conflict with the versions + def version_compare(left, right) + # due to the way zip works, we need the longer to be on the left of the split + left, right = right, left if left.split(".").length < right.split(".").length + + # reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c. + cmp = left.gsub(/^[=<>]+/, "").split(".").zip(right.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i <=> v2.to_i : acc } + + # ensure a >= b + left, right = right, left if cmp < 0 + + [left, right] + end + + # consolidate multiple versioning rules + def consolidate_vers(vers, lib: nil) + if vers.size > 1 + max_key, *all_keys = vers.keys + all_keys.each do |alt| + higher, lower = version_compare(alt, max_key) + # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0 + # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there) + warn("#{lib}: #{higher} > #{lower}") if lower != "" || verbose + vers[higher].concat(vers.delete(lower)) + max_key = higher + end + end + + ver = vers.keys.first + modules = vers[ver] + # Only display "legacy" for requirements: + # - Listed in the legacy requirements.txt + # - Not listed in any collection requirements.txt + modules.delete("legacy") if modules.size > 1 + + [ver, modules] + end +end + +# {"lib" => {ver => [module]}} +if $PROGRAM_NAME == __FILE__ + pr = ParseRequirements.new + warn("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"] + ARGV.each { |arg| pr.add_target(arg) } + pr.verbose! if ENV["VERBOSE"] + pr.parse.output +end diff --git a/config/options.yml b/config/options.yml index 59d55c29..9d019775 100644 --- a/config/options.yml +++ b/config/options.yml @@ -41,8 +41,8 @@ rpm_repository: - el9 :rpms: :kafka: !ruby/regexp /.+-3\.7.+/ - :manageiq: !ruby/regexp /.+-18\.\d\.\d-(alpha|beta|rc)?\d+(\.\d)?\.el.+/ - :manageiq-release: !ruby/regexp /.+-18\.0.+/ + :manageiq: !ruby/regexp /.+-19\.\d\.\d-(alpha|beta|rc)?\d+(\.\d)?\.el.+/ + :manageiq-release: !ruby/regexp /.+-19\.0.+/ :python-bambou: !ruby/regexp /.+-3\.1\.1.+/ :python-pylxca: !ruby/regexp /.+-2\.1\.1.+/ :python-unittest2: !ruby/regexp /.+-1\.1\.0.+/ diff --git a/config/requirements.txt b/config/requirements.txt index 194fe274..c3395c74 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -1,51 +1,85 @@ -aiohttp # vmware/vmware_rest -apache-libcloud==2.5.0 -asn1crypto==0.24.0 -azure-cli-core==2.0.35 -azure-graphrbac==0.40.0 -azure-keyvault==1.0.0 -azure-mgmt-authorization==0.51.1 -azure-mgmt-batch==5.0.1 -azure-mgmt-cdn==3.0.0 -azure-mgmt-compute==4.4.0 -azure-mgmt-containerinstance==1.4.0 -azure-mgmt-containerregistry==2.0.0 -azure-mgmt-containerservice==4.4.0 -azure-mgmt-cosmosdb==0.5.2 -azure-mgmt-devtestlabs==3.0.0 -azure-mgmt-dns==2.1.0 -azure-mgmt-hdinsight==0.1.0 -azure-mgmt-keyvault==1.1.0 -azure-mgmt-loganalytics==0.2.0 -azure-mgmt-marketplaceordering==0.1.0 -azure-mgmt-monitor==0.5.2 -azure-mgmt-network==2.3.0 -azure-mgmt-nspkg==2.0.0 -azure-mgmt-rdbms==1.4.1 -azure-mgmt-redis==5.0.0 -azure-mgmt-resource==2.1.0 -azure-mgmt-servicebus==0.5.3 -azure-mgmt-sql==0.10.0 -azure-mgmt-storage==3.1.0 -azure-mgmt-trafficmanager==0.50.0 -azure-mgmt-web==0.41.0 -azure-storage==0.35.1 -backports.ssl-match-hostname==3.5.0.1 +ansible-pylibssh>=0.2.0 # ansible/netcommon +apache-libcloud # legacy +awxkit # awx/awx +azure-cli-core>=2.34.0 # azure/azcollection +azure-common>=1.1.11 # azure/azcollection +azure-containerregistry>=1.1.0 # azure/azcollection +azure-graphrbac>=0.61.1 # azure/azcollection +azure-identity>=1.16.0 # azure/azcollection +azure-keyvault>=1.1.0 # azure/azcollection +azure-mgmt-apimanagement>=3.0.0 # azure/azcollection +azure-mgmt-authorization>=2.0.0 # azure/azcollection +azure-mgmt-automation>=1.0.0 # azure/azcollection +azure-mgmt-batch>=5.0.1 # azure/azcollection +azure-mgmt-cdn>=11.0.0 # azure/azcollection +azure-mgmt-compute>=26.1.0 # azure/azcollection +azure-mgmt-containerinstance>=9.0.0 # azure/azcollection +azure-mgmt-containerregistry>=9.1.0 # azure/azcollection +azure-mgmt-containerservice>=20.0.0 # azure/azcollection +azure-mgmt-core>=1.3.0 # azure/azcollection +azure-mgmt-cosmosdb>=6.4.0 # azure/azcollection +azure-mgmt-datafactory>=2.0.0 # azure/azcollection +azure-mgmt-datalake-store>=1.0.0 # azure/azcollection +azure-mgmt-devtestlabs>=9.0.0 # azure/azcollection +azure-mgmt-dns>=8.0.0 # azure/azcollection +azure-mgmt-eventhub>=10.1.0 # azure/azcollection +azure-mgmt-hdinsight>=9.0.0 # azure/azcollection +azure-mgmt-iothub>=2.2.0 # azure/azcollection +azure-mgmt-keyvault>=10.0.0 # azure/azcollection +azure-mgmt-loganalytics>=12.0.0 # azure/azcollection +azure-mgmt-managedservices>=6.0.0 # azure/azcollection +azure-mgmt-managementgroups>=1.0.0 # azure/azcollection +azure-mgmt-marketplaceordering>=1.1.0 # azure/azcollection +azure-mgmt-monitor>=3.0.0 # azure/azcollection +azure-mgmt-network>=19.1.0 # azure/azcollection +azure-mgmt-notificationhubs>=7.0.0 # azure/azcollection +azure-mgmt-nspkg>=2.0.0 # azure/azcollection +azure-mgmt-privatedns>=1.0.0 # azure/azcollection +azure-mgmt-rdbms>=10.0.0 # azure/azcollection +azure-mgmt-recoveryservices>=2.0.0 # azure/azcollection +azure-mgmt-recoveryservicesbackup>=3.0.0 # azure/azcollection +azure-mgmt-redis>=13.0.0 # azure/azcollection +azure-mgmt-resource>=21.1.0 # azure/azcollection +azure-mgmt-search>=8.0.0 # azure/azcollection +azure-mgmt-servicebus>=7.1.0 # azure/azcollection +azure-mgmt-sql>=3.0.1 # azure/azcollection +azure-mgmt-storage>=19.0.0 # azure/azcollection +azure-mgmt-trafficmanager>=1.0.0b1 # azure/azcollection +azure-mgmt-web>=6.1.0 # azure/azcollection +azure-nspkg>=2.0.0 # azure/azcollection +azure-storage-blob>=12.11.0 # azure/azcollection boto3>=1.18.0 # amazon/aws, community/aws botocore>=1.21.0 # amazon/aws, community/aws -deprecation>=2.0 -google-auth==1.6.2 -ipaddress==1.0.23 -monotonic==1.4 -ncclient>=0.6.3 -netaddr==0.7.19 -openstacksdk==0.23.0 -ovirt-engine-sdk-python==4.2.4 -pexpect==4.6.0 -psutil==5.6.6 # Match the version installed directly into the venv -pykerberos==1.2.1 +cryptography # cisco/intersight +deprecation # legacy +google-auth # google/cloud +google-cloud-storage # google/cloud +grpcio # ansible/netcommon +jsonpatch # kubernetes/core +jsonschema>=4.18.0 # ansible/utils +jxmlease # ansible/netcommon +kubernetes>=12.0.0 # community/okd, kubernetes/core +msrest>=0.7.1 # azure/azcollection +msrestazure>=0.6.4 # azure/azcollection +ncclient # ansible/netcommon +netaddr # ansible/netcommon, ansible/utils +openstacksdk>=0.36,<0.99.0 # openstack/cloud +ovirt-engine-sdk-python>=4.5.0 # ovirt/ovirt +ovirt-imageio # ovirt/ovirt +packaging # azure/azcollection +paramiko # ansible/netcommon +protobuf # ansible/netcommon +psutil # legacy +python-dateutil # awx/awx +pytz # awx/awx pyvmomi>=6.7.1 # community/vmware -pywinrm # general requirement -requests==2.25.1 -requests-credssp==0.1.0 -requests-kerberos==0.14.0 +pywinrm # legacy +pyyaml # theforeman/foreman +requests # google/cloud, theforeman/foreman +requests-credssp # legacy +requests-kerberos # legacy +requests-oauthlib # community/okd, kubernetes/core +requests[security] # azure/azcollection +textfsm # ansible/utils +ttp # ansible/utils +xmltodict # ansible/netcommon, ansible/utils, azure/azcollection diff --git a/rpm_spec/subpackages/manageiq-ansible-venv b/rpm_spec/subpackages/manageiq-ansible-venv index ac752c01..52c7ff4b 100644 --- a/rpm_spec/subpackages/manageiq-ansible-venv +++ b/rpm_spec/subpackages/manageiq-ansible-venv @@ -5,6 +5,10 @@ Summary: %{product_summary} Ansible Runner Virtual Environment Requires: ansible >= 1:7, ansible < 1:8 Requires: python3-virtualenv +# used by ansible-runner (pip installed) +Requires: python3-importlib-metadata +# used by azure, ncclient +Requires: python3-paramiko AutoReqProv: no %description ansible-venv diff --git a/spec/bin/parse_requirements_spec.rb b/spec/bin/parse_requirements_spec.rb new file mode 100644 index 00000000..97483e9f --- /dev/null +++ b/spec/bin/parse_requirements_spec.rb @@ -0,0 +1,47 @@ +require_relative "../../bin/parse_requirements" + +RSpec.describe ParseRequirements do + # tests just assume there is one python rpm installed + before { subject.os_packages = %w[paramiko] } + + describe "#parse_line" do + it "ignores blanks" do + expect(parse_line("")).to be_nil + expect(parse_line("# comment")).to be_nil + end + + it "parses non versions" do + expect(parse_line("a")).to eq(["a", ""]) + expect(parse_line("paramiko")).to eq(["paramiko", ""]) + end + + it "parses versions" do + expect(parse_line("a >= 5")).to eq(["a", ">=5"]) + expect(parse_line("b>= 5")).to eq(["b", ">=5"]) + end + + it "respects rpm libraries" do + expect(parse_line("a >= 5")).to eq(["a", ">=5"]) + expect(parse_line("paramiko>= 5")).to eq(["paramiko", ""]) + end + + it "convert == to >=" do + expect(parse_line("a == 5")).to eq(["a", ">=5"]) + end + end + + describe "#consolidate_vers" do + it "picks the higher comparison" do + expect(subject).to receive(:warn).with("b: >2 > >1") + expect(consolidate_vers({">1" => ["c1"], ">2" => ["legacy"]}, :lib => "b")).to eq([">2", ["c1"]]) + end + end + + def parse_line(line) + subject.send(:parse_line, line) + end + + def consolidate_vers(vers, lib: nil) + subject.send(:consolidate_vers, vers, :lib => lib) + end +end