From 85d63db0b2cba31cf388c66aeb142ba44e09a74a Mon Sep 17 00:00:00 2001
From: Jason Frey <fryguy9@gmail.com>
Date: Wed, 17 Jul 2024 16:38:12 -0400
Subject: [PATCH 01/24] Fix service ui productization symlinking

---
 lib/manageiq/rpm_build/generate_core.rb | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/manageiq/rpm_build/generate_core.rb b/lib/manageiq/rpm_build/generate_core.rb
index f2957aad..6c8a164d 100644
--- a/lib/manageiq/rpm_build/generate_core.rb
+++ b/lib/manageiq/rpm_build/generate_core.rb
@@ -53,7 +53,7 @@ def precompile_sti_loader
       end
 
       def build_service_ui
-        symlink_plugin_paths("manageiq-ui-service", ui_service_dir)
+        symlink_service_ui_productization
 
         Dir.chdir(ui_service_dir) do
           shell_cmd("yarn install") # TODO: Add --immutable once s390x doesn't change the checksums.
@@ -151,6 +151,23 @@ def symlink_plugin_paths(source_dir, target_path)
         end
       end
 
+      def symlink_service_ui_productization
+        plugin_path = plugin_paths.detect do |path|
+          path.join("manageiq-ui-service/client").exist?
+        end
+        return unless plugin_path
+
+        prod_path = plugin_path.join("manageiq-ui-service")
+
+        skin_dir = "client/skin"
+        skin_path = prod_path.join(skin_dir)
+        FileUtils.ln_s(skin_path, ui_service_dir.join(skin_dir)) if skin_path.exist?
+
+        lang_file = "client/gettext/json/supported_languages.json"
+        lang_path = prod_path.join(lang_file)
+        FileUtils.ln_s(lang_path, ui_service_dir.join(lang_file)) if lang_path.exist?
+      end
+
       def compile_locale_files
         miq_dir.glob("locale/*/*.po").each do |po|
           mo = po.dirname.join("LC_MESSAGES", "#{File.basename(po, '.po')}.mo")

From 853802cc45f04f1a4c747a7cb3417c52d4b433fb Mon Sep 17 00:00:00 2001
From: Brandon Dunne <brandondunne@hotmail.com>
Date: Wed, 21 Aug 2024 14:54:09 -0400
Subject: [PATCH 02/24] \s+ wasn't matching multiple spaces and failed to match
 the last line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The + needs to be escaped for it to match the last line

Example failure:
× rsyslog.service - System Logging Service
     Loaded: loaded (/usr/lib/systemd/system/rsyslog.service; enabled; preset: enabled)
     Active: failed (Result: exit-code) since Wed 2024-08-21 09:45:27 EDT; 18min ago
       Docs: man:rsyslogd(8)
             https://www.rsyslog.com/doc/
    Process: 37598 ExecStart=/usr/sbin/rsyslogd -n $SYSLOGD_OPTIONS (code=exited, status=1/FAILURE)
   Main PID: 37598 (code=exited, status=1/FAILURE)
        CPU: 69ms

Aug 21 09:45:27 mmim421.example.com systemd[1]: rsyslog.service: Scheduled restart job, restart counter is at 5.
Aug 21 09:45:27 mmim421.example.com systemd[1]: Stopped System Logging Service.
Aug 21 09:45:27 mmim421.example.com systemd[1]: rsyslog.service: Start request repeated too quickly.
Aug 21 09:45:27 mmim421.example.com systemd[1]: rsyslog.service: Failed with result 'exit-code'.
Aug 21 09:45:27 mmim421.example.com systemd[1]: Failed to start System Logging Service.
---
 rpm_spec/subpackages/manageiq-appliance | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rpm_spec/subpackages/manageiq-appliance b/rpm_spec/subpackages/manageiq-appliance
index 2c857063..57a41127 100644
--- a/rpm_spec/subpackages/manageiq-appliance
+++ b/rpm_spec/subpackages/manageiq-appliance
@@ -99,7 +99,7 @@ fi
 
 # Disable rsyslog duplicating systemd-journal output
 # This will comment out the multi-line module load from /etc/rsyslog.conf
-sed -i '/^module(load="imjournal"/, /^\s+StateFile="imjournal.state")/ s|^|#|' %{_sysconfdir}/rsyslog.conf
+sed -i '/^module(load="imjournal"/, /^\s\+StateFile="imjournal.state")/ s|^|#|' %{_sysconfdir}/rsyslog.conf
 if systemctl is-active --quiet rsyslog; then
   systemctl restart rsyslog
 fi

From c3be6d7714d89340dcd9810856373778ac96668b Mon Sep 17 00:00:00 2001
From: Brandon Dunne <brandondunne@hotmail.com>
Date: Thu, 22 Aug 2024 16:11:02 -0400
Subject: [PATCH 03/24] Fix already broken rsyslog on appliances

Followup to #484
If the load imklog line (first line after our commented section) is
double commented, remove the first comment from every line in the rest of
the file.  Fixes appliances that were broken before #484 due to the rest
of the file being commented out.

Example of the bad state:
# rsyslog configuration file

# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html

#### GLOBAL DIRECTIVES ####

# Where to place auxiliary files
global(workDirectory="/var/lib/rsyslog")

# Use default timestamp format
module(load="builtin:omfile" Template="RSYSLOG_TraditionalFileFormat")

#### MODULES ####

module(load="imuxsock" 	  # provides support for local system logging (e.g. via logger command)
       SysSock.Use="off") # Turn off message reception via local log socket;
			  # local messages are retrieved through imjournal now.
#module(load="imjournal" 	    # provides access to the systemd journal
#       UsePid="system" # PID nummber is retrieved as the ID of the process the journal entry originates from
#       FileCreateMode="0644" # Set the access permissions for the state file
#       StateFile="imjournal.state") # File to store the position in the journal
##module(load="imklog") # reads kernel messages (the same are read from journald)
##module(load="immark") # provides --MARK-- message capability
#
## Include all config files in /etc/rsyslog.d/
#include(file="/etc/rsyslog.d/*.conf" mode="optional")
#
## Provides UDP syslog reception
## for parameters see http://www.rsyslog.com/doc/imudp.html
##module(load="imudp") # needs to be done just once
##input(type="imudp" port="514")
#
## Provides TCP syslog reception
## for parameters see http://www.rsyslog.com/doc/imtcp.html
##module(load="imtcp") # needs to be done just once
##input(type="imtcp" port="514")
#
##### RULES ####
#
## Log all kernel messages to the console.
## Logging much else clutters up the screen.
##kern.*                                                 /dev/console
#
## Log anything (except mail) of level info or higher.
## Don't log private authentication messages!
#*.info;mail.none;authpriv.none;cron.none                /var/log/messages
#
## The authpriv file has restricted access.
#authpriv.*                                              /var/log/secure
#
## Log all the mail messages in one place.
#mail.*                                                  -/var/log/maillog
#
#
## Log cron stuff
#cron.*                                                  /var/log/cron
#
## Everybody gets emergency messages
#*.emerg                                                 :omusrmsg:*
#
## Save news errors of level crit and higher in a special file.
#uucp,news.crit                                          /var/log/spooler
#
## Save boot messages also to boot.log
#local7.*                                                /var/log/boot.log
#
#
## ### sample forwarding rule ###
##action(type="omfwd"
## # An on-disk queue is created for this action. If the remote host is
## # down, messages are spooled to disk and sent when it is up again.
##queue.filename="fwdRule1"       # unique name prefix for spool files
##queue.maxdiskspace="1g"         # 1gb space limit (use as much as possible)
##queue.saveonshutdown="on"       # save messages to disk on shutdown
##queue.type="LinkedList"         # run asynchronously
##action.resumeRetryCount="-1"    # infinite retries if host is down
## # Remote Logging (we use TCP for reliable delivery)
## # remote_host is: name/ip, e.g. 192.168.0.1, port optional e.g. 10514
##Target="remote_host" Port="XXX" Protocol="tcp")
---
 rpm_spec/subpackages/manageiq-appliance | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rpm_spec/subpackages/manageiq-appliance b/rpm_spec/subpackages/manageiq-appliance
index 57a41127..19b8e971 100644
--- a/rpm_spec/subpackages/manageiq-appliance
+++ b/rpm_spec/subpackages/manageiq-appliance
@@ -100,6 +100,7 @@ fi
 # Disable rsyslog duplicating systemd-journal output
 # This will comment out the multi-line module load from /etc/rsyslog.conf
 sed -i '/^module(load="imjournal"/, /^\s\+StateFile="imjournal.state")/ s|^|#|' %{_sysconfdir}/rsyslog.conf
+sed -i '/^##module(load="imklog")/,$ s|^#||' %{_sysconfdir}/rsyslog.conf
 if systemctl is-active --quiet rsyslog; then
   systemctl restart rsyslog
 fi

From 36aa939d0f5df7aa8a4edbce9dcb62c7ac27d90d Mon Sep 17 00:00:00 2001
From: Jason Frey <fryguy9@gmail.com>
Date: Thu, 22 Aug 2024 16:34:19 -0400
Subject: [PATCH 04/24] Merge pull request #482 from
 bdunne/allow_timestamped_release_builds

[quinteros] Allow timestamped release builds

(cherry picked from commit 7996f395cad74a3aff32e7669cd44981fefc9123)
---
 config/options.yml                 | 2 +-
 lib/manageiq/rpm_build/rpm_repo.rb | 2 +-
 spec/config/options_spec.rb        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/options.yml b/config/options.yml
index 0a0b7955..513cd0ba 100644
--- a/config/options.yml
+++ b/config/options.yml
@@ -41,7 +41,7 @@ rpm_repository:
       - el9
       :rpms:
         :kafka: !ruby/regexp /.+-3\.7.+/
-        :manageiq: !ruby/regexp /.+-19\.\d\.\d-(alpha|beta|rc)?\d(\.\d)?\.el.+/
+        :manageiq: !ruby/regexp /.+-19\.\d\.\d-(alpha|beta|rc)?\d+(\.\d)?\.el.+/
         :manageiq-release: !ruby/regexp /.+-19\.0.+/
         :python-bambou: !ruby/regexp /.+-3\.1\.1.+/
         :python-pylxca: !ruby/regexp /.+-2\.1\.1.+/
diff --git a/lib/manageiq/rpm_build/rpm_repo.rb b/lib/manageiq/rpm_build/rpm_repo.rb
index ad60269b..99def0b2 100644
--- a/lib/manageiq/rpm_build/rpm_repo.rb
+++ b/lib/manageiq/rpm_build/rpm_repo.rb
@@ -24,7 +24,7 @@ def update
           puts "Downloading required RPMs..."
           OPTIONS.rpm_repository.content.each do |branch, values|
             values[:rpms]&.each do |rpm, version_regex|
-              client.list_objects(:bucket => OPTIONS.rpm_repository.s3_api.bucket, :prefix => File.join("builds", rpm.to_s)).flat_map(&:contents).each do |object|
+              client.list_objects(:bucket => OPTIONS.rpm_repository.s3_api.bucket, :prefix => File.join("builds", rpm.to_s, "/")).flat_map(&:contents).each do |object|
                 file = object.key
                 name = File.basename(file)
                 *, target, arch, _rpm = name.split(".")
diff --git a/spec/config/options_spec.rb b/spec/config/options_spec.rb
index 19336735..3fef9fb2 100644
--- a/spec/config/options_spec.rb
+++ b/spec/config/options_spec.rb
@@ -73,7 +73,7 @@
 
         nightly_versions.each do |version|
           file = ["manageiq", suffix, version].compact.join("-") + ".el9.x86_64.rpm"
-          include_examples("expect_excluded_rpm", file, "manageiq")
+          include_examples("expect_included_rpm", file, "manageiq")
         end
       end
     end

From 9d30b59f3130c96bb4ca60db1ec129c6d5abac77 Mon Sep 17 00:00:00 2001
From: Brandon Dunne <brandondunne@hotmail.com>
Date: Wed, 28 Aug 2024 17:21:46 -0400
Subject: [PATCH 05/24] Revert "Merge pull request #458 from
 bdunne/openssl_fix_2"

This reverts commit e5f8e5fc01c2370b3ae16eb47846055deb453782, reversing
changes made to a3d5b42577749798f4f1e469fed007305dee906a.
---
 Dockerfile                           | 4 +++-
 rpm_spec/subpackages/manageiq-gemset | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a4f5f9bb..25fb816b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,10 +18,12 @@ RUN ARCH=$(uname -m) && \
     dnf -y install \
       https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
       https://rpm.manageiq.org/release/19-spassky/el9/noarch/manageiq-release-19.0-1.el9.noarch.rpm && \
+    dnf -y --disablerepo=ubi-9-baseos-rpms swap openssl-fips-provider openssl-libs && \
+    dnf -y update && \
     dnf -y module enable ruby:3.1 && \
     dnf -y module enable nodejs:18 && \
     dnf -y group install "development tools" && \
-    dnf config-manager --save --setopt=tsflags=nodocs --setopt=exclude=openssl*-3.2* && \
+    dnf config-manager --setopt=tsflags=nodocs --save && \
     dnf -y install \
       cmake \
       copr-cli \
diff --git a/rpm_spec/subpackages/manageiq-gemset b/rpm_spec/subpackages/manageiq-gemset
index beb4d113..30badfcd 100644
--- a/rpm_spec/subpackages/manageiq-gemset
+++ b/rpm_spec/subpackages/manageiq-gemset
@@ -12,7 +12,7 @@ Requires: libxml2
 Requires: libxslt
 Requires: nfs-utils
 Requires: openscap-scanner
-Requires: openssl >= 1:3.0, openssl < 1:3.2
+Requires: openssl
 
 # For Miq IPMI (gems-pending)
 Requires: OpenIPMI

From d19198c484a50000e0e6658b11f2651b813a9174 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 5 Sep 2024 19:37:58 -0400
Subject: [PATCH 06/24] Python requirements for ansible aws

---
 config/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 12427a20..e1db8fd9 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -30,8 +30,8 @@ azure-mgmt-trafficmanager==0.50.0
 azure-mgmt-web==0.41.0
 azure-storage==0.35.1
 backports.ssl-match-hostname==3.5.0.1
-boto==2.47.0
-boto3==1.6.2
+boto3>=1.18.0 # amazon/aws, community/aws
+botocore>=1.21.0 # amazon/aws, community/aws
 deprecation==2.0
 google-auth==1.6.2
 ipaddress==1.0.23

From d7850c75ed69cfd64d1e9bf7bbaa3cdee14c14f7 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 5 Sep 2024 19:35:37 -0400
Subject: [PATCH 07/24] Python requirements for vmware

adding support for vmware rest client
---
 config/requirements.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index e1db8fd9..0141e1ab 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -1,3 +1,4 @@
+aiohttp # vmware/vmware_rest
 apache-libcloud==2.5.0
 asn1crypto==0.24.0
 azure-cli-core==2.0.35
@@ -43,8 +44,9 @@ ovirt-engine-sdk-python==4.2.4
 pexpect==4.6.0
 psutil==5.6.6 # Match the version installed directly into the venv
 pykerberos==1.2.1
-pyvmomi==6.5
+pyvmomi>=6.7 # community/vmware
 pywinrm==0.3.0
 requests==2.25.1
 requests-credssp==0.1.0
 requests-kerberos==0.14.0
+git>= https://github.com/vmware/vsphere-automation-sdk-python.git # community/vmware

From 84ae74d04c75d1ffefa6055c4d8769f118552e3b Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 5 Sep 2024 20:26:14 -0400
Subject: [PATCH 08/24] ansible: ensure vmware includes json support

```
"The installed version of pyvmomi lacks JSON output support; need pyvmomi>6.7.1"
```
---
 config/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 0141e1ab..324cdd18 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -44,7 +44,7 @@ ovirt-engine-sdk-python==4.2.4
 pexpect==4.6.0
 psutil==5.6.6 # Match the version installed directly into the venv
 pykerberos==1.2.1
-pyvmomi>=6.7 # community/vmware
+pyvmomi>=6.7.1 # community/vmware
 pywinrm==0.3.0
 requests==2.25.1
 requests-credssp==0.1.0

From 2a89dfe00eaaecc4ee0d0c30b3be1c7fa6345cfd Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 5 Sep 2024 20:30:08 -0400
Subject: [PATCH 09/24] ansible: allow deprecation and pywinrm to be latest
 version

```
[WARNING]: Skipping plugin (/usr/lib/python3.9/site-
packages/ansible/plugins/connection/winrm.py), cannot load: deprecated() got an
unexpected keyword argument 'name'
```
---
 config/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 324cdd18..3b178bfc 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -33,7 +33,7 @@ azure-storage==0.35.1
 backports.ssl-match-hostname==3.5.0.1
 boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
-deprecation==2.0
+deprecation>=2.0
 google-auth==1.6.2
 ipaddress==1.0.23
 monotonic==1.4
@@ -45,7 +45,7 @@ pexpect==4.6.0
 psutil==5.6.6 # Match the version installed directly into the venv
 pykerberos==1.2.1
 pyvmomi>=6.7.1 # community/vmware
-pywinrm==0.3.0
+pywinrm # general requirement
 requests==2.25.1
 requests-credssp==0.1.0
 requests-kerberos==0.14.0

From 55ce6dfc31f8aa6c115429d3c12ebc1d4cc14827 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Fri, 6 Sep 2024 09:32:16 -0400
Subject: [PATCH 10/24] update azure keyvault to avoid parsing error

will upgrade azure version to a later one, but this alpha had a bad
file in it and isn't working correctly

Same goes for ncclient

the git should have been git+https://...
but that is currently not working for this version of ansible
---
 config/requirements.txt | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 3b178bfc..194fe274 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -3,7 +3,7 @@ apache-libcloud==2.5.0
 asn1crypto==0.24.0
 azure-cli-core==2.0.35
 azure-graphrbac==0.40.0
-azure-keyvault==1.0.0a1
+azure-keyvault==1.0.0
 azure-mgmt-authorization==0.51.1
 azure-mgmt-batch==5.0.1
 azure-mgmt-cdn==3.0.0
@@ -37,7 +37,7 @@ deprecation>=2.0
 google-auth==1.6.2
 ipaddress==1.0.23
 monotonic==1.4
-ncclient==0.6.3
+ncclient>=0.6.3
 netaddr==0.7.19
 openstacksdk==0.23.0
 ovirt-engine-sdk-python==4.2.4
@@ -49,4 +49,3 @@ pywinrm # general requirement
 requests==2.25.1
 requests-credssp==0.1.0
 requests-kerberos==0.14.0
-git>= https://github.com/vmware/vsphere-automation-sdk-python.git # community/vmware

From f991adf18d2fac52f2318a9972f7d08908d9cfc7 Mon Sep 17 00:00:00 2001
From: Adam Grare <adam@grare.com>
Date: Fri, 13 Sep 2024 11:53:43 -0400
Subject: [PATCH 11/24] Add manageiq_podman_cleanup oneshot service

---
 rpm_spec/subpackages/manageiq-appliance     | 4 ++++
 rpm_spec/subpackages/manageiq-core-services | 1 +
 2 files changed, 5 insertions(+)

diff --git a/rpm_spec/subpackages/manageiq-appliance b/rpm_spec/subpackages/manageiq-appliance
index 19b8e971..0ace8fed 100644
--- a/rpm_spec/subpackages/manageiq-appliance
+++ b/rpm_spec/subpackages/manageiq-appliance
@@ -97,6 +97,9 @@ if systemctl is-active --quiet evm-failover-monitor; then
   systemctl restart evm-failover-monitor
 fi
 
+# Enable the manageiq-podman-cleanup oneshot script on boot
+systemctl enable manageiq-podman-cleanup
+
 # Disable rsyslog duplicating systemd-journal output
 # This will comment out the multi-line module load from /etc/rsyslog.conf
 sed -i '/^module(load="imjournal"/, /^\s\+StateFile="imjournal.state")/ s|^|#|' %{_sysconfdir}/rsyslog.conf
@@ -117,6 +120,7 @@ fi
 %{_unitdir}/evmserverd.service
 %{_unitdir}/manageiq-db-ready.service
 %{_unitdir}/manageiq-messaging-ready.service
+%{_unitdir}/manageiq-podman-cleanup.service
 %{_unitdir}/miqtop.service
 %{_unitdir}/miqvmstat.service
 %{manifest_root}/BUILD_APPLIANCE
diff --git a/rpm_spec/subpackages/manageiq-core-services b/rpm_spec/subpackages/manageiq-core-services
index 29c0e1ae..67b76372 100644
--- a/rpm_spec/subpackages/manageiq-core-services
+++ b/rpm_spec/subpackages/manageiq-core-services
@@ -22,4 +22,5 @@ done
 %{_prefix}/lib/systemd/system/manageiq*
 %exclude %{_prefix}/lib/systemd/system/manageiq-db-ready.service
 %exclude %{_prefix}/lib/systemd/system/manageiq-messaging-ready.service
+%exclude %{_prefix}/lib/systemd/system/manageiq-podman-cleanup.service
 %exclude %{_prefix}/lib/systemd/system/manageiq-providers*

From 94c1370b160006a8e0c9c834c607709de9b2a862 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 12 Sep 2024 15:58:31 -0400
Subject: [PATCH 12/24] Update ansible versions

- adding comments for the library that requested this library/version
- cleared out versions for legacy libraries (and removed some)

The following libraries have changed (and the reason).
Most are "legacy", meaning they were in previous requirements.txt but no
current ansible modules reference them.

A few were kept as they were helpful for general ansible modules

apache-libcloud==2.5.0    # remove version (not sure used at all)
asn1crypto==0.24.0        # remove
azure-storage             # remove. Now called azure-storage-blob
deprecation               # no version (not sure used at all)
backports.ssl-match-hostname==3.5.0.1 # remove
ipaddress==1.0.23         # remove (only for python < 3)
monotonic==1.4            # remove (part of human friendly, and for python < 3)
pexpect==4.6.0            # remove version
psutil==5.6.6             # remove version
pykerberos==1.2.1         # remove
requests-credssp==0.1.0   # remove version
requests-kerberos==0.14.0 # remove version
requests                  # remove version (provided by platform)
requests[system]          # remove version (provided by platform)

pyspnego[kerberos]>=0.8.0 # remove version (provided by platform) For AD.
pyvmomi                   # requirements are 6.7, forcing 6.7.1 for json support

pyspnego[kerberos] is used by microsoft/ad
It caused all sorts of issues installing gssapi
Updating ensuring krb5-del is available

did reference https://github.com/ManageIQ/manageiq-rpm_build/commit/880aa2c1ea7d5b876a3cd356a39bd595a62bd822
This listed the reasoning for a lot of previously included python packages
---
 config/requirements.txt | 130 +++++++++++++++++++++++++---------------
 1 file changed, 83 insertions(+), 47 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 194fe274..12cbad99 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -1,51 +1,87 @@
 aiohttp # vmware/vmware_rest
-apache-libcloud==2.5.0
-asn1crypto==0.24.0
-azure-cli-core==2.0.35
-azure-graphrbac==0.40.0
-azure-keyvault==1.0.0
-azure-mgmt-authorization==0.51.1
-azure-mgmt-batch==5.0.1
-azure-mgmt-cdn==3.0.0
-azure-mgmt-compute==4.4.0
-azure-mgmt-containerinstance==1.4.0
-azure-mgmt-containerregistry==2.0.0
-azure-mgmt-containerservice==4.4.0
-azure-mgmt-cosmosdb==0.5.2
-azure-mgmt-devtestlabs==3.0.0
-azure-mgmt-dns==2.1.0
-azure-mgmt-hdinsight==0.1.0
-azure-mgmt-keyvault==1.1.0
-azure-mgmt-loganalytics==0.2.0
-azure-mgmt-marketplaceordering==0.1.0
-azure-mgmt-monitor==0.5.2
-azure-mgmt-network==2.3.0
-azure-mgmt-nspkg==2.0.0
-azure-mgmt-rdbms==1.4.1
-azure-mgmt-redis==5.0.0
-azure-mgmt-resource==2.1.0
-azure-mgmt-servicebus==0.5.3
-azure-mgmt-sql==0.10.0
-azure-mgmt-storage==3.1.0
-azure-mgmt-trafficmanager==0.50.0
-azure-mgmt-web==0.41.0
-azure-storage==0.35.1
-backports.ssl-match-hostname==3.5.0.1
+ansible-pylibssh>=0.2.0 # ansible/netcommon
+apache-libcloud # legacy
+awxkit # awx/awx
+azure-cli-core==2.34.0 # azure/azcollection
+azure-common==1.1.11 # azure/azcollection
+azure-containerregistry==1.1.0 # azure/azcollection
+azure-graphrbac==0.61.1 # azure/azcollection
+azure-identity==1.7.0 # azure/azcollection
+azure-keyvault==1.1.0 # azure/azcollection
+azure-mgmt-apimanagement==3.0.0 # azure/azcollection
+azure-mgmt-authorization==2.0.0 # azure/azcollection
+azure-mgmt-automation==1.0.0 # azure/azcollection
+azure-mgmt-batch==5.0.1 # azure/azcollection
+azure-mgmt-cdn==11.0.0 # azure/azcollection
+azure-mgmt-compute==26.1.0 # azure/azcollection
+azure-mgmt-containerinstance==9.0.0 # azure/azcollection
+azure-mgmt-containerregistry==9.1.0 # azure/azcollection
+azure-mgmt-containerservice==20.0.0 # azure/azcollection
+azure-mgmt-core==1.3.0 # azure/azcollection
+azure-mgmt-cosmosdb==6.4.0 # azure/azcollection
+azure-mgmt-datafactory==2.0.0 # azure/azcollection
+azure-mgmt-datalake-store==1.0.0 # azure/azcollection
+azure-mgmt-devtestlabs==9.0.0 # azure/azcollection
+azure-mgmt-dns==8.0.0 # azure/azcollection
+azure-mgmt-eventhub==10.1.0 # azure/azcollection
+azure-mgmt-hdinsight==9.0.0 # azure/azcollection
+azure-mgmt-iothub==2.2.0 # azure/azcollection
+azure-mgmt-keyvault==10.0.0 # azure/azcollection
+azure-mgmt-loganalytics==12.0.0 # azure/azcollection
+azure-mgmt-managedservices==6.0.0 # azure/azcollection
+azure-mgmt-managementgroups==1.0.0 # azure/azcollection
+azure-mgmt-marketplaceordering==1.1.0 # azure/azcollection
+azure-mgmt-monitor==3.0.0 # azure/azcollection
+azure-mgmt-network==19.1.0 # azure/azcollection
+azure-mgmt-notificationhubs==7.0.0 # azure/azcollection
+azure-mgmt-nspkg==2.0.0 # azure/azcollection
+azure-mgmt-privatedns==1.0.0 # azure/azcollection
+azure-mgmt-rdbms==10.0.0 # azure/azcollection
+azure-mgmt-recoveryservices==2.0.0 # azure/azcollection
+azure-mgmt-recoveryservicesbackup==3.0.0 # azure/azcollection
+azure-mgmt-redis==13.0.0 # azure/azcollection
+azure-mgmt-resource==21.1.0 # azure/azcollection
+azure-mgmt-search==8.0.0 # azure/azcollection
+azure-mgmt-servicebus==7.1.0 # azure/azcollection
+azure-mgmt-sql==3.0.1 # azure/azcollection
+azure-mgmt-storage==19.0.0 # azure/azcollection
+azure-mgmt-trafficmanager==1.0.0b1 # azure/azcollection
+azure-mgmt-web==6.1.0 # azure/azcollection
+azure-nspkg==2.0.0 # azure/azcollection
+azure-storage-blob==12.11.0 # azure/azcollection
 boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
-deprecation>=2.0
-google-auth==1.6.2
-ipaddress==1.0.23
-monotonic==1.4
-ncclient>=0.6.3
-netaddr==0.7.19
-openstacksdk==0.23.0
-ovirt-engine-sdk-python==4.2.4
-pexpect==4.6.0
-psutil==5.6.6 # Match the version installed directly into the venv
-pykerberos==1.2.1
+cryptography>=36.0.0 # cisco/intersight
+deprecation # legacy
+google-auth==1.6.2 # google/cloud
+google-cloud-storage # google/cloud
+grpcio # ansible/netcommon
+jsonpatch # kubernetes/core
+jsonschema # ansible/utils
+jxmlease # ansible/netcommon
+kubernetes>=12.0.0 # community/okd, kubernetes/core
+msrest==0.7.1 # azure/azcollection
+msrestazure==0.6.4 # azure/azcollection
+ncclient>=0.6.3 # ansible/netcommon
+netaddr==0.7.19 # ansible/netcommon, ansible/utils
+openstacksdk>=0.36,<0.99.0 # openstack/cloud
+ovirt-engine-sdk-python>=4.5.0 # ovirt/ovirt
+ovirt-imageio # ovirt/ovirt
+packaging # azure/azcollection
+paramiko==2.8.1 # ansible/netcommon
+pexpect # legacy
+protobuf # ansible/netcommon
+psutil # legacy
+python-dateutil>=2.7.0 # awx/awx
+pytz # awx/awx
 pyvmomi>=6.7.1 # community/vmware
-pywinrm # general requirement
-requests==2.25.1
-requests-credssp==0.1.0
-requests-kerberos==0.14.0
+pywinrm # legacy
+pyyaml # theforeman/foreman
+requests # google/cloud, theforeman/foreman
+requests-credssp # legacy
+requests-kerberos # legacy
+requests-oauthlib # community/okd, kubernetes/core
+requests[security] # azure/azcollection
+textfsm # ansible/utils
+ttp # ansible/utils
+xmltodict # ansible/netcommon, ansible/utils, azure/azcollection

From ea41304bc7e81cd4f9b40eb251223d8e7fb616f9 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Fri, 13 Sep 2024 15:18:21 -0400
Subject: [PATCH 13/24] Add requirements.rb to parse/update requirements.txt

---
 bin/requirements.rb | 207 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 207 insertions(+)
 create mode 100755 bin/requirements.rb

diff --git a/bin/requirements.rb b/bin/requirements.rb
new file mode 100755
index 00000000..54969dd7
--- /dev/null
+++ b/bin/requirements.rb
@@ -0,0 +1,207 @@
+#!/usr/bin/env ruby
+
+# This script takes the existing requirements.txt file
+# and updates it with the version for our supported packages
+#
+# USAGE:
+# 1. Setup environment
+#
+#     source /var/lib/manageiq/venv/bin/activate
+#     upload bin/requirements.rb and config/requirements.txt to /tmp
+#     chmod 755 requirements.rb
+#
+# 1. Get all module requirements (don't include documentation or testing ones)
+#
+#     ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
+#
+# 2. Resolve conflicts and determine if new one is correct
+#
+#     diff {,new_}requirements.txt
+#     # cp new_requirements.txt requirements.txt
+#
+
+class ParseRequirements
+  PACKAGES=%w[
+    amazon/aws/requirements.txt
+    ansible/netcommon/requirements.txt
+    ansible/utils/requirements.txt
+    awx/awx/requirements.txt
+    azure/azcollection/requirements-azure.txt
+    cisco/intersight/requirements.txt
+    community/aws/requirements.txt
+    community/okd/requirements.txt
+    community/vmware/requirements.txt
+    google/cloud/requirements.txt
+    kubernetes/core/requirements.txt
+    openstack/cloud/requirements.txt
+    ovirt/ovirt/requirements.txt
+    theforeman/foreman/requirements.txt
+    vmware/vmware_rest/requirements.txt
+  ]
+  attr_reader :filenames, :non_modules, :final
+
+  def initialize
+    @filenames = []
+    @non_modules = []
+
+    @final = {}
+  end
+
+  def add_target(filename)
+    if Dir.exist?(filename)
+      add_dir(filename)
+    elsif File.exist?(filename)
+      add_file(filename)
+    else
+      $stderr.puts("File not found: #{filename}")
+    end
+  end
+
+  def add_file(filename)
+    @filenames << filename
+    @non_modules << filename unless filename.include?("ansible_collections")
+
+    self
+  end
+
+  def add_dir(dirname)
+    dirname = dirname[0..-2] if dirname.end_with?("/")
+    PACKAGES.each do |package|
+      filename = "#{dirname}/#{package}"
+      if File.exist?(filename)
+        @filenames << filename
+      else
+        $stderr.puts("NOTICE: missing #{filename}")
+      end
+    end
+
+    self
+  end
+
+  def parse
+    filenames.each do |fn|
+      # the list of requirements-files can have items commented out - ignore those
+
+      mod = module_name_from_filename(fn)
+      IO.foreach(fn, chomp: true).each do |line|
+        lib, ver = parse_line(line)
+        next unless lib
+
+        # skip git libraries. git>= line from vsphere gave us problems
+        next if lib.start_with?("git")
+
+        # Do not version packages that are provided by the system: requests, requests[security], pyspnego[kerberos]
+        ver = "" if lib.match?(/^requests($|\[)/) || lib.start_with?("pyspnego")
+
+        final[lib] ||= {}
+        (final[lib][ver] ||= []) << mod
+      end
+    end
+
+    self
+  end
+
+  def output
+    result = final.flat_map do |lib, vers|
+      # consolidate multiple versioning rules
+      if vers.size > 1
+        max_key, *all_keys = vers.keys
+        all_keys.each do |alt|
+          higher, lower, conflict = version_compare(alt, max_key)
+          # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
+          # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
+          $stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != ""
+          vers[higher].concat(vers.delete(lower))
+          max_key = higher
+        end
+      end
+
+      vers.map do |(ver, modules)|
+        # if we pass in a previous requirements.txt, lets not mention it
+        # exception: display if it is only mentioned in a previous requirements.txt file
+        modules.delete("legacy") if modules.size > 1
+        "#{lib}#{ver} # #{modules.join(", ")}"
+      end
+    end.sort.join("\n")
+
+    puts result
+  end
+
+  private
+
+  def module_name_from_filename(fn)
+    if non_modules.include?(fn)
+      "legacy"
+    else
+      fn.gsub(%r{.*ansible_collections/}, "")
+        .gsub(%r{/requirements.*}, "")
+    end
+  end
+
+  def parse_line(line)
+    line.downcase!
+    # TODO: do we want to keep legacy comments? Only useful for our requirements.txt file
+    line.gsub!(/#.*/, "")
+    line.strip!
+    return if line.empty?
+
+    # Some libraries list "python" instead of "python_version"
+    # Dropping since just listing the python version isn't useful
+    return if line.match?(/^python([ <=>]|_version)/)
+
+    # Some libraries list version "5+" instead of ">=5"
+    line.gsub!(/\([0-9.]*\)\+/, '>=\1')
+    line.gsub!("= ", "=")
+    # Ignore package requirements for older version of pythons (assumption here)
+    return if line.match?(/python_version ?[=<]/)
+
+    lib, ver = split_lib_ver(line)
+
+    # Note: Already normalized for lowercase
+    # Normalize library name with dash. All these characters are treated the same.
+    lib.gsub!(/[-_.]+/, "-")
+    ver ||= ""
+
+   # TODO: split off ;python_version in split_lib_version - evaluate it properly
+   return if ver.match?(/python_version *[=<]/)
+
+    [lib, ver]
+  end
+
+  # ipaddress>=1.0,<=2.0;python_version<3.0
+  # currently returning  "ipaddress", ">=1.0,<=2.0;python_version<3.0"
+  # @return lib, version
+  def split_lib_ver(line)
+    # split on first space (or =)
+    # version can have multiple spaces
+    lib, ver = line.match(/([^ >=]*) ?(.*)/).captures
+
+    [lib, ver]
+  end
+
+  # @return [Numeric, Numeric, Boolean]
+  #   highest, lowest for version comparison
+  #   boolean is true if there is a conflict with the versions
+  def version_compare(a, b)
+    winner = a if a.start_with?("==")
+    winner = b if b.start_with?("==")
+    # due to the way zip works, we need the longer to be on the left of the split
+    a, b = b, a if a.split(".").length < b.split(".").length
+
+    # when comparing, drop off the >= or == stuff, just look at the numbers
+    # kinda assuming that we are dealing mostly with >=
+    # reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c.
+    cmp = a.gsub(/^[=<>]+/, "").split(".").zip(b.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i<=>v2.to_i : acc }
+
+    # ensure a >= b
+    a, b = b, a if cmp < 0
+
+    [a, b, winner && winner != a]
+  end
+end
+
+# {"lib" => {ver => [module]}}
+
+pr = ParseRequirements.new
+ARGV.each { |arg| pr.add_target(arg) }
+pr.parse.output

From 5ae796d8e051a5796a85beb6d3d105c9617e5e00 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Fri, 6 Sep 2024 10:20:44 -0400
Subject: [PATCH 14/24] remove aiohttp

This was added to support all possible vmware clients
But we don't use this client, none of our customers requested it
and it looks to have security issues
---
 bin/requirements.rb     | 1 -
 config/requirements.txt | 1 -
 2 files changed, 2 deletions(-)

diff --git a/bin/requirements.rb b/bin/requirements.rb
index 54969dd7..2e70dc89 100755
--- a/bin/requirements.rb
+++ b/bin/requirements.rb
@@ -36,7 +36,6 @@ class ParseRequirements
     openstack/cloud/requirements.txt
     ovirt/ovirt/requirements.txt
     theforeman/foreman/requirements.txt
-    vmware/vmware_rest/requirements.txt
   ]
   attr_reader :filenames, :non_modules, :final
 
diff --git a/config/requirements.txt b/config/requirements.txt
index 12cbad99..3c24d685 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -1,4 +1,3 @@
-aiohttp # vmware/vmware_rest
 ansible-pylibssh>=0.2.0 # ansible/netcommon
 apache-libcloud # legacy
 awxkit # awx/awx

From 754c19f899c9f95cb74d7c3a7bc0ceb4217bf9ae Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Wed, 25 Sep 2024 13:04:46 -0400
Subject: [PATCH 15/24] Upgrade requirements.rb for system packages

---
 bin/requirements.rb | 55 +++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 14 deletions(-)

diff --git a/bin/requirements.rb b/bin/requirements.rb
index 54969dd7..a3a1d294 100755
--- a/bin/requirements.rb
+++ b/bin/requirements.rb
@@ -4,23 +4,40 @@
 # and updates it with the version for our supported packages
 #
 # USAGE:
+#
 # 1. Setup environment
 #
-#     source /var/lib/manageiq/venv/bin/activate
 #     upload bin/requirements.rb and config/requirements.txt to /tmp
+#     source /var/lib/manageiq/venv/bin/activate
 #     chmod 755 requirements.rb
 #
-# 1. Get all module requirements (don't include documentation or testing ones)
+# 2. Determine all python packages provided by rpms (and compare with OS_PACKAGES)
+#
+#     TODO: may want to grep rpm contents with 'info$' to determine elgibility.
+#
+#     for pkg in $(rpm -qa | grep python3-) ; do echo "### $pkg" ; rpm -ql $pkg | awk  -F/ '/site-packages/ { print $6 }' | sort -u ; done
+#
+# 3. Get all module requirements (don't include documentation or testing ones)
 #
 #     ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
 #
-# 2. Resolve conflicts and determine if new one is correct
+# 4. Resolve conflicts and determine if new one is correct
 #
 #     diff {,new_}requirements.txt
 #     # cp new_requirements.txt requirements.txt
 #
-
+# 5. Update dev riles
+#
+#     download /tmp/requirements{.rb,.txt} to local machine
+#     create a PR with updates
+#
 class ParseRequirements
+  # this is the list of packages provided by rpms
+  # TODO: paramiko
+  OS_PACKAGES=%w[
+    six dateutil iniparse idna setuptools inotify libcomps chardet decorator pysocks urllib3 requests
+    cloud-what systemd dbus gobject gpg pyspnego
+  ]
   PACKAGES=%w[
     amazon/aws/requirements.txt
     ansible/netcommon/requirements.txt
@@ -38,13 +55,18 @@ class ParseRequirements
     theforeman/foreman/requirements.txt
     vmware/vmware_rest/requirements.txt
   ]
-  attr_reader :filenames, :non_modules, :final
+  attr_reader :filenames, :non_modules, :final, :verbose
 
   def initialize
     @filenames = []
     @non_modules = []
 
     @final = {}
+    @verbose = false
+  end
+
+  def verbose!
+    @verbose = true
   end
 
   def add_target(filename)
@@ -90,8 +112,8 @@ def parse
         # skip git libraries. git>= line from vsphere gave us problems
         next if lib.start_with?("git")
 
-        # Do not version packages that are provided by the system: requests, requests[security], pyspnego[kerberos]
-        ver = "" if lib.match?(/^requests($|\[)/) || lib.start_with?("pyspnego")
+        # system packages are versioned by rpms
+        ver = "" if lib.match?(/^(#{OS_PACKAGES.join("|")})($|\[)/)
 
         final[lib] ||= {}
         (final[lib][ver] ||= []) << mod
@@ -110,18 +132,22 @@ def output
           higher, lower, conflict = version_compare(alt, max_key)
           # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
           # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
-          $stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != ""
+          $stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != "" || verbose
           vers[higher].concat(vers.delete(lower))
           max_key = higher
         end
       end
 
-      vers.map do |(ver, modules)|
-        # if we pass in a previous requirements.txt, lets not mention it
-        # exception: display if it is only mentioned in a previous requirements.txt file
-        modules.delete("legacy") if modules.size > 1
-        "#{lib}#{ver} # #{modules.join(", ")}"
-      end
+      ver = vers.keys.first
+      modules = vers[ver]
+      # if we pass in a previous requirements.txt, lets not mention it
+      # exception: display if it is only mentioned in a previous requirements.txt file
+      modules.delete("legacy") if modules.size > 1
+
+      # clear out versions for packages provided by the operatingsystem like requests, requests[security], and pyspnego
+      ver = "" if OS_PACKAGES.include?(lib.gsub(/\[.*\]/))
+
+      "#{lib}#{ver} # #{modules.join(", ")}"
     end.sort.join("\n")
 
     puts result
@@ -204,4 +230,5 @@ def version_compare(a, b)
 
 pr = ParseRequirements.new
 ARGV.each { |arg| pr.add_target(arg) }
+pr.verbose! if ENV["VERBOSE"]
 pr.parse.output

From 82b43f8a03ce2c17dca6edb775b27e72e873992f Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Wed, 25 Sep 2024 10:40:50 -0400
Subject: [PATCH 16/24] upgrade required versions

- google auth does not need to be pegged, opening up
- json schema upgraded for security reasons
- netaddr does not need to be pegged, opening up
- ncclient - opening up
- paramiko - added minimum for security reasons
- pyvmomi kept changed minimum version to support json
- azure-identity - changed minimum for security reasons (see note below)

NOTE:

```
azure-identity: >=1.16.0 > ==1.7.0 CONFLICT
```

Azure requirements.txt pegs versions.
Loosened it up to resolve a security vulnerability on azure-identity
But when we run requirements.rb in the future, we will see the conflict.
---
 config/requirements.txt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 12cbad99..ac17925d 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -6,7 +6,7 @@ azure-cli-core==2.34.0 # azure/azcollection
 azure-common==1.1.11 # azure/azcollection
 azure-containerregistry==1.1.0 # azure/azcollection
 azure-graphrbac==0.61.1 # azure/azcollection
-azure-identity==1.7.0 # azure/azcollection
+azure-identity>=1.16.0 # azure/azcollection
 azure-keyvault==1.1.0 # azure/azcollection
 azure-mgmt-apimanagement==3.0.0 # azure/azcollection
 azure-mgmt-authorization==2.0.0 # azure/azcollection
@@ -53,22 +53,22 @@ boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
 cryptography>=36.0.0 # cisco/intersight
 deprecation # legacy
-google-auth==1.6.2 # google/cloud
+google-auth # google/cloud
 google-cloud-storage # google/cloud
 grpcio # ansible/netcommon
 jsonpatch # kubernetes/core
-jsonschema # ansible/utils
+jsonschema>=4.18.0 # ansible/utils
 jxmlease # ansible/netcommon
 kubernetes>=12.0.0 # community/okd, kubernetes/core
 msrest==0.7.1 # azure/azcollection
 msrestazure==0.6.4 # azure/azcollection
-ncclient>=0.6.3 # ansible/netcommon
-netaddr==0.7.19 # ansible/netcommon, ansible/utils
+ncclient # ansible/netcommon
+netaddr # ansible/netcommon, ansible/utils
 openstacksdk>=0.36,<0.99.0 # openstack/cloud
 ovirt-engine-sdk-python>=4.5.0 # ovirt/ovirt
 ovirt-imageio # ovirt/ovirt
 packaging # azure/azcollection
-paramiko==2.8.1 # ansible/netcommon
+paramiko>=2.9.3 # ansible/netcommon
 pexpect # legacy
 protobuf # ansible/netcommon
 psutil # legacy

From 35401105fb1c8535c0dedffb4b9bccb150b2f813 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 26 Sep 2024 17:51:49 -0400
Subject: [PATCH 17/24] Add 2 python packages

Better handle system packages

- Dropped pexpect dependency. It is required by ansible-runner and not by a Collection
- Converted paramiko to an rpm dependency
- Convert python3-importlib-metadata (and therefore zipp) to an rpm dependency
- Cryptography is installed by rpm. Dropping explicit version
- Dateutil is installed by rpm. Dropping explicit version
---
 config/requirements.txt                    | 7 +++----
 rpm_spec/subpackages/manageiq-ansible-venv | 4 ++++
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index c92f067e..2e1dcc78 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -50,7 +50,7 @@ azure-nspkg==2.0.0 # azure/azcollection
 azure-storage-blob==12.11.0 # azure/azcollection
 boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
-cryptography>=36.0.0 # cisco/intersight
+cryptography # cisco/intersight
 deprecation # legacy
 google-auth # google/cloud
 google-cloud-storage # google/cloud
@@ -67,11 +67,10 @@ openstacksdk>=0.36,<0.99.0 # openstack/cloud
 ovirt-engine-sdk-python>=4.5.0 # ovirt/ovirt
 ovirt-imageio # ovirt/ovirt
 packaging # azure/azcollection
-paramiko>=2.9.3 # ansible/netcommon
-pexpect # legacy
+paramiko # ansible/netcommon
 protobuf # ansible/netcommon
 psutil # legacy
-python-dateutil>=2.7.0 # awx/awx
+python-dateutil # awx/awx
 pytz # awx/awx
 pyvmomi>=6.7.1 # community/vmware
 pywinrm # legacy
diff --git a/rpm_spec/subpackages/manageiq-ansible-venv b/rpm_spec/subpackages/manageiq-ansible-venv
index ac752c01..52c7ff4b 100644
--- a/rpm_spec/subpackages/manageiq-ansible-venv
+++ b/rpm_spec/subpackages/manageiq-ansible-venv
@@ -5,6 +5,10 @@ Summary: %{product_summary} Ansible Runner Virtual Environment
 
 Requires: ansible >= 1:7, ansible < 1:8
 Requires: python3-virtualenv
+# used by ansible-runner (pip installed)
+Requires: python3-importlib-metadata
+# used by azure, ncclient
+Requires: python3-paramiko
 AutoReqProv: no
 
 %description ansible-venv

From d3fbfbadabacf1698202adf9c44df1bfa889444a Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 26 Sep 2024 18:12:56 -0400
Subject: [PATCH 18/24] Update requirements.rb to dynamically determine OS rpms

- Listing possible rpms by anything that includes python3-
- Using egg-info and package-info files to determine python packages installed

This is slower, only runs on an appliance/container
but the process has been simplified
---
 bin/requirements.rb | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/bin/requirements.rb b/bin/requirements.rb
index a2f01437..0bc1f778 100755
--- a/bin/requirements.rb
+++ b/bin/requirements.rb
@@ -11,33 +11,23 @@
 #     source /var/lib/manageiq/venv/bin/activate
 #     chmod 755 requirements.rb
 #
-# 2. Determine all python packages provided by rpms (and compare with OS_PACKAGES)
-#
-#     TODO: may want to grep rpm contents with 'info$' to determine elgibility.
-#
-#     for pkg in $(rpm -qa | grep python3-) ; do echo "### $pkg" ; rpm -ql $pkg | awk  -F/ '/site-packages/ { print $6 }' | sort -u ; done
-#
-# 3. Get all module requirements (don't include documentation or testing ones)
+# 2. Get all module requirements
 #
 #     ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
 #
-# 4. Resolve conflicts and determine if new one is correct
+# 3. Resolve conflicts and determine if new one is correct
+#    double check that the legacy ones are still needed
 #
 #     diff {,new_}requirements.txt
 #     # cp new_requirements.txt requirements.txt
 #
-# 5. Update dev riles
+# 4. Update dev files
 #
-#     download /tmp/requirements{.rb,.txt} to local machine
+#     download /tmp/requirements.txt to local machine
 #     create a PR with updates
 #
 class ParseRequirements
   # this is the list of packages provided by rpms
-  # TODO: paramiko
-  OS_PACKAGES=%w[
-    six dateutil iniparse idna setuptools inotify libcomps chardet decorator pysocks urllib3 requests
-    cloud-what systemd dbus gobject gpg pyspnego
-  ]
   PACKAGES=%w[
     amazon/aws/requirements.txt
     ansible/netcommon/requirements.txt
@@ -56,6 +46,17 @@ class ParseRequirements
   ]
   attr_reader :filenames, :non_modules, :final, :verbose
 
+  # These packages are installed via rpm
+  def os_packages
+    # Leaving his as pure bash so we can run from the command line to fix issues.
+    @os_packages ||=
+      `for pkg in $(rpm -qa | grep python3- | sort) ; do rpm -ql $pkg | awk  -F/ '/site-packages.*-info$/ { print $6 }' | sed 's/-[0-9].*//' | tr '_A-Z' '-a-z' | sort -u; done`.chomp.split
+  end
+
+  def os_package_regex
+    os_package_regex ||= Regexp.union(os_packages)
+  end
+
   def initialize
     @filenames = []
     @non_modules = []
@@ -101,18 +102,16 @@ def add_dir(dirname)
 
   def parse
     filenames.each do |fn|
-      # the list of requirements-files can have items commented out - ignore those
-
       mod = module_name_from_filename(fn)
       IO.foreach(fn, chomp: true).each do |line|
         lib, ver = parse_line(line)
         next unless lib
 
-        # skip git libraries. git>= line from vsphere gave us problems
+        # Skip git libraries. The 'git>=.*' line from vsphere gave us problems.
         next if lib.start_with?("git")
 
-        # system packages are versioned by rpms
-        ver = "" if lib.match?(/^(#{OS_PACKAGES.join("|")})($|\[)/)
+        # Defer to version requirements provided by rpm system packages.
+        ver = "" if lib.match?(/^(#{os_package_regex.to_s})($|\[)/)
 
         final[lib] ||= {}
         (final[lib][ver] ||= []) << mod
@@ -139,13 +138,11 @@ def output
 
       ver = vers.keys.first
       modules = vers[ver]
-      # if we pass in a previous requirements.txt, lets not mention it
-      # exception: display if it is only mentioned in a previous requirements.txt file
+      # Only display "legacy" for requirements:
+      #   - Listed in the legacy requirements.txt
+      #   - Not listed in any collection requirements.txt
       modules.delete("legacy") if modules.size > 1
 
-      # clear out versions for packages provided by the operatingsystem like requests, requests[security], and pyspnego
-      ver = "" if OS_PACKAGES.include?(lib.gsub(/\[.*\]/))
-
       "#{lib}#{ver} # #{modules.join(", ")}"
     end.sort.join("\n")
 
@@ -228,6 +225,7 @@ def version_compare(a, b)
 # {"lib" => {ver => [module]}}
 
 pr = ParseRequirements.new
+$stderr.puts("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"]
 ARGV.each { |arg| pr.add_target(arg) }
 pr.verbose! if ENV["VERBOSE"]
 pr.parse.output

From e68c6a72cb4a9a410956de7481c3e1b36bfa7ea4 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 26 Sep 2024 18:39:50 -0400
Subject: [PATCH 19/24] Fix performance issue of looking up rpms

```
time VERBOSE=true ./requirements.rb requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
```

Before
------
real	0m5.443s

After
-----

real	0m0.778s
---
 bin/requirements.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/requirements.rb b/bin/requirements.rb
index 0bc1f778..dfc994a9 100755
--- a/bin/requirements.rb
+++ b/bin/requirements.rb
@@ -48,9 +48,9 @@ class ParseRequirements
 
   # These packages are installed via rpm
   def os_packages
-    # Leaving his as pure bash so we can run from the command line to fix issues.
+    # Leaving this as pure bash so we can run from the command line to fix issues.
     @os_packages ||=
-      `for pkg in $(rpm -qa | grep python3- | sort) ; do rpm -ql $pkg | awk  -F/ '/site-packages.*-info$/ { print $6 }' | sed 's/-[0-9].*//' | tr '_A-Z' '-a-z' | sort -u; done`.chomp.split
+      `rpm -ql $(rpm -qa | grep python3- | sort) | awk  -F/ '/site-packages.*-info$/ { print $6 }' | sed 's/-[0-9].*//' | tr '_A-Z' '-a-z' | sort -u`.chomp.split
   end
 
   def os_package_regex
@@ -111,7 +111,7 @@ def parse
         next if lib.start_with?("git")
 
         # Defer to version requirements provided by rpm system packages.
-        ver = "" if lib.match?(/^(#{os_package_regex.to_s})($|\[)/)
+        ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)
 
         final[lib] ||= {}
         (final[lib][ver] ||= []) << mod

From d91c2773be20e7394281acc83b1c1fc490ff8fd3 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Mon, 7 Oct 2024 17:08:08 -0400
Subject: [PATCH 20/24] Upgrade azure-datalake dependency

Fixes issue where we're pedding datalake to a dependency that has been yanked

```
WARNING: The candidate selected for download or install is a yanked version: 'azure-mgmt-datalake-store' candidate [...]
(from https://pypi.org/simple/azure-mgmt-datalake-store/)

Reason for being yanked: Retiring this version of the SDK.
```
---
 config/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 2e1dcc78..5e5fd87c 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -19,7 +19,7 @@ azure-mgmt-containerservice==20.0.0 # azure/azcollection
 azure-mgmt-core==1.3.0 # azure/azcollection
 azure-mgmt-cosmosdb==6.4.0 # azure/azcollection
 azure-mgmt-datafactory==2.0.0 # azure/azcollection
-azure-mgmt-datalake-store==1.0.0 # azure/azcollection
+azure-mgmt-datalake-store>=1.0.0 # azure/azcollection
 azure-mgmt-devtestlabs==9.0.0 # azure/azcollection
 azure-mgmt-dns==8.0.0 # azure/azcollection
 azure-mgmt-eventhub==10.1.0 # azure/azcollection

From 4358673b9d210709e9fe103e3295fdcdf461dd7f Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 31 Oct 2024 13:41:15 -0400
Subject: [PATCH 21/24] Update azure requirements

- Update all azure requirements to use >=

legacy reminder:
- azure-identity upgraded for security
- pyvmomi>=6.7.1 to support json
- jsonschema upgraded for security reasons
---
 config/requirements.txt | 94 ++++++++++++++++++++---------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/config/requirements.txt b/config/requirements.txt
index 5e5fd87c..c3395c74 100644
--- a/config/requirements.txt
+++ b/config/requirements.txt
@@ -1,53 +1,53 @@
 ansible-pylibssh>=0.2.0 # ansible/netcommon
 apache-libcloud # legacy
 awxkit # awx/awx
-azure-cli-core==2.34.0 # azure/azcollection
-azure-common==1.1.11 # azure/azcollection
-azure-containerregistry==1.1.0 # azure/azcollection
-azure-graphrbac==0.61.1 # azure/azcollection
+azure-cli-core>=2.34.0 # azure/azcollection
+azure-common>=1.1.11 # azure/azcollection
+azure-containerregistry>=1.1.0 # azure/azcollection
+azure-graphrbac>=0.61.1 # azure/azcollection
 azure-identity>=1.16.0 # azure/azcollection
-azure-keyvault==1.1.0 # azure/azcollection
-azure-mgmt-apimanagement==3.0.0 # azure/azcollection
-azure-mgmt-authorization==2.0.0 # azure/azcollection
-azure-mgmt-automation==1.0.0 # azure/azcollection
-azure-mgmt-batch==5.0.1 # azure/azcollection
-azure-mgmt-cdn==11.0.0 # azure/azcollection
-azure-mgmt-compute==26.1.0 # azure/azcollection
-azure-mgmt-containerinstance==9.0.0 # azure/azcollection
-azure-mgmt-containerregistry==9.1.0 # azure/azcollection
-azure-mgmt-containerservice==20.0.0 # azure/azcollection
-azure-mgmt-core==1.3.0 # azure/azcollection
-azure-mgmt-cosmosdb==6.4.0 # azure/azcollection
-azure-mgmt-datafactory==2.0.0 # azure/azcollection
+azure-keyvault>=1.1.0 # azure/azcollection
+azure-mgmt-apimanagement>=3.0.0 # azure/azcollection
+azure-mgmt-authorization>=2.0.0 # azure/azcollection
+azure-mgmt-automation>=1.0.0 # azure/azcollection
+azure-mgmt-batch>=5.0.1 # azure/azcollection
+azure-mgmt-cdn>=11.0.0 # azure/azcollection
+azure-mgmt-compute>=26.1.0 # azure/azcollection
+azure-mgmt-containerinstance>=9.0.0 # azure/azcollection
+azure-mgmt-containerregistry>=9.1.0 # azure/azcollection
+azure-mgmt-containerservice>=20.0.0 # azure/azcollection
+azure-mgmt-core>=1.3.0 # azure/azcollection
+azure-mgmt-cosmosdb>=6.4.0 # azure/azcollection
+azure-mgmt-datafactory>=2.0.0 # azure/azcollection
 azure-mgmt-datalake-store>=1.0.0 # azure/azcollection
-azure-mgmt-devtestlabs==9.0.0 # azure/azcollection
-azure-mgmt-dns==8.0.0 # azure/azcollection
-azure-mgmt-eventhub==10.1.0 # azure/azcollection
-azure-mgmt-hdinsight==9.0.0 # azure/azcollection
-azure-mgmt-iothub==2.2.0 # azure/azcollection
-azure-mgmt-keyvault==10.0.0 # azure/azcollection
-azure-mgmt-loganalytics==12.0.0 # azure/azcollection
-azure-mgmt-managedservices==6.0.0 # azure/azcollection
-azure-mgmt-managementgroups==1.0.0 # azure/azcollection
-azure-mgmt-marketplaceordering==1.1.0 # azure/azcollection
-azure-mgmt-monitor==3.0.0 # azure/azcollection
-azure-mgmt-network==19.1.0 # azure/azcollection
-azure-mgmt-notificationhubs==7.0.0 # azure/azcollection
-azure-mgmt-nspkg==2.0.0 # azure/azcollection
-azure-mgmt-privatedns==1.0.0 # azure/azcollection
-azure-mgmt-rdbms==10.0.0 # azure/azcollection
-azure-mgmt-recoveryservices==2.0.0 # azure/azcollection
-azure-mgmt-recoveryservicesbackup==3.0.0 # azure/azcollection
-azure-mgmt-redis==13.0.0 # azure/azcollection
-azure-mgmt-resource==21.1.0 # azure/azcollection
-azure-mgmt-search==8.0.0 # azure/azcollection
-azure-mgmt-servicebus==7.1.0 # azure/azcollection
-azure-mgmt-sql==3.0.1 # azure/azcollection
-azure-mgmt-storage==19.0.0 # azure/azcollection
-azure-mgmt-trafficmanager==1.0.0b1 # azure/azcollection
-azure-mgmt-web==6.1.0 # azure/azcollection
-azure-nspkg==2.0.0 # azure/azcollection
-azure-storage-blob==12.11.0 # azure/azcollection
+azure-mgmt-devtestlabs>=9.0.0 # azure/azcollection
+azure-mgmt-dns>=8.0.0 # azure/azcollection
+azure-mgmt-eventhub>=10.1.0 # azure/azcollection
+azure-mgmt-hdinsight>=9.0.0 # azure/azcollection
+azure-mgmt-iothub>=2.2.0 # azure/azcollection
+azure-mgmt-keyvault>=10.0.0 # azure/azcollection
+azure-mgmt-loganalytics>=12.0.0 # azure/azcollection
+azure-mgmt-managedservices>=6.0.0 # azure/azcollection
+azure-mgmt-managementgroups>=1.0.0 # azure/azcollection
+azure-mgmt-marketplaceordering>=1.1.0 # azure/azcollection
+azure-mgmt-monitor>=3.0.0 # azure/azcollection
+azure-mgmt-network>=19.1.0 # azure/azcollection
+azure-mgmt-notificationhubs>=7.0.0 # azure/azcollection
+azure-mgmt-nspkg>=2.0.0 # azure/azcollection
+azure-mgmt-privatedns>=1.0.0 # azure/azcollection
+azure-mgmt-rdbms>=10.0.0 # azure/azcollection
+azure-mgmt-recoveryservices>=2.0.0 # azure/azcollection
+azure-mgmt-recoveryservicesbackup>=3.0.0 # azure/azcollection
+azure-mgmt-redis>=13.0.0 # azure/azcollection
+azure-mgmt-resource>=21.1.0 # azure/azcollection
+azure-mgmt-search>=8.0.0 # azure/azcollection
+azure-mgmt-servicebus>=7.1.0 # azure/azcollection
+azure-mgmt-sql>=3.0.1 # azure/azcollection
+azure-mgmt-storage>=19.0.0 # azure/azcollection
+azure-mgmt-trafficmanager>=1.0.0b1 # azure/azcollection
+azure-mgmt-web>=6.1.0 # azure/azcollection
+azure-nspkg>=2.0.0 # azure/azcollection
+azure-storage-blob>=12.11.0 # azure/azcollection
 boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
 cryptography # cisco/intersight
@@ -59,8 +59,8 @@ jsonpatch # kubernetes/core
 jsonschema>=4.18.0 # ansible/utils
 jxmlease # ansible/netcommon
 kubernetes>=12.0.0 # community/okd, kubernetes/core
-msrest==0.7.1 # azure/azcollection
-msrestazure==0.6.4 # azure/azcollection
+msrest>=0.7.1 # azure/azcollection
+msrestazure>=0.6.4 # azure/azcollection
 ncclient # ansible/netcommon
 netaddr # ansible/netcommon, ansible/utils
 openstacksdk>=0.36,<0.99.0 # openstack/cloud

From f56dd2b6da54af7458698c2a3d480f5763a2cd5d Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Wed, 30 Oct 2024 16:26:11 -0400
Subject: [PATCH 22/24] rename requirements to parse_requirements

---
 bin/{requirements.rb => parse_requirements.rb} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bin/{requirements.rb => parse_requirements.rb} (100%)

diff --git a/bin/requirements.rb b/bin/parse_requirements.rb
similarity index 100%
rename from bin/requirements.rb
rename to bin/parse_requirements.rb

From 36f5db63e88039e00062e35876d419e6fba9ac10 Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 31 Oct 2024 11:54:24 -0400
Subject: [PATCH 23/24] Introduce tests for parse_requirements

fix current rubocop issues
---
 bin/parse_requirements.rb           | 139 ++++++++++++++++------------
 spec/bin/parse_requirements_spec.rb |  47 ++++++++++
 2 files changed, 125 insertions(+), 61 deletions(-)
 create mode 100644 spec/bin/parse_requirements_spec.rb

diff --git a/bin/parse_requirements.rb b/bin/parse_requirements.rb
index dfc994a9..6dfec37c 100755
--- a/bin/parse_requirements.rb
+++ b/bin/parse_requirements.rb
@@ -7,13 +7,13 @@
 #
 # 1. Setup environment
 #
-#     upload bin/requirements.rb and config/requirements.txt to /tmp
+#     upload bin/parse_requirements.rb and config/requirements.txt to /tmp
 #     source /var/lib/manageiq/venv/bin/activate
-#     chmod 755 requirements.rb
+#     chmod 755 parse_requirements.rb
 #
 # 2. Get all module requirements
 #
-#     ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
+#     ./parse_requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
 #
 # 3. Resolve conflicts and determine if new one is correct
 #    double check that the legacy ones are still needed
@@ -27,8 +27,8 @@
 #     create a PR with updates
 #
 class ParseRequirements
-  # this is the list of packages provided by rpms
-  PACKAGES=%w[
+  # this is the list of supported collections
+  PACKAGES = %w[
     amazon/aws/requirements.txt
     ansible/netcommon/requirements.txt
     ansible/utils/requirements.txt
@@ -43,7 +43,7 @@ class ParseRequirements
     openstack/cloud/requirements.txt
     ovirt/ovirt/requirements.txt
     theforeman/foreman/requirements.txt
-  ]
+  ].freeze
   attr_reader :filenames, :non_modules, :final, :verbose
 
   # These packages are installed via rpm
@@ -54,7 +54,13 @@ def os_packages
   end
 
   def os_package_regex
-    os_package_regex ||= Regexp.union(os_packages)
+    @os_package_regex ||= Regexp.union(os_packages)
+  end
+
+  # for test
+  def os_packages=(values)
+    @os_packages = values
+    @os_package_regex = nil
   end
 
   def initialize
@@ -75,7 +81,7 @@ def add_target(filename)
     elsif File.exist?(filename)
       add_file(filename)
     else
-      $stderr.puts("File not found: #{filename}")
+      warn("File not found: #{filename}")
     end
   end
 
@@ -93,28 +99,26 @@ def add_dir(dirname)
       if File.exist?(filename)
         @filenames << filename
       else
-        $stderr.puts("NOTICE: missing #{filename}")
+        warn("NOTICE: missing #{filename}")
       end
     end
 
     self
   end
 
-  def parse
-    filenames.each do |fn|
-      mod = module_name_from_filename(fn)
-      IO.foreach(fn, chomp: true).each do |line|
-        lib, ver = parse_line(line)
-        next unless lib
+  def add_line(line, mod)
+    lib, ver = parse_line(line)
+    return unless lib
 
-        # Skip git libraries. The 'git>=.*' line from vsphere gave us problems.
-        next if lib.start_with?("git")
-
-        # Defer to version requirements provided by rpm system packages.
-        ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)
+    final[lib] ||= {}
+    (final[lib][ver] ||= []) << mod
+  end
 
-        final[lib] ||= {}
-        (final[lib][ver] ||= []) << mod
+  def parse
+    filenames.each do |filename|
+      mod = module_name_from_filename(filename)
+      File.foreach(filename, :chomp => true).each do |line|
+        add_line(line, mod)
       end
     end
 
@@ -123,25 +127,7 @@ def parse
 
   def output
     result = final.flat_map do |lib, vers|
-      # consolidate multiple versioning rules
-      if vers.size > 1
-        max_key, *all_keys = vers.keys
-        all_keys.each do |alt|
-          higher, lower, conflict = version_compare(alt, max_key)
-          # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
-          # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
-          $stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != "" || verbose
-          vers[higher].concat(vers.delete(lower))
-          max_key = higher
-        end
-      end
-
-      ver = vers.keys.first
-      modules = vers[ver]
-      # Only display "legacy" for requirements:
-      #   - Listed in the legacy requirements.txt
-      #   - Not listed in any collection requirements.txt
-      modules.delete("legacy") if modules.size > 1
+      ver, modules = consolidate_vers(vers, :lib => lib)
 
       "#{lib}#{ver} # #{modules.join(", ")}"
     end.sort.join("\n")
@@ -151,12 +137,12 @@ def output
 
   private
 
-  def module_name_from_filename(fn)
-    if non_modules.include?(fn)
+  def module_name_from_filename(filename)
+    if non_modules.include?(filename)
       "legacy"
     else
-      fn.gsub(%r{.*ansible_collections/}, "")
-        .gsub(%r{/requirements.*}, "")
+      filename.gsub(%r{.*ansible_collections/}, "")
+              .gsub(%r{/requirements.*}, "")
     end
   end
 
@@ -179,14 +165,20 @@ def parse_line(line)
 
     lib, ver = split_lib_ver(line)
 
-    # Note: Already normalized for lowercase
+    # NOTE: Already normalized for lowercase
     # Normalize library name with dash. All these characters are treated the same.
     lib.gsub!(/[-_.]+/, "-")
     ver ||= ""
 
-   # TODO: split off ;python_version in split_lib_version - evaluate it properly
-   return if ver.match?(/python_version *[=<]/)
+    # TODO: split off ;python_version in split_lib_version - evaluate it properly
+    return if ver.match?(/python_version *[=<]/)
+
+    # Skip git libraries. The 'git>=.*' line from vsphere gave us problems.
+    return if lib.start_with?("git")
 
+    # Defer to version requirements provided by rpm system packages.
+    ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)
+    
     [lib, ver]
   end
 
@@ -204,28 +196,53 @@ def split_lib_ver(line)
   # @return [Numeric, Numeric, Boolean]
   #   highest, lowest for version comparison
   #   boolean is true if there is a conflict with the versions
-  def version_compare(a, b)
-    winner = a if a.start_with?("==")
-    winner = b if b.start_with?("==")
+  def version_compare(left, right)
+    winner = left if left.start_with?("==")
+    winner = right if right.start_with?("==")
     # due to the way zip works, we need the longer to be on the left of the split
-    a, b = b, a if a.split(".").length < b.split(".").length
+    left, right = right, left if left.split(".").length < right.split(".").length
 
     # when comparing, drop off the >= or == stuff, just look at the numbers
     # kinda assuming that we are dealing mostly with >=
     # reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c.
-    cmp = a.gsub(/^[=<>]+/, "").split(".").zip(b.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i<=>v2.to_i : acc }
+    cmp = left.gsub(/^[=<>]+/, "").split(".").zip(right.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i <=> v2.to_i : acc }
 
     # ensure a >= b
-    a, b = b, a if cmp < 0
+    left, right = right, left if cmp < 0
 
-    [a, b, winner && winner != a]
+    [left, right, winner && winner != left]
+  end
+
+  # consolidate multiple versioning rules
+  def consolidate_vers(vers, lib: nil)
+    if vers.size > 1
+      max_key, *all_keys = vers.keys
+      all_keys.each do |alt|
+        higher, lower, conflict = version_compare(alt, max_key)
+        # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
+        # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
+        warn("#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}") if lower != "" || verbose
+        vers[higher].concat(vers.delete(lower))
+        max_key = higher
+      end
+    end
+
+    ver = vers.keys.first
+    modules = vers[ver]
+    # Only display "legacy" for requirements:
+    #   - Listed in the legacy requirements.txt
+    #   - Not listed in any collection requirements.txt
+    modules.delete("legacy") if modules.size > 1
+
+    [ver, modules]
   end
 end
 
 # {"lib" => {ver => [module]}}
-
-pr = ParseRequirements.new
-$stderr.puts("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"]
-ARGV.each { |arg| pr.add_target(arg) }
-pr.verbose! if ENV["VERBOSE"]
-pr.parse.output
+if $PROGRAM_NAME == __FILE__
+  pr = ParseRequirements.new
+  warn("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"]
+  ARGV.each { |arg| pr.add_target(arg) }
+  pr.verbose! if ENV["VERBOSE"]
+  pr.parse.output
+end
diff --git a/spec/bin/parse_requirements_spec.rb b/spec/bin/parse_requirements_spec.rb
new file mode 100644
index 00000000..1b4c152f
--- /dev/null
+++ b/spec/bin/parse_requirements_spec.rb
@@ -0,0 +1,47 @@
+require_relative "../../bin/parse_requirements"
+
+RSpec.describe ParseRequirements do
+  # tests just assume there is one python rpm installed
+  before { subject.os_packages = %w[paramiko] }
+
+  describe "#parse_line" do
+    it "ignores blanks" do
+      expect(parse_line("")).to be_nil
+      expect(parse_line("# comment")).to be_nil
+    end
+
+    it "parses non versions" do
+      expect(parse_line("a")).to eq(["a", ""])
+      expect(parse_line("paramiko")).to eq(["paramiko", ""])
+    end
+
+    it "parses versions" do
+      expect(parse_line("a >= 5")).to eq(["a", ">=5"])
+      expect(parse_line("b>= 5")).to eq(["b", ">=5"])
+    end
+
+    it "respects rpm libraries" do
+      expect(parse_line("a >= 5")).to eq(["a", ">=5"])
+      expect(parse_line("paramiko>= 5")).to eq(["paramiko", ""])
+    end
+
+    # it "convert == to >=" do
+    #     expect(parse_line("a == 5")).to eq(["a", ">=5"])
+    # end
+  end
+
+  describe "#consolidate_vers" do
+    it "picks the higher comparison" do
+      expect(subject).to receive(:warn).with("b: >2 > >1 ")
+      expect(consolidate_vers({">1" => ["c1"], ">2" => ["legacy"]}, :lib => "b")).to eq([">2", ["c1"]])
+    end
+  end
+
+  def parse_line(line)
+    subject.send(:parse_line, line)
+  end
+
+  def consolidate_vers(vers, lib: nil)
+    subject.send(:consolidate_vers, vers, :lib => lib)
+  end
+end

From db3f8b10ad4b94b3076bbb1cb97c8fd5aecd265a Mon Sep 17 00:00:00 2001
From: Keenan Brock <keenan@thebrocks.net>
Date: Thu, 31 Oct 2024 11:59:39 -0400
Subject: [PATCH 24/24] Treat == as >= for requirements.txt

All ansible collections use >= for requirements
azure uses ==

We just hit another CVE issue where the requirements-azure.txt are not up to date
Fixing this by allowing the azure libraries to be the latest so we get bug fixes
---
 bin/parse_requirements.rb           | 16 +++++++---------
 spec/bin/parse_requirements_spec.rb |  8 ++++----
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/bin/parse_requirements.rb b/bin/parse_requirements.rb
index 6dfec37c..7a47537a 100755
--- a/bin/parse_requirements.rb
+++ b/bin/parse_requirements.rb
@@ -178,7 +178,7 @@ def parse_line(line)
 
     # Defer to version requirements provided by rpm system packages.
     ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)
-    
+
     [lib, ver]
   end
 
@@ -189,28 +189,26 @@ def split_lib_ver(line)
     # split on first space (or =)
     # version can have multiple spaces
     lib, ver = line.match(/([^ >=]*) ?(.*)/).captures
+    # azure uses ==, we are instead using >=
+    ver.gsub!("==", ">=")
 
     [lib, ver]
   end
 
-  # @return [Numeric, Numeric, Boolean]
+  # @return [Numeric, Numeric]
   #   highest, lowest for version comparison
   #   boolean is true if there is a conflict with the versions
   def version_compare(left, right)
-    winner = left if left.start_with?("==")
-    winner = right if right.start_with?("==")
     # due to the way zip works, we need the longer to be on the left of the split
     left, right = right, left if left.split(".").length < right.split(".").length
 
-    # when comparing, drop off the >= or == stuff, just look at the numbers
-    # kinda assuming that we are dealing mostly with >=
     # reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c.
     cmp = left.gsub(/^[=<>]+/, "").split(".").zip(right.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i <=> v2.to_i : acc }
 
     # ensure a >= b
     left, right = right, left if cmp < 0
 
-    [left, right, winner && winner != left]
+    [left, right]
   end
 
   # consolidate multiple versioning rules
@@ -218,10 +216,10 @@ def consolidate_vers(vers, lib: nil)
     if vers.size > 1
       max_key, *all_keys = vers.keys
       all_keys.each do |alt|
-        higher, lower, conflict = version_compare(alt, max_key)
+        higher, lower = version_compare(alt, max_key)
         # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
         # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
-        warn("#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}") if lower != "" || verbose
+        warn("#{lib}: #{higher} > #{lower}") if lower != "" || verbose
         vers[higher].concat(vers.delete(lower))
         max_key = higher
       end
diff --git a/spec/bin/parse_requirements_spec.rb b/spec/bin/parse_requirements_spec.rb
index 1b4c152f..97483e9f 100644
--- a/spec/bin/parse_requirements_spec.rb
+++ b/spec/bin/parse_requirements_spec.rb
@@ -25,14 +25,14 @@
       expect(parse_line("paramiko>= 5")).to eq(["paramiko", ""])
     end
 
-    # it "convert == to >=" do
-    #     expect(parse_line("a == 5")).to eq(["a", ">=5"])
-    # end
+    it "convert == to >=" do
+      expect(parse_line("a == 5")).to eq(["a", ">=5"])
+    end
   end
 
   describe "#consolidate_vers" do
     it "picks the higher comparison" do
-      expect(subject).to receive(:warn).with("b: >2 > >1 ")
+      expect(subject).to receive(:warn).with("b: >2 > >1")
       expect(consolidate_vers({">1" => ["c1"], ">2" => ["legacy"]}, :lib => "b")).to eq([">2", ["c1"]])
     end
   end