Skip to content

Commit

Permalink
Merge pull request #519 from kbrock/minimum_azure_versions
Browse files Browse the repository at this point in the history 
Update ansible azure storage blob
  • Loading branch information
@Fryguy
Fryguy authored Oct 31, 2024
2 parents 562f8f2 + db3f8b1 commit 36c0c10
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 111 deletions.
143 changes: 79 additions & 64 deletions bin/requirements.rb → bin/parse_requirements.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
#
# 1. Setup environment
#
# upload bin/requirements.rb and config/requirements.txt to /tmp
# upload bin/parse_requirements.rb and config/requirements.txt to /tmp
# source /var/lib/manageiq/venv/bin/activate
# chmod 755 requirements.rb
# chmod 755 parse_requirements.rb
#
# 2. Get all module requirements
#
# ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
# ./parse_requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
#
# 3. Resolve conflicts and determine if new one is correct
# double check that the legacy ones are still needed
Expand All @@ -27,8 +27,8 @@
# create a PR with updates
#
class ParseRequirements
# this is the list of packages provided by rpms
PACKAGES=%w[
# this is the list of supported collections
PACKAGES = %w[
amazon/aws/requirements.txt
ansible/netcommon/requirements.txt
ansible/utils/requirements.txt
Expand All @@ -43,7 +43,7 @@ class ParseRequirements
openstack/cloud/requirements.txt
ovirt/ovirt/requirements.txt
theforeman/foreman/requirements.txt
]
].freeze
attr_reader :filenames, :non_modules, :final, :verbose

# These packages are installed via rpm
Expand All @@ -54,7 +54,13 @@ def os_packages
end

def os_package_regex
os_package_regex ||= Regexp.union(os_packages)
@os_package_regex ||= Regexp.union(os_packages)
end

# for test
def os_packages=(values)
@os_packages = values
@os_package_regex = nil
end

def initialize
Expand All @@ -75,7 +81,7 @@ def add_target(filename)
elsif File.exist?(filename)
add_file(filename)
else
$stderr.puts("File not found: #{filename}")
warn("File not found: #{filename}")
end
end

Expand All @@ -93,28 +99,26 @@ def add_dir(dirname)
if File.exist?(filename)
@filenames << filename
else
$stderr.puts("NOTICE: missing #{filename}")
warn("NOTICE: missing #{filename}")
end
end

self
end

def parse
filenames.each do |fn|
mod = module_name_from_filename(fn)
IO.foreach(fn, chomp: true).each do |line|
lib, ver = parse_line(line)
next unless lib
def add_line(line, mod)
lib, ver = parse_line(line)
return unless lib

# Skip git libraries. The 'git>=.*' line from vsphere gave us problems.
next if lib.start_with?("git")

# Defer to version requirements provided by rpm system packages.
ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)
final[lib] ||= {}
(final[lib][ver] ||= []) << mod
end

final[lib] ||= {}
(final[lib][ver] ||= []) << mod
def parse
filenames.each do |filename|
mod = module_name_from_filename(filename)
File.foreach(filename, :chomp => true).each do |line|
add_line(line, mod)
end
end

Expand All @@ -123,25 +127,7 @@ def parse

def output
result = final.flat_map do |lib, vers|
# consolidate multiple versioning rules
if vers.size > 1
max_key, *all_keys = vers.keys
all_keys.each do |alt|
higher, lower, conflict = version_compare(alt, max_key)
# There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
# We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
$stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != "" || verbose
vers[higher].concat(vers.delete(lower))
max_key = higher
end
end

ver = vers.keys.first
modules = vers[ver]
# Only display "legacy" for requirements:
# - Listed in the legacy requirements.txt
# - Not listed in any collection requirements.txt
modules.delete("legacy") if modules.size > 1
ver, modules = consolidate_vers(vers, :lib => lib)

"#{lib}#{ver} # #{modules.join(", ")}"
end.sort.join("\n")
Expand All @@ -151,12 +137,12 @@ def output

private

def module_name_from_filename(fn)
if non_modules.include?(fn)
def module_name_from_filename(filename)
if non_modules.include?(filename)
"legacy"
else
fn.gsub(%r{.*ansible_collections/}, "")
.gsub(%r{/requirements.*}, "")
filename.gsub(%r{.*ansible_collections/}, "")
.gsub(%r{/requirements.*}, "")
end
end

Expand All @@ -179,13 +165,19 @@ def parse_line(line)

lib, ver = split_lib_ver(line)

# Note: Already normalized for lowercase
# NOTE: Already normalized for lowercase
# Normalize library name with dash. All these characters are treated the same.
lib.gsub!(/[-_.]+/, "-")
ver ||= ""

# TODO: split off ;python_version in split_lib_version - evaluate it properly
return if ver.match?(/python_version *[=<]/)
# TODO: split off ;python_version in split_lib_version - evaluate it properly
return if ver.match?(/python_version *[=<]/)

# Skip git libraries. The 'git>=.*' line from vsphere gave us problems.
return if lib.start_with?("git")

# Defer to version requirements provided by rpm system packages.
ver = "" if lib.match?(/^(#{os_package_regex})($|\[)/)

[lib, ver]
end
Expand All @@ -197,35 +189,58 @@ def split_lib_ver(line)
# split on first space (or =)
# version can have multiple spaces
lib, ver = line.match(/([^ >=]*) ?(.*)/).captures
# azure uses ==, we are instead using >=
ver.gsub!("==", ">=")

[lib, ver]
end

# @return [Numeric, Numeric, Boolean]
# @return [Numeric, Numeric]
# highest, lowest for version comparison
# boolean is true if there is a conflict with the versions
def version_compare(a, b)
winner = a if a.start_with?("==")
winner = b if b.start_with?("==")
def version_compare(left, right)
# due to the way zip works, we need the longer to be on the left of the split
a, b = b, a if a.split(".").length < b.split(".").length
left, right = right, left if left.split(".").length < right.split(".").length

# when comparing, drop off the >= or == stuff, just look at the numbers
# kinda assuming that we are dealing mostly with >=
# reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c.
cmp = a.gsub(/^[=<>]+/, "").split(".").zip(b.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i<=>v2.to_i : acc }
cmp = left.gsub(/^[=<>]+/, "").split(".").zip(right.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i <=> v2.to_i : acc }

# ensure a >= b
a, b = b, a if cmp < 0
left, right = right, left if cmp < 0

[a, b, winner && winner != a]
[left, right]
end

# consolidate multiple versioning rules
def consolidate_vers(vers, lib: nil)
if vers.size > 1
max_key, *all_keys = vers.keys
all_keys.each do |alt|
higher, lower = version_compare(alt, max_key)
# There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
# We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
warn("#{lib}: #{higher} > #{lower}") if lower != "" || verbose
vers[higher].concat(vers.delete(lower))
max_key = higher
end
end

ver = vers.keys.first
modules = vers[ver]
# Only display "legacy" for requirements:
# - Listed in the legacy requirements.txt
# - Not listed in any collection requirements.txt
modules.delete("legacy") if modules.size > 1

[ver, modules]
end
end

# {"lib" => {ver => [module]}}

pr = ParseRequirements.new
$stderr.puts("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"]
ARGV.each { |arg| pr.add_target(arg) }
pr.verbose! if ENV["VERBOSE"]
pr.parse.output
if $PROGRAM_NAME == __FILE__
pr = ParseRequirements.new
warn("system packages:", pr.os_packages.join(" "), "") if ENV["VERBOSE"]
ARGV.each { |arg| pr.add_target(arg) }
pr.verbose! if ENV["VERBOSE"]
pr.parse.output
end
94 changes: 47 additions & 47 deletions config/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,53 @@
ansible-pylibssh>=0.2.0 # ansible/netcommon
apache-libcloud # legacy
awxkit # awx/awx
azure-cli-core==2.34.0 # azure/azcollection
azure-common==1.1.11 # azure/azcollection
azure-containerregistry==1.1.0 # azure/azcollection
azure-graphrbac==0.61.1 # azure/azcollection
azure-cli-core>=2.34.0 # azure/azcollection
azure-common>=1.1.11 # azure/azcollection
azure-containerregistry>=1.1.0 # azure/azcollection
azure-graphrbac>=0.61.1 # azure/azcollection
azure-identity>=1.16.0 # azure/azcollection
azure-keyvault==1.1.0 # azure/azcollection
azure-mgmt-apimanagement==3.0.0 # azure/azcollection
azure-mgmt-authorization==2.0.0 # azure/azcollection
azure-mgmt-automation==1.0.0 # azure/azcollection
azure-mgmt-batch==5.0.1 # azure/azcollection
azure-mgmt-cdn==11.0.0 # azure/azcollection
azure-mgmt-compute==26.1.0 # azure/azcollection
azure-mgmt-containerinstance==9.0.0 # azure/azcollection
azure-mgmt-containerregistry==9.1.0 # azure/azcollection
azure-mgmt-containerservice==20.0.0 # azure/azcollection
azure-mgmt-core==1.3.0 # azure/azcollection
azure-mgmt-cosmosdb==6.4.0 # azure/azcollection
azure-mgmt-datafactory==2.0.0 # azure/azcollection
azure-keyvault>=1.1.0 # azure/azcollection
azure-mgmt-apimanagement>=3.0.0 # azure/azcollection
azure-mgmt-authorization>=2.0.0 # azure/azcollection
azure-mgmt-automation>=1.0.0 # azure/azcollection
azure-mgmt-batch>=5.0.1 # azure/azcollection
azure-mgmt-cdn>=11.0.0 # azure/azcollection
azure-mgmt-compute>=26.1.0 # azure/azcollection
azure-mgmt-containerinstance>=9.0.0 # azure/azcollection
azure-mgmt-containerregistry>=9.1.0 # azure/azcollection
azure-mgmt-containerservice>=20.0.0 # azure/azcollection
azure-mgmt-core>=1.3.0 # azure/azcollection
azure-mgmt-cosmosdb>=6.4.0 # azure/azcollection
azure-mgmt-datafactory>=2.0.0 # azure/azcollection
azure-mgmt-datalake-store>=1.0.0 # azure/azcollection
azure-mgmt-devtestlabs==9.0.0 # azure/azcollection
azure-mgmt-dns==8.0.0 # azure/azcollection
azure-mgmt-eventhub==10.1.0 # azure/azcollection
azure-mgmt-hdinsight==9.0.0 # azure/azcollection
azure-mgmt-iothub==2.2.0 # azure/azcollection
azure-mgmt-keyvault==10.0.0 # azure/azcollection
azure-mgmt-loganalytics==12.0.0 # azure/azcollection
azure-mgmt-managedservices==6.0.0 # azure/azcollection
azure-mgmt-managementgroups==1.0.0 # azure/azcollection
azure-mgmt-marketplaceordering==1.1.0 # azure/azcollection
azure-mgmt-monitor==3.0.0 # azure/azcollection
azure-mgmt-network==19.1.0 # azure/azcollection
azure-mgmt-notificationhubs==7.0.0 # azure/azcollection
azure-mgmt-nspkg==2.0.0 # azure/azcollection
azure-mgmt-privatedns==1.0.0 # azure/azcollection
azure-mgmt-rdbms==10.0.0 # azure/azcollection
azure-mgmt-recoveryservices==2.0.0 # azure/azcollection
azure-mgmt-recoveryservicesbackup==3.0.0 # azure/azcollection
azure-mgmt-redis==13.0.0 # azure/azcollection
azure-mgmt-resource==21.1.0 # azure/azcollection
azure-mgmt-search==8.0.0 # azure/azcollection
azure-mgmt-servicebus==7.1.0 # azure/azcollection
azure-mgmt-sql==3.0.1 # azure/azcollection
azure-mgmt-storage==19.0.0 # azure/azcollection
azure-mgmt-trafficmanager==1.0.0b1 # azure/azcollection
azure-mgmt-web==6.1.0 # azure/azcollection
azure-nspkg==2.0.0 # azure/azcollection
azure-storage-blob==12.11.0 # azure/azcollection
azure-mgmt-devtestlabs>=9.0.0 # azure/azcollection
azure-mgmt-dns>=8.0.0 # azure/azcollection
azure-mgmt-eventhub>=10.1.0 # azure/azcollection
azure-mgmt-hdinsight>=9.0.0 # azure/azcollection
azure-mgmt-iothub>=2.2.0 # azure/azcollection
azure-mgmt-keyvault>=10.0.0 # azure/azcollection
azure-mgmt-loganalytics>=12.0.0 # azure/azcollection
azure-mgmt-managedservices>=6.0.0 # azure/azcollection
azure-mgmt-managementgroups>=1.0.0 # azure/azcollection
azure-mgmt-marketplaceordering>=1.1.0 # azure/azcollection
azure-mgmt-monitor>=3.0.0 # azure/azcollection
azure-mgmt-network>=19.1.0 # azure/azcollection
azure-mgmt-notificationhubs>=7.0.0 # azure/azcollection
azure-mgmt-nspkg>=2.0.0 # azure/azcollection
azure-mgmt-privatedns>=1.0.0 # azure/azcollection
azure-mgmt-rdbms>=10.0.0 # azure/azcollection
azure-mgmt-recoveryservices>=2.0.0 # azure/azcollection
azure-mgmt-recoveryservicesbackup>=3.0.0 # azure/azcollection
azure-mgmt-redis>=13.0.0 # azure/azcollection
azure-mgmt-resource>=21.1.0 # azure/azcollection
azure-mgmt-search>=8.0.0 # azure/azcollection
azure-mgmt-servicebus>=7.1.0 # azure/azcollection
azure-mgmt-sql>=3.0.1 # azure/azcollection
azure-mgmt-storage>=19.0.0 # azure/azcollection
azure-mgmt-trafficmanager>=1.0.0b1 # azure/azcollection
azure-mgmt-web>=6.1.0 # azure/azcollection
azure-nspkg>=2.0.0 # azure/azcollection
azure-storage-blob>=12.11.0 # azure/azcollection
boto3>=1.18.0 # amazon/aws, community/aws
botocore>=1.21.0 # amazon/aws, community/aws
cryptography # cisco/intersight
Expand All @@ -59,8 +59,8 @@ jsonpatch # kubernetes/core
jsonschema>=4.18.0 # ansible/utils
jxmlease # ansible/netcommon
kubernetes>=12.0.0 # community/okd, kubernetes/core
msrest==0.7.1 # azure/azcollection
msrestazure==0.6.4 # azure/azcollection
msrest>=0.7.1 # azure/azcollection
msrestazure>=0.6.4 # azure/azcollection
ncclient # ansible/netcommon
netaddr # ansible/netcommon, ansible/utils
openstacksdk>=0.36,<0.99.0 # openstack/cloud
Expand Down
Loading

0 comments on commit 36c0c10

Please sign in to comment.