Skip to content

Commit

Permalink
fix(build_wheels): Platform and version conversion logic
Browse files Browse the repository at this point in the history
  • Loading branch information
jakub-kocka committed Nov 21, 2023
1 parent 4de7f92 commit 79af633
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 46 deletions.
25 changes: 18 additions & 7 deletions .github/workflows/build-wheels-platforms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ name: platforms-dispatches
# TODO schedule for workflow and delete pull_request/push on final merge request
on:
pull_request:
push:
workflow_dispatch:

env:
Expand All @@ -12,23 +11,35 @@ env:

jobs:
build-python-wheels:
name: Build Python Wheels for windows-latest
name: Build Python Wheels for ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
# TODO
# macos-M1 (macos-m1-self-hosted) has no 3.8 PYthon available
# linux-armv7-self-hosted: version `GLIBCXX_3.4.26' not found (required by /opt/actions-runner/externals/node20/bin/node)
os: [windows-latest, ubuntu-latest, macos-latest]
os:
- windows-latest
- ubuntu-latest
- macos-latest
- macos-m1-self-hosted
- linux-armv7-self-hosted
- BrnoRPI-GH004 # linux ARM64
python-version: ['3.8']

steps:
- name: Checkout repository
# ARM7 and ARM64 needs upgrade the system to use newer version of Node.js
if: matrix.os != 'linux-armv7-self-hosted' || 'BrnoRPI-GH004'
uses: actions/checkout@v4

- name: Checkout repository ARM7 and ARM64
if: matrix.os == 'linux-armv7-self-hosted' || 'BrnoRPI-GH004'
uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v3
# Temporary solution until the MacOS-M1 is mentained by GitHub
# Suppose the environement on self-hosted runner is correct
if: matrix.os != 'macos-m1-self-hosted'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ This YAML file is converted to Requirement from packaging.requirements because p
The opposite logic of exclude_list is handled by the function itself, which means it is supposed to be easy to use for developers, this is also the reason YAML format is used.

For every `package_name` there are options:
* `version` - supports all logic operators defined by PEP508 for versions (<, >, !=, etc.)
* `version`
- supports all logic operators defined by PEP508 for versions (<, >, !=, etc.)
* `platform`

which could be a string or a list of strings.
Expand Down
98 changes: 60 additions & 38 deletions build_wheels.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,6 @@ def _change_specifier_logic(specifier: str) -> str:
"""Change specifier logic to opposite
- e.g. "<3" will be ">3"
"""
#print(f"Original specifier: {specifier}")
new_spec = specifier
replacements = {'<': '>',
'>': '<',
Expand All @@ -202,7 +201,6 @@ def _change_specifier_logic(specifier: str) -> str:
for old, new in replacements.items():
if old in specifier and '===' not in specifier:
new_spec = specifier.replace(old, new)
#print(f"Specifier changed: {new_spec}")
return new_spec


Expand All @@ -213,46 +211,63 @@ def yaml_to_requirement(yaml_file:str, exclude: bool = False) -> set:
with open(yaml_file, 'r') as f:
yaml_list = yaml.load(f, yaml.Loader)

exclude_list_set: set[Requirement] = set()
requirements_set: set[Requirement] = set()

if not yaml_list:
return exclude_list_set
return requirements_set

for package in yaml_list:
req_txt = f"{package['package_name']}"
requirement_txt = f"{package['package_name']}"

if 'version' in package:
if not isinstance(package['version'], list):
requirement_txt += _change_specifier_logic(package['version']) if exclude else package['version']

if isinstance(package['version'], list):
ver_txt = ''
for ver_str in package['version']:
if exclude:
ver_txt += f'{_change_specifier_logic(ver_str)},'
else:
ver_txt += f'{ver_str},'
req_txt += ver_txt[:-1]
else:
version_list = (
[f'{_change_specifier_logic(ver)}' if exclude else f'{ver}' for ver in package['version']]
)

requirement_txt += ','.join(version_list)


if 'platform' in package and 'version' not in package:
if not isinstance(package['platform'], list):
requirement_txt += (
f"; sys_platform != '{package['platform']}'" if exclude
else f"; sys_platform == '{package['platform']}'"
)

if isinstance(package['platform'], list):
platform_list = (
[f"sys_platform != '{plf}'" if exclude
else f"sys_platform == '{plf}'" for plf in package['platform']]
)

requirement_txt += '; ' + ' or '.join(platform_list)


if 'platform' in package and 'version' in package:
requirement_old = f"{package['package_name']}"

if not isinstance(package['platform'], list):
requirement_txt += f"; sys_platform == '{package['platform']}'"
if exclude:
req_txt += _change_specifier_logic(package['version'])
else:
req_txt += package['version']
requirement_old += f"; sys_platform != '{package['platform']}'"
requirements_set.add(Requirement(requirement_old))

if 'platform' in package:
if isinstance(package['platform'], list):
plf_txt = '; '
for plf_str in package['platform']:
if exclude:
plf_txt += f"sys_platform != '{plf_str}' or "
else:
plf_txt += f"sys_platform == '{plf_str}' or "
req_txt += plf_txt[:-4]
else:
platform_list = [f"sys_platform == '{plf}'" for plf in package['platform']]
requirement_txt += '; ' + ' or '.join(platform_list)

if exclude:
req_txt += f"; sys_platform != '{package['platform']}'"
else:
req_txt += f"; sys_platform == '{package['platform']}'"
platform_list_old = [f"sys_platform != '{plf}'" for plf in package['platform']]
requirement_old += '; ' + ' or '.join(platform_list_old)
requirements_set.add(Requirement(requirement_old))


exclude_list_set.add(Requirement(req_txt))
return exclude_list_set
requirements_set.add(Requirement(requirement_txt))
return requirements_set


def exclude_from_requirements(assembled_requirements:set, exclude_list: set, print_requirements: bool = True) -> set:
Expand All @@ -265,9 +280,18 @@ def exclude_from_requirements(assembled_requirements:set, exclude_list: set, pri
print(Fore.BLUE + '---------- REQUIREMENTS ----------')

for requirement in assembled_requirements:
printed = False
for req in exclude_list:
if req.name not in requirement.name:
not_in_exclude.append(True)

if req.name in requirement.name:

if not req.specifier and not req.marker:
if print_requirements:
print(Fore.RED + f'-- {requirement}')
continue

if requirement.specifier and req.specifier:
new_specifier = requirement.specifier & req.specifier
elif requirement.specifier and not req.specifier:
Expand All @@ -286,10 +310,6 @@ def exclude_from_requirements(assembled_requirements:set, exclude_list: set, pri
else:
new_markers = ''

if not req.specifier and not req.marker:
if print_requirements:
print(Fore.RED + f'-- {requirement}')
continue

if new_markers:
new_requirement = Requirement(f'{requirement.name}{new_specifier}; {new_markers}')
Expand All @@ -299,16 +319,18 @@ def exclude_from_requirements(assembled_requirements:set, exclude_list: set, pri
new_assembled_requirements.add(new_requirement)

if print_requirements:
print(Fore.RED + f'-- {requirement}')
if not printed:
print(Fore.RED + f'-- {requirement}')
printed = True
print(Fore.GREEN + f'++ {new_requirement}')
else:
not_in_exclude.append(True)


if len(not_in_exclude) == len(exclude_list):
if print_requirements:
print(Style.RESET_ALL + str(requirement))
new_assembled_requirements.add(requirement)

not_in_exclude = []
not_in_exclude.clear()

if print_requirements:
print(Fore.BLUE + '---------- END OF REQUIREMENTS ----------' + Style.RESET_ALL)
Expand Down

0 comments on commit 79af633

Please sign in to comment.