Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

wip: parallel unpackers #49

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft

wip: parallel unpackers #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions probe_builder/builder/distro/base_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from probe_builder.kernel_crawler.download import download_batch
from probe_builder.py23 import make_bytes, make_string

from concurrent.futures import ThreadPoolExecutor

logger = logging.getLogger(__name__)


Expand All @@ -23,6 +25,8 @@ def to_s(s):

class DistroBuilder(object):

executor = ThreadPoolExecutor(len(os.sched_getaffinity(0)))

@staticmethod
def md5sum(path):
from hashlib import md5
Expand Down
26 changes: 16 additions & 10 deletions probe_builder/builder/distro/centos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os
import re

import click
import traceback

from probe_builder.builder import toolkit
from .base_builder import DistroBuilder
Expand All @@ -15,15 +15,21 @@ class CentosBuilder(DistroBuilder):

def unpack_kernels(self, workspace, distro, kernels):
kernel_dirs = list()

for release, rpms in kernels.items():
target = workspace.subdir('build', distro, release)
kernel_dirs.append((release, target))

for rpm in rpms:
rpm_basename = os.path.basename(rpm)
marker = os.path.join(target, '.' + rpm_basename)
toolkit.unpack_rpm(workspace, rpm, target, marker)
def unpack_release_packages(release, target, debs):
try:
for rpm in rpms:
rpm_basename = os.path.basename(rpm)
marker = os.path.join(target, '.' + rpm_basename)
toolkit.unpack_rpm(workspace, rpm, target, marker)
except:
traceback.print_exc()

with self.executor as executor:
for release, rpms in kernels.items():
target = workspace.subdir('build', distro, release)
# FIXME this should be done inspecting the future
kernel_dirs.append((release, target))
executor.submit(unpack_release_packages, release, target, rpms)

return kernel_dirs

Expand Down
24 changes: 15 additions & 9 deletions probe_builder/builder/distro/debian.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,24 +44,30 @@ def _reparent_link(base_path, release, link_name):
def unpack_kernels(self, workspace, distro, kernels):
kernel_dirs = list()

for release, debs in kernels.items():
# we can no longer use '-' as the separator, since now also have variant
# (e.g. cloud-amd64)
version, vararch = release.rsplit(':', 1)
# restore the original composite e.g. 5.16.0-1-cloud-amd64
release = release.replace(':', '-')

target = workspace.subdir('build', distro, version)

def unpack_release_packages(release, target, debs):
try:
for deb in debs:
deb_basename = os.path.basename(deb)
marker = os.path.join(target, '.' + deb_basename)
toolkit.unpack_deb(workspace, deb, target, marker)
# FIXME here we should probably just return success, and have the future handle this
kernel_dirs.append((release, target))
except:
traceback.print_exc()

# unpack packages in parallel
with self.executor as executor:
for release, debs in kernels.items():
# we can no longer use '-' as the separator, since now also have variant
# (e.g. cloud-amd64)
version, vararch = release.rsplit(':', 1)
# restore the original composite e.g. 5.16.0-1-cloud-amd64
release = release.replace(':', '-')

target = workspace.subdir('build', distro, version)
logger.debug("Scheduling unpacking for {}".format(version))
executor.submit(unpack_release_packages, release, target, debs)

for release, target in kernel_dirs:
kerneldir = self.get_kernel_dir(workspace, release, target)

Expand Down
2 changes: 1 addition & 1 deletion probe_builder/builder/distro/flatcar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def build_kernel_impl(cls, config_hash, container_name, image_name, kernel_dir,
logger.info('Skipping build of {} probe {}-{} ({}): {}'.format(label, coreos_kernel_release, config_hash,
release, skip_reason))

docker.rm(container_name)
#docker.rm(container_name)
try:
builder_image.run(workspace, probe, kernel_dir, coreos_kernel_release, config_hash, container_name, image_name, args)
except subprocess.CalledProcessError:
Expand Down
65 changes: 36 additions & 29 deletions probe_builder/builder/distro/ubuntu.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,40 +29,47 @@ def crawl(self, workspace, distro, crawler_distro, download_config=None, filter=
def unpack_kernels(self, workspace, distro, kernels):
kernel_dirs = list()

# notice how here kernels is a list of tuples
# ( '5.4.0-1063-aws', [".._5.4.0-1063.66_amd64.deb"] )
for release, debs in kernels:
# we don't have the version handy, so gather it from all the package
# names in the release. these all should match but at this point we can
# only validate that it is so
version = None

for deb in debs:
deb_basename = os.path.basename(deb)
m = self.KERNEL_VERSION_RE.search(deb_basename)
if not m:
raise ValueError("{} doesn't look like a kernel package".format(deb))
if version is None:
version = (m.group('version'), m.group('update'))
else:
new_version = (m.group('version'), m.group('update'))
if version[0] != new_version[0] and not new_version[1].startswith(version[1]):
raise ValueError("Unexpected version {}/{} from package {} (expected {}/{})".format(
new_version[0], new_version[1], deb,
version[0], version[1]
))
# extracted files will end up in a directory derived from the version
# e.g. 5.4.0-1063/66
target = workspace.subdir('build', distro, version[0], version[1])
# which we will address by release
# ( '5.4.0-1063-aws', '/path/to/5.4.0-1063/66' )
kernel_dirs.append((release, target))

def unpack_release_packages(target, debs):
for deb in debs:
deb_basename = os.path.basename(deb)
marker = os.path.join(target, '.' + deb_basename)
toolkit.unpack_deb(workspace, deb, target, marker)


with self.executor as executor:
# notice how here kernels is a list of tuples
# ( '5.4.0-1063-aws', [".._5.4.0-1063.66_amd64.deb"] )
for release, debs in kernels:
# we don't have the version handy, so gather it from all the package
# names in the release. these all should match but at this point we can
# only validate that it is so
version = None

for deb in debs:
deb_basename = os.path.basename(deb)
m = self.KERNEL_VERSION_RE.search(deb_basename)
if not m:
raise ValueError("{} doesn't look like a kernel package".format(deb))
if version is None:
version = (m.group('version'), m.group('update'))
else:
new_version = (m.group('version'), m.group('update'))
if version[0] != new_version[0] and not new_version[1].startswith(version[1]):
raise ValueError("Unexpected version {}/{} from package {} (expected {}/{})".format(
new_version[0], new_version[1], deb,
version[0], version[1]
))
# extracted files will end up in a directory derived from the version
# e.g. 5.4.0-1063/66
target = workspace.subdir('build', distro, version[0], version[1])
# which we will address by release
# ( '5.4.0-1063-aws', '/path/to/5.4.0-1063/66' )
# FIXME this should be done only after checking the return value of the future
kernel_dirs.append((release, target))

executor.submit(unpack_release_packages, target, debs)
# end of with

return kernel_dirs

def hash_config(self, release, target):
Expand Down