diff --git a/probe_builder/builder/distro/base_builder.py b/probe_builder/builder/distro/base_builder.py index c987d47..5ff2ab9 100644 --- a/probe_builder/builder/distro/base_builder.py +++ b/probe_builder/builder/distro/base_builder.py @@ -11,6 +11,8 @@ from probe_builder.kernel_crawler.download import download_batch from probe_builder.py23 import make_bytes, make_string +from concurrent.futures import ThreadPoolExecutor + logger = logging.getLogger(__name__) @@ -23,6 +25,8 @@ def to_s(s): class DistroBuilder(object): + executor = ThreadPoolExecutor(len(os.sched_getaffinity(0))) + @staticmethod def md5sum(path): from hashlib import md5 diff --git a/probe_builder/builder/distro/centos.py b/probe_builder/builder/distro/centos.py index 98c94f9..6627e79 100644 --- a/probe_builder/builder/distro/centos.py +++ b/probe_builder/builder/distro/centos.py @@ -2,7 +2,7 @@ import os import re -import click +import traceback from probe_builder.builder import toolkit from .base_builder import DistroBuilder @@ -15,15 +15,21 @@ class CentosBuilder(DistroBuilder): def unpack_kernels(self, workspace, distro, kernels): kernel_dirs = list() - - for release, rpms in kernels.items(): - target = workspace.subdir('build', distro, release) - kernel_dirs.append((release, target)) - - for rpm in rpms: - rpm_basename = os.path.basename(rpm) - marker = os.path.join(target, '.' + rpm_basename) - toolkit.unpack_rpm(workspace, rpm, target, marker) + def unpack_release_packages(release, target, debs): + try: + for rpm in rpms: + rpm_basename = os.path.basename(rpm) + marker = os.path.join(target, '.' + rpm_basename) + toolkit.unpack_rpm(workspace, rpm, target, marker) + except: + traceback.print_exc() + + with self.executor as executor: + for release, rpms in kernels.items(): + target = workspace.subdir('build', distro, release) + # FIXME this should be done inspecting the future + kernel_dirs.append((release, target)) + executor.submit(unpack_release_packages, release, target, rpms) return kernel_dirs diff --git a/probe_builder/builder/distro/debian.py b/probe_builder/builder/distro/debian.py index f84c15f..db8fb50 100644 --- a/probe_builder/builder/distro/debian.py +++ b/probe_builder/builder/distro/debian.py @@ -44,24 +44,30 @@ def _reparent_link(base_path, release, link_name): def unpack_kernels(self, workspace, distro, kernels): kernel_dirs = list() - for release, debs in kernels.items(): - # we can no longer use '-' as the separator, since now also have variant - # (e.g. cloud-amd64) - version, vararch = release.rsplit(':', 1) - # restore the original composite e.g. 5.16.0-1-cloud-amd64 - release = release.replace(':', '-') - - target = workspace.subdir('build', distro, version) - + def unpack_release_packages(release, target, debs): try: for deb in debs: deb_basename = os.path.basename(deb) marker = os.path.join(target, '.' + deb_basename) toolkit.unpack_deb(workspace, deb, target, marker) + # FIXME here we should probably just return success, and have the future handle this kernel_dirs.append((release, target)) except: traceback.print_exc() + # unpack packages in parallel + with self.executor as executor: + for release, debs in kernels.items(): + # we can no longer use '-' as the separator, since now also have variant + # (e.g. cloud-amd64) + version, vararch = release.rsplit(':', 1) + # restore the original composite e.g. 5.16.0-1-cloud-amd64 + release = release.replace(':', '-') + + target = workspace.subdir('build', distro, version) + logger.debug("Scheduling unpacking for {}".format(version)) + executor.submit(unpack_release_packages, release, target, debs) + for release, target in kernel_dirs: kerneldir = self.get_kernel_dir(workspace, release, target) diff --git a/probe_builder/builder/distro/flatcar.py b/probe_builder/builder/distro/flatcar.py index a07f722..b31b0d5 100644 --- a/probe_builder/builder/distro/flatcar.py +++ b/probe_builder/builder/distro/flatcar.py @@ -61,7 +61,7 @@ def build_kernel_impl(cls, config_hash, container_name, image_name, kernel_dir, logger.info('Skipping build of {} probe {}-{} ({}): {}'.format(label, coreos_kernel_release, config_hash, release, skip_reason)) - docker.rm(container_name) + #docker.rm(container_name) try: builder_image.run(workspace, probe, kernel_dir, coreos_kernel_release, config_hash, container_name, image_name, args) except subprocess.CalledProcessError: diff --git a/probe_builder/builder/distro/ubuntu.py b/probe_builder/builder/distro/ubuntu.py index fc17ba9..8c1b165 100644 --- a/probe_builder/builder/distro/ubuntu.py +++ b/probe_builder/builder/distro/ubuntu.py @@ -29,40 +29,47 @@ def crawl(self, workspace, distro, crawler_distro, download_config=None, filter= def unpack_kernels(self, workspace, distro, kernels): kernel_dirs = list() - # notice how here kernels is a list of tuples - # ( '5.4.0-1063-aws', [".._5.4.0-1063.66_amd64.deb"] ) - for release, debs in kernels: - # we don't have the version handy, so gather it from all the package - # names in the release. these all should match but at this point we can - # only validate that it is so - version = None - - for deb in debs: - deb_basename = os.path.basename(deb) - m = self.KERNEL_VERSION_RE.search(deb_basename) - if not m: - raise ValueError("{} doesn't look like a kernel package".format(deb)) - if version is None: - version = (m.group('version'), m.group('update')) - else: - new_version = (m.group('version'), m.group('update')) - if version[0] != new_version[0] and not new_version[1].startswith(version[1]): - raise ValueError("Unexpected version {}/{} from package {} (expected {}/{})".format( - new_version[0], new_version[1], deb, - version[0], version[1] - )) - # extracted files will end up in a directory derived from the version - # e.g. 5.4.0-1063/66 - target = workspace.subdir('build', distro, version[0], version[1]) - # which we will address by release - # ( '5.4.0-1063-aws', '/path/to/5.4.0-1063/66' ) - kernel_dirs.append((release, target)) - + def unpack_release_packages(target, debs): for deb in debs: deb_basename = os.path.basename(deb) marker = os.path.join(target, '.' + deb_basename) toolkit.unpack_deb(workspace, deb, target, marker) + + with self.executor as executor: + # notice how here kernels is a list of tuples + # ( '5.4.0-1063-aws', [".._5.4.0-1063.66_amd64.deb"] ) + for release, debs in kernels: + # we don't have the version handy, so gather it from all the package + # names in the release. these all should match but at this point we can + # only validate that it is so + version = None + + for deb in debs: + deb_basename = os.path.basename(deb) + m = self.KERNEL_VERSION_RE.search(deb_basename) + if not m: + raise ValueError("{} doesn't look like a kernel package".format(deb)) + if version is None: + version = (m.group('version'), m.group('update')) + else: + new_version = (m.group('version'), m.group('update')) + if version[0] != new_version[0] and not new_version[1].startswith(version[1]): + raise ValueError("Unexpected version {}/{} from package {} (expected {}/{})".format( + new_version[0], new_version[1], deb, + version[0], version[1] + )) + # extracted files will end up in a directory derived from the version + # e.g. 5.4.0-1063/66 + target = workspace.subdir('build', distro, version[0], version[1]) + # which we will address by release + # ( '5.4.0-1063-aws', '/path/to/5.4.0-1063/66' ) + # FIXME this should be done only after checking the return value of the future + kernel_dirs.append((release, target)) + + executor.submit(unpack_release_packages, target, debs) + # end of with + return kernel_dirs def hash_config(self, release, target):