Traceback (most recent call last): File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 657, in cli_main main() File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 352, in main run_compose( File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 488, in run_compose essentials_phase.stop() File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop self.pool.stop() File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process phase.stop() File "/usr/lib/python3.12/site-packages/pungi/phases/base.py", line 111, in stop self.pool.stop() File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py", line 558, in process self.worker(compose, arch, variant, cmd, pkgset_phase, num) File "/usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py", line 862, in worker task_id = runroot.run( ^^^^^^^^^^^^ File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 289, in run self._run_koji( File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 137, in _run_koji raise RuntimeError( RuntimeError: Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more details. Frame _run_koji in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 137 130 max_retries = 3 131 while True: 132 output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) 133 if output["retcode"] == 0: 134 self._result = output 135 return 136 elif attempt >= max_retries or not self._has_losetup_error(log_dir): --> 137 raise RuntimeError( 138 "Runroot task failed: %s. See %s for more details." 139 % (output["task_id"], log_file) 140 ) 141 attempt += 1 142 143 def _ssh_run(self, hostname, user, command, fmt_dict=None, log_file=None): arch = 'x86_64' attempt = 0 command = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo koji_cmd = ['koji', '--profile=koji', 'runroot', '--nowait', '--task-id', '--use-shell', '--channel-override=runroot', '--package=lorax', '--mount=/mnt/centos/composes/stream-9/production/CentOS-Stream-9-2026081 koji_wrapper = kwargs = {'mounts': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1'], 'weight': None, 'chown_paths': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/bui log_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS-logs' log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log' max_retries = 3 output = {'retcode': 1, 'output': 'Watching tasks (this may be safely interrupted)...\n6414164 runroot (x86_64): free\n6414164 runroot (x86_64): free -> assigned (x86-06.stream.rdu2.redhat.com)\n6414164 runroo packages = ['lorax'] runroot_channel = 'runroot' runroot_tag = 'c9s-build' self = self._logger = None self._result = None self.compose = self.runroot_method = 'koji' Frame run in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 289 282 See `KojiWrapper.get_runroot_cmd()` for more information. 283 """ 284 if self.runroot_method == "local": 285 self._run_local( 286 command, log_file=log_file, packages=packages, arch=arch, **kwargs 287 ) 288 elif self.runroot_method == "koji": --> 289 self._run_koji( 290 command, log_file=log_file, packages=packages, arch=arch, **kwargs 291 ) 292 elif self.runroot_method == "openssh": 293 self._run_openssh( 294 command, log_file=log_file, packages=packages, arch=arch, **kwargs 295 ) arch = 'x86_64' command = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo kwargs = {'mounts': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1'], 'weight': None, 'chown_paths': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/bui log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log' packages = ['lorax'] self = self._logger = None self._result = None self.compose = self.runroot_method = 'koji' Frame worker in /usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py at line 862 855 weight=compose.conf["runroot_weights"].get("buildinstall"), 856 ) 857 else: 858 try: 859 lorax_log_dir = _get_log_dir(compose, variant, arch) 860 except Exception: 861 lorax_log_dir = None --> 862 task_id = runroot.run( 863 cmd, 864 log_file=log_file, 865 arch=arch, 866 packages=packages, 867 mounts=[compose.topdir], 868 weight=compose.conf["runroot_weights"].get("buildinstall"), arch = 'x86_64' buildinstall_method = 'lorax' chown_paths = ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS', '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-Ba cmd = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo compose = final_output_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS' log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log' log_filename = 'buildinstall-BaseOS' lorax_log_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS-logs' lorax_use_koji_plugin = False msg = 'Running buildinstall for arch x86_64, variant BaseOS' num = 4 output_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS' packages = ['lorax'] pkgset_phase = runroot = self = self._args = self._daemonic = False self._ident = 140267853346496 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-659' self._native_id = 185575 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.deliverable = 'buildinstall' self.failed = True self.get_timeout = 1 self.ident = 140267853346496 self.kill = True self.name = 'Thread-659' self.native_id = 185575 self.pool = self.running = False self.traceparent = None task_id = None variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None) Frame process in /usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py at line 558 551 self.deliverable = "buildinstall" 552 553 def process(self, item, num): 554 compose, arch, variant, cmd, pkgset_phase = item 555 can_fail = compose.can_fail(variant, arch, self.deliverable) 556 with failable(compose, can_fail, variant, arch, self.deliverable, num): 557 try: --> 558 self.worker(compose, arch, variant, cmd, pkgset_phase, num) 559 except RuntimeError: 560 self._print_depsolve_error(compose, arch, variant) 561 raise 562 563 def _print_depsolve_error(self, compose, arch, variant): 564 try: arch = 'x86_64' can_fail = False cmd = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo compose = item = num = 4 pkgset_phase = self = self._args = self._daemonic = False self._ident = 140267853346496 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-659' self._native_id = 185575 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.deliverable = 'buildinstall' self.failed = True self.get_timeout = 1 self.ident = 140267853346496 self.kill = True self.name = 'Thread-659' self.native_id = 185575 self.pool = self.running = False self.traceparent = None variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None) Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = num = 4 self = self._args = self._daemonic = False self._ident = 140267853346496 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-659' self._native_id = 185575 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.deliverable = 'buildinstall' self.failed = True self.get_timeout = 1 self.ident = 140267853346496 self.kill = True self.name = 'Thread-659' self.native_id = 185575 self.pool = self.running = False self.traceparent = None Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more detail self.finished_tasks = {('BaseOS', 's390x'), ('BaseOS', 'aarch64'), ('BaseOS', 'ppc64le')} self.kickstart_file = None self.queue = self.queue_get_lock = self.queue_processed = 4 self.queue_put_lock = self.queue_total = 4 self.reused_tasks = set() self.threads = [, , , Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/base.py at line 111 104 "Note that variants can be excluded in configuration file" 105 ) 106 107 def stop(self): 108 if self.finished: 109 return 110 if hasattr(self, "pool"): --> 111 self.pool.stop() 112 self.finished = True 113 self.compose.log_info("[DONE ] %s" % self.msg) 114 115 if hasattr(self, "_start_time"): 116 self.compose.log_info( 117 "PHASE %s took %d seconds" self = self._skipped = False self._start_time = 1787159314.322116 self.buildinstall_method = 'lorax' self.compose = self.finished = False self.lorax_use_koji_plugin = False self.msg = '---------- PHASE: BUILDINSTALL ----------' self.name = 'buildinstall' self.pkgset_phase = self.pool = self.used_patterns = None self.warned_skipped = False Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81 74 self.pool.queue_total, 75 phases_names, 76 ) 77 self.pool.log_info("[BEGIN] %s" % (msg,)) 78 79 for phase in pipeline: 80 phase.start() --> 81 phase.stop() 82 83 self.pool.log_info("[DONE ] %s" % (msg,)) item = [] msg = 'Running pipeline (1/4). Phases: buildinstall' num = 1 phase = phases_names = 'buildinstall' pipeline = [] self = self._args = self._daemonic = False self._ident = 140269073536704 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-648' self._native_id = 185563 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140269073536704 self.kill = True self.name = 'Thread-648' self.native_id = 185563 self.pool = self.running = False self.traceparent = None Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = [] num = 1 self = self._args = self._daemonic = False self._ident = 140269073536704 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-648' self._native_id = 185563 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140269073536704 self.kill = True self.name = 'Thread-648' self.native_id = 185563 self.pool = self.running = False self.traceparent = None Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more detail self.queue = self.queue_get_lock = self.queue_processed = 4 self.queue_put_lock = self.queue_total = 4 self.threads = [, , , Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59 52 53 self.pool.start() 54 55 def stop(self): 56 if self.finished: 57 return 58 if hasattr(self, "pool"): --> 59 self.pool.stop() 60 self.finished = True 61 self.compose.log_info("[DONE ] %s" % self.msg) 62 63 64 class PipelineThread(WorkerThread): 65 """ self = self._phases_schema = self.compose = self.finished = False self.msg = '---------- PHASE: WEAVER ----------' self.name = 'weaver' self.pool = Frame run_compose in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 488 481 extrafiles_phase, 482 ostree_phase, 483 ) 484 essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema) 485 essentials_phase.start() 486 ostree_container_phase.start() 487 try: --> 488 essentials_phase.stop() 489 490 # write treeinfo before ISOs are created 491 for variant in compose.get_variants(): 492 for arch in variant.arches + ["src"]: 493 pungi.metadata.write_tree_info( 494 compose, arch, variant, bi=buildinstall_phase buildinstall_phase = compose = config_copy_path = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-copy' config_dump_full = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-dump.global.log' config_file = '/home/composer/jenkins/workspace/stream9-prod-compose/pungi-centos/centos/variables.conf' create_latest_link = True createiso_phase = createrepo_phase = errors = [] essentials_phase = essentials_schema = extra_isos_phase = extrafiles_phase = f = <_io.TextIOWrapper name='/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'> gather_phase = image_build_phase = image_checksum_phase = imagebuilder_phase = init_phase = installed_pkgs_log = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/installed-pkgs.global.log' kiwibuild_phase = latest_link_components = -1 latest_link_status = None livemedia_phase = osbuild_phase = ostree_container_phase = ostree_installer_phase = ostree_phase = phase = pkgset_phase = pungi = repoclosure_phase = rpm_repo_mapping_phase = setup_phase = setup_schema = test_phase = trigger_konflux_pipeline = Frame main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 352 345 logger.error("Config validation failed with the error: %s" % error) 346 fail_to_start("Config validation failed", errors=errors) 347 sys.exit(1) 348 349 notifier.compose = compose 350 COMPOSE = compose 351 try: --> 352 run_compose( 353 compose, 354 create_latest_link=create_latest_link, 355 latest_link_status=latest_link_status, 356 latest_link_components=latest_link_components, 357 ) 358 except UnsignedPackagesError: Compose = PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'imagebuilder', 'image_checksum', 'init', 'kiwibuild', 'live_media', 'osbuild', 'ostree_container', 'o abort = .abort at 0x7f92f0961b20> compose = compose_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1' compose_type = 'production' conf = {'RELEASE_NAME': 'CentOS Stream', 'RELEASE_SHORT': 'CentOS-Stream', 'RELEASE_VERSION': '9', 'PKGSET_KOJI_TAG': 'c9s-compose', 'PKGSET_KOJI_MODULE_TAG': 'el9-modules-pending-signed', 'RUNROOT_CHANNEL': create_latest_link = True errors = [] fail_to_start = .fail_to_start at 0x7f92f09619e0> group = kobo = latest_link_components = -1 latest_link_status = None logger = notifier = opts = Namespace(target_dir='/mnt/centos/composes/stream-9/production', compose_dir=None, label='RC-0.1', no_label=False, supported=False, old_composes=['/mnt/centos/composes/stream-9/production'], config='/ parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=, conflict_handler='error', add_help=True) productmd = pungi = rv = None warning = 'WARNING: Unrecognized config option: RELEASE_VERSION_Y.' warnings = ["WARNING: Config option release_is_layered was removed and has no effect; remove it. It's layered if there's configuration for base product.", "WARNING: Config option runroot was removed and has no e Frame cli_main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 657 650 signal.signal(signal.SIGINT, sigterm_handler) 651 signal.signal(signal.SIGTERM, sigterm_handler) 652 653 tracing.setup() 654 655 with tracing.span("run-compose"): 656 try: --> 657 main() 658 except (Exception, KeyboardInterrupt) as ex: 659 tracing.record_exception(ex) 660 if COMPOSE: 661 COMPOSE.log_error("Compose run failed: %s" % ex) 662 COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True)) 663 COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) ex = RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more details.')