Traceback (most recent call last):
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 657, in cli_main
main()
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 352, in main
run_compose(
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 488, in run_compose
essentials_phase.stop()
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop
self.pool.stop()
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
six.reraise(exc_info[0], exc_info[1], exc_info[2])
File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
self.process(item, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process
phase.stop()
File "/usr/lib/python3.12/site-packages/pungi/phases/base.py", line 111, in stop
self.pool.stop()
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
six.reraise(exc_info[0], exc_info[1], exc_info[2])
File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
self.process(item, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py", line 558, in process
self.worker(compose, arch, variant, cmd, pkgset_phase, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py", line 862, in worker
task_id = runroot.run(
^^^^^^^^^^^^
File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 289, in run
self._run_koji(
File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 137, in _run_koji
raise RuntimeError(
RuntimeError: Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more details.
Frame _run_koji in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 137
130 max_retries = 3
131 while True:
132 output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
133 if output["retcode"] == 0:
134 self._result = output
135 return
136 elif attempt >= max_retries or not self._has_losetup_error(log_dir):
--> 137 raise RuntimeError(
138 "Runroot task failed: %s. See %s for more details."
139 % (output["task_id"], log_file)
140 )
141 attempt += 1
142
143 def _ssh_run(self, hostname, user, command, fmt_dict=None, log_file=None):
arch = 'x86_64'
attempt = 0
command = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo
koji_cmd = ['koji', '--profile=koji', 'runroot', '--nowait', '--task-id', '--use-shell', '--channel-override=runroot', '--package=lorax', '--mount=/mnt/centos/composes/stream-9/production/CentOS-Stream-9-2026081
koji_wrapper =
kwargs = {'mounts': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1'], 'weight': None, 'chown_paths': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/bui
log_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS-logs'
log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log'
max_retries = 3
output = {'retcode': 1, 'output': 'Watching tasks (this may be safely interrupted)...\n6414164 runroot (x86_64): free\n6414164 runroot (x86_64): free -> assigned (x86-06.stream.rdu2.redhat.com)\n6414164 runroo
packages = ['lorax']
runroot_channel = 'runroot'
runroot_tag = 'c9s-build'
self =
self._logger = None
self._result = None
self.compose =
self.runroot_method = 'koji'
Frame run in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 289
282 See `KojiWrapper.get_runroot_cmd()` for more information.
283 """
284 if self.runroot_method == "local":
285 self._run_local(
286 command, log_file=log_file, packages=packages, arch=arch, **kwargs
287 )
288 elif self.runroot_method == "koji":
--> 289 self._run_koji(
290 command, log_file=log_file, packages=packages, arch=arch, **kwargs
291 )
292 elif self.runroot_method == "openssh":
293 self._run_openssh(
294 command, log_file=log_file, packages=packages, arch=arch, **kwargs
295 )
arch = 'x86_64'
command = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo
kwargs = {'mounts': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1'], 'weight': None, 'chown_paths': ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/bui
log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log'
packages = ['lorax']
self =
self._logger = None
self._result = None
self.compose =
self.runroot_method = 'koji'
Frame worker in /usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py at line 862
855 weight=compose.conf["runroot_weights"].get("buildinstall"),
856 )
857 else:
858 try:
859 lorax_log_dir = _get_log_dir(compose, variant, arch)
860 except Exception:
861 lorax_log_dir = None
--> 862 task_id = runroot.run(
863 cmd,
864 log_file=log_file,
865 arch=arch,
866 packages=packages,
867 mounts=[compose.topdir],
868 weight=compose.conf["runroot_weights"].get("buildinstall"),
arch = 'x86_64'
buildinstall_method = 'lorax'
chown_paths = ['/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS', '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-Ba
cmd = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo
compose =
final_output_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS'
log_file = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log'
log_filename = 'buildinstall-BaseOS'
lorax_log_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS-logs'
lorax_use_koji_plugin = False
msg = 'Running buildinstall for arch x86_64, variant BaseOS'
num = 4
output_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS'
packages = ['lorax']
pkgset_phase =
runroot =
self =
self._args =
self._daemonic = False
self._ident = 140267853346496
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-659'
self._native_id = 185575
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.deliverable = 'buildinstall'
self.failed = True
self.get_timeout = 1
self.ident = 140267853346496
self.kill = True
self.name = 'Thread-659'
self.native_id = 185575
self.pool =
self.running = False
self.traceparent = None
task_id = None
variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None)
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/buildinstall.py at line 558
551 self.deliverable = "buildinstall"
552
553 def process(self, item, num):
554 compose, arch, variant, cmd, pkgset_phase = item
555 can_fail = compose.can_fail(variant, arch, self.deliverable)
556 with failable(compose, can_fail, variant, arch, self.deliverable, num):
557 try:
--> 558 self.worker(compose, arch, variant, cmd, pkgset_phase, num)
559 except RuntimeError:
560 self._print_depsolve_error(compose, arch, variant)
561 raise
562
563 def _print_depsolve_error(self, compose, arch, variant):
564 try:
arch = 'x86_64'
can_fail = False
cmd = "rm -rf /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/work/x86_64/buildinstall/BaseOS && lorax '--product=CentOS Stream' --version=9 --release=9 --source=file:///mnt/centos/compo
compose =
item =
num = 4
pkgset_phase =
self =
self._args =
self._daemonic = False
self._ident = 140267853346496
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-659'
self._native_id = 185575
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.deliverable = 'buildinstall'
self.failed = True
self.get_timeout = 1
self.ident = 140267853346496
self.kill = True
self.name = 'Thread-659'
self.native_id = 185575
self.pool =
self.running = False
self.traceparent = None
variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None)
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
60
61 self.pool.queue_get_lock.acquire()
62 self.pool.queue_processed += 1
63 num = self.pool.queue_processed
64 self.pool.queue_get_lock.release()
65
66 try:
--> 67 self.process(item, num)
68 except:
69 self.failed = True
70 self.pool.exceptions.append(sys.exc_info())
71 self.pool.kill()
72
73 def process(self, item, num):
item =
num = 4
self =
self._args =
self._daemonic = False
self._ident = 140267853346496
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-659'
self._native_id = 185575
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.deliverable = 'buildinstall'
self.failed = True
self.get_timeout = 1
self.ident = 140267853346496
self.kill = True
self.name = 'Thread-659'
self.native_id = 185575
self.pool =
self.running = False
self.traceparent = None
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
712
713 def reraise(tp, value, tb=None):
714 try:
715 if value is None:
716 value = tp()
717 if value.__traceback__ is not tb:
718 raise value.with_traceback(tb)
--> 719 raise value
720 finally:
721 value = None
722 tb = None
723
724 else:
725 def exec_(_code_, _globs_=None, _locs_=None):
tb = None
tp =
value = None
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
113 """
114 for i in self.threads:
115 i.running = False
116 for i in self.threads:
117 i.join()
118 if self.exceptions:
119 exc_info = self.exceptions[0]
--> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2])
121
122 def kill(self):
123 for i in self.threads:
124 i.kill = True
125 i.running = False
126
exc_info =
i =
self =
self._logger =
self.exceptions = [(, RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more detail
self.finished_tasks = {('BaseOS', 's390x'), ('BaseOS', 'aarch64'), ('BaseOS', 'ppc64le')}
self.kickstart_file = None
self.queue =
self.queue_get_lock =
self.queue_processed = 4
self.queue_put_lock =
self.queue_total = 4
self.reused_tasks = set()
self.threads = [, , ,
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/base.py at line 111
104 "Note that variants can be excluded in configuration file"
105 )
106
107 def stop(self):
108 if self.finished:
109 return
110 if hasattr(self, "pool"):
--> 111 self.pool.stop()
112 self.finished = True
113 self.compose.log_info("[DONE ] %s" % self.msg)
114
115 if hasattr(self, "_start_time"):
116 self.compose.log_info(
117 "PHASE %s took %d seconds"
self =
self._skipped = False
self._start_time = 1787159314.322116
self.buildinstall_method = 'lorax'
self.compose =
self.finished = False
self.lorax_use_koji_plugin = False
self.msg = '---------- PHASE: BUILDINSTALL ----------'
self.name = 'buildinstall'
self.pkgset_phase =
self.pool =
self.used_patterns = None
self.warned_skipped = False
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81
74 self.pool.queue_total,
75 phases_names,
76 )
77 self.pool.log_info("[BEGIN] %s" % (msg,))
78
79 for phase in pipeline:
80 phase.start()
--> 81 phase.stop()
82
83 self.pool.log_info("[DONE ] %s" % (msg,))
item = []
msg = 'Running pipeline (1/4). Phases: buildinstall'
num = 1
phase =
phases_names = 'buildinstall'
pipeline = []
self =
self._args =
self._daemonic = False
self._ident = 140269073536704
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-648'
self._native_id = 185563
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140269073536704
self.kill = True
self.name = 'Thread-648'
self.native_id = 185563
self.pool =
self.running = False
self.traceparent = None
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
60
61 self.pool.queue_get_lock.acquire()
62 self.pool.queue_processed += 1
63 num = self.pool.queue_processed
64 self.pool.queue_get_lock.release()
65
66 try:
--> 67 self.process(item, num)
68 except:
69 self.failed = True
70 self.pool.exceptions.append(sys.exc_info())
71 self.pool.kill()
72
73 def process(self, item, num):
item = []
num = 1
self =
self._args =
self._daemonic = False
self._ident = 140269073536704
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-648'
self._native_id = 185563
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140269073536704
self.kill = True
self.name = 'Thread-648'
self.native_id = 185563
self.pool =
self.running = False
self.traceparent = None
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
712
713 def reraise(tp, value, tb=None):
714 try:
715 if value is None:
716 value = tp()
717 if value.__traceback__ is not tb:
718 raise value.with_traceback(tb)
--> 719 raise value
720 finally:
721 value = None
722 tb = None
723
724 else:
725 def exec_(_code_, _globs_=None, _locs_=None):
tb = None
tp =
value = None
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
113 """
114 for i in self.threads:
115 i.running = False
116 for i in self.threads:
117 i.join()
118 if self.exceptions:
119 exc_info = self.exceptions[0]
--> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2])
121
122 def kill(self):
123 for i in self.threads:
124 i.kill = True
125 i.running = False
126
exc_info =
i =
self =
self._logger =
self.exceptions = [(, RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more detail
self.queue =
self.queue_get_lock =
self.queue_processed = 4
self.queue_put_lock =
self.queue_total = 4
self.threads = [, , ,
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59
52
53 self.pool.start()
54
55 def stop(self):
56 if self.finished:
57 return
58 if hasattr(self, "pool"):
--> 59 self.pool.stop()
60 self.finished = True
61 self.compose.log_info("[DONE ] %s" % self.msg)
62
63
64 class PipelineThread(WorkerThread):
65 """
self =
self._phases_schema =
self.compose =
self.finished = False
self.msg = '---------- PHASE: WEAVER ----------'
self.name = 'weaver'
self.pool =
Frame run_compose in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 488
481 extrafiles_phase,
482 ostree_phase,
483 )
484 essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
485 essentials_phase.start()
486 ostree_container_phase.start()
487 try:
--> 488 essentials_phase.stop()
489
490 # write treeinfo before ISOs are created
491 for variant in compose.get_variants():
492 for arch in variant.arches + ["src"]:
493 pungi.metadata.write_tree_info(
494 compose, arch, variant, bi=buildinstall_phase
buildinstall_phase =
compose =
config_copy_path = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-copy'
config_dump_full = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-dump.global.log'
config_file = '/home/composer/jenkins/workspace/stream9-prod-compose/pungi-centos/centos/variables.conf'
create_latest_link = True
createiso_phase =
createrepo_phase =
errors = []
essentials_phase =
essentials_schema =
extra_isos_phase =
extrafiles_phase =
f = <_io.TextIOWrapper name='/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'>
gather_phase =
image_build_phase =
image_checksum_phase =
imagebuilder_phase =
init_phase =
installed_pkgs_log = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/global/installed-pkgs.global.log'
kiwibuild_phase =
latest_link_components = -1
latest_link_status = None
livemedia_phase =
osbuild_phase =
ostree_container_phase =
ostree_installer_phase =
ostree_phase =
phase =
pkgset_phase =
pungi =
repoclosure_phase =
rpm_repo_mapping_phase =
setup_phase =
setup_schema =
test_phase =
trigger_konflux_pipeline =
Frame main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 352
345 logger.error("Config validation failed with the error: %s" % error)
346 fail_to_start("Config validation failed", errors=errors)
347 sys.exit(1)
348
349 notifier.compose = compose
350 COMPOSE = compose
351 try:
--> 352 run_compose(
353 compose,
354 create_latest_link=create_latest_link,
355 latest_link_status=latest_link_status,
356 latest_link_components=latest_link_components,
357 )
358 except UnsignedPackagesError:
Compose =
PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'imagebuilder', 'image_checksum', 'init', 'kiwibuild', 'live_media', 'osbuild', 'ostree_container', 'o
abort = .abort at 0x7f92f0961b20>
compose =
compose_dir = '/mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1'
compose_type = 'production'
conf = {'RELEASE_NAME': 'CentOS Stream', 'RELEASE_SHORT': 'CentOS-Stream', 'RELEASE_VERSION': '9', 'PKGSET_KOJI_TAG': 'c9s-compose', 'PKGSET_KOJI_MODULE_TAG': 'el9-modules-pending-signed', 'RUNROOT_CHANNEL':
create_latest_link = True
errors = []
fail_to_start = .fail_to_start at 0x7f92f09619e0>
group =
kobo =
latest_link_components = -1
latest_link_status = None
logger =
notifier =
opts = Namespace(target_dir='/mnt/centos/composes/stream-9/production', compose_dir=None, label='RC-0.1', no_label=False, supported=False, old_composes=['/mnt/centos/composes/stream-9/production'], config='/
parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=, conflict_handler='error', add_help=True)
productmd =
pungi =
rv = None
warning = 'WARNING: Unrecognized config option: RELEASE_VERSION_Y.'
warnings = ["WARNING: Config option release_is_layered was removed and has no effect; remove it. It's layered if there's configuration for base product.", "WARNING: Config option runroot was removed and has no e
Frame cli_main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 657
650 signal.signal(signal.SIGINT, sigterm_handler)
651 signal.signal(signal.SIGTERM, sigterm_handler)
652
653 tracing.setup()
654
655 with tracing.span("run-compose"):
656 try:
--> 657 main()
658 except (Exception, KeyboardInterrupt) as ex:
659 tracing.record_exception(ex)
660 if COMPOSE:
661 COMPOSE.log_error("Compose run failed: %s" % ex)
662 COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True))
663 COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
ex = RuntimeError('Runroot task failed: 6414164. See /mnt/centos/composes/stream-9/production/CentOS-Stream-9-20260819.1/logs/x86_64/buildinstall-BaseOS.x86_64.log for more details.')