From 15a799b52155a47568f4b049ff8487a2718b270c Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 3 Oct 2012 02:31:41 -0700 Subject: egencache: add --update-manifests, bug #436918 Update manifest files, and sign them if signing is enabled. This supports parallelization if enabled via the --jobs option. The --thin-manifests and --sign-manifests options may be used to manually override layout.conf settings. There's also a new --strict-manifests option that may be used to manually override the "strict" FEATURES setting, a --gpg-key option to override PORTAGE_GPG_KEY, and a --gpg-dir option to override PORTAGE_GPG_DIR. --- pym/portage/util/_async/AsyncScheduler.py | 88 +++++++++++++++++++++++++++++++ pym/portage/util/_async/ForkProcess.py | 48 +++++++++++++++++ pym/portage/util/_async/__init__.py | 2 + 3 files changed, 138 insertions(+) create mode 100644 pym/portage/util/_async/AsyncScheduler.py create mode 100644 pym/portage/util/_async/ForkProcess.py create mode 100644 pym/portage/util/_async/__init__.py (limited to 'pym/portage/util') diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py new file mode 100644 index 000000000..cae45fd90 --- /dev/null +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -0,0 +1,88 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from _emerge.AsynchronousTask import AsynchronousTask +from _emerge.PollScheduler import PollScheduler + +class AsyncScheduler(AsynchronousTask, PollScheduler): + + __slots__ = ('_error_count', '_loadavg_check_id', + '_max_jobs', '_max_load', + '_remaining_tasks', '_running_tasks', '_term_check_id') + + def __init__(self, max_jobs=None, max_load=None, **kwargs): + AsynchronousTask.__init__(self) + PollScheduler.__init__(self, **kwargs) + + if max_jobs is None: + max_jobs = 1 + self._max_jobs = max_jobs + self._max_load = max_load + self._error_count = 0 + self._running_tasks = set() + self._remaining_tasks = True + self._term_check_id = None + self._loadavg_check_id = None + + def _cancel(self): + self._terminated.set() + self._terminate_tasks() + + def _terminate_tasks(self): + for task in list(self._running_tasks): + task.cancel() + + def _next_task(self): + raise NotImplementedError(self) + + def _keep_scheduling(self): + return self._remaining_tasks and not self._terminated_tasks + + def _running_job_count(self): + return len(self._running_tasks) + + def _schedule_tasks(self): + while self._keep_scheduling() and self._can_add_job(): + try: + task = self._next_task() + except StopIteration: + self._remaining_tasks = False + else: + self._running_tasks.add(task) + task.scheduler = self.sched_iface + task.addExitListener(self._task_exit) + task.start() + + def _task_exit(self, task): + self._running_tasks.discard(task) + if task.returncode != os.EX_OK: + self._error_count += 1 + self._schedule() + + def _start(self): + self._term_check_id = self.sched_iface.idle_add(self._termination_check) + if self._max_load is not None: + # We have to schedule periodically, in case the load + # average has changed since the last call. + self._loadavg_check_id = self.sched_iface.timeout_add( + self._loadavg_latency, self._schedule) + self._schedule() + + def _wait(self): + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self.sched_iface.iteration() + + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self.sched_iface.iteration() + + if self._error_count > 0: + self.returncode = 1 + else: + self.returncode = os.EX_OK + + return self.returncode diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py new file mode 100644 index 000000000..607d0ff57 --- /dev/null +++ b/pym/portage/util/_async/ForkProcess.py @@ -0,0 +1,48 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import signal +import traceback + +import portage +from portage import os +from _emerge.SpawnProcess import SpawnProcess + +class ForkProcess(SpawnProcess): + + def _spawn(self, args, fd_pipes=None, **kwargs): + """ + Fork a subprocess, apply local settings, and call fetch(). + """ + + pid = os.fork() + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + portage.process.spawned_pids.append(pid) + return [pid] + + portage.locks._close_fds() + # Disable close_fds since we don't exec (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + rval = 1 + try: + rval = self._run() + except SystemExit: + raise + except: + traceback.print_exc() + finally: + # Call os._exit() from finally block, in order to suppress any + # finally blocks from earlier in the call stack. See bug #345289. + os._exit(rval) + + def _run(self): + raise NotImplementedError(self) diff --git a/pym/portage/util/_async/__init__.py b/pym/portage/util/_async/__init__.py new file mode 100644 index 000000000..418ad862b --- /dev/null +++ b/pym/portage/util/_async/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 -- cgit v1.2.3-1-g7c22