From dffb2901d349a66bdbba30423c358da7c9938e47 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 6 May 2011 21:53:31 -0700 Subject: Use finer grained locks for install. Narrow scope of merge locks to improve performance. Instead of locking the DB for the entire package merge, just lock it when we actually need to do so. Also add locks around conf_mem_file updating and pkg_* phases. Locking in pkg_* phases can be turned off with FEATURES="no-ebuild-locks" if you use ebuilds that are careful not to mess with each other during theses phases. The default is to leave this locking enabled. Given this new locking, I've improved the scheduler to run merge jobs in parallel. Time required for merging 348 packages with --usepkgonly: - Before patch: 29m50s - After patch: 10m2s - After patch w/o locks: 7m9s Change-Id: I63588c4cc59fa6fe2f8327ea1e4a9e71b241d4fe Review URL: http://gerrit.chromium.org/gerrit/498 --- pym/_emerge/EbuildPhase.py | 15 ++++++++++++++- pym/_emerge/Scheduler.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'pym/_emerge') diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index 07fb69ca7..77b3a4d88 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -10,6 +10,8 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask from portage.util import writemsg +from portage.locks import lockdir +from portage.locks import unlockdir from portage.xml.metadata import MetaDataXML import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -28,7 +30,7 @@ from portage import _unicode_encode class EbuildPhase(CompositeTask): - __slots__ = ("actionmap", "phase", "settings") + __slots__ = ("actionmap", "ebuild_lock", "phase", "settings") # FEATURES displayed prior to setup phase _features_display = ("ccache", "distcc", "fakeroot", @@ -37,6 +39,9 @@ class EbuildPhase(CompositeTask): "splitdebug", "suidctl", "test", "userpriv", "usersandbox") + # Locked phases + _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") + def _start(self): need_builddir = self.phase not in EbuildProcess._phases_without_builddir @@ -138,10 +143,18 @@ class EbuildPhase(CompositeTask): phase=self.phase, scheduler=self.scheduler, settings=self.settings) + if (self.phase in self._locked_phases and + "no-ebuild-locks" not in self.settings.features): + root = self.settings["ROOT"] + lock_path = os.path.join(root, portage.VDB_PATH + "-ebuild") + self.ebuild_lock = lockdir(lock_path) self._start_task(ebuild_process, self._ebuild_exit) def _ebuild_exit(self, ebuild_process): + if self.ebuild_lock: + unlockdir(self.ebuild_lock) + fail = False if self._default_exit(ebuild_process) != os.EX_OK: if self.phase == "test" and \ diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index df13b6b84..fc69d48ae 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -387,6 +387,7 @@ class Scheduler(PollScheduler): def _set_max_jobs(self, max_jobs): self._max_jobs = max_jobs self._task_queues.jobs.max_jobs = max_jobs + self._task_queues.merge.max_jobs = max_jobs def _background_mode(self): """ -- cgit v1.2.3-1-g7c22