summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid James <davidjames@chromium.org>2011-05-09 21:11:47 -0700
committerZac Medico <zmedico@gentoo.org>2011-05-09 21:47:42 -0700
commit928bc527a8c3eab3784df54226df2f2d83c6c85c (patch)
tree78372e2d8c9c01fbe4050b87c1e35bf8c3e995b3
parent57b3d395aa582e798f008240ebd635fd0f269bed (diff)
downloadportage-928bc527a8c3eab3784df54226df2f2d83c6c85c.tar.gz
portage-928bc527a8c3eab3784df54226df2f2d83c6c85c.tar.bz2
portage-928bc527a8c3eab3784df54226df2f2d83c6c85c.zip
Cache counter values, avoiding I/O when it doesn't change.
This improves merge times by up to 25%, since looping over the vardb for each package install is slow. TEST=Emerge a bunch of packages, notice 25% speed improvement. BUG=chromium-os:15112 Change-Id: I51dd617219cd1820ceeb702291bd790990995be4
-rw-r--r--pym/portage/dbapi/_MergeProcess.py3
-rw-r--r--pym/portage/dbapi/vartree.py71
2 files changed, 40 insertions, 34 deletions
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 12a0baf55..05f45d51a 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -143,6 +143,7 @@ class MergeProcess(SpawnProcess):
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.register(elog_reader_fd,
self._registered_events, self._elog_output_handler)
+ counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
@@ -196,7 +197,7 @@ class MergeProcess(SpawnProcess):
try:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
- prev_mtimes=self.prev_mtimes)
+ prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c81e99b9e..7c6602752 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -166,6 +166,8 @@ class vardbapi(dbapi):
self._linkmap = LinkageMap(self)
self._owners = self._owners_db(self)
+ self._cached_counter = None
+
def getpath(self, mykey, filename=None):
# This is an optimized hotspot, so don't use unicode-wrapped
# os module and don't use os.path.join().
@@ -724,17 +726,6 @@ class vardbapi(dbapi):
@param myroot: ignored, self._eroot is used instead
"""
myroot = None
- cp_list = self.cp_list
- max_counter = 0
- for cp in self.cp_all():
- for cpv in cp_list(cp):
- try:
- counter = int(self.aux_get(cpv, ["COUNTER"])[0])
- except (KeyError, OverflowError, ValueError):
- continue
- if counter > max_counter:
- max_counter = counter
-
new_vdb = False
counter = -1
try:
@@ -762,16 +753,27 @@ class vardbapi(dbapi):
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
- # We must ensure that we return a counter
- # value that is at least as large as the
- # highest one from the installed packages,
- # since having a corrupt value that is too low
- # can trigger incorrect AUTOCLEAN behavior due
- # to newly installed packages having lower
- # COUNTERs than the previous version in the
- # same slot.
- if counter > max_counter:
+ if self._cached_counter == counter:
max_counter = counter
+ else:
+ # We must ensure that we return a counter
+ # value that is at least as large as the
+ # highest one from the installed packages,
+ # since having a corrupt value that is too low
+ # can trigger incorrect AUTOCLEAN behavior due
+ # to newly installed packages having lower
+ # COUNTERs than the previous version in the
+ # same slot.
+ cp_list = self.cp_list
+ max_counter = counter
+ for cp in self.cp_all():
+ for cpv in cp_list(cp):
+ try:
+ pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0])
+ except (KeyError, OverflowError, ValueError):
+ continue
+ if pkg_counter > max_counter:
+ max_counter = pkg_counter
if counter < 0 and not new_vdb:
writemsg(_("!!! Initializing COUNTER to " \
@@ -789,18 +791,19 @@ class vardbapi(dbapi):
"""
myroot = None
mycpv = None
-
self.lock()
try:
counter = self.get_counter_tick_core() - 1
- if incrementing:
- #increment counter
- counter += 1
- # use same permissions as config._init_dirs()
- ensure_dirs(os.path.dirname(self._counter_path),
- gid=portage_gid, mode=0o2750, mask=0o2)
- # update new global counter file
- write_atomic(self._counter_path, str(counter))
+ if self._cached_counter != counter:
+ if incrementing:
+ #increment counter
+ counter += 1
+ # use same permissions as config._init_dirs()
+ ensure_dirs(os.path.dirname(self._counter_path),
+ gid=portage_gid, mode=0o2750, mask=0o2)
+ # update new global counter file
+ write_atomic(self._counter_path, str(counter))
+ self._cached_counter = counter
finally:
self.unlock()
@@ -2860,7 +2863,7 @@ class dblink(object):
os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
- mydbapi=None, prev_mtimes=None):
+ mydbapi=None, prev_mtimes=None, counter=None):
"""
This function does the following:
@@ -3276,7 +3279,8 @@ class dblink(object):
self.copyfile(inforoot+"/"+x)
# write local package counter for recording
- counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
+ if counter is not None:
+ counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
encoding=_encodings['fs'], errors='strict'),
'w', encoding=_encodings['repo.content'], errors='backslashreplace'
@@ -3906,7 +3910,7 @@ class dblink(object):
showMessage(zing + " " + mydest + "\n")
def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
- mydbapi=None, prev_mtimes=None):
+ mydbapi=None, prev_mtimes=None, counter=None):
"""
@param myroot: ignored, self._eroot is used instead
"""
@@ -3918,7 +3922,8 @@ class dblink(object):
self.vartree.dbapi._bump_mtime(self.mycpv)
try:
retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
- cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
+ cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes,
+ counter=counter)
# If PORTAGE_BUILDDIR doesn't exist, then it probably means
# fail-clean is enabled, and the success/die hooks have