Add scheduler._init_installed_graph() and _destroy_installed_graph()

methods and use them to free memory when calling _calc_resume_list()
for emerge --keep-going.

1 files changed, 39 insertions, 18 deletions

diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index 86d815c8c..89e5f235f 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -3,6 +3,7 @@
 
 from __future__ import print_function
 
+import gc
 import gzip
 import logging
 import shutil
@@ -38,11 +39,11 @@ from _emerge.clear_caches import clear_caches
 from _emerge.create_depgraph_params import create_depgraph_params
 from _emerge.create_world_atom import create_world_atom
 from _emerge.DepPriority import DepPriority
-from _emerge.depgraph import depgraph, resume_depgraph, \
-	_frozen_depgraph_config
+from _emerge.depgraph import depgraph, resume_depgraph
 from _emerge.EbuildFetcher import EbuildFetcher
 from _emerge.EbuildPhase import EbuildPhase
 from _emerge.emergelog import emergelog, _emerge_log_dir
+from _emerge.FakeVartree import FakeVartree
 from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps
 from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo
 from _emerge.JobStatusDisplay import JobStatusDisplay
@@ -197,23 +198,12 @@ class Scheduler(PollScheduler):
 		self.edebug = 0
 		if settings.get("PORTAGE_DEBUG", "") == "1":
 			self.edebug = 1
+		self.pkgsettings = {}
 		self._config_pool = {}
-
-		# TODO: Replace the BlockerDB with a depgraph of installed packages
-		# that's updated incrementally with each upgrade/uninstall operation
-		# This will be useful for making quick and safe decisions with respect
-		# to aggressive parallelization discussed in bug #279623.
-		frozen_config = None
-		if frozen_config is None:
-			frozen_config = _frozen_depgraph_config(settings, trees,
-				myopts, None)
-		self._frozen_config = frozen_config
-		self.pkgsettings = frozen_config.pkgsettings
-		self._blocker_db = {}
-		for root in trees:
+		for root in self.trees:
 			self._config_pool[root] = []
-			self._blocker_db[root] = BlockerDB(
-				frozen_config.trees[root]["vartree"])
+
+		self._init_installed_graph()
 
 		fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
 			schedule=self._schedule_fetch)
@@ -293,6 +283,30 @@ class Scheduler(PollScheduler):
 			self._running_portage = self._pkg(cpv, "installed",
 				self._running_root, installed=True)
 
+	def _init_installed_graph(self):
+		"""
+		Initialization structures used for dependency calculations
+		involving currently installed packages.
+		"""
+		# TODO: Replace the BlockerDB with a depgraph of installed packages
+		# that's updated incrementally with each upgrade/uninstall operation
+		# This will be useful for making quick and safe decisions with respect
+		# to aggressive parallelization discussed in bug #279623.
+		self._blocker_db = {}
+		for root in self.trees:
+			self._blocker_db[root] = \
+				BlockerDB(FakeVartree(self.trees[root]["root_config"]))
+
+	def _destroy_installed_graph(self):
+		"""
+		Use this to free memory before calling _calc_resume_list().
+		After _calc_resume_list(), the _init_installed_graph() needs
+		to be called in order to re-generate the structures that this
+		method destroys.
+		"""
+		self._blocker_db = None
+		gc.collect()
+
 	def _poll(self, timeout=None):
 		self._schedule()
 		PollScheduler._poll(self, timeout=timeout)
@@ -549,7 +563,6 @@ class Scheduler(PollScheduler):
 		# Call gc.collect() here to avoid heap overflow that
 		# triggers 'Cannot allocate memory' errors (reported
 		# with python-2.5).
-		import gc
 		gc.collect()
 
 		blocker_db = self._blocker_db[new_pkg.root]
@@ -1064,6 +1077,10 @@ class Scheduler(PollScheduler):
 			if not mergelist:
 				break
 
+			# free some memory before creating
+			# the resume depgraph
+			self._destroy_installed_graph()
+
 			if not self._calc_resume_list():
 				break
 
@@ -1071,6 +1088,10 @@ class Scheduler(PollScheduler):
 			if not self._mergelist:
 				break
 
+			# Initialize the installed graph again
+			# since it was destroyed above in order
+			# to free memory.
+			self._init_installed_graph()
 			self._save_resume_list()
 			self._pkg_count.curval = 0
 			self._pkg_count.maxval = len([x for x in self._mergelist \