Move common file functionality into a FileLoader class and delegate parsing to subclasses via a lineParser func, no idea how this will work performance wise. Fix errors in parser (blank lines are not errors ;))

svn path=/main/trunk/; revision=7365

2 files changed, 140 insertions, 112 deletions

diff --git a/pym/portage/env/config.py b/pym/portage/env/config.py
index a354dcf7d..bb63e35e8 100644
--- a/pym/portage/env/config.py
+++ b/pym/portage/env/config.py
@@ -6,7 +6,7 @@
 from UserDict import UserDict
 from portage.env.loaders import KeyListFileLoader, KeyValuePairFileLoader, ItemFileLoader
 
-class UserConfigKlass(UserDict,object):
+class UserConfigKlass(UserDict, object):
 	"""
 	A base class stub for things to inherit from.
 	Users may want a non-file backend.
@@ -19,6 +19,7 @@ class UserConfigKlass(UserDict,object):
 		@param loader: A class that has a load() that returns two dicts
 			the first being a data dict, the second being a dict of errors.
 		"""
+		UserDict.__init__(self)
 		self._loader = loader
 
 	def load(self):
diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py
index 03d3f2264..e80bce6de 100644
--- a/pym/portage/env/loaders.py
+++ b/pym/portage/env/loaders.py
@@ -15,11 +15,13 @@ class LoaderError(Exception):
 		@type error_msg: String
 		"""
 
-		self.resource
+		self.resource = resource
+		self.error_msg = error_msg
 	
 	def __str__(self):
 		return "Failed while loading resource: %s, error was: %s" % (
-			resource, error_msg)
+			self.resource, self.error_msg)
+
 
 def RecursiveFileLoader(filename):
 	"""
@@ -36,16 +38,17 @@ def RecursiveFileLoader(filename):
 	"""
 
 	if os.path.isdir(filename):
-		for root, dirs, files in os.walk(self.fname):
+		for root, dirs, files in os.walk(filename):
 			if 'CVS' in dirs:
 				dirs.remove('CVS')
-			files = filter(files, startswith('.'))
-			files = filter(files, endswith('~'))
-			for file in files:
-				yield file
+			files = filter(files, str.startswith('.'))
+			files = filter(files, str.endswith('~'))
+			for f in files:
+				yield f
 	else:
 		yield filename
 
+
 class DataLoader(object):
 
 	def __init__(self, validator):
@@ -63,9 +66,58 @@ class DataLoader(object):
 		"""
 		Function to do the actual work of a Loader
 		"""
-		pass
+		raise NotImplementedError("Please override in a subclass")
+
+
+class FileLoader(DataLoader):
+	""" Class to access data in files """
+
+	def __init__(self, filename, validator):
+		"""
+			Args:
+				filename : Name of file or directory to open
+				validator : class with validate() method to validate data.
+		"""
+		DataLoader.__init__(self, validator)
+		self.fname = filename
+
+	def load(self):
+		"""
+		Return the {source: {key: value}} pairs from a file
+		Return the {source: [list of errors] from a load
+
+		@param recursive: If set and self.fname is a directory; 
+			load all files in self.fname
+		@type: Boolean
+		@rtype: tuple
+		@returns:
+		Returns (data,errors), both may be empty dicts or populated.
+		"""
+		data = {}
+		errors = {}
+		# I tried to save a nasty lookup on lineparser by doing the lookup
+		# once, which may be expensive due to digging in child classes.
+		func = self.lineParser
+		for fn in RecursiveFileLoader(self.fname):
+			f = open(fn, 'rb')
+			for line_num, line in enumerate(f):
+				func(line, line_num, data, errors)
+		return (data, errors)
+
+	def lineParser(self, line, line_num, data, errors):
+		""" This function parses 1 line at a time
+			Args:
+				line: a string representing 1 line of a file
+				line_num: an integer representing what line we are processing
+				data: a dict that contains the data we have extracted from the file
+				      already
+				errors: a dict representing parse errors.
+			Returns:
+				Nothing (None).  Writes to data and errors
+		"""
+		raise NotImplementedError("Please over-ride this in a child class")
 
-class ItemFileLoader(DataLoader):
+class ItemFileLoader(FileLoader):
 	"""
 	Class to load data from a file full of items one per line
 	
@@ -78,36 +130,31 @@ class ItemFileLoader(DataLoader):
 	Note that due to the data store being a dict, duplicates
 	are removed.
 	"""
-	
-	_recursive = False
 
 	def __init__(self, filename, validator):
-		DataLoader.__init__(self, validator)
-		self.fname = filename
+		FileLoader.__init__(self, filename, validator)
 	
-	def load(self):
-		data = {}
-		errors = {}
-		for file in RecursiveFileLoader(self.fname):
-			f = open(file, 'rb')
-			for line_num, line in enumerate(f):
-				if line.startswith('#'):
-					continue
-				split = line.strip().split()
-				if not len(split):
-					errors.setdefault(self.fname,[]).append(
-					"Malformed data at line: %s, data: %s"
-					% (line_num + 1, split))
-				key = split[0]
-				if not self._validator.validate(key):
-					errors.setdefault(self.fname,[]).append(
-					"Validation failed at line: %s, data %s"
-					% (line_num + 1, split))
-					continue
-				data[key] = None
-		return (data, errors)
-
-class KeyListFileLoader(DataLoader):
+	def lineParser(self, line, line_num, data, errors):
+		line = line.strip()
+		if line.startswith('#'): # Skip commented lines
+			return
+		if not len(line): # skip empty lines
+			return
+		split = line.split()
+		if not len(split):
+			errors.setdefault(self.fname, []).append(
+				"Malformed data at line: %s, data: %s"
+				% (line_num + 1, line))
+			return
+		key = split[0]
+		if not self._validator.validate(key):
+			errors.setdefault(self.fname, []).append(
+			"Validation failed at line: %s, data %s"
+			% (line_num + 1, key))
+			return
+		data[key] = None
+
+class KeyListFileLoader(FileLoader):
 	"""
 	Class to load data from a file full of key [list] tuples
 	
@@ -116,40 +163,36 @@ class KeyListFileLoader(DataLoader):
 	{'key':['foo1','foo2','foo3']}
 	"""
 
-	_recursive = False
-
 	def __init__(self, filename, validator):
-		DataLoader.__init__(self, validator)
-		self.fname = filename
-
-	def load(self):
-		data = {}
-		errors = {}
-		for file in RecursiveFileLoader(self.fname):
-			f = open(file, 'rb')
-			for line_num, line in enumerate(f):
-				if line.startswith('#'):
-					continue
-				split = line.strip().split()
-				if len(split) < 2:
-					errors.setdefault(self.fname,[]).append(
-					"Malformed data at line: %s, data: %s"
-					% (line_num + 1, split))
-					continue
-				key = split[0]
-				value = split[1:]
-				if not self._validator.validate(key):
-					errors.setdefault(self.fname,[]).append(
-					"Validation failed at line: %s, data %s"
-					% (line_num + 1, split))
-					continue
-				if key in data:
-					data[key].append(value)
-				else:
-					data[key] = value
-		return (data, errors)
-
-class KeyValuePairFileLoader(DataLoader):
+		FileLoader.__init__(self, filename, validator)
+
+
+	def lineParser(self, line, line_num, data, errors):
+		line = line.strip()
+		if line.startswith('#'): # Skip commented lines
+			return
+		if not len(line): # skip empty lines
+			return
+		split = line.split()
+		if len(split) < 2:
+			errors.setdefault(self.fname, []).append(
+			"Malformed data at line: %s, data: %s"
+			% (line_num + 1, line))
+			return
+		key = split[0]
+		value = split[1:]
+		if not self._validator.validate(key):
+			errors.setdefault(self.fname, []).append(
+			"Validation failed at line: %s, data %s"
+			% (line_num + 1, key))
+			return
+		if key in data:
+			data[key].append(value)
+		else:
+			data[key] = value
+
+
+class KeyValuePairFileLoader(FileLoader):
 	"""
 	Class to load data from a file full of key=value pairs
 	
@@ -160,47 +203,31 @@ class KeyValuePairFileLoader(DataLoader):
 	 'foo':'bar'}
 	"""
 
-	_recursive = False
-
 	def __init__(self, filename, validator):
-		DataLoader.__init__(self, validator)
-		self.fname = filename
+		FileLoader.__init__(self, filename, validator)
+
+
+	def lineParser(self, line, line_num, data, errors):
+		line = line.strip()
+		if line.startswith('#'): # skip commented lines
+			return
+		if not len(line): # skip empty lines
+			return
+		split = line.split('=')
+		if len(split) < 2:
+			errors.setdefault(self.fname, []).append(
+			"Malformed data at line: %s, data %s"
+			% (line_num + 1, line))
+			return
+		key = split[0]
+		value = split[1:]
+		if not self._validator.validate(key):
+			errors.setdefault(self.fname, []).append(
+			"Validation failed at line: %s, data %s"
+			% (line_num + 1, key))
+			return
+		if key in data:
+			data[key].append(value)
+		else:
+			data[key] = value
 
-	def load(self):
-		"""
-		Return the {source: {key: value}} pairs from a file
-		Return the {source: [list of errors] from a load
-
-		@param recursive: If set and self.fname is a directory; 
-			load all files in self.fname
-		@type: Boolean
-		@rtype: tuple
-		@returns:
-		Returns (data,errors), both may be empty dicts or populated.
-		"""
-
-		DataLoader.load(self)
-		data = {}
-		errors = {}
-		for file in RecursiveFileLoader(self.fname):
-			f = open(file, 'rb')
-			for line_num, line in enumerate(f):
-				if line.startswith('#'):
-					continue
-				split = line.strip().split('=')
-				if len(split) < 2:
-					errors.setdefault(self.fname, []).append(
-					"Malformed data at line: %s, data %s"
-					% (line_num + 1, split))
-				key = split[0]
-				value = split[1:]
-				if not self._validator.validate(key):
-					errors.setdefault(self.fname, []).append(
-					"Validation failed at line: %s, data %s"
-					% (line_num + 1, split))
-					continue
-				if key in data:
-					data[key].append(value)
-				else:
-					data[key] = value
-		return (data, errors)