Enable elog functionality for the python side of portage

svn path=/main/trunk/; revision=6548

10 files changed, 213 insertions, 119 deletions

diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py
index be7eb5b66..77bdd1b94 100644
--- a/pym/portage/elog/__init__.py
+++ b/pym/portage/elog/__init__.py
@@ -8,111 +8,99 @@ from portage.exception import PortageException
 from portage.process import atexit_register
 from portage.util import writemsg
 
-from portage import listdir
+from portage.elog.messages import collect_ebuild_messages, collect_messages
+from portage.elog.filtering import filter_loglevels
 
 import os
 
+def _merge_logentries(a, b):
+	rValue = {}
+	phases = set(a.keys()+b.keys())
+	for p in phases:
+		rValue[p] = []
+		if a.has_key(p):
+			for x in a[p]:
+				rValue[p].append(x)
+		if b.has_key(p):
+			for x in b[p]:
+				rValue[p].append(x)
+	return rValue
+
+def _combine_logentries(logentries):
+	# generate a single string with all log messages
+	rValue = ""
+	for phase in EBUILD_PHASES:
+		if not phase in logentries:
+			continue
+		for msgtype, msgcontent in logentries[phase]:
+			rValue += "%s: %s\n" % (msgtype, phase)
+			for line in msgcontent:
+				rValue += line
+			rValue += "\n"
+	return rValue
+
 _elog_atexit_handlers = []
 def elog_process(cpv, mysettings):
-	mylogfiles = listdir(mysettings["T"]+"/logging/")
-	# shortcut for packages without any messages
-	if len(mylogfiles) == 0:
-		return
-	# exploit listdir() file order so we process log entries in chronological order
-	mylogfiles.reverse()
-	all_logentries = {}
-	for f in mylogfiles:
-		msgfunction, msgtype = f.split(".")
-		if msgfunction not in EBUILD_PHASES:
-			writemsg("!!! can't process invalid log file: %s\n" % f,
-				noiselevel=-1)
-			continue
-		if not msgfunction in all_logentries:
-			all_logentries[msgfunction] = []
-		msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines()
-		all_logentries[msgfunction].append((msgtype, msgcontent))
+	ebuild_logentries = collect_ebuild_messages(os.path.join(mysettings["T"], "logging"))
+	all_logentries = collect_messages()
+	if all_logentries.has_key(cpv):
+		all_logentries[cpv] = _merge_logentries(ebuild_logentries, all_logentries[cpv])
+	else:
+		all_logentries[cpv] = ebuild_logentries
 
-	def filter_loglevels(logentries, loglevels):
-		# remove unwanted entries from all logentries
-		rValue = {}
-		loglevels = map(str.upper, loglevels)
-		for phase in logentries.keys():
-			for msgtype, msgcontent in logentries[phase]:
-				if msgtype.upper() in loglevels or "*" in loglevels:
-					if not rValue.has_key(phase):
-						rValue[phase] = []
-					rValue[phase].append((msgtype, msgcontent))
-		return rValue
-	
 	my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split())
-	default_logentries = filter_loglevels(all_logentries, my_elog_classes)
 
-	# in case the filters matched all messages and no module overrides exist
-	if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")):
-		return
 
-	def combine_logentries(logentries):
-		# generate a single string with all log messages
-		rValue = ""
-		for phase in EBUILD_PHASES:
-			if not phase in logentries:
-				continue
-			for msgtype, msgcontent in logentries[phase]:
-				rValue += "%s: %s\n" % (msgtype, phase)
-				for line in msgcontent:
-					rValue += line
-				rValue += "\n"
-		return rValue
-	
-	default_fulllog = combine_logentries(default_logentries)
+	for key in all_logentries.keys():
+		default_logentries = filter_loglevels(all_logentries[key], my_elog_classes)
 
-	# pass the processing to the individual modules
-	logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split()
-	for s in logsystems:
-		# allow per module overrides of PORTAGE_ELOG_CLASSES
-		if ":" in s:
-			s, levels = s.split(":", 1)
-			levels = levels.split(",")
-			mod_logentries = filter_loglevels(all_logentries, levels)
-			mod_fulllog = combine_logentries(mod_logentries)
-		else:
-			mod_logentries = default_logentries
-			mod_fulllog = default_fulllog
-		if len(mod_logentries) == 0:
-			continue
-		# - is nicer than _ for module names, so allow people to use it.
-		s = s.replace("-", "_")
-		try:
-			# FIXME: ugly ad.hoc import code
-			# TODO:  implement a common portage module loader
-			name = "portage.elog.mod_" + s
-			m = __import__(name)
-			for comp in name.split(".")[1:]:
-				m = getattr(m, comp)
-			def timeout_handler(signum, frame):
-				raise PortageException("Timeout in elog_process for system '%s'" % s)
-			import signal
-			signal.signal(signal.SIGALRM, timeout_handler)
-			# Timeout after one minute (in case something like the mail
-			# module gets hung).
-			signal.alarm(60)
+		# in case the filters matched all messages and no module overrides exist
+		if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")):
+			return
+
+		default_fulllog = _combine_logentries(default_logentries)
+
+		# pass the processing to the individual modules
+		logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split()
+		for s in logsystems:
+			# allow per module overrides of PORTAGE_ELOG_CLASSES
+			if ":" in s:
+				s, levels = s.split(":", 1)
+				levels = levels.split(",")
+				mod_logentries = filter_loglevels(all_logentries[key], levels)
+				mod_fulllog = combine_logentries(mod_logentries)
+			else:
+				mod_logentries = default_logentries
+				mod_fulllog = default_fulllog
+			if len(mod_logentries) == 0:
+				continue
+			# - is nicer than _ for module names, so allow people to use it.
+			s = s.replace("-", "_")
 			try:
-				m.process(mysettings, cpv, mod_logentries, mod_fulllog)
-			finally:
-				signal.alarm(0)
-			if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers:
-				_elog_atexit_handlers.append(m.finalize)
-				atexit_register(m.finalize, mysettings)
-		except (ImportError, AttributeError), e:
-			writemsg("!!! Error while importing logging modules " + \
-				"while loading \"mod_%s\":\n" % str(s))
-			writemsg("%s\n" % str(e), noiselevel=-1)
-		except PortageException, e:
-			writemsg("%s\n" % str(e), noiselevel=-1)
+				# FIXME: ugly ad.hoc import code
+				# TODO:  implement a common portage module loader
+				name = "portage.elog.mod_" + s
+				m = __import__(name)
+				for comp in name.split(".")[1:]:
+					m = getattr(m, comp)
+				def timeout_handler(signum, frame):
+					raise PortageException("Timeout in elog_process for system '%s'" % s)
+				import signal
+				signal.signal(signal.SIGALRM, timeout_handler)
+				# Timeout after one minute (in case something like the mail
+				# module gets hung).
+				signal.alarm(60)
+				try:
+					m.process(mysettings, str(key), mod_logentries, mod_fulllog)
+				finally:
+					signal.alarm(0)
+				if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers:
+					_elog_atexit_handlers.append(m.finalize)
+					atexit_register(m.finalize, mysettings)
+			except (ImportError, AttributeError), e:
+				writemsg("!!! Error while importing logging modules " + \
+					"while loading \"mod_%s\":\n" % str(s))
+				writemsg("%s\n" % str(e), noiselevel=-1)
+			except PortageException, e:
+				writemsg("%s\n" % str(e), noiselevel=-1)
 
-	# clean logfiles to avoid repetitions
-	for f in mylogfiles:
-		try:
-			os.unlink(os.path.join(mysettings["T"], "logging", f))
-		except OSError:
-			pass
diff --git a/pym/portage/elog/filtering.py b/pym/portage/elog/filtering.py
new file mode 100644
index 000000000..f4748fede
--- /dev/null
+++ b/pym/portage/elog/filtering.py
@@ -0,0 +1,17 @@
+# elog/messages.py - elog core functions
+# Copyright 2006-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $
+
+def filter_loglevels(logentries, loglevels):
+	# remove unwanted entries from all logentries
+	rValue = {}
+	loglevels = map(str.upper, loglevels)
+	for phase in logentries.keys():
+		for msgtype, msgcontent in logentries[phase]:
+			if msgtype.upper() in loglevels or "*" in loglevels:
+				if not rValue.has_key(phase):
+					rValue[phase] = []
+				rValue[phase].append((msgtype, msgcontent))
+	return rValue
+	
diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py
new file mode 100644
index 000000000..bfe428030
--- /dev/null
+++ b/pym/portage/elog/messages.py
@@ -0,0 +1,89 @@
+# elog/messages.py - elog core functions
+# Copyright 2006-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $
+
+from portage.output import colorize
+from portage.const import EBUILD_PHASES
+from portage.util import writemsg
+
+from portage import listdir
+
+import os
+
+def collect_ebuild_messages(path):
+	""" Collect elog messages generated by the bash logging function stored 
+		at 'path'.
+	"""
+	mylogfiles = listdir(path)
+	# shortcut for packages without any messages
+	if len(mylogfiles) == 0:
+		return {}
+	# exploit listdir() file order so we process log entries in chronological order
+	mylogfiles.reverse()
+	logentries = {}
+	for f in mylogfiles:
+		msgfunction, msgtype = f.split(".")
+		if msgfunction not in EBUILD_PHASES:
+			writemsg("!!! can't process invalid log file: %s\n" % f,
+				noiselevel=-1)
+			continue
+		if not msgfunction in logentries:
+			logentries[msgfunction] = []
+		msgcontent = open(os.path.join(path, f), "r").readlines()
+		logentries[msgfunction].append((msgtype, msgcontent))
+	# clean logfiles to avoid repetitions
+	for f in mylogfiles:
+		try:
+			os.unlink(os.path.join(path, f))
+		except OSError:
+			pass
+	return logentries
+
+_msgbuffer = {}
+def _elog_base(level, msg, phase="other", key=None, color=None):
+	""" Backend for the other messaging functions, should not be called 
+	    directly.
+	"""
+	if color == None:
+		color = "GOOD"
+	print colorize(color, " * ")+msg
+	if not _msgbuffer.has_key(key):
+		_msgbuffer[key] = {}
+	if not _msgbuffer[key].has_key(phase):
+		_msgbuffer[key][phase] = []
+	_msgbuffer[key][phase].append((level, msg))
+
+	#raise NotImplementedError()
+
+def collect_messages():
+	rValue = _msgbuffer
+	_reset_buffer()
+	return rValue
+
+def _reset_buffer():
+	""" Reset the internal message buffer when it has been processed, 
+	    should not be called directly.
+	"""
+	_msgbuffer = {}
+
+# creating and exporting the actual messaging functions
+_functions = { "einfo": ("INFO", "GOOD"),
+		"elog": ("LOG", "GOOD"),
+		"ewarn": ("WARN", "WARN"),
+		"eqawarn": ("QA", "WARN"),
+		"eerror": ("ERROR", "ERROR"),
+}
+
+def _make_msgfunction(level, color):
+	def _elog(msg, phase="other", key=None):
+		""" Display and log a message assigned to the given key/cpv 
+		    (or unassigned if no key is given).
+		""" 
+		_elog_base(level, msg,  phase=phase, key=key, color=color)
+	return _elog
+
+import sys
+for f in _functions.keys():
+	setattr(sys.modules[__name__], f, _make_msgfunction(_functions[f][0], _functions[f][1]))
+del f, _functions
diff --git a/pym/portage/elog/mod_custom.py b/pym/portage/elog/mod_custom.py
index 3433bbbf1..a486c8931 100644
--- a/pym/portage/elog/mod_custom.py
+++ b/pym/portage/elog/mod_custom.py
@@ -5,8 +5,8 @@
 
 import portage.elog_modules.mod_save, portage.process, portage.exception
 
-def process(mysettings, cpv, logentries, fulltext):
-	elogfilename = portage.elog_modules.mod_save.process(mysettings, cpv, logentries, fulltext)
+def process(mysettings, key, logentries, fulltext):
+	elogfilename = portage.elog_modules.mod_save.process(mysettings, key, logentries, fulltext)
 	
 	if (not "PORTAGE_ELOG_COMMAND" in mysettings.keys()) \
 			or len(mysettings["PORTAGE_ELOG_COMMAND"]) == 0:
@@ -14,7 +14,7 @@ def process(mysettings, cpv, logentries, fulltext):
 	else:
 		mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"]
 		mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename)
-		mylogcmd = mylogcmd.replace("${PACKAGE}", cpv)
+		mylogcmd = mylogcmd.replace("${PACKAGE}", key)
 		retval = portage.process.spawn_bash(mylogcmd)
 		if retval != 0:
 			raise portage.exception.PortageException("!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval)
diff --git a/pym/portage/elog/mod_echo.py b/pym/portage/elog/mod_echo.py
index 73928b437..125a8c40e 100644
--- a/pym/portage/elog/mod_echo.py
+++ b/pym/portage/elog/mod_echo.py
@@ -7,19 +7,19 @@ from portage.output import EOutput
 from portage.const import EBUILD_PHASES
 
 _items = {}
-def process(mysettings, cpv, logentries, fulltext):
-	_items[cpv] = logentries
+def process(mysettings, key, logentries, fulltext):
+	_items[key] = logentries
 
 def finalize(mysettings):
 	printer = EOutput()
-	for cpv in _items.keys():
+	for key in _items.keys():
 		print
-		printer.einfo("Messages for package %s:" % cpv)
+		printer.einfo("Messages for package %s:" % key)
 		print
 		for phase in EBUILD_PHASES:
-			if not phase in _items[cpv]:
+			if not phase in _items[key]:
 				continue
-			for msgtype, msgcontent in _items[cpv][phase]:
+			for msgtype, msgcontent in _items[key][phase]:
 				fmap = {"INFO": printer.einfo,
 						"WARN": printer.ewarn,
 						"ERROR": printer.eerror,
diff --git a/pym/portage/elog/mod_mail.py b/pym/portage/elog/mod_mail.py
index 4695e8ec4..d0d2b33ac 100644
--- a/pym/portage/elog/mod_mail.py
+++ b/pym/portage/elog/mod_mail.py
@@ -5,7 +5,7 @@
 
 import portage.mail, socket
 
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
 	if mysettings.has_key("PORTAGE_ELOG_MAILURI"):
 		myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0]
 	else:
@@ -13,7 +13,7 @@ def process(mysettings, cpv, logentries, fulltext):
 	
 	myfrom = mysettings["PORTAGE_ELOG_MAILFROM"]
 	mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"]
-	mysubject = mysubject.replace("${PACKAGE}", cpv)
+	mysubject = mysubject.replace("${PACKAGE}", key)
 	mysubject = mysubject.replace("${HOST}", socket.getfqdn())
 
 	mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, fulltext)
diff --git a/pym/portage/elog/mod_mail_summary.py b/pym/portage/elog/mod_mail_summary.py
index a1f4ed57f..a8e694935 100644
--- a/pym/portage/elog/mod_mail_summary.py
+++ b/pym/portage/elog/mod_mail_summary.py
@@ -7,10 +7,10 @@ import portage.mail, socket, os, time
 from email.MIMEText import MIMEText as TextMessage
 
 _items = {}
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
 	header = ">>> Messages generated for package %s by process %d on %s:\n\n" % \
-		(cpv, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())))
-	_items[cpv] = header + fulltext
+		(key, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())))
+	_items[key] = header + fulltext
 
 def finalize(mysettings):
 	if len(_items) == 0:
@@ -31,8 +31,8 @@ def finalize(mysettings):
 
 	mybody = "elog messages for the following packages generated by " + \
 		"process %d on host %s:\n" % (os.getpid(), socket.getfqdn())
-	for cpv in _items.keys():
-		 mybody += "- %s\n" % cpv
+	for key in _items.keys():
+		 mybody += "- %s\n" % key
 
 	mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, mybody, attachments=_items.values())
 	portage.mail.send_mail(mysettings, mymessage)
diff --git a/pym/portage/elog/mod_save.py b/pym/portage/elog/mod_save.py
index eff40edd9..31c410fac 100644
--- a/pym/portage/elog/mod_save.py
+++ b/pym/portage/elog/mod_save.py
@@ -6,8 +6,8 @@
 import os, time
 from portage.data import portage_uid, portage_gid
 
-def process(mysettings, cpv, logentries, fulltext):
-	cpv_path = cpv.replace("/", ":")
+def process(mysettings, key, logentries, fulltext):
+	path = key.replace("/", ":")
 
 	if mysettings["PORT_LOGDIR"] != "":
 		elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
@@ -18,7 +18,7 @@ def process(mysettings, cpv, logentries, fulltext):
 	os.chown(elogdir, portage_uid, portage_gid)
 	os.chmod(elogdir, 02770)
 
-	elogfilename = elogdir+"/"+cpv_path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
+	elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
 	elogfile = open(elogfilename, "w")
 	elogfile.write(fulltext)
 	elogfile.close()
diff --git a/pym/portage/elog/mod_save_summary.py b/pym/portage/elog/mod_save_summary.py
index aebc6f9aa..25f8ae77a 100644
--- a/pym/portage/elog/mod_save_summary.py
+++ b/pym/portage/elog/mod_save_summary.py
@@ -6,7 +6,7 @@
 import os, time
 from portage.data import portage_uid, portage_gid
 
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
 	if mysettings["PORT_LOGDIR"] != "":
 		elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
 	else:
@@ -20,7 +20,7 @@ def process(mysettings, cpv, logentries, fulltext):
 	elogfilename = elogdir+"/summary.log"
 	elogfile = open(elogfilename, "a")
 	elogfile.write(">>> Messages generated by process %d on %s for package %s:\n\n" % \
-			(os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), cpv))
+			(os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), key))
 	elogfile.write(fulltext)
 	elogfile.write("\n")
 	elogfile.close()
diff --git a/pym/portage/elog/mod_syslog.py b/pym/portage/elog/mod_syslog.py
index 28aa15b3c..bb66b2b74 100644
--- a/pym/portage/elog/mod_syslog.py
+++ b/pym/portage/elog/mod_syslog.py
@@ -6,7 +6,7 @@
 import syslog
 from portage.const import EBUILD_PHASES
 
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
 	syslog.openlog("portage", syslog.LOG_ERR | syslog.LOG_WARNING | syslog.LOG_INFO | syslog.LOG_NOTICE, syslog.LOG_LOCAL5)
 	for phase in EBUILD_PHASES:
 		if not phase in logentries:
@@ -18,5 +18,5 @@ def process(mysettings, cpv, logentries, fulltext):
 				"LOG": syslog.LOG_NOTICE,
 				"QA": syslog.LOG_WARNING}
 			msgtext = "".join(msgcontent)
-			syslog.syslog(pri[msgtype], "%s: %s: %s" % (cpv, phase, msgtext))
+			syslog.syslog(pri[msgtype], "%s: %s: %s" % (key, phase, msgtext))
 	syslog.closelog()