fixed bug related to pinging stale hosts

2005/06/30 14:16:36-05:00 anl.gov!joey
fixed problem with pinging nonexistant hosts

2005/06/27 14:04:46-05:00 anl.gov!joey
Major cleanup: changed tabs to spaces, PyLinted the code round 2, fixed all conflicting variable names, got rid of unnecessary code

(Logical change 1.249)


git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1032 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 171 insertions, 124 deletions

diff --git a/src/sbin/StatReports.py b/src/sbin/StatReports.py
index df515a43b..df83df611 100644
--- a/src/sbin/StatReports.py
+++ b/src/sbin/StatReports.py
@@ -2,6 +2,7 @@
 
 #Jun 7 2005
 #StatReports
+'''Generates & distributes reports of statistic information for bcfg2'''
 
 from ConfigParser import ConfigParser
 from elementtree.ElementTree import *
@@ -9,13 +10,18 @@ from xml.parsers.expat import ExpatError
 from xml.sax.saxutils import escape
 from smtplib import SMTP
 from time import asctime, strftime, strptime, ctime, gmtime
+from socket import gethostbyname, gethostbyaddr, gaierror
 from sys import exit, argv
 from getopt import getopt, GetoptError
 import re, string, os
 
 
-def generateReport(report, delivery, deliverytype, statdata):
-    reportSections = []
+def generatereport(report, delivery, deliverytype, statdata):
+    '''generatereport creates and returns a report consisting
+     list of tuples contining (title,body) pairs'''
+
+
+    reportsections = []
 
     deliverytype = delivery.get("type", default = "nodes-individual")
     reportgood = report.get("good", default = 'Y')
@@ -28,151 +34,194 @@ def generateReport(report, delivery, deliverytype, statdata):
     mheader = ''
     dirty = ''
     clean = ''
+
+
+    '''build fqdn cache'''
+    
+    domain_list=['mcs.anl.gov', 'bgl.mcs.anl.gov', 'anchor.anl.gov', 'globus.org']
+    fqdncache = {}
+    allnodes = statdata.findall("Node") #this code is duplicated please remove...
+    regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|')
+    pattern = re.compile(regex)
+    for node in allnodes:
+        nodename = node.get("name")
+        fqdncache[nodename] = ""
+        if pattern.match(node.get("name")):
+            for domain in domain_list:
+                try:
+                    fqdn = "%s.%s" % (nodename, domain)
+                    ipaddr = gethostbyname(fqdn)
+                    fqdncache[nodename] = fqdn
+                    break
+                except gaierror:
+                    continue
+
+        if fqdncache[nodename] == "":
+            statdata.remove(node);
+            del fqdncache[nodename]
+
+
+
     for machine in report.findall('Machine'):
         for node in statdata.findall('Node'):
-	    if node.attrib['name'] == machine.attrib['name']:
-	        if deliverytype == 'nodes-digest':
-		    mheader = "Machine: %s\n"%machine.attrib['name']
-		for stats in node.findall('Statistics'):
-		    if stats.attrib['state'] == 'clean' and current_date in stats.attrib['time']:
-		        clean += "%s\n"%machine.attrib['name']
-		    if reportmodified == 'Y':
-		        for modxml in stats.findall('Modified'):
-			    if current_date in stats.attrib['time']:
-			        modified+="\n%s\n"%tostring(modxml)
-		    for bad in stats.findall('Bad'):
-		        srtd = bad.findall('*')
-		        srtd.sort(lambda x, y:cmp(tostring(x), tostring(y)))
-		        strongbad = Element("Bad")
-		        map(lambda x:strongbad.append(x), srtd)
-		        baddata += "Time Ran:%s\n%s\n"%(stats.attrib['time'], tostring(strongbad))
-		        dirty += "%s\n"%machine.attrib['name']
-		        strongbad = ''
-	if deliverytype == 'nodes-individual':
+            if node.attrib['name'] == machine.attrib['name']:
+                if deliverytype == 'nodes-digest':
+                    mheader = "Machine: %s\n" % machine.attrib['name']
+                for stats in node.findall('Statistics'):
+                    if stats.attrib['state'] == 'clean' \
+                    and current_date in stats.attrib['time']:
+                        clean += "%s\n" % machine.attrib['name']
+                    if reportmodified == 'Y':
+                        for modxml in stats.findall('Modified'):
+                            if current_date in stats.attrib['time']:
+                                modified += "\n%s\n" % tostring(modxml)
+                    for bad in stats.findall('Bad'):
+                        srtd = bad.findall('*')
+                        srtd.sort(lambda x, y:cmp(tostring(x), tostring(y)))
+                        strongbad = Element("Bad")
+                        map(lambda x:strongbad.append(x), srtd)
+                        baddata += "Time Ran:%s\n%s\n" % (stats.attrib['time'], tostring(strongbad))
+                        dirty += "%s\n" % machine.attrib['name']
+                        strongbad = ''
+        if deliverytype == 'nodes-individual':
             if baddata != '':
-                reportSections.append(("%s: Bcfg Nightly Errors"%machine.attrib['name'], "%s%s"%(modified, baddata)))
+                reportsections.append(("%s: Bcfg Nightly Errors" % machine.attrib['name'], \
+                                       "%s%s" % (modified, baddata)))
             else:
                 if reportgood == 'Y':
-                    reportSections.append(("%s: Bcfg Nightly Good"%machine.attrib['name'], "%s%s"%(modified, baddata)))
-	    baddata = ''
-	    modified = ''
-	else:
-	    if not (modified == '' and baddata == ''):
-	        msg += "%s %s %s\n"%(mheader, modified, baddata)
-	    baddata = ''
-	    modified = ''
+                    reportsections.append(("%s: Bcfg Nightly Good"%machine.attrib['name'], \
+                                           "%s%s" % (modified, baddata)))
+            baddata = ''
+            modified = ''
+        else:
+            if not (modified == '' and baddata == ''):
+                msg += "%s %s %s\n" % (mheader, modified, baddata)
+            baddata = ''
+            modified = ''
 
     if deliverytype == 'nodes-digest':
         if msg != '':
-            reportSections.append(("Bcfg Nightly Errors", "DIRTY:\n%s\nCLEAN:\n%s\nDETAILS:\n%s"%(dirty, clean, msg)))
+            reportsections.append(("Bcfg Nightly Errors", \
+                                       "DIRTY:\n%s\nCLEAN:\n%s\nDETAILS:\n%s" % (dirty, clean, msg)))
         else:
             if delivery.attrib['good'] == 'Y':
-                reportSections.append(("Bcfg Nightly All Machines Good", "All Machines Nomnial"))
+                reportsections.append(("Bcfg Nightly All Machines Good", "All Machines Nomnial"))
+
+
 
 
     if deliverytype == 'overview-stats':
-	children = statdata.findall("Node")
-	regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|')
-	p = re.compile(regex)
-	childstates = []
-	for child in children:
-	    if p.match(child.get("name")):
-	        child.states = []
-		for state in child.findall("Statistics"):
-		    child.states.append((child.get("name"), state.get("state"), state.get("time")))
-		if child.states != []:
-		    childstates.append(child.states[len(child.states)-1])
-		    childstates.sort(lambda x, y:cmp(x[0], y[0]))
-
-	staleones = []
+        children = statdata.findall("Node")
+        regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|')
+        pattern = re.compile(regex)
+        childstates = []
+        for child in children:
+            if pattern.match(child.get("name")):
+                child.states = []
+                for state in child.findall("Statistics"):
+                    child.states.append((child.get("name"), state.get("state"), state.get("time")))
+                if child.states != []:
+                    childstates.append(child.states[len(child.states)-1])
+                    childstates.sort(lambda x, y:cmp(x[0], y[0]))
+
+        staleones = []
         cleanones = []
         dirtyones = []
         unpingableones = []
 
-	for instance in childstates:
-	    if instance[1] == "dirty":
-	        dirtyones.append(instance)
-	    elif instance[1] == "clean":
-	        cleanones.append(instance)
-	    if strptime(instance[2])[0] != strptime(ctime())[0] \
-	    or strptime(instance[2])[1] != strptime(ctime())[1] \
-	    or strptime(instance[2])[2] != strptime(ctime())[2]:
-	        staleones.append(instance)
-
-	if staleones != []:
-	    print "Pinging hosts that didn't run today. Please wait"
-	for instance in staleones:
-	    if os.system( 'ping -c 1 '+instance[0]+'.mcs.anl.gov &>/dev/null'):
-	        staleones.remove(instance)
-	        unpingableones.append(instance)
+        for instance in childstates:
+            if instance[1] == "dirty":
+                dirtyones.append(instance)
+            elif instance[1] == "clean":
+                cleanones.append(instance)
+            if strptime(instance[2])[0] != strptime(ctime())[0] \
+            or strptime(instance[2])[1] != strptime(ctime())[1] \
+            or strptime(instance[2])[2] != strptime(ctime())[2]:
+                staleones.append(instance)
+
+        removableones = []
+        
+        if staleones != []:
+            print "Pinging hosts that didn't run today. Please wait"
+        for instance in staleones:
+            if os.system( 'ping -c 1 ' + fqdncache[instance[0]] + ' &>/dev/null') != 0:
+                removableones.append(instance)
+                unpingableones.append(instance)
+
+
+        for item in unpingableones:
+            staleones.remove(item)
 
         statmsg = ''
-	statmsg += "SUMMARY INFORMATION:\n"
-	statmsg += "Up & Not Running Nightly:     %d\n"%len(staleones)
-	statmsg += "Unpingable:                   %d\n"%len(unpingableones)
-	statmsg += "Dirty:                        %d\n"%len(dirtyones)
-	statmsg += "Clean:                        %d\n"%len(cleanones)
-	statmsg += "---------------------------------\n"
-	total = len(cleanones)+len(dirtyones)
-	statmsg += "Total:                        %d\n\n\n"%len(childstates)
-
-	statmsg += "\n UP AND NOT RUNNING NIGHTLY:\n"
-	for one in staleones:
-	    statmsg += one[0] + ".mcs.anl.gov\n"
-	statmsg += "\nDIRTY:\n"
-	for one in dirtyones:
-	    statmsg += one[0] + ".mcs.anl.gov\n"
-	statmsg += "\nCLEAN:\n"
-	for one in cleanones:
-	    statmsg += one[0] + ".mcs.anl.gov\n"
-	statmsg += "\nUNPINGABLE:\n"
-	for one in unpingableones:
-	    statmsg += one[0] + ".mcs.anl.gov\n"
-
-        reportSections.append(("Bcfg Nightly Errors", "%s"%(statmsg)))
-
-    return reportSections
+        statmsg += "SUMMARY INFORMATION:\n"
+        statmsg += "Up & Not Running Nightly:     %d\n" % len(staleones)
+        statmsg += "Unpingable:                   %d\n" % len(unpingableones)
+        statmsg += "Dirty:                        %d\n" % len(dirtyones)
+        statmsg += "Clean:                        %d\n" % len(cleanones)
+        statmsg += "---------------------------------\n"
+        #total = len(cleanones) + len(dirtyones)
+        statmsg += "Total:                        %d\n\n\n" % len(childstates)
+
+        statmsg += "\n UP AND NOT RUNNING NIGHTLY:\n"
+        for one in staleones:
+            statmsg += fqdncache[one[0]] + "\n"
+        statmsg += "\nDIRTY:\n"
+        for one in dirtyones:
+            statmsg += fqdncache[one[0]] + "\n"
+        statmsg += "\nCLEAN:\n"
+        for one in cleanones:
+            statmsg += fqdncache[one[0]] + "\n"
+        statmsg += "\nUNPINGABLE:\n"
+        for one in unpingableones:
+            statmsg += fqdncache[one[0]] + "\n"
+
+        reportsections.append(("Bcfg Nightly Errors", "%s" % (statmsg)))
+
+    return reportsections
 
 
-
-    
 def mail(reportsections, delivery):
-    current_date = asctime()[:10]
+    '''mail mails a previously generated report'''
+    
     mailer = SMTP('localhost')
     fromaddr = "root@netzero.mcs.anl.gov"
 
     for destination in delivery.findall('Destination'):
         toaddr = destination.attrib['address']
         for section in reportsections:
-            msg="To: %s\nFrom: %s\nSubject: %s\n\n\n%s"%(toaddr, fromaddr, section[0], section[1])
+            msg = "To: %s\nFrom: %s\nSubject: %s\n\n\n%s" % \
+                  (toaddr, fromaddr, section[0], section[1])
+
             mailer.sendmail(fromaddr, toaddr, msg)
     mailer.quit()
 
 
-
-def rss(reportsections, delivery):
-    #need Report, Delivery, ReportSections(list of tuples)
-	
-    #check and see if rss file xists
+def rss(reportsections, delivery, report):
+    '''rss appends a new report to the specified rss file
+     keeping the last 9 articles'''
+    #check and see if rss file exists
     for destination in delivery.findall('Destination'):
         try:
-	    fil = open(destination.attrib['address'], 'r')
-	    olddoc = XML(fil.read())
-	    #read array of 9 most recent items out
-            items = olddoc.find("channel").findall("item")[0:9]#defines the number of recent articles to keep
+            fil = open(destination.attrib['address'], 'r')
+            olddoc = XML(fil.read())
+
+            #defines the number of recent articles to keep
+            items = olddoc.find("channel").findall("item")[0:9]
             fil.close()
-	    fil = open(destination.attrib['address'], 'w')
+            fil = open(destination.attrib['address'], 'w')
         except (IOError, ExpatError):
-	    fil = open(destination.attrib['address'], 'w')
-	    items = []
+            fil = open(destination.attrib['address'], 'w')
+            items = []
 
-        rss = Element("rss")
+        rssdata = Element("rss")
         channel = SubElement(rss, "channel")
-        rss.set("version", "2.0")
+        rssdata.set("version", "2.0")
         chantitle = SubElement(channel, "title")
         chantitle.text = report.attrib['name']
         chanlink = SubElement(channel, "link")
-        chanlink.text = "http://www.mcs.anl.gov/"#this can later link to WWW report if one gets published with this one
+        
+        #this can later link to WWW report if one gets published simultaneously
+        chanlink.text = "http://www.mcs.anl.gov/"
         chandesc = SubElement(channel, "description")
         chandesc.text = "Information regarding the 10 most recent bcfg2 runs."
 
@@ -189,18 +238,17 @@ def rss(reportsections, delivery):
             for item in items:
                 channel.append(item)
 
-        tree = "<?xml version=\"1.0\"?>" + tostring(rss)
+        tree = "<?xml version=\"1.0\"?>" + tostring(rssdata)
         fil.write(tree)
         fil.close()
 
-
 def www(reportsections, delivery):
-    #need Report, Delivery, ReportSections(list of tuples)
-	
+    '''www outputs report to simple HTML'''
+        
     #check and see if rss file xists
     for destination in delivery.findall('Destination'):
         fil = open(destination.attrib['address'], 'w')
-	
+        
         html = Element("HTML")
         body = SubElement(html, "BODY")
         for section in reportsections:
@@ -225,9 +273,9 @@ if __name__ == '__main__':
     statpath = "%s/statistics.xml" % c.get('server', 'metadata')
     try:
         opts, args = getopt(argv[1:], "hc:s:", ["help", "config=", "stats="])
-    except GetoptError, msg:
+    except GetoptError, mesg:
         # print help information and exit:
-        print "%s\nUsage:\nStatReports.py [-h] [-c <configuration-file>] [-s <statistics-file>]"%(msg) 
+        print "%s\nUsage:\nStatReports.py [-h] [-c <configuration-file>] [-s <statistics-file>]" % (mesg) 
         exit(2)
     for o, a in opts:
         if o in ("-h", "--help"):
@@ -239,9 +287,8 @@ if __name__ == '__main__':
             statpath = a
 
 
-
     try:
-        statdata = XML(open(statpath).read())
+        statsdata = XML(open(statpath).read())
     except (IOError, ExpatError):
         print("StatReports: Failed to parse %s"%(statpath))
         exit(1)
@@ -253,20 +300,20 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s"%(configpath))
         exit(1)
 
-    for report in configdata.findall('Report'):
-        for delivery in report.findall('Delivery'):
-            deliverytype = delivery.get('type', default='nodes-digest')
-            deliverymechanism = delivery.get('mechanism', default='invalid')
+    for reprt in configdata.findall('Report'):
+        for deliv in reprt.findall('Delivery'):
+            delivtype = deliv.get('type', default='nodes-digest')
+            deliverymechanism = deliv.get('mechanism', default='invalid')
 
-            reportsections = generateReport(report, delivery, deliverytype, statdata)
+            reportsects = generatereport(reprt, deliv, delivtype, statsdata)
             
             if deliverymechanism == 'mail':
-                mail(reportsections, delivery)
+                mail(reportsects, deliv)
             elif deliverymechanism == 'rss':
-                rss(reportsections, delivery)
+                rss(reportsects, deliv, reprt)
             elif deliverymechanism == 'www':
-                www(reportsections, delivery)
+                www(reportsects, deliv)
             else:
                 print("StatReports: Invalid delivery mechanism in report-configuration!")
             deliverymechanism = ''
-            deliverytype = ''
+            delivtype = ''