From 33afd46c72c1863df5efda8a0855001399c8c49c Mon Sep 17 00:00:00 2001 From: Joey Hagedorn Date: Tue, 12 Jul 2005 20:18:17 +0000 Subject: fixed bug related to pinging stale hosts 2005/06/30 14:16:36-05:00 anl.gov!joey fixed problem with pinging nonexistant hosts 2005/06/27 14:04:46-05:00 anl.gov!joey Major cleanup: changed tabs to spaces, PyLinted the code round 2, fixed all conflicting variable names, got rid of unnecessary code (Logical change 1.249) git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1032 ce84e21b-d406-0410-9b95-82705330c041 --- src/sbin/StatReports.py | 295 ++++++++++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 124 deletions(-) (limited to 'src') diff --git a/src/sbin/StatReports.py b/src/sbin/StatReports.py index df515a43b..df83df611 100644 --- a/src/sbin/StatReports.py +++ b/src/sbin/StatReports.py @@ -2,6 +2,7 @@ #Jun 7 2005 #StatReports +'''Generates & distributes reports of statistic information for bcfg2''' from ConfigParser import ConfigParser from elementtree.ElementTree import * @@ -9,13 +10,18 @@ from xml.parsers.expat import ExpatError from xml.sax.saxutils import escape from smtplib import SMTP from time import asctime, strftime, strptime, ctime, gmtime +from socket import gethostbyname, gethostbyaddr, gaierror from sys import exit, argv from getopt import getopt, GetoptError import re, string, os -def generateReport(report, delivery, deliverytype, statdata): - reportSections = [] +def generatereport(report, delivery, deliverytype, statdata): + '''generatereport creates and returns a report consisting + list of tuples contining (title,body) pairs''' + + + reportsections = [] deliverytype = delivery.get("type", default = "nodes-individual") reportgood = report.get("good", default = 'Y') @@ -28,151 +34,194 @@ def generateReport(report, delivery, deliverytype, statdata): mheader = '' dirty = '' clean = '' + + + '''build fqdn cache''' + + domain_list=['mcs.anl.gov', 'bgl.mcs.anl.gov', 'anchor.anl.gov', 'globus.org'] + fqdncache = {} + allnodes = statdata.findall("Node") #this code is duplicated please remove... + regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|') + pattern = re.compile(regex) + for node in allnodes: + nodename = node.get("name") + fqdncache[nodename] = "" + if pattern.match(node.get("name")): + for domain in domain_list: + try: + fqdn = "%s.%s" % (nodename, domain) + ipaddr = gethostbyname(fqdn) + fqdncache[nodename] = fqdn + break + except gaierror: + continue + + if fqdncache[nodename] == "": + statdata.remove(node); + del fqdncache[nodename] + + + for machine in report.findall('Machine'): for node in statdata.findall('Node'): - if node.attrib['name'] == machine.attrib['name']: - if deliverytype == 'nodes-digest': - mheader = "Machine: %s\n"%machine.attrib['name'] - for stats in node.findall('Statistics'): - if stats.attrib['state'] == 'clean' and current_date in stats.attrib['time']: - clean += "%s\n"%machine.attrib['name'] - if reportmodified == 'Y': - for modxml in stats.findall('Modified'): - if current_date in stats.attrib['time']: - modified+="\n%s\n"%tostring(modxml) - for bad in stats.findall('Bad'): - srtd = bad.findall('*') - srtd.sort(lambda x, y:cmp(tostring(x), tostring(y))) - strongbad = Element("Bad") - map(lambda x:strongbad.append(x), srtd) - baddata += "Time Ran:%s\n%s\n"%(stats.attrib['time'], tostring(strongbad)) - dirty += "%s\n"%machine.attrib['name'] - strongbad = '' - if deliverytype == 'nodes-individual': + if node.attrib['name'] == machine.attrib['name']: + if deliverytype == 'nodes-digest': + mheader = "Machine: %s\n" % machine.attrib['name'] + for stats in node.findall('Statistics'): + if stats.attrib['state'] == 'clean' \ + and current_date in stats.attrib['time']: + clean += "%s\n" % machine.attrib['name'] + if reportmodified == 'Y': + for modxml in stats.findall('Modified'): + if current_date in stats.attrib['time']: + modified += "\n%s\n" % tostring(modxml) + for bad in stats.findall('Bad'): + srtd = bad.findall('*') + srtd.sort(lambda x, y:cmp(tostring(x), tostring(y))) + strongbad = Element("Bad") + map(lambda x:strongbad.append(x), srtd) + baddata += "Time Ran:%s\n%s\n" % (stats.attrib['time'], tostring(strongbad)) + dirty += "%s\n" % machine.attrib['name'] + strongbad = '' + if deliverytype == 'nodes-individual': if baddata != '': - reportSections.append(("%s: Bcfg Nightly Errors"%machine.attrib['name'], "%s%s"%(modified, baddata))) + reportsections.append(("%s: Bcfg Nightly Errors" % machine.attrib['name'], \ + "%s%s" % (modified, baddata))) else: if reportgood == 'Y': - reportSections.append(("%s: Bcfg Nightly Good"%machine.attrib['name'], "%s%s"%(modified, baddata))) - baddata = '' - modified = '' - else: - if not (modified == '' and baddata == ''): - msg += "%s %s %s\n"%(mheader, modified, baddata) - baddata = '' - modified = '' + reportsections.append(("%s: Bcfg Nightly Good"%machine.attrib['name'], \ + "%s%s" % (modified, baddata))) + baddata = '' + modified = '' + else: + if not (modified == '' and baddata == ''): + msg += "%s %s %s\n" % (mheader, modified, baddata) + baddata = '' + modified = '' if deliverytype == 'nodes-digest': if msg != '': - reportSections.append(("Bcfg Nightly Errors", "DIRTY:\n%s\nCLEAN:\n%s\nDETAILS:\n%s"%(dirty, clean, msg))) + reportsections.append(("Bcfg Nightly Errors", \ + "DIRTY:\n%s\nCLEAN:\n%s\nDETAILS:\n%s" % (dirty, clean, msg))) else: if delivery.attrib['good'] == 'Y': - reportSections.append(("Bcfg Nightly All Machines Good", "All Machines Nomnial")) + reportsections.append(("Bcfg Nightly All Machines Good", "All Machines Nomnial")) + + if deliverytype == 'overview-stats': - children = statdata.findall("Node") - regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|') - p = re.compile(regex) - childstates = [] - for child in children: - if p.match(child.get("name")): - child.states = [] - for state in child.findall("Statistics"): - child.states.append((child.get("name"), state.get("state"), state.get("time"))) - if child.states != []: - childstates.append(child.states[len(child.states)-1]) - childstates.sort(lambda x, y:cmp(x[0], y[0])) - - staleones = [] + children = statdata.findall("Node") + regex = string.join(map(lambda x:x.get("name"), report.findall('Machine')), '|') + pattern = re.compile(regex) + childstates = [] + for child in children: + if pattern.match(child.get("name")): + child.states = [] + for state in child.findall("Statistics"): + child.states.append((child.get("name"), state.get("state"), state.get("time"))) + if child.states != []: + childstates.append(child.states[len(child.states)-1]) + childstates.sort(lambda x, y:cmp(x[0], y[0])) + + staleones = [] cleanones = [] dirtyones = [] unpingableones = [] - for instance in childstates: - if instance[1] == "dirty": - dirtyones.append(instance) - elif instance[1] == "clean": - cleanones.append(instance) - if strptime(instance[2])[0] != strptime(ctime())[0] \ - or strptime(instance[2])[1] != strptime(ctime())[1] \ - or strptime(instance[2])[2] != strptime(ctime())[2]: - staleones.append(instance) - - if staleones != []: - print "Pinging hosts that didn't run today. Please wait" - for instance in staleones: - if os.system( 'ping -c 1 '+instance[0]+'.mcs.anl.gov &>/dev/null'): - staleones.remove(instance) - unpingableones.append(instance) + for instance in childstates: + if instance[1] == "dirty": + dirtyones.append(instance) + elif instance[1] == "clean": + cleanones.append(instance) + if strptime(instance[2])[0] != strptime(ctime())[0] \ + or strptime(instance[2])[1] != strptime(ctime())[1] \ + or strptime(instance[2])[2] != strptime(ctime())[2]: + staleones.append(instance) + + removableones = [] + + if staleones != []: + print "Pinging hosts that didn't run today. Please wait" + for instance in staleones: + if os.system( 'ping -c 1 ' + fqdncache[instance[0]] + ' &>/dev/null') != 0: + removableones.append(instance) + unpingableones.append(instance) + + + for item in unpingableones: + staleones.remove(item) statmsg = '' - statmsg += "SUMMARY INFORMATION:\n" - statmsg += "Up & Not Running Nightly: %d\n"%len(staleones) - statmsg += "Unpingable: %d\n"%len(unpingableones) - statmsg += "Dirty: %d\n"%len(dirtyones) - statmsg += "Clean: %d\n"%len(cleanones) - statmsg += "---------------------------------\n" - total = len(cleanones)+len(dirtyones) - statmsg += "Total: %d\n\n\n"%len(childstates) - - statmsg += "\n UP AND NOT RUNNING NIGHTLY:\n" - for one in staleones: - statmsg += one[0] + ".mcs.anl.gov\n" - statmsg += "\nDIRTY:\n" - for one in dirtyones: - statmsg += one[0] + ".mcs.anl.gov\n" - statmsg += "\nCLEAN:\n" - for one in cleanones: - statmsg += one[0] + ".mcs.anl.gov\n" - statmsg += "\nUNPINGABLE:\n" - for one in unpingableones: - statmsg += one[0] + ".mcs.anl.gov\n" - - reportSections.append(("Bcfg Nightly Errors", "%s"%(statmsg))) - - return reportSections + statmsg += "SUMMARY INFORMATION:\n" + statmsg += "Up & Not Running Nightly: %d\n" % len(staleones) + statmsg += "Unpingable: %d\n" % len(unpingableones) + statmsg += "Dirty: %d\n" % len(dirtyones) + statmsg += "Clean: %d\n" % len(cleanones) + statmsg += "---------------------------------\n" + #total = len(cleanones) + len(dirtyones) + statmsg += "Total: %d\n\n\n" % len(childstates) + + statmsg += "\n UP AND NOT RUNNING NIGHTLY:\n" + for one in staleones: + statmsg += fqdncache[one[0]] + "\n" + statmsg += "\nDIRTY:\n" + for one in dirtyones: + statmsg += fqdncache[one[0]] + "\n" + statmsg += "\nCLEAN:\n" + for one in cleanones: + statmsg += fqdncache[one[0]] + "\n" + statmsg += "\nUNPINGABLE:\n" + for one in unpingableones: + statmsg += fqdncache[one[0]] + "\n" + + reportsections.append(("Bcfg Nightly Errors", "%s" % (statmsg))) + + return reportsections - - def mail(reportsections, delivery): - current_date = asctime()[:10] + '''mail mails a previously generated report''' + mailer = SMTP('localhost') fromaddr = "root@netzero.mcs.anl.gov" for destination in delivery.findall('Destination'): toaddr = destination.attrib['address'] for section in reportsections: - msg="To: %s\nFrom: %s\nSubject: %s\n\n\n%s"%(toaddr, fromaddr, section[0], section[1]) + msg = "To: %s\nFrom: %s\nSubject: %s\n\n\n%s" % \ + (toaddr, fromaddr, section[0], section[1]) + mailer.sendmail(fromaddr, toaddr, msg) mailer.quit() - -def rss(reportsections, delivery): - #need Report, Delivery, ReportSections(list of tuples) - - #check and see if rss file xists +def rss(reportsections, delivery, report): + '''rss appends a new report to the specified rss file + keeping the last 9 articles''' + #check and see if rss file exists for destination in delivery.findall('Destination'): try: - fil = open(destination.attrib['address'], 'r') - olddoc = XML(fil.read()) - #read array of 9 most recent items out - items = olddoc.find("channel").findall("item")[0:9]#defines the number of recent articles to keep + fil = open(destination.attrib['address'], 'r') + olddoc = XML(fil.read()) + + #defines the number of recent articles to keep + items = olddoc.find("channel").findall("item")[0:9] fil.close() - fil = open(destination.attrib['address'], 'w') + fil = open(destination.attrib['address'], 'w') except (IOError, ExpatError): - fil = open(destination.attrib['address'], 'w') - items = [] + fil = open(destination.attrib['address'], 'w') + items = [] - rss = Element("rss") + rssdata = Element("rss") channel = SubElement(rss, "channel") - rss.set("version", "2.0") + rssdata.set("version", "2.0") chantitle = SubElement(channel, "title") chantitle.text = report.attrib['name'] chanlink = SubElement(channel, "link") - chanlink.text = "http://www.mcs.anl.gov/"#this can later link to WWW report if one gets published with this one + + #this can later link to WWW report if one gets published simultaneously + chanlink.text = "http://www.mcs.anl.gov/" chandesc = SubElement(channel, "description") chandesc.text = "Information regarding the 10 most recent bcfg2 runs." @@ -189,18 +238,17 @@ def rss(reportsections, delivery): for item in items: channel.append(item) - tree = "" + tostring(rss) + tree = "" + tostring(rssdata) fil.write(tree) fil.close() - def www(reportsections, delivery): - #need Report, Delivery, ReportSections(list of tuples) - + '''www outputs report to simple HTML''' + #check and see if rss file xists for destination in delivery.findall('Destination'): fil = open(destination.attrib['address'], 'w') - + html = Element("HTML") body = SubElement(html, "BODY") for section in reportsections: @@ -225,9 +273,9 @@ if __name__ == '__main__': statpath = "%s/statistics.xml" % c.get('server', 'metadata') try: opts, args = getopt(argv[1:], "hc:s:", ["help", "config=", "stats="]) - except GetoptError, msg: + except GetoptError, mesg: # print help information and exit: - print "%s\nUsage:\nStatReports.py [-h] [-c ] [-s ]"%(msg) + print "%s\nUsage:\nStatReports.py [-h] [-c ] [-s ]" % (mesg) exit(2) for o, a in opts: if o in ("-h", "--help"): @@ -239,9 +287,8 @@ if __name__ == '__main__': statpath = a - try: - statdata = XML(open(statpath).read()) + statsdata = XML(open(statpath).read()) except (IOError, ExpatError): print("StatReports: Failed to parse %s"%(statpath)) exit(1) @@ -253,20 +300,20 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s"%(configpath)) exit(1) - for report in configdata.findall('Report'): - for delivery in report.findall('Delivery'): - deliverytype = delivery.get('type', default='nodes-digest') - deliverymechanism = delivery.get('mechanism', default='invalid') + for reprt in configdata.findall('Report'): + for deliv in reprt.findall('Delivery'): + delivtype = deliv.get('type', default='nodes-digest') + deliverymechanism = deliv.get('mechanism', default='invalid') - reportsections = generateReport(report, delivery, deliverytype, statdata) + reportsects = generatereport(reprt, deliv, delivtype, statsdata) if deliverymechanism == 'mail': - mail(reportsections, delivery) + mail(reportsects, deliv) elif deliverymechanism == 'rss': - rss(reportsections, delivery) + rss(reportsects, deliv, reprt) elif deliverymechanism == 'www': - www(reportsections, delivery) + www(reportsects, deliv) else: print("StatReports: Invalid delivery mechanism in report-configuration!") deliverymechanism = '' - deliverytype = '' + delivtype = '' -- cgit v1.2.3-1-g7c22