From 40367f6cb0e69b8b993317149d824d9bf489a4f2 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Fri, 15 Sep 2006 02:44:58 +0000 Subject: Rename StatReports to bcfg2-build-reports git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@2257 ce84e21b-d406-0410-9b95-82705330c041 --- src/sbin/StatReports | 290 ------------------------------------------- src/sbin/bcfg2-build-reports | 290 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 290 deletions(-) delete mode 100755 src/sbin/StatReports create mode 100755 src/sbin/bcfg2-build-reports (limited to 'src/sbin') diff --git a/src/sbin/StatReports b/src/sbin/StatReports deleted file mode 100755 index 722c563c0..000000000 --- a/src/sbin/StatReports +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python - -'''bcfg2-build-reports Generates & distributes reports of statistic information -for bcfg2''' -__revision__ = '$Revision$' - -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from lxml.etree import XML, XSLT, parse, Element, ElementTree, SubElement, tostring, XMLSyntaxError -from time import asctime, strptime -import copy, getopt, re, os, socket, sys - -def generatereport(rspec, nrpt): - '''generatereport creates and returns an ElementTree representation - of a report adhering to the XML spec for intermediate reports''' - reportspec = copy.deepcopy(rspec) - nodereprt = copy.deepcopy(nrpt) - - reportgood = reportspec.get("good", default = 'Y') - reportmodified = reportspec.get("modified", default = 'Y') - current_date = asctime()[:10] - - '''build regex of all the nodes we are reporting about''' - pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) - - for node in nodereprt.findall('Node'): - if not (node.findall("Statistics") and pattern.match(node.get('name'))): - # don't know enough about node - nodereprt.remove(node) - continue - - #reduce to most recent Statistics entry - statisticslist = node.findall('Statistics') - #this line actually sorts from most recent to oldest - statisticslist.sort(lambda y, x: cmp(strptime(x.get("time")), strptime(y.get("time")))) - stats = statisticslist[0] - - [node.remove(item) for item in node.findall('Statistics')] - - #add a good tag if node is good and we wnat to report such - if reportgood == 'Y' and stats.get('state') == 'clean': - SubElement(stats,"Good") - - [stats.remove(item) for item in stats.findall("Bad") + stats.findall("Modified") if \ - item.getchildren() == None] - [stats.remove(item) for item in stats.findall("Modified") if reportmodified == 'N'] - - #test for staleness -if stale add Stale tag - if stats.get("time").find(current_date) == -1: - SubElement(stats,"Stale") - node.append(stats) - return nodereprt - -def mail(mailbody, confi): - '''mail mails a previously generated report''' - - try: - mailer = confi.get('statistics', 'sendmailpath') - except (NoSectionError, NoOptionError): - mailer = "/usr/sbin/sendmail" - # open a pipe to the mail program and - # write the data to the pipe - pipe = os.popen("%s -t" % mailer, 'w') - pipe.write(mailbody) - exitcode = pipe.close() - if exitcode: - print "Exit code: %s" % exitcode - -def rss(reportxml, delivery, report): - '''rss appends a new report to the specified rss file - keeping the last 9 articles''' - #check and see if rss file exists - for destination in delivery.findall('Destination'): - try: - fil = open(destination.attrib['address'], 'r') - olddoc = XML(fil.read()) - - #defines the number of recent articles to keep - items = olddoc.find("channel").findall("item")[0:9] - fil.close() - fil = open(destination.attrib['address'], 'w') - except (IOError, XMLSyntaxError): - fil = open(destination.attrib['address'], 'w') - items = [] - - rssdata = Element("rss") - channel = SubElement(rssdata, "channel") - rssdata.set("version", "2.0") - chantitle = SubElement(channel, "title") - chantitle.text = report.attrib['name'] - chanlink = SubElement(channel, "link") - - #this can later link to WWW report if one gets published simultaneously? - chanlink.text = "http://www.mcs.anl.gov/cobalt/bcfg2" - chandesc = SubElement(channel, "description") - chandesc.text = "Information regarding the 10 most recent bcfg2 runs." - - channel.append(XML(reportxml)) - - if items != []: - for item in items: - channel.append(item) - - tree = "" + tostring(rssdata) - fil.write(tree) - fil.close() - -def www(reportxml, delivery): - '''www outputs report to''' - - #this can later link to WWW report if one gets published simultaneously? - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def fileout(reportxml, delivery): - '''outputs to plain text file''' - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def pretty_print(element, level=0): - '''Produce a pretty-printed text representation of element''' - if element.text: - fmt = "%s<%%s %%s>%%s" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])), - element.text, element.tag) - if element._children: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(element._children) * "%s") + "%s\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()]),) - data += tuple([pretty_print(entry, level+2) for entry in element._children]) + (element.tag, ) - else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])) - return fmt % data - - -if __name__ == '__main__': - if '-C' in sys.argv: - cfpath = sys.argv[sys.argv.index('-C') + 1] - else: - cfpath = '/etc/bcfg2.conf' - c = ConfigParser() - c.read([cfpath]) - configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') - statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') - clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') - try: - prefix = c.get('server', 'prefix') - except (NoSectionError, NoOptionError): - prefix = '/usr' - - transformpath = "/%s/share/bcfg2/xsl-transforms/" % (prefix) - #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" - - try: - opts, args = getopt.getopt(sys.argv[1:], "C:hc:s:", ["help", "config=", "stats="]) - except getopt.GetoptError, mesg: - # print help information and exit: - print "%s\nUsage:\nbcfg2-build-reports [-h] [-c ] [-s ]" % (mesg) - raise SystemExit, 2 - for o, a in opts: - if o in ("-h", "--help"): - print "Usage:\nbcfg2-build-reports [-h] [-c ] [-s ]" - raise SystemExit - if o in ("-c", "--config"): - configpath = a - if o in ("-s", "--stats"): - statpath = a - - - #See if hostinfo.xml exists, and is less than 23.5 hours old - #try: - #hostinstat = os.stat(hostinfopath) - #if (time() - hostinstat[9])/(60*60) > 23.5: - os.system('bcfg2-ping-sweep') # bcfg2-ping-sweep needs to be in path - #except OSError: - # os.system('GenerateHostInfo')#Generate HostInfo needs to be in path - - - '''Reads Data & Config files''' - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(statpath)) - raise SystemExit, 1 - try: - configdata = XML(open(configpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(configpath)) - raise SystemExit, 1 - try: - clientsdata = XML(open(clientsdatapath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(clientsdatapath)) - raise SystemExit, 1 - - #Merge data from three sources - nodereport = Element("Report", attrib={"time" : asctime()}) - #should all of the other info in Metadata be appended? - #What about all of the package stuff for other types of reports? - - for client in clientsdata.findall("Client"): - nodel = Element("Node", attrib={"name" : client.get("name")}) - nodel.append(client) - - for nod in statsdata.findall("Node"): - if client.get('name').find(nod.get('name')) == 0: - for statel in nod.findall("Statistics"): - nodel.append(statel) - nodereport.append(nodel) - - - for reprt in configdata.findall('Report'): - nodereport.set("name", reprt.get("name", default="BCFG Report")) - - if reprt.get('refresh-time') != None: - nodereport.set("refresh-time", reprt.get("refresh-time", default="600")) - - - procnodereport = generatereport(reprt, nodereport) - - for deliv in reprt.findall('Delivery'): - #is a deepcopy of procnodereport necessary? - - delivtype = deliv.get('type', default='nodes-digest') - deliverymechanism = deliv.get('mechanism', default='www') - - #apply XSLT, different ones based on report type, and options - if deliverymechanism == 'null-operator': #Special Cases - fileout(tostring(ElementTree(procnodereport).getroot()), deliv) - break - transform = delivtype + '-' + deliverymechanism + '.xsl' - - try: #make sure valid stylesheet is selected - os.stat(transformpath + transform) - except: - print("bcfg2-build-reports: Invalid report type or delivery mechanism.\n Can't find: "\ - + transformpath + transform) - raise SystemExit, 1 - - try: #try to parse stylesheet - stylesheet = XSLT(parse(transformpath + transform)) - except: - print("bcfg2-build-reports: invalid XSLT transform file.") - raise SystemExit, 1 - - if deliverymechanism == 'mail': - if delivtype == 'nodes-individual': - reportdata = copy.deepcopy(procnodereport) - for noden in reportdata.findall("Node"): - [reportdata.remove(y) for y in reportdata.findall("Node")] - reportdata.append(noden) - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - #prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - - else: - reportdata = copy.deepcopy(procnodereport) - - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - #prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - else: - outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot()) - if deliverymechanism == 'rss': - rss(outputstring, deliv, reprt) - else: # must be deliverymechanism == 'www': - www(outputstring, deliv) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports new file mode 100755 index 000000000..722c563c0 --- /dev/null +++ b/src/sbin/bcfg2-build-reports @@ -0,0 +1,290 @@ +#!/usr/bin/env python + +'''bcfg2-build-reports Generates & distributes reports of statistic information +for bcfg2''' +__revision__ = '$Revision$' + +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from lxml.etree import XML, XSLT, parse, Element, ElementTree, SubElement, tostring, XMLSyntaxError +from time import asctime, strptime +import copy, getopt, re, os, socket, sys + +def generatereport(rspec, nrpt): + '''generatereport creates and returns an ElementTree representation + of a report adhering to the XML spec for intermediate reports''' + reportspec = copy.deepcopy(rspec) + nodereprt = copy.deepcopy(nrpt) + + reportgood = reportspec.get("good", default = 'Y') + reportmodified = reportspec.get("modified", default = 'Y') + current_date = asctime()[:10] + + '''build regex of all the nodes we are reporting about''' + pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) + + for node in nodereprt.findall('Node'): + if not (node.findall("Statistics") and pattern.match(node.get('name'))): + # don't know enough about node + nodereprt.remove(node) + continue + + #reduce to most recent Statistics entry + statisticslist = node.findall('Statistics') + #this line actually sorts from most recent to oldest + statisticslist.sort(lambda y, x: cmp(strptime(x.get("time")), strptime(y.get("time")))) + stats = statisticslist[0] + + [node.remove(item) for item in node.findall('Statistics')] + + #add a good tag if node is good and we wnat to report such + if reportgood == 'Y' and stats.get('state') == 'clean': + SubElement(stats,"Good") + + [stats.remove(item) for item in stats.findall("Bad") + stats.findall("Modified") if \ + item.getchildren() == None] + [stats.remove(item) for item in stats.findall("Modified") if reportmodified == 'N'] + + #test for staleness -if stale add Stale tag + if stats.get("time").find(current_date) == -1: + SubElement(stats,"Stale") + node.append(stats) + return nodereprt + +def mail(mailbody, confi): + '''mail mails a previously generated report''' + + try: + mailer = confi.get('statistics', 'sendmailpath') + except (NoSectionError, NoOptionError): + mailer = "/usr/sbin/sendmail" + # open a pipe to the mail program and + # write the data to the pipe + pipe = os.popen("%s -t" % mailer, 'w') + pipe.write(mailbody) + exitcode = pipe.close() + if exitcode: + print "Exit code: %s" % exitcode + +def rss(reportxml, delivery, report): + '''rss appends a new report to the specified rss file + keeping the last 9 articles''' + #check and see if rss file exists + for destination in delivery.findall('Destination'): + try: + fil = open(destination.attrib['address'], 'r') + olddoc = XML(fil.read()) + + #defines the number of recent articles to keep + items = olddoc.find("channel").findall("item")[0:9] + fil.close() + fil = open(destination.attrib['address'], 'w') + except (IOError, XMLSyntaxError): + fil = open(destination.attrib['address'], 'w') + items = [] + + rssdata = Element("rss") + channel = SubElement(rssdata, "channel") + rssdata.set("version", "2.0") + chantitle = SubElement(channel, "title") + chantitle.text = report.attrib['name'] + chanlink = SubElement(channel, "link") + + #this can later link to WWW report if one gets published simultaneously? + chanlink.text = "http://www.mcs.anl.gov/cobalt/bcfg2" + chandesc = SubElement(channel, "description") + chandesc.text = "Information regarding the 10 most recent bcfg2 runs." + + channel.append(XML(reportxml)) + + if items != []: + for item in items: + channel.append(item) + + tree = "" + tostring(rssdata) + fil.write(tree) + fil.close() + +def www(reportxml, delivery): + '''www outputs report to''' + + #this can later link to WWW report if one gets published simultaneously? + for destination in delivery.findall('Destination'): + fil = open(destination.attrib['address'], 'w') + + fil.write(reportxml) + fil.close() + +def fileout(reportxml, delivery): + '''outputs to plain text file''' + for destination in delivery.findall('Destination'): + fil = open(destination.attrib['address'], 'w') + + fil.write(reportxml) + fil.close() + +def pretty_print(element, level=0): + '''Produce a pretty-printed text representation of element''' + if element.text: + fmt = "%s<%%s %%s>%%s" % (level*" ") + data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])), + element.text, element.tag) + if element._children: + fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(element._children) * "%s") + "%s\n" % (level*" ") + data = (element.tag, ) + (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()]),) + data += tuple([pretty_print(entry, level+2) for entry in element._children]) + (element.tag, ) + else: + fmt = "%s<%%s %%s/>\n" % (level * " ") + data = (element.tag, " ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])) + return fmt % data + + +if __name__ == '__main__': + if '-C' in sys.argv: + cfpath = sys.argv[sys.argv.index('-C') + 1] + else: + cfpath = '/etc/bcfg2.conf' + c = ConfigParser() + c.read([cfpath]) + configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') + statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') + clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') + try: + prefix = c.get('server', 'prefix') + except (NoSectionError, NoOptionError): + prefix = '/usr' + + transformpath = "/%s/share/bcfg2/xsl-transforms/" % (prefix) + #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" + + try: + opts, args = getopt.getopt(sys.argv[1:], "C:hc:s:", ["help", "config=", "stats="]) + except getopt.GetoptError, mesg: + # print help information and exit: + print "%s\nUsage:\nbcfg2-build-reports [-h] [-c ] [-s ]" % (mesg) + raise SystemExit, 2 + for o, a in opts: + if o in ("-h", "--help"): + print "Usage:\nbcfg2-build-reports [-h] [-c ] [-s ]" + raise SystemExit + if o in ("-c", "--config"): + configpath = a + if o in ("-s", "--stats"): + statpath = a + + + #See if hostinfo.xml exists, and is less than 23.5 hours old + #try: + #hostinstat = os.stat(hostinfopath) + #if (time() - hostinstat[9])/(60*60) > 23.5: + os.system('bcfg2-ping-sweep') # bcfg2-ping-sweep needs to be in path + #except OSError: + # os.system('GenerateHostInfo')#Generate HostInfo needs to be in path + + + '''Reads Data & Config files''' + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("bcfg2-build-reports: Failed to parse %s"%(statpath)) + raise SystemExit, 1 + try: + configdata = XML(open(configpath).read()) + except (IOError, XMLSyntaxError): + print("bcfg2-build-reports: Failed to parse %s"%(configpath)) + raise SystemExit, 1 + try: + clientsdata = XML(open(clientsdatapath).read()) + except (IOError, XMLSyntaxError): + print("bcfg2-build-reports: Failed to parse %s"%(clientsdatapath)) + raise SystemExit, 1 + + #Merge data from three sources + nodereport = Element("Report", attrib={"time" : asctime()}) + #should all of the other info in Metadata be appended? + #What about all of the package stuff for other types of reports? + + for client in clientsdata.findall("Client"): + nodel = Element("Node", attrib={"name" : client.get("name")}) + nodel.append(client) + + for nod in statsdata.findall("Node"): + if client.get('name').find(nod.get('name')) == 0: + for statel in nod.findall("Statistics"): + nodel.append(statel) + nodereport.append(nodel) + + + for reprt in configdata.findall('Report'): + nodereport.set("name", reprt.get("name", default="BCFG Report")) + + if reprt.get('refresh-time') != None: + nodereport.set("refresh-time", reprt.get("refresh-time", default="600")) + + + procnodereport = generatereport(reprt, nodereport) + + for deliv in reprt.findall('Delivery'): + #is a deepcopy of procnodereport necessary? + + delivtype = deliv.get('type', default='nodes-digest') + deliverymechanism = deliv.get('mechanism', default='www') + + #apply XSLT, different ones based on report type, and options + if deliverymechanism == 'null-operator': #Special Cases + fileout(tostring(ElementTree(procnodereport).getroot()), deliv) + break + transform = delivtype + '-' + deliverymechanism + '.xsl' + + try: #make sure valid stylesheet is selected + os.stat(transformpath + transform) + except: + print("bcfg2-build-reports: Invalid report type or delivery mechanism.\n Can't find: "\ + + transformpath + transform) + raise SystemExit, 1 + + try: #try to parse stylesheet + stylesheet = XSLT(parse(transformpath + transform)) + except: + print("bcfg2-build-reports: invalid XSLT transform file.") + raise SystemExit, 1 + + if deliverymechanism == 'mail': + if delivtype == 'nodes-individual': + reportdata = copy.deepcopy(procnodereport) + for noden in reportdata.findall("Node"): + [reportdata.remove(y) for y in reportdata.findall("Node")] + reportdata.append(noden) + result = stylesheet.apply(ElementTree(reportdata)) + outputstring = stylesheet.tostring(result) + + if not outputstring == None: + toastring = '' + for desti in deliv.findall("Destination"): + toastring = "%s%s " % \ + (toastring, desti.get('address')) + #prepend To: and From: + outputstring = "To: %s\nFrom: root@%s\n%s"% \ + (toastring, socket.getfqdn(), outputstring) + mail(outputstring, c) #call function to send + + else: + reportdata = copy.deepcopy(procnodereport) + + result = stylesheet.apply(ElementTree(reportdata)) + outputstring = stylesheet.tostring(result) + + if not outputstring == None: + toastring = '' + for desti in deliv.findall("Destination"): + toastring = "%s%s " % \ + (toastring, desti.get('address')) + #prepend To: and From: + outputstring = "To: %s\nFrom: root@%s\n%s"% \ + (toastring, socket.getfqdn(), outputstring) + mail(outputstring, c) #call function to send + else: + outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot()) + if deliverymechanism == 'rss': + rss(outputstring, deliv, reprt) + else: # must be deliverymechanism == 'www': + www(outputstring, deliv) -- cgit v1.2.3-1-g7c22