diff options
Diffstat (limited to 'build/lib/Bcfg2/Server/Reports/importscript.py')
-rw-r--r-- | build/lib/Bcfg2/Server/Reports/importscript.py | 270 |
1 files changed, 270 insertions, 0 deletions
diff --git a/build/lib/Bcfg2/Server/Reports/importscript.py b/build/lib/Bcfg2/Server/Reports/importscript.py new file mode 100644 index 000000000..cdfd8079c --- /dev/null +++ b/build/lib/Bcfg2/Server/Reports/importscript.py @@ -0,0 +1,270 @@ +#! /usr/bin/env python +'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine''' +__revision__ = '$Revision$' + +import os, sys, binascii +try: + import Bcfg2.Server.Reports.settings +except Exception, e: + sys.stderr.write("Failed to load configuration settings. %s\n" % e) + sys.exit(1) + +project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) +project_name = os.path.basename(project_directory) +sys.path.append(os.path.join(project_directory, '..')) +project_module = __import__(project_name, '', '', ['']) +sys.path.pop() +# Set DJANGO_SETTINGS_MODULE appropriately. +os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name + +from Bcfg2.Server.Reports.reports.models import * +from lxml.etree import XML, XMLSyntaxError +from getopt import getopt, GetoptError +from datetime import datetime +from time import strptime +from django.db import connection +from Bcfg2.Server.Reports.updatefix import update_database +import ConfigParser +import logging +import Bcfg2.Logger +import platform + +def build_reason_kwargs(r_ent): + binary_file=False + if r_ent.get('current_bfile', False): + binary_file=True + rc_diff = r_ent.get('current_bfile') + if len(rc_diff) > 1024*1024: + rc_diff = '' + elif len(rc_diff) == 0: + # No point in flagging binary if we have no data + binary_file=False + elif r_ent.get('current_bdiff', False): + rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) + elif r_ent.get('current_diff', False): + rc_diff = r_ent.get('current_diff') + else: + rc_diff = '' + return dict(owner=r_ent.get('owner', default=""), + current_owner=r_ent.get('current_owner', default=""), + group=r_ent.get('group', default=""), + current_group=r_ent.get('current_group', default=""), + perms=r_ent.get('perms', default=""), + current_perms=r_ent.get('current_perms', default=""), + status=r_ent.get('status', default=""), + current_status=r_ent.get('current_status', default=""), + to=r_ent.get('to', default=""), + current_to=r_ent.get('current_to', default=""), + version=r_ent.get('version', default=""), + current_version=r_ent.get('current_version', default=""), + current_exists=r_ent.get('current_exists', default="True").capitalize()=="True", + current_diff=rc_diff, + is_binary=binary_file) + + +def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): + clients = {} + [clients.__setitem__(c.name, c) \ + for c in Client.objects.all()] + + pingability = {} + [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \ + for n in cdata.findall('Client')] + + for node in sdata.findall('Node'): + name = node.get('name') + c_inst, created = Client.objects.get_or_create(name=name) + if vlevel > 0: + logger.info("Client %s added to db" % name) + clients[name] = c_inst + try: + pingability[name] + except KeyError: + pingability[name] = 'N' + for statistics in node.findall('Statistics'): + timestamp = datetime(*strptime(statistics.get('time'))[0:6]) + ilist = Interaction.objects.filter(client=c_inst, + timestamp=timestamp) + if ilist: + current_interaction = ilist[0] + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s already exists" % \ + (c_inst.id, timestamp, current_interaction.id)) + continue + else: + newint = Interaction(client=c_inst, + timestamp = timestamp, + state = statistics.get('state', default="unknown"), + repo_rev_code = statistics.get('revision',default="unknown"), + client_version = statistics.get('client_version',default="unknown"), + goodcount = statistics.get('good',default="0"), + totalcount = statistics.get('total',default="0"), + server = location) + newint.save() + current_interaction = newint + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s INSERTED in to db"%(c_inst.id, + timestamp, current_interaction.id)) + + + counter_fields = { TYPE_CHOICES[0]: 0, TYPE_CHOICES[1]: 0, TYPE_CHOICES[2]: 0 } + pattern = [('Bad/*', TYPE_CHOICES[0]), + ('Extra/*', TYPE_CHOICES[2]), + ('Modified/*', TYPE_CHOICES[1]),] + for (xpath, type) in pattern: + for x in statistics.findall(xpath): + counter_fields[type] = counter_fields[type] + 1 + kargs = build_reason_kwargs(x) + + try: + rr = None + if not quick: + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: + pass + if not rr: + rr = Reason(**kargs) + rr.save() + if vlevel > 0: + logger.info("Created reason: %s" % rr.id) + except Exception, ex: + logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) + rr = Reason(current_exists=x.get('current_exists', + default="True").capitalize()=="True") + rr.save() + + entry, created = Entries.objects.get_or_create(\ + name=x.get('name'), kind=x.tag) + + Entries_interactions(entry=entry, reason=rr, + interaction=current_interaction, + type=type[0]).save() + if vlevel > 0: + logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) + + # Update interaction counters + current_interaction.bad_entries = counter_fields[TYPE_CHOICES[0]] + current_interaction.modified_entries = counter_fields[TYPE_CHOICES[1]] + current_interaction.extra_entries = counter_fields[TYPE_CHOICES[2]] + current_interaction.save() + + mperfs = [] + for times in statistics.findall('OpStamps'): + for metric, value in times.items(): + mmatch = [] + if not quick: + mmatch = Performance.objects.filter(metric=metric, value=value) + + if mmatch: + mperf = mmatch[0] + else: + mperf = Performance(metric=metric, value=value) + mperf.save() + mperfs.append(mperf) + current_interaction.performance_items.add(*mperfs) + + for key in pingability.keys(): + if key not in clients: + continue + try: + pmatch = Ping.objects.filter(client=clients[key]).order_by('-endtime')[0] + if pmatch.status == pingability[key]: + pmatch.endtime = datetime.now() + pmatch.save() + continue + except IndexError: + pass + Ping(client=clients[key], status=pingability[key], + starttime=datetime.now(), + endtime=datetime.now()).save() + + if vlevel > 1: + logger.info("---------------PINGDATA SYNCED---------------------") + + #Clients are consistent + +if __name__ == '__main__': + from sys import argv + verb = 0 + cpath = "/etc/bcfg2.conf" + clientpath = False + statpath = False + syslog = False + + try: + opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", "verbose", "updates" , + "debug", "clients=", "stats=", + "config=", "syslog"]) + except GetoptError, mesg: + # print help information and exit: + print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) + raise SystemExit, 2 + + for o, a in opts: + if o in ("-h", "--help"): + print "Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n" + print "h : help; this message" + print "v : verbose; print messages on record insertion/skip" + print "u : updates; print status messages as items inserted semi-verbose" + print "d : debug; print most SQL used to manipulate database" + print "C : path to bcfg2.conf config file." + print "c : clients.xml file" + print "s : statistics.xml file" + print "S : syslog; output to syslog" + raise SystemExit + if o in ["-C", "--config"]: + cpath = a + + if o in ("-v", "--verbose"): + verb = 1 + if o in ("-u", "--updates"): + verb = 2 + if o in ("-d", "--debug"): + verb = 3 + if o in ("-c", "--clients"): + clientspath = a + + if o in ("-s", "--stats"): + statpath = a + if o in ("-S", "--syslog"): + syslog = True + + logger = logging.getLogger('importscript.py') + logging.getLogger().setLevel(logging.INFO) + Bcfg2.Logger.setup_logging('importscript.py', + True, + syslog) + + cf = ConfigParser.ConfigParser() + cf.read([cpath]) + + if not statpath: + try: + statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print "Could not read bcfg2.conf; exiting" + raise SystemExit, 1 + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(statpath)) + raise SystemExit, 1 + + if not clientpath: + try: + clientspath = "%s/Metadata/clients.xml" % \ + cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print "Could not read bcfg2.conf; exiting" + raise SystemExit, 1 + try: + clientsdata = XML(open(clientspath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(clientspath)) + raise SystemExit, 1 + + q = '-O3' in sys.argv + # Be sure the database is ready for new schema + update_database() + load_stats(clientsdata, statsdata, verb, logger, quick=q, location=platform.node()) |