diff options
Diffstat (limited to 'reports/brpt/importscript.py')
| -rwxr-xr-x | reports/brpt/importscript.py | 285 |
1 files changed, 0 insertions, 285 deletions
diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py deleted file mode 100755 index 252fce886..000000000 --- a/reports/brpt/importscript.py +++ /dev/null @@ -1,285 +0,0 @@ -#! /usr/bin/env python -'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine''' -__revision__ = '$Revision$' - -import os, sys -try: - import settings -except ImportError: - try: - import brpt.settings - except ImportError: - sys.stderr.write("Failed to locate settings.py. Is brpt python module installed?") - sys.exit(1) - -project_directory = os.path.dirname(settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason -from lxml.etree import XML, XMLSyntaxError -from sys import argv -from getopt import getopt, GetoptError -from datetime import datetime -from time import strptime, sleep -from django.db import connection, backend -import ConfigParser - -if __name__ == '__main__': - somewhatverbose = False - verbose = False - veryverbose = False - - try: - opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" ,"debug", "clients=", "stats="]) - except GetoptError, mesg: - # print help information and exit: - print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) - raise SystemExit, 2 - opts.append(("1","1"))#this requires the loop run at least once - for o, a in opts: - if o in ("-h", "--help"): - print "Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n" - print "h : help; this message" - print "v : verbose; print messages on record insertion/skip" - print "u : updates; print status messages as items inserted semi-verbose" - print "d : debug; print most SQL used to manipulate database" - print "C : path to bcfg2.conf config file." - print "c : clients.xml file" - print "s : statistics.xml file" - raise SystemExit - if o in ("-C"): - cpath = a - else: - cpath = "/etc/bcfg2.conf" - - cf = ConfigParser.ConfigParser() - cf.read([cpath]) - - if o in ("-v", "--verbose"): - verbose = True - if o in ("-u", "--updates"): - somewhatverbose = True - if o in ("-d", "--debug"): - veryverbose = True - if o in ("-c", "--clients"): - clientspath = a - else: - try: - clientspath = "%s/Metadata/clients.xml"%cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 - - if o in ("-s", "--stats"): - statpath = a - else: - try: - statpath = "%s/etc/statistics.xml"%cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 - - '''Reads Data & Config files''' - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(statpath)) - raise SystemExit, 1 - try: - clientsdata = XML(open(clientspath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(clientspath)) - raise SystemExit, 1 - - cursor = connection.cursor() - clients = {} - cursor.execute("SELECT name, id from reports_client;") - [clients.__setitem__(a,b) for a,b in cursor.fetchall()] - - for node in statsdata.findall('Node'): - name = node.get('name') - if not clients.has_key(name): - cursor.execute("INSERT INTO reports_client VALUES (NULL, %s, %s, NULL, NULL)", [datetime.now(),name]) - clients[name] = cursor.lastrowid - if verbose: - print("Client %s added to db"%name) - else: - if verbose: - print("Client %s already exists in db"%name) - - - cursor.execute("SELECT client_id, timestamp, id from reports_interaction") - interactions_hash = {} - [interactions_hash.__setitem__(str(x[0])+"-"+x[1].isoformat(),x[2]) for x in cursor.fetchall()]#possibly change str to tuple - pingability = {} - [pingability.__setitem__(n.get('name'),n.get('pingable',default='N')) for n in clientsdata.findall('Client')] - - - cursor.execute("SELECT id, owner, current_owner, %s, current_group, perms, current_perms, status, current_status, %s, current_to, version, current_version, current_exists, current_diff from reports_reason"%(backend.quote_name("group"),backend.quote_name("to"))) - reasons_hash = {} - [reasons_hash.__setitem__(tuple(n[1:]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, name, kind, reason_id from reports_bad") - bad_hash = {} - [bad_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, name, kind, reason_id from reports_extra") - extra_hash = {} - [extra_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, name, kind, reason_id from reports_modified") - modified_hash = {} - [modified_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, metric, value from reports_performance") - performance_hash = {} - [performance_hash.__setitem__((n[1],n[2]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT x.client_id, reports_ping.status from (SELECT client_id, MAX(endtime) from reports_ping GROUP BY client_id) x, reports_ping WHERE x.client_id = reports_ping.client_id") - ping_hash = {} - [ping_hash.__setitem__(n[0],n[1]) for n in cursor.fetchall()] - - - for r in statsdata.findall('.//Bad/*')+statsdata.findall('.//Extra/*')+statsdata.findall('.//Modified/*'): - arguments = [r.get('owner', default=""), r.get('current_owner', default=""), - r.get('group', default=""), r.get('current_group', default=""), - r.get('perms', default=""), r.get('current_perms', default=""), - r.get('status', default=""), r.get('current_status', default=""), - r.get('to', default=""), r.get('current_to', default=""), - r.get('version', default=""), r.get('current_version', default=""), - (r.get('current_exists', default="True").capitalize()=="True"), - r.get('current_diff', default="")] - if reasons_hash.has_key(tuple(arguments)): - current_reason_id = reasons_hash[tuple(arguments)] - if verbose: - print("Reason already exists..... It's ID is: %s"%current_reason_id) - else: - cursor.execute("INSERT INTO reports_reason VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", - arguments) - current_reason_id = cursor.lastrowid - reasons_hash[tuple(arguments)] = current_reason_id - if verbose: - print("Reason inserted with id %s"%current_reason_id) - if (somewhatverbose or verbose): - print "----------------REASONS SYNCED---------------------" - - for node in statsdata.findall('Node'): - name = node.get('name') - try: - pingability[name] - except KeyError: - pingability[name] = 'N' - for statistics in node.findall('Statistics'): - t = strptime(statistics.get('time')) - timestamp = datetime(t[0],t[1],t[2],t[3],t[4],t[5])#Maybe replace with django.core.db typecasts typecast_timestamp()? import from django.backends util - if interactions_hash.has_key(str(clients[name]) +"-"+ timestamp.isoformat()): - current_interaction_id = interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] - if verbose: - print("Interaction for %s at %s with id %s already exists"%(clients[name], - datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) - else: - cursor.execute("INSERT INTO reports_interaction VALUES (NULL, %s, %s, %s, %s, %s, %s, %s);", - [clients[name], timestamp, - statistics.get('state', default="unknown"), statistics.get('revision',default="unknown"), - statistics.get('client_version',default="unknown"), - statistics.get('good',default="0"), statistics.get('total',default="0")]) - current_interaction_id = cursor.lastrowid - interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] = current_interaction_id - if verbose: - print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], - timestamp, current_interaction_id)) - - - for (xpath, hashname, tablename) in [('Bad/*', bad_hash, 'reports_bad'), - ('Extra/*', extra_hash, 'reports_extra'), - ('Modified/*', modified_hash, 'reports_modified')]: - for x in statistics.findall(xpath): - arguments = [x.get('owner', default=""), x.get('current_owner', default=""), - x.get('group', default=""), x.get('current_group', default=""), - x.get('perms', default=""), x.get('current_perms', default=""), - x.get('status', default=""), x.get('current_status', default=""), - x.get('to', default=""), x.get('current_to', default=""), - x.get('version', default=""), x.get('current_version', default=""), - (x.get('current_exists', default="True").capitalize()=="True"), - x.get('current_diff', default="")] - - if not hashname.has_key((x.get('name'), x.tag, reasons_hash[tuple(arguments)])): - cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s, %s);", - [x.get('name'), - x.tag, - (x.get('critical', default="False").capitalize()=="True"), - reasons_hash[tuple(arguments)]]) - item_id = cursor.lastrowid - hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])] = item_id - if verbose: - print "Bad item INSERTED having reason id %s and ID %s"%(reasons_hash[tuple(arguments)], - hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])]) - else: - item_id = hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])] - if verbose: - print "Bad item exists, has reason id %s and ID %s"%(reasons_hash[tuple(arguments)], - hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])]) - try: - cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", - [item_id, current_interaction_id]) - except: - pass - - for times in statistics.findall('OpStamps'): - for tags in times.items(): - if not performance_hash.has_key((tags[0],float(tags[1]))): - cursor.execute("INSERT INTO reports_performance VALUES (NULL, %s, %s)",[tags[0],tags[1]]) - performance_hash[(tags[0],tags[1])] = cursor.lastrowid - else: - item_id = performance_hash[(tags[0],float(tags[1]))] - #already exists - try: - cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);", - [item_id, current_interaction_id]) - except: - pass - - if (somewhatverbose or verbose): - print("----------------INTERACTIONS SYNCED----------------") - cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer") - for row in cursor.fetchall(): - cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s", - [row[0],row[1]]) - if (somewhatverbose or verbose): - print("------------LATEST INTERACTION SET----------------") - - for key in pingability.keys(): - if clients.has_key(key): - if ping_hash.has_key(clients[key]): - if ping_hash[clients[key]] == pingability[key]: - cursor.execute("UPDATE reports_ping SET endtime = %s where reports_ping.client_id = %s", - [datetime.now(),clients[key]]) - else: - ping_hash[clients[key]] = pingability[key] - cursor.execute("INSERT INTO reports_ping VALUES (NULL, %s, %s, %s, %s)", - [clients[key], datetime.now(), datetime.now(), pingability[key]]) - else: - ping_hash[clients[key]] = pingability[key] - cursor.execute("INSERT INTO reports_ping VALUES (NULL, %s, %s, %s, %s)", - [clients[key], datetime.now(), datetime.now(), pingability[key]]) - - if (somewhatverbose or verbose): - print "---------------PINGDATA SYNCED---------------------" - - connection._commit() - #Clients are consistent - if veryverbose: - for q in connection.queries: - if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')| - q['sql'].startswith('INSERT INTO reports_extra_interactions')| - q['sql'].startswith('INSERT INTO reports_performance_interaction')| - q['sql'].startswith('INSERT INTO reports_modified_interactions')| - q['sql'].startswith('UPDATE reports_client SET current_interaction_id ')): - print q - - raise SystemExit, 0 |