summaryrefslogtreecommitdiffstats
path: root/src/lib/Server/Reports/importscript.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Server/Reports/importscript.py')
-rwxr-xr-xsrc/lib/Server/Reports/importscript.py285
1 files changed, 285 insertions, 0 deletions
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
new file mode 100755
index 000000000..252fce886
--- /dev/null
+++ b/src/lib/Server/Reports/importscript.py
@@ -0,0 +1,285 @@
+#! /usr/bin/env python
+'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine'''
+__revision__ = '$Revision$'
+
+import os, sys
+try:
+ import settings
+except ImportError:
+ try:
+ import brpt.settings
+ except ImportError:
+ sys.stderr.write("Failed to locate settings.py. Is brpt python module installed?")
+ sys.exit(1)
+
+project_directory = os.path.dirname(settings.__file__)
+project_name = os.path.basename(project_directory)
+sys.path.append(os.path.join(project_directory, '..'))
+project_module = __import__(project_name, '', '', [''])
+sys.path.pop()
+# Set DJANGO_SETTINGS_MODULE appropriately.
+os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
+
+from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason
+from lxml.etree import XML, XMLSyntaxError
+from sys import argv
+from getopt import getopt, GetoptError
+from datetime import datetime
+from time import strptime, sleep
+from django.db import connection, backend
+import ConfigParser
+
+if __name__ == '__main__':
+ somewhatverbose = False
+ verbose = False
+ veryverbose = False
+
+ try:
+ opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" ,"debug", "clients=", "stats="])
+ except GetoptError, mesg:
+ # print help information and exit:
+ print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)
+ raise SystemExit, 2
+ opts.append(("1","1"))#this requires the loop run at least once
+ for o, a in opts:
+ if o in ("-h", "--help"):
+ print "Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n"
+ print "h : help; this message"
+ print "v : verbose; print messages on record insertion/skip"
+ print "u : updates; print status messages as items inserted semi-verbose"
+ print "d : debug; print most SQL used to manipulate database"
+ print "C : path to bcfg2.conf config file."
+ print "c : clients.xml file"
+ print "s : statistics.xml file"
+ raise SystemExit
+ if o in ("-C"):
+ cpath = a
+ else:
+ cpath = "/etc/bcfg2.conf"
+
+ cf = ConfigParser.ConfigParser()
+ cf.read([cpath])
+
+ if o in ("-v", "--verbose"):
+ verbose = True
+ if o in ("-u", "--updates"):
+ somewhatverbose = True
+ if o in ("-d", "--debug"):
+ veryverbose = True
+ if o in ("-c", "--clients"):
+ clientspath = a
+ else:
+ try:
+ clientspath = "%s/Metadata/clients.xml"%cf.get('server', 'repository')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ print "Could not read bcfg2.conf; exiting"
+ raise SystemExit, 1
+
+ if o in ("-s", "--stats"):
+ statpath = a
+ else:
+ try:
+ statpath = "%s/etc/statistics.xml"%cf.get('server', 'repository')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ print "Could not read bcfg2.conf; exiting"
+ raise SystemExit, 1
+
+ '''Reads Data & Config files'''
+ try:
+ statsdata = XML(open(statpath).read())
+ except (IOError, XMLSyntaxError):
+ print("StatReports: Failed to parse %s"%(statpath))
+ raise SystemExit, 1
+ try:
+ clientsdata = XML(open(clientspath).read())
+ except (IOError, XMLSyntaxError):
+ print("StatReports: Failed to parse %s"%(clientspath))
+ raise SystemExit, 1
+
+ cursor = connection.cursor()
+ clients = {}
+ cursor.execute("SELECT name, id from reports_client;")
+ [clients.__setitem__(a,b) for a,b in cursor.fetchall()]
+
+ for node in statsdata.findall('Node'):
+ name = node.get('name')
+ if not clients.has_key(name):
+ cursor.execute("INSERT INTO reports_client VALUES (NULL, %s, %s, NULL, NULL)", [datetime.now(),name])
+ clients[name] = cursor.lastrowid
+ if verbose:
+ print("Client %s added to db"%name)
+ else:
+ if verbose:
+ print("Client %s already exists in db"%name)
+
+
+ cursor.execute("SELECT client_id, timestamp, id from reports_interaction")
+ interactions_hash = {}
+ [interactions_hash.__setitem__(str(x[0])+"-"+x[1].isoformat(),x[2]) for x in cursor.fetchall()]#possibly change str to tuple
+ pingability = {}
+ [pingability.__setitem__(n.get('name'),n.get('pingable',default='N')) for n in clientsdata.findall('Client')]
+
+
+ cursor.execute("SELECT id, owner, current_owner, %s, current_group, perms, current_perms, status, current_status, %s, current_to, version, current_version, current_exists, current_diff from reports_reason"%(backend.quote_name("group"),backend.quote_name("to")))
+ reasons_hash = {}
+ [reasons_hash.__setitem__(tuple(n[1:]),n[0]) for n in cursor.fetchall()]
+
+ cursor.execute("SELECT id, name, kind, reason_id from reports_bad")
+ bad_hash = {}
+ [bad_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()]
+
+ cursor.execute("SELECT id, name, kind, reason_id from reports_extra")
+ extra_hash = {}
+ [extra_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()]
+
+ cursor.execute("SELECT id, name, kind, reason_id from reports_modified")
+ modified_hash = {}
+ [modified_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()]
+
+ cursor.execute("SELECT id, metric, value from reports_performance")
+ performance_hash = {}
+ [performance_hash.__setitem__((n[1],n[2]),n[0]) for n in cursor.fetchall()]
+
+ cursor.execute("SELECT x.client_id, reports_ping.status from (SELECT client_id, MAX(endtime) from reports_ping GROUP BY client_id) x, reports_ping WHERE x.client_id = reports_ping.client_id")
+ ping_hash = {}
+ [ping_hash.__setitem__(n[0],n[1]) for n in cursor.fetchall()]
+
+
+ for r in statsdata.findall('.//Bad/*')+statsdata.findall('.//Extra/*')+statsdata.findall('.//Modified/*'):
+ arguments = [r.get('owner', default=""), r.get('current_owner', default=""),
+ r.get('group', default=""), r.get('current_group', default=""),
+ r.get('perms', default=""), r.get('current_perms', default=""),
+ r.get('status', default=""), r.get('current_status', default=""),
+ r.get('to', default=""), r.get('current_to', default=""),
+ r.get('version', default=""), r.get('current_version', default=""),
+ (r.get('current_exists', default="True").capitalize()=="True"),
+ r.get('current_diff', default="")]
+ if reasons_hash.has_key(tuple(arguments)):
+ current_reason_id = reasons_hash[tuple(arguments)]
+ if verbose:
+ print("Reason already exists..... It's ID is: %s"%current_reason_id)
+ else:
+ cursor.execute("INSERT INTO reports_reason VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);",
+ arguments)
+ current_reason_id = cursor.lastrowid
+ reasons_hash[tuple(arguments)] = current_reason_id
+ if verbose:
+ print("Reason inserted with id %s"%current_reason_id)
+ if (somewhatverbose or verbose):
+ print "----------------REASONS SYNCED---------------------"
+
+ for node in statsdata.findall('Node'):
+ name = node.get('name')
+ try:
+ pingability[name]
+ except KeyError:
+ pingability[name] = 'N'
+ for statistics in node.findall('Statistics'):
+ t = strptime(statistics.get('time'))
+ timestamp = datetime(t[0],t[1],t[2],t[3],t[4],t[5])#Maybe replace with django.core.db typecasts typecast_timestamp()? import from django.backends util
+ if interactions_hash.has_key(str(clients[name]) +"-"+ timestamp.isoformat()):
+ current_interaction_id = interactions_hash[str(clients[name])+"-"+timestamp.isoformat()]
+ if verbose:
+ print("Interaction for %s at %s with id %s already exists"%(clients[name],
+ datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id))
+ else:
+ cursor.execute("INSERT INTO reports_interaction VALUES (NULL, %s, %s, %s, %s, %s, %s, %s);",
+ [clients[name], timestamp,
+ statistics.get('state', default="unknown"), statistics.get('revision',default="unknown"),
+ statistics.get('client_version',default="unknown"),
+ statistics.get('good',default="0"), statistics.get('total',default="0")])
+ current_interaction_id = cursor.lastrowid
+ interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] = current_interaction_id
+ if verbose:
+ print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name],
+ timestamp, current_interaction_id))
+
+
+ for (xpath, hashname, tablename) in [('Bad/*', bad_hash, 'reports_bad'),
+ ('Extra/*', extra_hash, 'reports_extra'),
+ ('Modified/*', modified_hash, 'reports_modified')]:
+ for x in statistics.findall(xpath):
+ arguments = [x.get('owner', default=""), x.get('current_owner', default=""),
+ x.get('group', default=""), x.get('current_group', default=""),
+ x.get('perms', default=""), x.get('current_perms', default=""),
+ x.get('status', default=""), x.get('current_status', default=""),
+ x.get('to', default=""), x.get('current_to', default=""),
+ x.get('version', default=""), x.get('current_version', default=""),
+ (x.get('current_exists', default="True").capitalize()=="True"),
+ x.get('current_diff', default="")]
+
+ if not hashname.has_key((x.get('name'), x.tag, reasons_hash[tuple(arguments)])):
+ cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s, %s);",
+ [x.get('name'),
+ x.tag,
+ (x.get('critical', default="False").capitalize()=="True"),
+ reasons_hash[tuple(arguments)]])
+ item_id = cursor.lastrowid
+ hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])] = item_id
+ if verbose:
+ print "Bad item INSERTED having reason id %s and ID %s"%(reasons_hash[tuple(arguments)],
+ hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])])
+ else:
+ item_id = hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])]
+ if verbose:
+ print "Bad item exists, has reason id %s and ID %s"%(reasons_hash[tuple(arguments)],
+ hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])])
+ try:
+ cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);",
+ [item_id, current_interaction_id])
+ except:
+ pass
+
+ for times in statistics.findall('OpStamps'):
+ for tags in times.items():
+ if not performance_hash.has_key((tags[0],float(tags[1]))):
+ cursor.execute("INSERT INTO reports_performance VALUES (NULL, %s, %s)",[tags[0],tags[1]])
+ performance_hash[(tags[0],tags[1])] = cursor.lastrowid
+ else:
+ item_id = performance_hash[(tags[0],float(tags[1]))]
+ #already exists
+ try:
+ cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);",
+ [item_id, current_interaction_id])
+ except:
+ pass
+
+ if (somewhatverbose or verbose):
+ print("----------------INTERACTIONS SYNCED----------------")
+ cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer")
+ for row in cursor.fetchall():
+ cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s",
+ [row[0],row[1]])
+ if (somewhatverbose or verbose):
+ print("------------LATEST INTERACTION SET----------------")
+
+ for key in pingability.keys():
+ if clients.has_key(key):
+ if ping_hash.has_key(clients[key]):
+ if ping_hash[clients[key]] == pingability[key]:
+ cursor.execute("UPDATE reports_ping SET endtime = %s where reports_ping.client_id = %s",
+ [datetime.now(),clients[key]])
+ else:
+ ping_hash[clients[key]] = pingability[key]
+ cursor.execute("INSERT INTO reports_ping VALUES (NULL, %s, %s, %s, %s)",
+ [clients[key], datetime.now(), datetime.now(), pingability[key]])
+ else:
+ ping_hash[clients[key]] = pingability[key]
+ cursor.execute("INSERT INTO reports_ping VALUES (NULL, %s, %s, %s, %s)",
+ [clients[key], datetime.now(), datetime.now(), pingability[key]])
+
+ if (somewhatverbose or verbose):
+ print "---------------PINGDATA SYNCED---------------------"
+
+ connection._commit()
+ #Clients are consistent
+ if veryverbose:
+ for q in connection.queries:
+ if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')|
+ q['sql'].startswith('INSERT INTO reports_extra_interactions')|
+ q['sql'].startswith('INSERT INTO reports_performance_interaction')|
+ q['sql'].startswith('INSERT INTO reports_modified_interactions')|
+ q['sql'].startswith('UPDATE reports_client SET current_interaction_id ')):
+ print q
+
+ raise SystemExit, 0