From 921dd44a03856b81972d46dcab747f0951ca09fd Mon Sep 17 00:00:00 2001 From: Joey Hagedorn Date: Mon, 10 Jul 2006 16:00:35 +0000 Subject: Preliminary import script working version, imports Reasons, Interactions, Clients git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1921 ce84e21b-d406-0410-9b95-82705330c041 --- reports/brpt/importscript.py | 104 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) (limited to 'reports/brpt') diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py index 381bd9cdb..56c1def1d 100755 --- a/reports/brpt/importscript.py +++ b/reports/brpt/importscript.py @@ -22,7 +22,7 @@ from lxml.etree import XML, XMLSyntaxError from sys import argv from getopt import getopt, GetoptError from datetime import datetime -from time import strptime +from time import strptime, sleep if __name__ == '__main__': @@ -53,6 +53,108 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s"%(clientspath)) raise SystemExit, 1 + from django.db import connection, backend + cursor = connection.cursor() + + cursor.execute("SELECT name, id from reports_client;") + clients = {} + [clients.__setitem__(a,b) for a,b in cursor.fetchall()] + + for node in statsdata.findall('Node'): + name = node.get('name') + if not clients.has_key(name): + cursor.execute("INSERT INTO reports_client VALUES (NULL, %s, %s, NULL)", [datetime.now(),name]) + clients[name] = cursor.lastrowid +# print("Client %s added to db"%name) +# else: +# print("Client %s already exists in db"%name) + + + cursor.execute("SELECT client_id, timestamp, id from reports_interaction") + interactions = cursor.fetchall() + #interactions_slice = [x[0:2] for x in interactions] + interactions_hash = {} + [interactions_hash.__setitem__(str(x[0])+"-"+x[1].isoformat(),x[2]) for x in interactions] + pingability = {} + [pingability.__setitem__(n.get('name'),n.get('pingable',default='N')) for n in clientsdata.findall('Client')] + + cursor.execute("SELECT id, owner, current_owner, %s, current_group, perms, current_perms, status, current_status, %s, current_to, version, current_version, current_exists, current_diff from reports_reason"%(backend.quote_name("group"),backend.quote_name("to"))) + reasons = cursor.fetchall() + + cursor.execute("SELECT id, name, kind, reason_id from reports_bad") + bad_things = cursor.fetchall() + cursor.execute("SELECT id, name, kind, reason_id from reports_extra") + extra_things = cursor.fetchall() + cursor.execute("SELECT id, name, kind, reason_id from reports_modified") + modified_things = cursor.fetchall() + cursor.execute("SELECT id, metric, value from reports_performance") + performance_things = cursor.fetchall() + + reasons_hash = {} + [reasons_hash.__setitem__(tuple(n[1:]),n[0]) for n in reasons] + for r in statsdata.findall('.//Bad/*')+statsdata.findall('.//Extra/*')+statsdata.findall('.//Modified/*'): + arguments = [r.get('owner', default=""), r.get('current_owner', default=""), + r.get('group', default=""), r.get('current_group', default=""), + eval(r.get('perms', default="''")), eval(r.get('current_perms', default="''")), + r.get('status', default=""), r.get('current_status', default=""), + r.get('to', default=""), r.get('current_to', default=""), + r.get('version', default=""), r.get('current_version', default=""), + eval(r.get('current_exists', default="True").capitalize()), r.get('current_diff', default="")] + if reasons_hash.has_key(tuple(arguments)): + current_reason_id = reasons_hash[tuple(arguments)] +# print("Reason already exists..... It's ID is: %s"%current_reason_id) + else: + cursor.execute("INSERT INTO reports_reason VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", + arguments) + current_reason_id = cursor.lastrowid + reasons.append([current_reason_id]+arguments) + reasons_hash[tuple(arguments)] = current_reason_id + +# print("Reason inserted with id %s"%current_reason_id) + print "----------------REASONS SYNCED------------------" + for node in statsdata.findall('Node'): + name = node.get('name') + try: + pingability[name] + except KeyError: + pingabilty[name] = 'N' + for statistics in node.findall('Statistics'): + t = strptime(statistics.get('time')) + timestamp = datetime(t[0],t[1],t[2],t[3],t[4],t[5]) + if interactions_hash.has_key(str(clients[name]) +"-"+ timestamp.isoformat()): + current_interaction_id = interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] +# print("Interaction for %s at %s with id %s already exists"%(clients[name], +# datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) + else: + cursor.execute("INSERT INTO reports_interaction VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s);", + [clients[name], + timestamp, + statistics.get('state', default="unknown"), + statistics.get('revision',default="unknown"), + statistics.get('client_version',default="unknown"), + pingability[name], + statistics.get('good',default="0"), + statistics.get('total',default="0")]) + current_interaction_id = cursor.lastrowid + interactions.append([clients[name], timestamp, current_interaction_id]) + interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] = current_interaction_id +# print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], +# timestamp, current_interaction_id)) + + #insert bad children, extra children, modified children, performance items for this iteration of Client/Interaction + + #use that crazy query to update all the latest client_interaction records. + + + connection._commit() + #Clients are consistent + for q in connection.queries: + print q + + raise SystemExit, 0 + + + #------------------------------- for node in statsdata.findall('Node'): (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'), defaults={'name': node.get('name'), 'creation': datetime.now()}) -- cgit v1.2.3-1-g7c22