From c46b40cb6f11ba9a7a2f62a41a52a6d6a143f5af Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Wed, 10 Oct 2007 20:36:02 +0000 Subject: Fix importscript.py performance problems git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3786 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Reports/importscript.py | 83 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 45 deletions(-) (limited to 'src/lib/Server/Reports/importscript.py') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 23d233379..59c29ed64 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -29,6 +29,26 @@ from time import strptime, sleep from django.db import connection, backend import ConfigParser +def build_reason_kwargs(r_ent): + if r_ent.get('current_bdiff', False): + rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) + else: + rc_diff = r_ent.get('current_diff', '') + return dict(owner=r_ent.get('owner', default=""), + current_owner=r_ent.get('current_owner', default=""), + group=r_ent.get('group', default=""), + current_group=r_ent.get('current_group', default=""), + perms=r_ent.get('perms', default=""), + current_perms=r_ent.get('current_perms', default=""), + status=r_ent.get('status', default=""), + current_status=r_ent.get('current_status', default=""), + to=r_ent.get('to', default=""), + current_to=r_ent.get('current_to', default=""), + version=r_ent.get('version', default=""), + current_version=r_ent.get('current_version', default=""), + current_exists=r_ent.get('current_exists', default="True").capitalize()=="True", + current_diff=rc_diff) + if __name__ == '__main__': somewhatverbose = False verbose = False @@ -120,10 +140,6 @@ if __name__ == '__main__': [pingability.__setitem__(n.get('name'),n.get('pingable',default='N')) for n in clientsdata.findall('Client')] - cursor.execute("SELECT id, owner, current_owner, %s, current_group, perms, current_perms, status, current_status, %s, current_to, version, current_version, current_exists, current_diff from reports_reason"%(backend.quote_name("group"),backend.quote_name("to"))) - reasons_hash = {} - [reasons_hash.__setitem__(tuple(n[1:]),n[0]) for n in cursor.fetchall()] - cursor.execute("SELECT id, name, kind, reason_id from reports_bad") bad_hash = {} [bad_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] @@ -146,29 +162,20 @@ if __name__ == '__main__': for r in statsdata.findall('.//Bad/*')+statsdata.findall('.//Extra/*')+statsdata.findall('.//Modified/*'): - if r.get('current_bdiff', False): - rc_diff = binascii.a2b_base64(r.get('current_bdiff')) - else: - rc_diff = r.get('current_diff', '') - arguments = [r.get('owner', default=""), r.get('current_owner', default=""), - r.get('group', default=""), r.get('current_group', default=""), - r.get('perms', default=""), r.get('current_perms', default=""), - r.get('status', default=""), r.get('current_status', default=""), - r.get('to', default=""), r.get('current_to', default=""), - r.get('version', default=""), r.get('current_version', default=""), - (r.get('current_exists', default="True").capitalize()=="True"), - rc_diff] - if reasons_hash.has_key(tuple(arguments)): - current_reason_id = reasons_hash[tuple(arguments)] + kargs = build_reason_kwargs(r) + rlist = \ + Reason.objects.filter(**kargs) + if rlist: + current_reason_id = rlist[0].id if verbose: - print("Reason already exists..... It's ID is: %s"%current_reason_id) + print("Reason already exists. It's ID is: %s"%current_reason_id) else: - cursor.execute("INSERT INTO reports_reason VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", - arguments) - current_reason_id = cursor.lastrowid - reasons_hash[tuple(arguments)] = current_reason_id + reason = Reason(**kargs) + reason.save() + current_reason_id = reason.id if verbose: print("Reason inserted with id %s"%current_reason_id) + if (somewhatverbose or verbose): print "----------------REASONS SYNCED---------------------" @@ -203,36 +210,22 @@ if __name__ == '__main__': ('Extra/*', extra_hash, 'reports_extra'), ('Modified/*', modified_hash, 'reports_modified')]: for x in statistics.findall(xpath): - if x.get('current_bdiff', False): - xc_diff = binascii.a2b_base64(x.get('current_bdiff')) - else: - xc_diff = x.get('current_diff', '') - - arguments = [x.get('owner', default=""), x.get('current_owner', default=""), - x.get('group', default=""), x.get('current_group', default=""), - x.get('perms', default=""), x.get('current_perms', default=""), - x.get('status', default=""), x.get('current_status', default=""), - x.get('to', default=""), x.get('current_to', default=""), - x.get('version', default=""), x.get('current_version', default=""), - (x.get('current_exists', default="True").capitalize()=="True"), - xc_diff] - - if not hashname.has_key((x.get('name'), x.tag, reasons_hash[tuple(arguments)])): + kargs = build_reason_kwargs(x) + rr = Reason.objects.filter(**kargs)[0] + if not hashname.has_key((x.get('name'), x.tag, rr.id)): cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s, %s);", [x.get('name'), x.tag, (x.get('critical', default="False").capitalize()=="True"), - reasons_hash[tuple(arguments)]]) + rr.id]) item_id = cursor.lastrowid - hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])] = item_id + hashname[(x.get('name'), x.tag, rr.id)] = item_id if verbose: - print "Bad item INSERTED having reason id %s and ID %s"%(reasons_hash[tuple(arguments)], - hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])]) + print "Bad item INSERTED having reason id %s and ID %s"%(rr.id, hashname[(x.get('name'),x.tag, rr.id)]) else: - item_id = hashname[(x.get('name'), x.tag, reasons_hash[tuple(arguments)])] + item_id = hashname[(x.get('name'), x.tag, rr.id)] if verbose: - print "Bad item exists, has reason id %s and ID %s"%(reasons_hash[tuple(arguments)], - hashname[(x.get('name'),x.tag, reasons_hash[tuple(arguments)])]) + print "Bad item exists, has reason id %s and ID %s"%(rr.id, hashname[(x.get('name'),x.tag, rr.id)]) try: cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", [item_id, current_interaction_id]) -- cgit v1.2.3-1-g7c22