From 27aa20d54c54537670a7e2a0aca9d2b1e68c1ce7 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Wed, 10 Oct 2007 23:35:41 +0000 Subject: Continue changes to use django datatypes (further reduces resource consumption) git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3787 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Reports/importscript.py | 70 +++++++++++++++------------------- 1 file changed, 30 insertions(+), 40 deletions(-) (limited to 'src/lib/Server/Reports/importscript.py') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 59c29ed64..8edb31614 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -132,26 +132,10 @@ if __name__ == '__main__': if verbose: print("Client %s already exists in db"%name) - - cursor.execute("SELECT client_id, timestamp, id from reports_interaction") - interactions_hash = {} - [interactions_hash.__setitem__(str(x[0])+"-"+x[1].isoformat(),x[2]) for x in cursor.fetchall()]#possibly change str to tuple pingability = {} [pingability.__setitem__(n.get('name'),n.get('pingable',default='N')) for n in clientsdata.findall('Client')] - cursor.execute("SELECT id, name, kind, reason_id from reports_bad") - bad_hash = {} - [bad_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, name, kind, reason_id from reports_extra") - extra_hash = {} - [extra_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - - cursor.execute("SELECT id, name, kind, reason_id from reports_modified") - modified_hash = {} - [modified_hash.__setitem__((n[1],n[2],n[3]),n[0]) for n in cursor.fetchall()] - cursor.execute("SELECT id, metric, value from reports_performance") performance_hash = {} [performance_hash.__setitem__((n[1],n[2]),n[0]) for n in cursor.fetchall()] @@ -181,6 +165,7 @@ if __name__ == '__main__': for node in statsdata.findall('Node'): name = node.get('name') + c_inst = Client.objects.filter(id=clients[name])[0] try: pingability[name] except KeyError: @@ -188,49 +173,54 @@ if __name__ == '__main__': for statistics in node.findall('Statistics'): t = strptime(statistics.get('time')) timestamp = datetime(t[0],t[1],t[2],t[3],t[4],t[5])#Maybe replace with django.core.db typecasts typecast_timestamp()? import from django.backends util - if interactions_hash.has_key(str(clients[name]) +"-"+ timestamp.isoformat()): - current_interaction_id = interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] + ilist = Interaction.objects.filter(client=c_inst, + timestamp=timestamp) + if ilist: + current_interaction_id = ilist[0].id if verbose: print("Interaction for %s at %s with id %s already exists"%(clients[name], datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) else: - cursor.execute("INSERT INTO reports_interaction VALUES (NULL, %s, %s, %s, %s, %s, %s, %s);", - [clients[name], timestamp, - statistics.get('state', default="unknown"), statistics.get('revision',default="unknown"), - statistics.get('client_version',default="unknown"), - statistics.get('good',default="0"), statistics.get('total',default="0")]) - current_interaction_id = cursor.lastrowid - interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] = current_interaction_id + newint = Interaction(client=c_inst, + timestamp=timestamp, + state=statistics.get('state', default="unknown"), + repo_revision=statistics.get('revision',default="unknown"), + client_version=statistics.get('client_version',default="unknown"), + goodcount=statistics.get('good',default="0"), + totalcount=statistics.get('total',default="0")) + newint.save() + current_interaction_id = newint.id if verbose: print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], timestamp, current_interaction_id)) - for (xpath, hashname, tablename) in [('Bad/*', bad_hash, 'reports_bad'), - ('Extra/*', extra_hash, 'reports_extra'), - ('Modified/*', modified_hash, 'reports_modified')]: + pattern = [('Bad/*', Bad), ('Extra/*', Extra), + ('Modified/*', Modified)] + for (xpath, obj) in pattern: for x in statistics.findall(xpath): kargs = build_reason_kwargs(x) rr = Reason.objects.filter(**kargs)[0] - if not hashname.has_key((x.get('name'), x.tag, rr.id)): - cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s, %s);", - [x.get('name'), - x.tag, - (x.get('critical', default="False").capitalize()=="True"), - rr.id]) - item_id = cursor.lastrowid - hashname[(x.get('name'), x.tag, rr.id)] = item_id + links = obj.objects.filter(name=x.get('name'), + kind=x.tag, + reason=rr.id) + if links: + item_id = links[0].id if verbose: - print "Bad item INSERTED having reason id %s and ID %s"%(rr.id, hashname[(x.get('name'),x.tag, rr.id)]) + print "%s item exists, has reason id %s and ID %s"%(xpath, rr.id, item_id) else: - item_id = hashname[(x.get('name'), x.tag, rr.id)] + newitem = obj(name=x.get('name'), + kind=x.tag, + reason=rr.id) + newitem.save() + item_id = newitem.id if verbose: - print "Bad item exists, has reason id %s and ID %s"%(rr.id, hashname[(x.get('name'),x.tag, rr.id)]) + print "Bad item INSERTED having reason id %s and ID %s"%(rr.id, item_id) try: cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", [item_id, current_interaction_id]) except: - pass + pass for times in statistics.findall('OpStamps'): for tags in times.items(): -- cgit v1.2.3-1-g7c22