From a31b5a2bec0733aa4c69778f4d09afe447c750c0 Mon Sep 17 00:00:00 2001 From: Joey Hagedorn Date: Wed, 12 Jul 2006 19:38:31 +0000 Subject: import script added verbose/debug options; defaults to silent operation git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1928 ce84e21b-d406-0410-9b95-82705330c041 --- reports/brpt/importscript.py | 245 ++++++++++------------------------------- reports/brpt/reports/models.py | 15 ++- 2 files changed, 70 insertions(+), 190 deletions(-) (limited to 'reports/brpt') diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py index 4de369f12..d1277e723 100755 --- a/reports/brpt/importscript.py +++ b/reports/brpt/importscript.py @@ -25,17 +25,27 @@ from datetime import datetime from time import strptime, sleep if __name__ == '__main__': - + verbose = False + veryverbose = False try: - opts, args = getopt(argv[1:], "hc:s:", ["help", "clients=", "stats="]) + opts, args = getopt(argv[1:], "hvdc:s:", ["help", "verbose", "debug", "clients=", "stats="]) except GetoptError, mesg: # print help information and exit: - print "%s\nUsage:\nStatReports.py [-h] -c -s " % (mesg) + print "%s\nUsage:\nStatReports.py [-h] [-v] [-d] -c -s " % (mesg) raise SystemExit, 2 for o, a in opts: if o in ("-h", "--help"): - print "Usage:\nStatReports.py [-h] -c -s " + print "Usage:\nStatReports.py [-h] [-v] -c -s \n" + print "h : help; this message" + print "v : verbose; print messages on record insertion/skip" + print "d : debug; print all SQL used to manipulatee database" + print "c : clients.xml file" + print "s : statistics.xml file" raise SystemExit + if o in ("-v", "--verbose"): + verbose = True + if o in ("-d", "--debug"): + veryverbose = True if o in ("-c", "--clients"): clientspath = a if o in ("-s", "--stats"): @@ -65,9 +75,11 @@ if __name__ == '__main__': if not clients.has_key(name): cursor.execute("INSERT INTO reports_client VALUES (NULL, %s, %s, NULL)", [datetime.now(),name]) clients[name] = cursor.lastrowid -# print("Client %s added to db"%name) -# else: -# print("Client %s already exists in db"%name) + if verbose: + print("Client %s added to db"%name) + else: + if verbose: + print("Client %s already exists in db"%name) cursor.execute("SELECT client_id, timestamp, id from reports_interaction") @@ -103,19 +115,21 @@ if __name__ == '__main__': r.get('status', default=""), r.get('current_status', default=""), r.get('to', default=""), r.get('current_to', default=""), r.get('version', default=""), r.get('current_version', default=""), - eval(r.get('current_exists', default="True").capitalize()), r.get('current_diff', default="")] + (r.get('current_exists', default="True").capitalize()=="True"), + r.get('current_diff', default="")] if reasons_hash.has_key(tuple(arguments)): current_reason_id = reasons_hash[tuple(arguments)] -# print("Reason already exists..... It's ID is: %s"%current_reason_id) + if verbose: + print("Reason already exists..... It's ID is: %s"%current_reason_id) else: cursor.execute("INSERT INTO reports_reason VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", arguments) current_reason_id = cursor.lastrowid reasons_hash[tuple(arguments)] = current_reason_id - -# print("Reason inserted with id %s"%current_reason_id) - - print "----------------REASONS SYNCED---------------------" + if verbose: + print("Reason inserted with id %s"%current_reason_id) + if verbose: + print "----------------REASONS SYNCED---------------------" for node in statsdata.findall('Node'): name = node.get('name') @@ -128,8 +142,9 @@ if __name__ == '__main__': timestamp = datetime(t[0],t[1],t[2],t[3],t[4],t[5]) if interactions_hash.has_key(str(clients[name]) +"-"+ timestamp.isoformat()): current_interaction_id = interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] -# print("Interaction for %s at %s with id %s already exists"%(clients[name], -# datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) + if verbose: + print("Interaction for %s at %s with id %s already exists"%(clients[name], + datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) else: cursor.execute("INSERT INTO reports_interaction VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s);", [clients[name], timestamp, @@ -138,8 +153,9 @@ if __name__ == '__main__': statistics.get('good',default="0"), statistics.get('total',default="0")]) current_interaction_id = cursor.lastrowid interactions_hash[str(clients[name])+"-"+timestamp.isoformat()] = current_interaction_id - #print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], - # timestamp, current_interaction_id)) + if verbose: + print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], + timestamp, current_interaction_id)) for (xpath, hashname, tablename) in [('Bad/*', bad_hash, 'reports_bad'), ('Extra/*', extra_hash, 'reports_extra'), ('Modified/*', modified_hash, 'reports_modified')]: @@ -151,17 +167,23 @@ if __name__ == '__main__': x.get('status', default=""), x.get('current_status', default=""), x.get('to', default=""), x.get('current_to', default=""), x.get('version', default=""), x.get('current_version', default=""), - eval(x.get('current_exists', default="True").capitalize()), x.get('current_diff', default="")] - cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s);", - [x.get('name'), x.tag, reasons_hash[tuple(arguments)]]) + (x.get('current_exists', default="True").capitalize()=="True"), + x.get('current_diff', default="")] + cursor.execute("INSERT INTO "+tablename+" VALUES (NULL, %s, %s, %s, %s);", + [x.get('name'), + x.tag, + (x.get('critical', default="False").capitalize()=="True"), + reasons_hash[tuple(arguments)]]) item_id = cursor.lastrowid hashname[(x.get('name'), x.tag)] = (item_id, current_interaction_id) - #print "Bad item INSERTED having reason id %s and ID %s"%(hashname[(x.get('name'),x.tag)][1], - # hashname[(x.get('name'),x.tag)][0]) + if verbose: + print "Bad item INSERTED having reason id %s and ID %s"%(hashname[(x.get('name'),x.tag)][1], + hashname[(x.get('name'),x.tag)][0]) else: item_id = hashname[(x.get('name'), x.tag)][0] - #print "Bad item exists, has reason id %s and ID %s"%(hashname[(x.get('name'),x.tag)][1], - # hashname[(x.get('name'),x.tag)][0]) + if verbose: + print "Bad item exists, has reason id %s and ID %s"%(hashname[(x.get('name'),x.tag)][1], + hashname[(x.get('name'),x.tag)][0]) try: cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", [item_id, current_interaction_id]) @@ -170,11 +192,11 @@ if __name__ == '__main__': for times in statistics.findall('OpStamps'): for tags in times.items(): - if not performance_hash.has_key((tags[0],eval(tags[1]))): + if not performance_hash.has_key((tags[0],float(tags[1]))): cursor.execute("INSERT INTO reports_performance VALUES (NULL, %s, %s)",[tags[0],tags[1]]) performance_hash[(tags[0],tags[1])] = cursor.lastrowid else: - item_id = performance_hash[(tags[0],eval(tags[1]))] + item_id = performance_hash[(tags[0],float(tags[1]))] #already exists try: cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);", @@ -182,172 +204,23 @@ if __name__ == '__main__': except: pass - - print("----------------INTERACTIONS SYNCED----------------") - connection._commit() + if verbose: + print("----------------INTERACTIONS SYNCED----------------") cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer") for row in cursor.fetchall(): cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s", [row[0],row[1]]) - print "setting current_interaction_id for client ID %s to %s"%(row[1],row[0]) - - print("------------LATEST INTERACTION SET----------------") - - #use that crazy query to update all the latest client_interaction records. - + if verbose: + print("------------LATEST INTERACTION SET----------------") connection._commit() #Clients are consistent - for q in connection.queries: - if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')| - q['sql'].startswith('INSERT INTO reports_extra_interactions')| - q['sql'].startswith('INSERT INTO reports_performance_interaction')| - q['sql'].startswith('INSERT INTO reports_modified_interactions')| - q['sql'].startswith('UPDATE reports_client SET current_interaction_id')): - print q + if veryverbose: + for q in connection.queries: + if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')| + q['sql'].startswith('INSERT INTO reports_extra_interactions')| + q['sql'].startswith('INSERT INTO reports_performance_interaction')| + q['sql'].startswith('INSERT INTO reports_modified_interactions')): + print q raise SystemExit, 0 - - - #------------------------------- - for node in statsdata.findall('Node'): - (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'), - defaults={'name': node.get('name'), 'creation': datetime.now()}) - - for statistics in node.findall('Statistics'): - t = strptime(statistics.get('time')) - (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id, - timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]), - defaults={'client':client_rec, - 'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]), - 'state':statistics.get('state', default="unknown"), - 'repo_revision':statistics.get('revision', default="unknown"), - 'client_version':statistics.get('client_version'), - 'goodcount':statistics.get('good', default="unknown"), - 'totalcount':statistics.get('total', default="unknown")}) - for bad in statistics.findall('Bad'): - for ele in bad.getchildren(): - (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''), - current_owner=ele.get('current_owner',default=''), - group=ele.get('group',default=''), - current_group=ele.get('current_group',default=''), - perms=ele.get('perms',default=''), - current_perms=ele.get('current_perms',default=''), - status=ele.get('status',default=''), - current_status=ele.get('current_status',default=''), - to=ele.get('to',default=''), - current_to=ele.get('current_to',default=''), - version=ele.get('version',default=''), - current_version=ele.get('current_version',default=''), - current_exists=ele.get('current_exists',default='True'), - current_diff=ele.get('current_diff',default=''), - defaults={'owner':ele.get('owner',default=''), - 'current_owner':ele.get('current_owner',default=''), - 'group':ele.get('group',default=''), - 'current_group':ele.get('current_group',default=''), - 'perms':ele.get('perms',default=''), - 'current_perms':ele.get('current_perms',default=''), - 'status':ele.get('status',default=''), - 'current_status':ele.get('current_status',default=''), - 'to':ele.get('to',default=''), - 'current_to':ele.get('current_to',default=''), - 'version':ele.get('version',default=''), - 'current_version':ele.get('current_version',default=''), - 'current_exists':ele.get('current_exists',default='True'), - 'current_diff':ele.get('current_diff',default='')}) - - - (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag, - defaults={'name':ele.get('name'), - 'kind':ele.tag, - 'reason':reason_rec}) - - if not ele_rec in interaction_rec.bad_items.all(): - interaction_rec.bad_items.add(ele_rec) - - for modified in statistics.findall('Modified'): - for ele in modified.getchildren(): - (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''), - current_owner=ele.get('current_owner',default=''), - group=ele.get('group',default=''), - current_group=ele.get('current_group',default=''), - perms=ele.get('perms',default=''), - current_perms=ele.get('current_perms',default=''), - status=ele.get('status',default=''), - current_status=ele.get('current_status',default=''), - to=ele.get('to',default=''), - current_to=ele.get('current_to',default=''), - version=ele.get('version',default=''), - current_version=ele.get('current_version',default=''), - current_exists=ele.get('current_exists',default='True'), - current_diff=ele.get('current_diff',default=''), - defaults={'owner':ele.get('owner',default=''), - 'current_owner':ele.get('current_owner',default=''), - 'group':ele.get('group',default=''), - 'current_group':ele.get('current_group',default=''), - 'perms':ele.get('perms',default=''), - 'current_perms':ele.get('current_perms',default=''), - 'status':ele.get('status',default=''), - 'current_status':ele.get('current_status',default=''), - 'to':ele.get('to',default=''), - 'current_to':ele.get('current_to',default=''), - 'version':ele.get('version',default=''), - 'current_version':ele.get('current_version',default=''), - 'current_exists':ele.get('current_exists',default='True'), - 'current_diff':ele.get('current_diff',default='')}) - - - (ele_rec, er_created) = Modified.objects.get_or_create(name=ele.get('name'), kind=ele.tag, - defaults={'name':ele.get('name'), - 'kind':ele.tag, - 'reason':reason_rec}) - if not ele_rec in interaction_rec.modified_items.all(): - interaction_rec.modified_items.add(ele_rec) - - for extra in statistics.findall('Extra'): - for ele in extra.getchildren(): - (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''), - current_owner=ele.get('current_owner',default=''), - group=ele.get('group',default=''), - current_group=ele.get('current_group',default=''), - perms=ele.get('perms',default=''), - current_perms=ele.get('current_perms',default=''), - status=ele.get('status',default=''), - current_status=ele.get('current_status',default=''), - to=ele.get('to',default=''), - current_to=ele.get('current_to',default=''), - version=ele.get('version',default=''), - current_version=ele.get('current_version',default=''), - current_exists=ele.get('current_exists',default='True'), - current_diff=ele.get('current_diff',default=''), - defaults={'owner':ele.get('owner',default=''), - 'current_owner':ele.get('current_owner',default=''), - 'group':ele.get('group',default=''), - 'current_group':ele.get('current_group',default=''), - 'perms':ele.get('perms',default=''), - 'current_perms':ele.get('current_perms',default=''), - 'status':ele.get('status',default=''), - 'current_status':ele.get('current_status',default=''), - 'to':ele.get('to',default=''), - 'current_to':ele.get('current_to',default=''), - 'version':ele.get('version',default=''), - 'current_version':ele.get('current_version',default=''), - 'current_exists':ele.get('current_exists',default='True'), - 'current_diff':ele.get('current_diff',default='')}) - - - (ele_rec, er_created) = Extra.objects.get_or_create(name=ele.get('name'), kind=ele.tag, - defaults={'name':ele.get('name'), - 'kind':ele.tag, - 'reason':reason_rec}) - - if not ele_rec in interaction_rec.extra_items.all(): - interaction_rec.extra_items.add(ele_rec) - - for times in statistics.findall('OpStamps'): - for tags in times.items(): - (time_rec, tr_created) = Performance.objects.get_or_create(metric=tags[0], value=tags[1], - defaults={'metric':tags[0], - 'value':tags[1]}) - if not ele_rec in interaction_rec.extra_items.all(): - interaction_rec.performance_items.add(time_rec) diff --git a/reports/brpt/reports/models.py b/reports/brpt/reports/models.py index 1cc0a7a1e..cea6b5e0d 100644 --- a/reports/brpt/reports/models.py +++ b/reports/brpt/reports/models.py @@ -46,11 +46,15 @@ class InteractiveManager(models.Manager): from django.db import connection cursor = connection.cursor() if (maxdate == 'now' or maxdate == None): - cursor.execute("select id, client_id, MAX(timestamp) AS maxtimestamp from reports_interaction GROUP BY client_id") + cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) "+ + "as timer from reports_interaction GROUP BY client_id) x, reports_interaction where "+ + "reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer") else: - cursor.execute("select id, client_id, timestamp, MAX(timestamp) AS maxtimestamp from reports_interaction where timestamp < %s GROUP BY client_id", [maxdate]) - #rows = cursor.fetchall() - #return rows + cursor.execute("select reports_interaction.id, x.client_id from (select client_id, timestamp, MAX(timestamp) "+ + "as timer from reports_interaction WHERE timestamp < %s GROUP BY client_id) x, reports_interaction where "+ + "reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer", [maxdate]) + +# cursor.execute("select id, client_id, timestamp, MAX(timestamp) AS maxtimestamp from reports_interaction where timestamp < %s GROUP BY client_id", [maxdate]) in_idents = [item[0] for item in cursor.fetchall()] return self.filter(id__in = in_idents) @@ -139,6 +143,7 @@ class Modified(models.Model): interactions = models.ManyToManyField(Interaction, related_name="modified_items") name = models.CharField(maxlength=128, core=True)#name of modified thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... + critical = models.BooleanField() reason = models.ForeignKey(Reason) def __str__(self): return self.name @@ -147,6 +152,7 @@ class Extra(models.Model): interactions = models.ManyToManyField(Interaction, related_name="extra_items") name = models.CharField(maxlength=128, core=True)#name of Extra thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... + critical = models.BooleanField() reason = models.ForeignKey(Reason) def __str__(self): return self.name @@ -155,6 +161,7 @@ class Bad(models.Model): interactions = models.ManyToManyField(Interaction, related_name="bad_items") name = models.CharField(maxlength=128, core=True)#name of bad thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... + critical = models.BooleanField() reason = models.ForeignKey(Reason) def __str__(self): return self.name -- cgit v1.2.3-1-g7c22