From af88fd95bfbba987798012edd431d4150e777b66 Mon Sep 17 00:00:00 2001 From: Joey Hagedorn Date: Wed, 28 Jun 2006 23:14:26 +0000 Subject: New reporting system's Performance report is ultra fast due to new SQL queries. Also, importscript now properly imports OpStamps from xml statistics file. It is very slow still though... git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1902 ce84e21b-d406-0410-9b95-82705330c041 --- reports/brpt/importscript.py | 34 ++++++++++++++-------------------- reports/brpt/reports/models.py | 38 +++++++++++++++++++++++++++++++++++--- reports/brpt/reports/views.py | 39 ++++++++++++++++++++++----------------- 3 files changed, 71 insertions(+), 40 deletions(-) (limited to 'reports/brpt') diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py index 585018f0e..3e37e9dba 100755 --- a/reports/brpt/importscript.py +++ b/reports/brpt/importscript.py @@ -76,8 +76,6 @@ if __name__ == '__main__': 'client_version':statistics.get('client_version'), 'goodcount':statistics.get('good', default="unknown"), 'totalcount':statistics.get('total', default="unknown")}) -# if ir_created: -# interaction_rec.save() for bad in statistics.findall('Bad'): for ele in bad.getchildren(): (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag, @@ -85,11 +83,9 @@ if __name__ == '__main__': 'kind':ele.tag, 'problemcode':'', 'reason':'Unknown'}) -# if er_created: -# ele_rec.save() - if not ele_rec in interaction_rec.bad_items.all(): - interaction_rec.bad_items.add(ele_rec) + if not ele_rec in interaction_rec.bad_items.all(): + interaction_rec.bad_items.add(ele_rec) for modified in statistics.findall('Modified'): for ele in modified.getchildren(): @@ -98,10 +94,8 @@ if __name__ == '__main__': 'kind':ele.tag, 'problemcode':'', 'reason':'Unknown'}) - # if er_created: - # ele_rec.save() - if not ele_rec in interaction_rec.modified_items.all(): - interaction_rec.modified_items.add(ele_rec) + if not ele_rec in interaction_rec.modified_items.all(): + interaction_rec.modified_items.add(ele_rec) for extra in statistics.findall('Extra'): for ele in extra.getchildren(): @@ -110,11 +104,9 @@ if __name__ == '__main__': 'kind':ele.tag, 'problemcode':'', 'reason':'Unknown'}) - # if er_created: - # ele_rec.save() - if not ele_rec in interaction_rec.extra_items.all(): - interaction_rec.extra_items.add(ele_rec) + if not ele_rec in interaction_rec.extra_items.all(): + interaction_rec.extra_items.add(ele_rec) #try to find extra element with given name and type and problemcode and reason #if ones doesn't exist create it @@ -122,12 +114,14 @@ if __name__ == '__main__': #if one is not associated, associate it - -''' - -probefetch='22.5812318325' config='13.1471130848' parse='0.104132890701' inventory='0.685513019562' install='0.487507104874' total='304.099304914' - -''' + for times in statistics.findall('OpStamps'): + for tags in times.items(): + (time_rec, tr_created) = Performance.objects.get_or_create(metric=tags[0], value=tags[1], + defaults={'metric':tags[0], + 'value':tags[1]}) + if not ele_rec in interaction_rec.extra_items.all(): + interaction_rec.performance_items.add(time_rec) + #print Client.objects.all().order_by('-name')[0].name diff --git a/reports/brpt/reports/models.py b/reports/brpt/reports/models.py index 23ad65045..cba78a5b9 100644 --- a/reports/brpt/reports/models.py +++ b/reports/brpt/reports/models.py @@ -57,10 +57,13 @@ class Repository(models.Model): return self.timestamp class InteractiveManager(models.Manager): - def interaction_per_client(self, maxdate): + + '''returns most recent interaction as of specified timestamp in format: + '2006-01-01 00:00:00' or 'now' or None->'now' ''' + def interaction_per_client(self, maxdate = None): from django.db import connection cursor = connection.cursor() - if maxdate == 'now': + if (maxdate == 'now' or maxdate == None): cursor.execute("select id, client_id, MAX(timestamp) AS maxtimestamp from reports_interaction GROUP BY client_id") else: cursor.execute("select id, client_id, timestamp, MAX(timestamp) AS maxtimestamp from reports_interaction where timestamp < %s GROUP BY client_id", [maxdate]) @@ -159,6 +162,35 @@ class Bad(models.Model): return self.name +class PerformanceManager(models.Manager): + + #Date format for maxdate: '2006-01-01 00:00:00' + def performance_per_client(self, maxdate = None): + from django.db import connection + cursor = connection.cursor() + if (maxdate == 'now' or maxdate == None): + cursor.execute("SELECT reports_client.name, reports_performance.metric, reports_performance.value "+ + "FROM reports_performance, reports_performance_interaction, reports_client WHERE ( "+ + "reports_client.current_interaction_id = reports_performance_interaction.interaction_id AND "+ + "reports_performance.id = reports_performance_interaction.performance_id)") + else: + cursor.execute("SELECT reports_client.name, reports_performance.metric, reports_performance.value, "+ + "MAX(reports_interaction.timestamp) FROM reports_performance, reports_performance_interaction, "+ + "reports_interaction, reports_client WHERE reports_interaction.id = "+ + "reports_performance_interaction.interaction_id AND reports_client.id = "+ + "reports_interaction.client_id AND reports_performance.id = "+ + "reports_performance_interaction.performance_id AND reports_interaction.timestamp < %s GROUP BY "+ + "reports_performance.id", [maxdate]) + + results = {} + for row in cursor.fetchall(): + try: + results[row[0]].__setitem__(row[1],row[2]) + except KeyError: + results[row[0]] = {row[1]:row[2]} + + return results + #performance metrics, models a performance-metric-item class Performance(models.Model): interaction = models.ManyToManyField(Interaction, related_name="performance_items") @@ -167,5 +199,5 @@ class Performance(models.Model): def __str__(self): return self.metric - + objects = PerformanceManager() diff --git a/reports/brpt/reports/views.py b/reports/brpt/reports/views.py index 6dbc86c82..274a5cec9 100644 --- a/reports/brpt/reports/views.py +++ b/reports/brpt/reports/views.py @@ -14,13 +14,13 @@ def client_index(request): client_list = Client.objects.all().order_by('name') return render_to_response('clients/index.html',{'client_list': client_list}) -def client_detail(request, hostname = -1, pk = -1): +def client_detail(request, hostname = None, pk = None): #SETUP error pages for when you specify a client or interaction that doesn't exist client = get_object_or_404(Client, name=hostname) - if(pk == -1): + if(pk == None): interaction = client.current_interaction else: - interaction = client.interactions.get(pk=pk) + interaction = client.interactions.get(pk=pk)#can this be a get object or 404? return render_to_response('clients/detail.html',{'client': client, 'interaction': interaction}) @@ -43,47 +43,52 @@ def display_summary(request): return render_to_response('displays/summary.html', client_lists) -def display_timing(request): +def display_timing(request, timestamp = None): #We're going to send a list of dictionaries. Each dictionary will be a row in the table #+------+-------+----------------+-----------+---------+----------------+-------+ #| name | parse | probe download | inventory | install | cfg dl & parse | total | #+------+-------+----------------+-----------+---------+----------------+-------+ - client_list = Client.objects.all().order_by('-name') + client_list = Client.objects.all().order_by('name') stats_list = [] - #if we have stats for a client, go ahead and add it to the list(wrap in TRY) + #Try to parse timestamp, if it has an @ symbol, replace it with a space and pass it. + #sanity check it too. + #else, justcall it with nothing.... + #use a popup calendar ! + results = Performance.objects.performance_per_client('2006-07-07 00:00:00') + for client in client_list:#Go explicitly to an interaction ID! (new item in dictionary) - #performance_items = client.interactions.latest().performance_items.all()#allow this to be selectable(hist) - d = {} - #[d.update({x:y}) for x,y in [a.values() for a in client.interactions.latest().performance_items.all().values('metric','value')]] - [d.update({x["metric"]:x["value"]}) for x in client.current_interaction.performance_items.all().values('metric','value')] + try: + d = results[client.name] + except KeyError: + d = {} + dict_unit = {} - try: dict_unit["name"] = client.name #node name except: dict_unit["name"] = "n/a" try: - dict_unit["parse"] = d["config_parse"] - d["config_download"] #parse + dict_unit["parse"] = round(d["config_parse"] - d["config_download"],4) #parse except: dict_unit["parse"] = "n/a" try: - dict_unit["probe"] = d["probe_upload"] - d["start"] #probe + dict_unit["probe"] = round(d["probe_upload"] - d["start"],4) #probe except: dict_unit["probe"] = "n/a" try: - dict_unit["inventory"] = d["inventory"] - d["initialization"] #inventory + dict_unit["inventory"] = round(d["inventory"] - d["initialization"],4) #inventory except: dict_unit["inventory"] = "n/a" try: - dict_unit["install"] = d["install"] - d["inventory"] #install + dict_unit["install"] = round(d["install"] - d["inventory"],4) #install except: dict_unit["install"] = "n/a" try: - dict_unit["config"] = d["config_parse"] - d["probe_upload"]#config download & parse + dict_unit["config"] = round(d["config_parse"] - d["probe_upload"],4)#config download & parse except: dict_unit["config"] = "n/a" try: - dict_unit["total"] = d["finished"] - d["start"] #total + dict_unit["total"] = round(d["finished"] - d["start"],4) #total except: dict_unit["total"] = "n/a" -- cgit v1.2.3-1-g7c22