New reporting system's Performance report is ultra fast due to new SQL queries. Also, importscript now properly imports OpStamps from xml statistics file. It is very slow still though...

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1902 ce84e21b-d406-0410-9b95-82705330c041

3 files changed, 71 insertions, 40 deletions

diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py
index 585018f0e..3e37e9dba 100755
--- a/reports/brpt/importscript.py
+++ b/reports/brpt/importscript.py
@@ -76,8 +76,6 @@ if __name__ == '__main__':
                                                                                         'client_version':statistics.get('client_version'),
                                                                                         'goodcount':statistics.get('good', default="unknown"),
                                                                                         'totalcount':statistics.get('total', default="unknown")})
-#            if ir_created:
-#                interaction_rec.save()
             for bad in statistics.findall('Bad'):
                 for ele in bad.getchildren():
                     (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
@@ -85,11 +83,9 @@ if __name__ == '__main__':
                                                                                 'kind':ele.tag,
                                                                                 'problemcode':'',
                                                                                 'reason':'Unknown'})
-#                    if er_created:
-#                        ele_rec.save()
                         
-                if not ele_rec in interaction_rec.bad_items.all():
-                    interaction_rec.bad_items.add(ele_rec)
+                    if not ele_rec in interaction_rec.bad_items.all():
+                        interaction_rec.bad_items.add(ele_rec)
 
             for modified in statistics.findall('Modified'):
                 for ele in modified.getchildren():
@@ -98,10 +94,8 @@ if __name__ == '__main__':
                                                                                      'kind':ele.tag,
                                                                                      'problemcode':'',
                                                                                      'reason':'Unknown'})
- #                   if er_created:
- #                       ele_rec.save()
-                if not ele_rec in interaction_rec.modified_items.all():
-                    interaction_rec.modified_items.add(ele_rec)
+                    if not ele_rec in interaction_rec.modified_items.all():
+                        interaction_rec.modified_items.add(ele_rec)
 
             for extra in statistics.findall('Extra'):
                 for ele in extra.getchildren():
@@ -110,11 +104,9 @@ if __name__ == '__main__':
                                                                                   'kind':ele.tag,
                                                                                   'problemcode':'',
                                                                                   'reason':'Unknown'})
- #                   if er_created:
- #                       ele_rec.save()
                         
-                if not ele_rec in interaction_rec.extra_items.all():
-                    interaction_rec.extra_items.add(ele_rec)
+                    if not ele_rec in interaction_rec.extra_items.all():
+                        interaction_rec.extra_items.add(ele_rec)
                                 
                         #try to find extra element with given name and type and problemcode and reason
                         #if ones doesn't exist create it
@@ -122,12 +114,14 @@ if __name__ == '__main__':
                         #if one is not associated, associate it
 
 
-
-'''
-
-probefetch='22.5812318325' config='13.1471130848' parse='0.104132890701' inventory='0.685513019562' install='0.487507104874' total='304.099304914'
-
-'''
+            for times in statistics.findall('OpStamps'):
+                for tags in times.items():
+                    (time_rec, tr_created) = Performance.objects.get_or_create(metric=tags[0], value=tags[1],
+                                                                               defaults={'metric':tags[0],
+                                                                                         'value':tags[1]})
+                    if not ele_rec in interaction_rec.extra_items.all():
+                        interaction_rec.performance_items.add(time_rec)
+                    
 
 
 #print Client.objects.all().order_by('-name')[0].name
diff --git a/reports/brpt/reports/models.py b/reports/brpt/reports/models.py
index 23ad65045..cba78a5b9 100644
--- a/reports/brpt/reports/models.py
+++ b/reports/brpt/reports/models.py
@@ -57,10 +57,13 @@ class Repository(models.Model):
         return self.timestamp
 
 class InteractiveManager(models.Manager):
-    def interaction_per_client(self, maxdate):
+
+    '''returns most recent interaction as of specified timestamp in format:
+    '2006-01-01 00:00:00' or 'now' or None->'now'  '''
+    def interaction_per_client(self, maxdate = None):
         from django.db import connection
         cursor = connection.cursor()
-        if maxdate == 'now':
+        if (maxdate == 'now' or maxdate == None): 
             cursor.execute("select id, client_id, MAX(timestamp) AS maxtimestamp from reports_interaction GROUP BY client_id")
         else:
             cursor.execute("select id, client_id, timestamp, MAX(timestamp) AS maxtimestamp from reports_interaction where timestamp < %s GROUP BY client_id", [maxdate])
@@ -159,6 +162,35 @@ class Bad(models.Model):
         return self.name
  
 
+class PerformanceManager(models.Manager):
+
+    #Date format for maxdate: '2006-01-01 00:00:00'            
+    def performance_per_client(self, maxdate = None):
+        from django.db import connection
+        cursor = connection.cursor()
+        if (maxdate == 'now' or maxdate == None):
+            cursor.execute("SELECT reports_client.name, reports_performance.metric, reports_performance.value "+
+            "FROM reports_performance, reports_performance_interaction, reports_client WHERE ( "+
+            "reports_client.current_interaction_id = reports_performance_interaction.interaction_id AND "+
+            "reports_performance.id = reports_performance_interaction.performance_id)")
+        else:
+            cursor.execute("SELECT reports_client.name, reports_performance.metric, reports_performance.value, "+
+            "MAX(reports_interaction.timestamp) FROM reports_performance, reports_performance_interaction, "+
+            "reports_interaction, reports_client WHERE reports_interaction.id = "+
+            "reports_performance_interaction.interaction_id AND reports_client.id = "+
+            "reports_interaction.client_id AND reports_performance.id = "+
+            "reports_performance_interaction.performance_id AND reports_interaction.timestamp < %s GROUP BY "+
+            "reports_performance.id", [maxdate])
+
+        results = {}
+        for row in cursor.fetchall():
+            try:
+                results[row[0]].__setitem__(row[1],row[2])
+            except KeyError:
+                results[row[0]] = {row[1]:row[2]}
+                                
+        return results
+    
 #performance metrics, models a performance-metric-item
 class Performance(models.Model):
     interaction = models.ManyToManyField(Interaction, related_name="performance_items")
@@ -167,5 +199,5 @@ class Performance(models.Model):
     def __str__(self):
         return self.metric
  
-
+    objects = PerformanceManager()
  
diff --git a/reports/brpt/reports/views.py b/reports/brpt/reports/views.py
index 6dbc86c82..274a5cec9 100644
--- a/reports/brpt/reports/views.py
+++ b/reports/brpt/reports/views.py
@@ -14,13 +14,13 @@ def client_index(request):
     client_list = Client.objects.all().order_by('name')
     return render_to_response('clients/index.html',{'client_list': client_list})
 
-def client_detail(request, hostname = -1, pk = -1):
+def client_detail(request, hostname = None, pk = None):
     #SETUP error pages for when you specify a client or interaction that doesn't exist
     client = get_object_or_404(Client, name=hostname)
-    if(pk == -1):
+    if(pk == None):
         interaction = client.current_interaction
     else:
-        interaction = client.interactions.get(pk=pk)
+        interaction = client.interactions.get(pk=pk)#can this be a get object or 404?
 
     return render_to_response('clients/detail.html',{'client': client, 'interaction': interaction})
 
@@ -43,47 +43,52 @@ def display_summary(request):
 
     return render_to_response('displays/summary.html', client_lists)
 
-def display_timing(request):
+def display_timing(request, timestamp = None):
     #We're going to send a list of dictionaries. Each dictionary will be a row in the table
     #+------+-------+----------------+-----------+---------+----------------+-------+
     #| name | parse | probe download | inventory | install | cfg dl & parse | total |
     #+------+-------+----------------+-----------+---------+----------------+-------+
-    client_list = Client.objects.all().order_by('-name')
+    client_list = Client.objects.all().order_by('name')
     stats_list = []
-    #if we have stats for a client, go ahead and add it to the list(wrap in TRY)
+    #Try to parse timestamp, if it has an @ symbol, replace it with a space and pass it.
+    #sanity check it too.
+    #else, justcall it with nothing....
+    #use a popup calendar !
+    results = Performance.objects.performance_per_client('2006-07-07 00:00:00')
+
     for client in client_list:#Go explicitly to an interaction ID! (new item in dictionary)
-        #performance_items = client.interactions.latest().performance_items.all()#allow this to be selectable(hist)
-        d = {}
-        #[d.update({x:y}) for x,y in [a.values() for a in client.interactions.latest().performance_items.all().values('metric','value')]]
-        [d.update({x["metric"]:x["value"]}) for x in client.current_interaction.performance_items.all().values('metric','value')]
+        try:
+            d = results[client.name]
+        except KeyError:
+            d = {}
+
         dict_unit = {}
-        
         try:
             dict_unit["name"] = client.name #node name
         except:
             dict_unit["name"] = "n/a"
         try:
-            dict_unit["parse"] = d["config_parse"] - d["config_download"] #parse
+            dict_unit["parse"] = round(d["config_parse"] - d["config_download"],4) #parse
         except:
             dict_unit["parse"] = "n/a"
         try:
-            dict_unit["probe"] = d["probe_upload"] - d["start"] #probe
+            dict_unit["probe"] = round(d["probe_upload"] - d["start"],4) #probe
         except:
             dict_unit["probe"] = "n/a"
         try:
-            dict_unit["inventory"] = d["inventory"] - d["initialization"] #inventory
+            dict_unit["inventory"] = round(d["inventory"] - d["initialization"],4) #inventory
         except:
             dict_unit["inventory"] = "n/a"
         try:
-            dict_unit["install"] = d["install"] - d["inventory"] #install
+            dict_unit["install"] = round(d["install"] - d["inventory"],4) #install
         except:
             dict_unit["install"] = "n/a"
         try:
-            dict_unit["config"] = d["config_parse"] - d["probe_upload"]#config download & parse
+            dict_unit["config"] = round(d["config_parse"] - d["probe_upload"],4)#config download & parse
         except:
             dict_unit["config"] = "n/a"
         try:
-            dict_unit["total"] = d["finished"] - d["start"] #total
+            dict_unit["total"] = round(d["finished"] - d["start"],4) #total
         except:
             dict_unit["total"] = "n/a"