added final support for reasons in new reporting system; edited importer script to bring that data in as well

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1910 ce84e21b-d406-0410-9b95-82705330c041

3 files changed, 141 insertions, 81 deletions

diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py
index 3e37e9dba..381bd9cdb 100755
--- a/reports/brpt/importscript.py
+++ b/reports/brpt/importscript.py
@@ -1,9 +1,8 @@
 #! /usr/bin/env python
-'''Imports statistics.xml and clients.xml files in to database backend for statistics'''
+'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine'''
 __revision__ = '$Revision$'
 
 import os, sys
-#i can clean all of this up to be like two lines...
 try:    # Add this project to sys.path so that it's importable
     import settings # Assumed to be in the same directory.
 except ImportError:
@@ -15,12 +14,10 @@ project_name = os.path.basename(project_directory)
 sys.path.append(os.path.join(project_directory, '..'))
 project_module = __import__(project_name, '', '', [''])
 sys.path.pop()
-
 # Set DJANGO_SETTINGS_MODULE appropriately.
 os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
-#got everything ready; lets do stuff
 
-from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance
+from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason
 from lxml.etree import XML, XMLSyntaxError
 from sys import argv
 from getopt import getopt, GetoptError
@@ -28,8 +25,6 @@ from datetime import datetime
 from time import strptime
 
 if __name__ == '__main__':
-#need clients.xml
-#need statistics.xml
 
     try:
         opts, args = getopt(argv[1:], "hc:s:", ["help", "clients=", "stats="])
@@ -46,7 +41,6 @@ if __name__ == '__main__':
         if o in ("-s", "--stats"):
             statpath = a
 
-
     '''Reads Data & Config files'''
     try:
         statsdata = XML(open(statpath).read())
@@ -59,70 +53,144 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s"%(clientspath))
         raise SystemExit, 1
 
-    
-    #for client in clientsdata.findall('Client'):
     for node in statsdata.findall('Node'):
-        #if client_rec.name == node.get('name'):
-        (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'), defaults={'name': node.get('name'), 'creation': datetime.now()})
-#        if cr_created:
-#            client_rec.save()
+        (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'),
+                                        defaults={'name': node.get('name'), 'creation': datetime.now()})
+
         for statistics in node.findall('Statistics'):
             t = strptime(statistics.get('time'))
-            (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id,timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
-                                                                              defaults={'client':client_rec,
-                                                                                        'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
-                                                                                        'state':statistics.get('state', default="unknown"),
-                                                                                        'repo_revision':statistics.get('revision', default="unknown"),
-                                                                                        'client_version':statistics.get('client_version'),
-                                                                                        'goodcount':statistics.get('good', default="unknown"),
-                                                                                        'totalcount':statistics.get('total', default="unknown")})
+            (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id,
+                                                timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
+                                                defaults={'client':client_rec,
+                                                          'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
+                                                          'state':statistics.get('state', default="unknown"),
+                                                          'repo_revision':statistics.get('revision', default="unknown"),
+                                                          'client_version':statistics.get('client_version'),
+                                                          'goodcount':statistics.get('good', default="unknown"),
+                                                          'totalcount':statistics.get('total', default="unknown")})
             for bad in statistics.findall('Bad'):
                 for ele in bad.getchildren():
-                    (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                      defaults={'name':ele.get('name'),
-                                                                                'kind':ele.tag,
-                                                                                'problemcode':'',
-                                                                                'reason':'Unknown'})
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
                         
+
+                    (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
+
                     if not ele_rec in interaction_rec.bad_items.all():
                         interaction_rec.bad_items.add(ele_rec)
 
             for modified in statistics.findall('Modified'):
                 for ele in modified.getchildren():
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
+                        
+
                     (ele_rec, er_created) = Modified.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                           defaults={'name':ele.get('name'),
-                                                                                     'kind':ele.tag,
-                                                                                     'problemcode':'',
-                                                                                     'reason':'Unknown'})
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
                     if not ele_rec in interaction_rec.modified_items.all():
                         interaction_rec.modified_items.add(ele_rec)
 
             for extra in statistics.findall('Extra'):
                 for ele in extra.getchildren():
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
+                        
+
                     (ele_rec, er_created) = Extra.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                        defaults={'name':ele.get('name'),
-                                                                                  'kind':ele.tag,
-                                                                                  'problemcode':'',
-                                                                                  'reason':'Unknown'})
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
                         
                     if not ele_rec in interaction_rec.extra_items.all():
                         interaction_rec.extra_items.add(ele_rec)
-                                
-                        #try to find extra element with given name and type and problemcode and reason
-                        #if ones doesn't exist create it
-                        #try to get associated bad element
-                        #if one is not associated, associate it
-
 
             for times in statistics.findall('OpStamps'):
                 for tags in times.items():
                     (time_rec, tr_created) = Performance.objects.get_or_create(metric=tags[0], value=tags[1],
-                                                                               defaults={'metric':tags[0],
-                                                                                         'value':tags[1]})
+                                                    defaults={'metric':tags[0],
+                                                              'value':tags[1]})
                     if not ele_rec in interaction_rec.extra_items.all():
                         interaction_rec.performance_items.add(time_rec)
-                    
-
-
-#print Client.objects.all().order_by('-name')[0].name
-
diff --git a/reports/brpt/reports/models.py b/reports/brpt/reports/models.py
index cba78a5b9..c9c656ab0 100644
--- a/reports/brpt/reports/models.py
+++ b/reports/brpt/reports/models.py
@@ -11,25 +11,7 @@ KIND_CHOICES = (
     ('Directory', 'Directory'),
     ('Permissions','Permissions'),
 )
-REASON_CHOICES = (
-    #these are the possible reasons there can be a problem with a node:
-    ('', 'No Reason'),
-    ('O','Owner'),
-    ('P','Permissions'),
-    ('E','Existence'),
-    ('C','Content'),
-    ('OP','Owner, Permissions'),
-    ('OE','Owner, Existence'),
-    ('OC','Owner, Content'),
-    ('PE','Permissions, Existence'),
-    ('PC','Permissions, Content'),
-    ('EC','Existence, Content'),
-    ('OPE','Owner, Permissions, Existence'),
-    ('OPC','Owner, Permissions, Content'),
-    ('OEC','Owner, Existence, Content'),
-    ('PEC','Permissions, Existence, Content'),
-    ('OPEC','Owner, Permissions, Existence, Content'),
-)
+
 class Client(models.Model):
     #This exists for clients that are no longer in the repository even! (timeless)
     creation = models.DateTimeField()
@@ -129,39 +111,48 @@ class Interaction(models.Model):
     class Meta:
         get_latest_by = 'timestamp'
 
+class Reason(models.Model):
+    owner = models.TextField(maxlength=128, blank=True)
+    current_owner = models.TextField(maxlength=128, blank=True)
+    group = models.TextField(maxlength=128, blank=True)
+    current_group = models.TextField(maxlength=128, blank=True)
+    perms =  models.IntegerField(blank=True)
+    current_perms = models.IntegerField(blank=True)
+    status = models.TextField(maxlength=3, blank=True)#on/off/(None)
+    current_status = models.TextField(maxlength=1, blank=True)#on/off/(None)
+    to = models.TextField(maxlength=256, blank=True)
+    current_to = models.TextField(maxlength=256, blank=True)
+    version = models.TextField(maxlength=128, blank=True)
+    current_version = models.TextField(maxlength=128, blank=True)
+    current_exists = models.BooleanField()#False means its missing!, only display if its False, true is default..
+    current_diff = models.TextField(maxlength=1280, blank=True) #diff
+    def _str_(self):
+        return "Reason"
 
 class Modified(models.Model):
     interactions = models.ManyToManyField(Interaction, related_name="modified_items")
     name = models.CharField(maxlength=128, core=True)#name of modified thing.
     kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile...
-    problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES)
-    reason = models.TextField(maxlength=1280)
+    reason = models.ForeignKey(Reason)
     def __str__(self):
         return self.name
- 
-
-    
+   
 class Extra(models.Model):
     interactions = models.ManyToManyField(Interaction, related_name="extra_items")
     name = models.CharField(maxlength=128, core=True)#name of Extra thing.
     kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile...
-    problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES)
-    reason = models.TextField(maxlength=1280)
+    reason = models.ForeignKey(Reason)
     def __str__(self):
         return self.name
- 
-
     
 class Bad(models.Model):
     interactions = models.ManyToManyField(Interaction, related_name="bad_items")
     name = models.CharField(maxlength=128, core=True)#name of bad thing.
     kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile...
-    problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES)
-    reason = models.TextField(maxlength=1280)
+    reason = models.ForeignKey(Reason)
     def __str__(self):
         return self.name
  
-
 class PerformanceManager(models.Manager):
 
     #Date format for maxdate: '2006-01-01 00:00:00'            
diff --git a/reports/brpt/reports/templates/displays/timing.html b/reports/brpt/reports/templates/displays/timing.html
index 758eb4cc9..b734d9836 100644
--- a/reports/brpt/reports/templates/displays/timing.html
+++ b/reports/brpt/reports/templates/displays/timing.html
@@ -15,16 +15,17 @@
   <br/>
 <div>
 <span class="mini-date">
+
+<b>Enter date or use calendar popup: </b>
 <form name="timestamp-select">
 <input type="text" name="date1" value="{{timestamp_date}}" size=10>@
-<!--#If there is a timestamp-date and timestamp-time, set it to that, otherwise, now-->
 <input type="text" name="time" value="{{timestamp_time}}" size=8>
 <a href="" onClick="cal.select(document.forms['timestamp-select'].date1,'anchor1','yyyy-MM-dd'); return false;"
    name="anchor1" ID="anchor1">Calendar</A>
 <input type="button" name="go" value="Go" onClick="location.href='/displays/timing/'+document.forms['timestamp-select'].date1.value+'@'+document.forms['timestamp-select'].time.value;" />
  | <input type="button" name="now" value="Now" onClick="location.href='/displays/timing/';"/>
 </form>
-</span><br/><br/></div>
+</span><br/><br/><br/></div>
   <center>
       <table id="t1" class="sortable">
           <tr>