added final support for reasons in new reporting system; edited importer script to bring that data in as well

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1910 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 116 insertions, 48 deletions

diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py
index 3e37e9dba..381bd9cdb 100755
--- a/reports/brpt/importscript.py
+++ b/reports/brpt/importscript.py
@@ -1,9 +1,8 @@
 #! /usr/bin/env python
-'''Imports statistics.xml and clients.xml files in to database backend for statistics'''
+'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine'''
 __revision__ = '$Revision$'
 
 import os, sys
-#i can clean all of this up to be like two lines...
 try:    # Add this project to sys.path so that it's importable
     import settings # Assumed to be in the same directory.
 except ImportError:
@@ -15,12 +14,10 @@ project_name = os.path.basename(project_directory)
 sys.path.append(os.path.join(project_directory, '..'))
 project_module = __import__(project_name, '', '', [''])
 sys.path.pop()
-
 # Set DJANGO_SETTINGS_MODULE appropriately.
 os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
-#got everything ready; lets do stuff
 
-from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance
+from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason
 from lxml.etree import XML, XMLSyntaxError
 from sys import argv
 from getopt import getopt, GetoptError
@@ -28,8 +25,6 @@ from datetime import datetime
 from time import strptime
 
 if __name__ == '__main__':
-#need clients.xml
-#need statistics.xml
 
     try:
         opts, args = getopt(argv[1:], "hc:s:", ["help", "clients=", "stats="])
@@ -46,7 +41,6 @@ if __name__ == '__main__':
         if o in ("-s", "--stats"):
             statpath = a
 
-
     '''Reads Data & Config files'''
     try:
         statsdata = XML(open(statpath).read())
@@ -59,70 +53,144 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s"%(clientspath))
         raise SystemExit, 1
 
-    
-    #for client in clientsdata.findall('Client'):
     for node in statsdata.findall('Node'):
-        #if client_rec.name == node.get('name'):
-        (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'), defaults={'name': node.get('name'), 'creation': datetime.now()})
-#        if cr_created:
-#            client_rec.save()
+        (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'),
+                                        defaults={'name': node.get('name'), 'creation': datetime.now()})
+
         for statistics in node.findall('Statistics'):
             t = strptime(statistics.get('time'))
-            (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id,timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
-                                                                              defaults={'client':client_rec,
-                                                                                        'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
-                                                                                        'state':statistics.get('state', default="unknown"),
-                                                                                        'repo_revision':statistics.get('revision', default="unknown"),
-                                                                                        'client_version':statistics.get('client_version'),
-                                                                                        'goodcount':statistics.get('good', default="unknown"),
-                                                                                        'totalcount':statistics.get('total', default="unknown")})
+            (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id,
+                                                timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
+                                                defaults={'client':client_rec,
+                                                          'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]),
+                                                          'state':statistics.get('state', default="unknown"),
+                                                          'repo_revision':statistics.get('revision', default="unknown"),
+                                                          'client_version':statistics.get('client_version'),
+                                                          'goodcount':statistics.get('good', default="unknown"),
+                                                          'totalcount':statistics.get('total', default="unknown")})
             for bad in statistics.findall('Bad'):
                 for ele in bad.getchildren():
-                    (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                      defaults={'name':ele.get('name'),
-                                                                                'kind':ele.tag,
-                                                                                'problemcode':'',
-                                                                                'reason':'Unknown'})
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
                         
+
+                    (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
+
                     if not ele_rec in interaction_rec.bad_items.all():
                         interaction_rec.bad_items.add(ele_rec)
 
             for modified in statistics.findall('Modified'):
                 for ele in modified.getchildren():
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
+                        
+
                     (ele_rec, er_created) = Modified.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                           defaults={'name':ele.get('name'),
-                                                                                     'kind':ele.tag,
-                                                                                     'problemcode':'',
-                                                                                     'reason':'Unknown'})
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
                     if not ele_rec in interaction_rec.modified_items.all():
                         interaction_rec.modified_items.add(ele_rec)
 
             for extra in statistics.findall('Extra'):
                 for ele in extra.getchildren():
+                    (reason_rec, rr_created) = Reason.objects.get_or_create(owner=ele.get('owner',default=''),
+                                                        current_owner=ele.get('current_owner',default=''),
+                                                        group=ele.get('group',default=''),
+                                                        current_group=ele.get('current_group',default=''),
+                                                        perms=ele.get('perms',default=''),
+                                                        current_perms=ele.get('current_perms',default=''),
+                                                        status=ele.get('status',default=''),
+                                                        current_status=ele.get('current_status',default=''),
+                                                        to=ele.get('to',default=''),
+                                                        current_to=ele.get('current_to',default=''),
+                                                        version=ele.get('version',default=''),
+                                                        current_version=ele.get('current_version',default=''),
+                                                        current_exists=ele.get('current_exists',default='True'),
+                                                        current_diff=ele.get('current_diff',default=''),
+                                                        defaults={'owner':ele.get('owner',default=''),
+                                                                  'current_owner':ele.get('current_owner',default=''),
+                                                                  'group':ele.get('group',default=''),
+                                                                  'current_group':ele.get('current_group',default=''),
+                                                                  'perms':ele.get('perms',default=''),
+                                                                  'current_perms':ele.get('current_perms',default=''),
+                                                                  'status':ele.get('status',default=''),
+                                                                  'current_status':ele.get('current_status',default=''),
+                                                                  'to':ele.get('to',default=''),
+                                                                  'current_to':ele.get('current_to',default=''),
+                                                                  'version':ele.get('version',default=''),
+                                                                  'current_version':ele.get('current_version',default=''),
+                                                                  'current_exists':ele.get('current_exists',default='True'),
+                                                                  'current_diff':ele.get('current_diff',default='')})
+                        
+
                     (ele_rec, er_created) = Extra.objects.get_or_create(name=ele.get('name'), kind=ele.tag,
-                                                                        defaults={'name':ele.get('name'),
-                                                                                  'kind':ele.tag,
-                                                                                  'problemcode':'',
-                                                                                  'reason':'Unknown'})
+                                                defaults={'name':ele.get('name'),
+                                                          'kind':ele.tag,
+                                                          'reason':reason_rec})
                         
                     if not ele_rec in interaction_rec.extra_items.all():
                         interaction_rec.extra_items.add(ele_rec)
-                                
-                        #try to find extra element with given name and type and problemcode and reason
-                        #if ones doesn't exist create it
-                        #try to get associated bad element
-                        #if one is not associated, associate it
-
 
             for times in statistics.findall('OpStamps'):
                 for tags in times.items():
                     (time_rec, tr_created) = Performance.objects.get_or_create(metric=tags[0], value=tags[1],
-                                                                               defaults={'metric':tags[0],
-                                                                                         'value':tags[1]})
+                                                    defaults={'metric':tags[0],
+                                                              'value':tags[1]})
                     if not ele_rec in interaction_rec.extra_items.all():
                         interaction_rec.performance_items.add(time_rec)
-                    
-
-
-#print Client.objects.all().order_by('-name')[0].name
-