From e55ef35c002f4106a96f52fd0de1d7b13126fcc1 Mon Sep 17 00:00:00 2001 From: Joey Hagedorn Date: Tue, 20 Jun 2006 19:41:24 +0000 Subject: New models with Reasons; added Import script, to bring old data in to the database git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1888 ce84e21b-d406-0410-9b95-82705330c041 --- reports/brpt/importscript.py | 122 ++++++++++++++++++++++++++++++++++++++++ reports/brpt/reports/models.py | 36 +++++++++--- reports/brpt/reports/models.pyc | Bin 5369 -> 6518 bytes 3 files changed, 150 insertions(+), 8 deletions(-) create mode 100755 reports/brpt/importscript.py diff --git a/reports/brpt/importscript.py b/reports/brpt/importscript.py new file mode 100755 index 000000000..9122e7846 --- /dev/null +++ b/reports/brpt/importscript.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python +'''Imports statistics.xml and clients.xml files in to database backend for statistics''' +__revision__ = '$Revision$' + +import os, sys +#i can clean all of this up to be like two lines... +try: # Add this project to sys.path so that it's importable + import settings # Assumed to be in the same directory. +except ImportError: + sys.stderr.write("Failed to locate settings.py") + sys.exit(1) + +project_directory = os.path.dirname(settings.__file__) +project_name = os.path.basename(project_directory) +sys.path.append(os.path.join(project_directory, '..')) +project_module = __import__(project_name, '', '', ['']) +sys.path.pop() + +# Set DJANGO_SETTINGS_MODULE appropriately. +os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name +#got everything ready; lets do stuff + +from brpt.reports.models import Client, Interaction, Bad, Modified, Extra, Performance +from lxml.etree import XML, XMLSyntaxError +from sys import argv +from getopt import getopt, GetoptError +from datetime import datetime +from time import strptime + +if __name__ == '__main__': +#need clients.xml +#need statistics.xml + + try: + opts, args = getopt(argv[1:], "hc:s:", ["help", "clients=", "stats="]) + except GetoptError, mesg: + # print help information and exit: + print "%s\nUsage:\nStatReports.py [-h] -c -s " % (mesg) + raise SystemExit, 2 + for o, a in opts: + if o in ("-h", "--help"): + print "Usage:\nStatReports.py [-h] -c -s " + raise SystemExit + if o in ("-c", "--clients"): + clientspath = a + if o in ("-s", "--stats"): + statpath = a + + + '''Reads Data & Config files''' + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(statpath)) + raise SystemExit, 1 + try: + clientsdata = XML(open(clientspath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(clientspath)) + raise SystemExit, 1 + + + #for client in clientsdata.findall('Client'): + for node in statsdata.findall('Node'): + #if client_rec.name == node.get('name'): + (client_rec, cr_created) = Client.objects.get_or_create(name=node.get('name'), defaults={'name': node.get('name'), 'creation': datetime.now()}) + for statistics in node.findall('Statistics'): + t = strptime(statistics.get('time')) + (interaction_rec, ir_created) = Interaction.objects.get_or_create(client=client_rec.id,timestamp=datetime(t[0],t[1],t[2],t[3],t[4],t[5]), + defaults={'client':client_rec, + 'timestamp':datetime(t[0],t[1],t[2],t[3],t[4],t[5]), + 'state':statistics.get('state', default="unknown"), + 'repo_revision':statistics.get('revision', default="unknown"), + 'client_version':statistics.get('client_version'), + 'goodcount':statistics.get('good', default="unknown"), + 'totalcount':statistics.get('total', default="unknown")}) + for bad in statistics.findall('Bad'): + for ele in bad.getchildren(): + (ele_rec, er_created) = Bad.objects.get_or_create(name=ele.get('name'), kind=ele.tag, + defaults={'name':ele.get('name'), + 'kind':ele.tag, + 'problemcode':'', + 'reason':'Unknown'}) + if not ele_rec in interaction_rec.bad_items.all(): + interaction_rec.bad_items.add(ele_rec) + + for modified in statistics.findall('Modified'): + for ele in modified.getchildren(): + (ele_rec, er_created) = Modified.objects.get_or_create(name=ele.get('name'), kind=ele.tag, + defaults={'name':ele.get('name'), + 'kind':ele.tag, + 'problemcode':'', + 'reason':'Unknown'}) + if not ele_rec in interaction_rec.modified_items.all(): + interaction_rec.modified_items.add(ele_rec) + + for extra in statistics.findall('Extra'): + for ele in extra.getchildren(): + (ele_rec, er_created) = Extra.objects.get_or_create(name=ele.get('name'), kind=ele.tag, + defaults={'name':ele.get('name'), + 'kind':ele.tag, + 'problemcode':'', + 'reason':'Unknown'}) + if not ele_rec in interaction_rec.extra_items.all(): + interaction_rec.extra_items.add(ele_rec) + + #try to find extra element with given name and type and problemcode and reason + #if ones doesn't exist create it + #try to get associated bad element + #if one is not associated, associate it + + + +''' + +probefetch='22.5812318325' config='13.1471130848' parse='0.104132890701' inventory='0.685513019562' install='0.487507104874' total='304.099304914' + +''' + + +#print Client.objects.all().order_by('-name')[0].name + diff --git a/reports/brpt/reports/models.py b/reports/brpt/reports/models.py index ac3506214..63c86fee0 100644 --- a/reports/brpt/reports/models.py +++ b/reports/brpt/reports/models.py @@ -10,7 +10,24 @@ KIND_CHOICES = ( ('Directory', 'Directory'), ('Permissions','Permissions'), ) - +REASON_CHOICES = ( + ('', 'No Reason'), + ('O','Owner'), + ('P','Permissions'), + ('E','Existence'), + ('C','Content'), + ('OP','Owner, Permissions'), + ('OE','Owner, Existence'), + ('OC','Owner, Content'), + ('PE','Permissions, Existence'), + ('PC','Permissions, Content'), + ('EC','Existence, Content'), + ('OPE','Owner, Permissions, Existence'), + ('OPC','Owner, Permissions, Content'), + ('OEC','Owner, Existence, Content'), + ('PEC','Permissions, Existence, Content'), + ('OPEC','Owner, Permissions, Existence, Content'), +) class Client(models.Model): #This exists for clients that are no longer in the repository even! (timeless) creation = models.DateTimeField() @@ -88,37 +105,40 @@ class Interaction(models.Model): class Modified(models.Model): - interaction = models.ForeignKey(Interaction, related_name="modified_items", edit_inline=models.STACKED) + interactions = models.ManyToManyField(Interaction, related_name="modified_items") name = models.CharField(maxlength=128, core=True)#name of modified thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... - how = models.CharField(maxlength=256) + problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES) + reason = models.TextField(maxlength=1280) def __str__(self): return self.name class Extra(models.Model): - interaction = models.ForeignKey(Interaction, related_name="extra_items", edit_inline=models.STACKED) + interactions = models.ManyToManyField(Interaction, related_name="extra_items") name = models.CharField(maxlength=128, core=True)#name of Extra thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... - why = models.CharField(maxlength=256)#current state of some thing... + problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES) + reason = models.TextField(maxlength=1280) def __str__(self): return self.name class Bad(models.Model): - interaction = models.ForeignKey(Interaction, related_name="bad_items", edit_inline=models.STACKED) + interactions = models.ManyToManyField(Interaction, related_name="bad_items") name = models.CharField(maxlength=128, core=True)#name of bad thing. kind = models.CharField(maxlength=16, choices=KIND_CHOICES)#Service/Package/ConfgFile... - reason = models.CharField(maxlength=256)#that its bad... + problemcode = models.CharField(maxlength=8, choices=REASON_CHOICES) + reason = models.TextField(maxlength=1280) def __str__(self): return self.name #performance metrics, models a performance-metric-item class Performance(models.Model): - interaction = models.ForeignKey(Interaction, related_name="performance_items", edit_inline=models.STACKED) + interaction = models.ManyToManyField(Interaction, related_name="performance_items") metric = models.CharField(maxlength=128, core=True) value = models.FloatField(max_digits=32, decimal_places=16) def __str__(self): diff --git a/reports/brpt/reports/models.pyc b/reports/brpt/reports/models.pyc index 5d2c3cd75..935de1c5e 100644 Binary files a/reports/brpt/reports/models.pyc and b/reports/brpt/reports/models.pyc differ -- cgit v1.2.3-1-g7c22