summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTim Laszlo <tim.laszlo@gmail.com>2010-06-03 14:59:40 +0000
committerSol Jerome <solj@ices.utexas.edu>2010-06-03 10:07:57 -0500
commite7746bf420f1d536cee9198a533383b8bb4edca3 (patch)
tree83fae8f66e39ec13d2727e954f893cc5e1c96a7e
parent8170fac20bedde567bf9cf482bca894ef97f8a07 (diff)
downloadbcfg2-e7746bf420f1d536cee9198a533383b8bb4edca3.tar.gz
bcfg2-e7746bf420f1d536cee9198a533383b8bb4edca3.tar.bz2
bcfg2-e7746bf420f1d536cee9198a533383b8bb4edca3.zip
Initial version of admin tool for reports
git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5886 ce84e21b-d406-0410-9b95-82705330c041
-rwxr-xr-xsrc/lib/Server/Admin/Reports.py189
-rw-r--r--src/lib/Server/Admin/__init__.py2
2 files changed, 190 insertions, 1 deletions
diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
new file mode 100755
index 000000000..e4cbae8e1
--- /dev/null
+++ b/src/lib/Server/Admin/Reports.py
@@ -0,0 +1,189 @@
+'''Admin interface for dynamic reports'''
+import Bcfg2.Logger
+import Bcfg2.Server.Admin
+import ConfigParser
+import os
+import logging
+import pickle
+import sys
+from Bcfg2.Server.Reports.importscript import load_stats
+from Bcfg2.Server.Reports.updatefix import update_database
+from lxml.etree import XML, XMLSyntaxError
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+
+# Load django
+import django.core.management
+
+# FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf
+try:
+ import Bcfg2.Server.Reports.settings
+except Exception, e:
+ sys.stderr.write("Failed to load configuration settings. %s\n" % e)
+ sys.exit(1)
+
+project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__)
+project_name = os.path.basename(project_directory)
+sys.path.append(os.path.join(project_directory, '..'))
+project_module = __import__(project_name, '', '', [''])
+sys.path.pop()
+
+# Set DJANGO_SETTINGS_MODULE appropriately.
+os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
+from django.db import connection
+
+from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \
+ Entries_interactions, Performance, \
+ Reason, Ping, TYPE_CHOICES, InternalDatabaseVersion
+
+class Reports(Bcfg2.Server.Admin.Mode):
+ '''Admin interface for dynamic reports'''
+ __shorthelp__ = "Manage dynamic reports"
+ __longhelp__ = (__shorthelp__)
+ __usage__ = ("bcfg2-admin reports [command] [options]\n"
+ " -v|--verbose Be verbose\n"
+ " -q|--quiet Print only errors\n"
+ "\n"
+ " Commands:\n"
+ " load_stats Load statistics data\n"
+ " -s|--stats Path to statistics.xml file\n"
+ " -c|--clients-file Path to clients.xml file\n"
+ " -O3 Fast mode. Duplicates data!\n"
+ " scrub Scrub the database for duplicate reasons\n"
+ " update Apply any updates to the reporting database\n"
+ "\n")
+
+ def __init__(self, cfile):
+ Bcfg2.Server.Admin.Mode.__init__(self, cfile)
+ self.log.setLevel(logging.INFO)
+ self.django_commands = [ 'syncdb', 'sqlall', 'validate' ]
+ self.__usage__ = self.__usage__ + " Django commands:\n " + \
+ "\n ".join(self.django_commands)
+
+ def __call__(self, args):
+ Bcfg2.Server.Admin.Mode.__call__(self, args)
+ if len(args) == 0 or args[0] == '-h':
+ print(self.__usage__)
+ raise SystemExit(0)
+
+ verb = 0
+
+ if '-v' in args or '--verbose' in args:
+ self.log.setLevel(logging.DEBUG)
+ verb = 1
+ if '-q' in args or '--quiet' in args:
+ self.log.setLevel(logging.WARNING)
+
+ # FIXME - dry run
+
+ if args[0] in self.django_commands:
+ self.django_command_proxy(args[0])
+ elif args[0] == 'scrub':
+ self.scrub()
+ elif args[0] == 'update':
+ update_database()
+ elif args[0] == 'load_stats':
+ quick = '-O3' in args
+ stats_file=None
+ clients_file=None
+ i=1
+ while i < len(args):
+ if args[i] == '-s' or args[i] == '--stats':
+ stats_file = args[i+1]
+ if stats_file[0] == '-':
+ self.errExit("Invalid statistics file: %s" % stats_file)
+ elif args[i] == '-c' or args[i] == '--clients-file':
+ clients_file = args[i+1]
+ if clients_file[0] == '-':
+ self.errExit("Invalid clients file: %s" % clients_file)
+ i = i + 1
+ self.load_stats(stats_file, clients_file, verb, quick)
+ else:
+ print "Unknown command: %s" % args[0]
+
+ def scrub(self):
+ ''' Perform a thorough scrub and cleanup of the database '''
+
+ # Currently only reasons are a problem
+ try:
+ start_count = Reason.objects.count()
+ except Exception, e:
+ self.log.error("Failed to load reason objects: %s" % e)
+ return
+ dup_reasons = []
+
+ cmp_reasons = dict()
+ for reason in Reason.objects.all():
+ ''' Loop through each reason and create a key out of the data. \
+ This lets us take advantage of a fast hash lookup for \
+ comparisons '''
+ id = reason.id
+ reason.id = None
+ key=md5(pickle.dumps(reason)).hexdigest()
+ reason.id = id
+
+ if key in cmp_reasons:
+ self.log.debug("Update interactions from %d to %d" \
+ % (reason.id, cmp_reasons[key]))
+ try:
+ Entries_interactions.objects.filter(reason=reason).\
+ update(reason=cmp_reasons[key])
+ dup_reasons.append(reason.id)
+ except Exception, ex:
+ self.log.error("Failed to update interactions for %d : %s" \
+ % (reason.id, ex))
+ else:
+ cmp_reasons[key] = reason.id
+ self.log.debug("key %d" % reason.id)
+
+ self.log.debug("Done with updates, deleting dupes")
+ for dup in dup_reasons:
+ self.log.debug("Deleting %d" % dup)
+ try:
+ Reason.objects.get(id=dup).delete()
+ except Exception, ex:
+ self.log.error("Failed to delete reason %d: %s" % (dup, ex))
+
+ self.log.info("Found %d dupes out of %d" % (len(dup_reasons), start_count))
+
+ def django_command_proxy(self, command):
+ '''Call a django command'''
+ if command == 'sqlall':
+ django.core.management.call_command(command, 'reports')
+ else:
+ django.core.management.call_command(command)
+
+ def load_stats(self, stats_file=None, clientspath=None, verb=0, quick=False):
+ '''Load statistics data into the database'''
+ location = ''
+
+ if not stats_file:
+ try:
+ stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ self.errExit("Could not read bcfg2.conf; exiting")
+ try:
+ statsdata = XML(open(stats_file).read())
+ except (IOError, XMLSyntaxError):
+ self.errExit("StatReports: Failed to parse %s"%(stats_file))
+
+ if not clientspath:
+ try:
+ clientspath = "%s/Metadata/clients.xml" % \
+ self.cfp.get('server', 'repository')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ self.errExit("Could not read bcfg2.conf; exiting")
+ try:
+ clientsdata = XML(open(clientspath).read())
+ except (IOError, XMLSyntaxError):
+ self.errExit("StatReports: Failed to parse %s"%(clientspath))
+
+ try:
+ location = self.cfp.get('components', 'bcfg2')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ self.errExit("Could not read bcfg2 from bcfg2.conf; exiting")
+ load_stats(clientsdata, statsdata, verb, self.log, quick=quick, location=location)
+
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 89415cf9f..99f52bda4 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -1,7 +1,7 @@
__revision__ = '$Revision$'
__all__ = ['Mode', 'Client', 'Compare', 'Init', 'Minestruct', 'Perf',
- 'Pull', 'Query', 'Snapshots', 'Tidy', 'Viz', 'Xcmd']
+ 'Pull', 'Query', 'Reports', 'Snapshots', 'Tidy', 'Viz', 'Xcmd']
import ConfigParser
import logging