Reporting: Merge new reporting data

Move reporting data to a new schema
Use south for django migrations
Add bcfg2-report-collector daemon

Conflicts:
	doc/development/index.txt
	doc/server/plugins/connectors/properties.txt
	doc/server/plugins/generators/packages.txt
	setup.py
	src/lib/Bcfg2/Client/Tools/SELinux.py
	src/lib/Bcfg2/Compat.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Server/Admin/Init.py
	src/lib/Bcfg2/Server/Admin/Reports.py
	src/lib/Bcfg2/Server/BuiltinCore.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/Inotify.py
	src/lib/Bcfg2/Server/Plugin/base.py
	src/lib/Bcfg2/Server/Plugin/interfaces.py
	src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py
	src/lib/Bcfg2/Server/Plugins/FileProbes.py
	src/lib/Bcfg2/Server/Plugins/Ohai.py
	src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
	src/lib/Bcfg2/Server/Plugins/Packages/Source.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/lib/Bcfg2/Server/Plugins/Properties.py
	src/lib/Bcfg2/Server/Reports/backends.py
	src/lib/Bcfg2/Server/Reports/manage.py
	src/lib/Bcfg2/Server/Reports/nisauth.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py

2 files changed, 36 insertions, 71 deletions

diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py
index 63a0092d5..815d34e97 100644
--- a/src/lib/Bcfg2/Server/Admin/Reports.py
+++ b/src/lib/Bcfg2/Server/Admin/Reports.py
@@ -8,19 +8,16 @@ import pickle
 import platform
 import sys
 import traceback
-from lxml.etree import XML, XMLSyntaxError
 
-from Bcfg2.Compat import ConfigParser, md5
+from Bcfg2.Compat import md5
 
-import Bcfg2.settings
+from Bcfg2 import settings
 
 # Load django and reports stuff _after_ we know we can load settings
-import django.core.management
-from Bcfg2.Server.Reports.importscript import load_stats
-from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
-from Bcfg2.Server.Reports.utils import *
+from django.core import management
+from Bcfg2.Reporting.utils import *
 
-project_directory = os.path.dirname(Bcfg2.settings.__file__)
+project_directory = os.path.dirname(settings.__file__)
 project_name = os.path.basename(project_directory)
 sys.path.append(os.path.join(project_directory, '..'))
 project_module = __import__(project_name, '', '', [''])
@@ -30,9 +27,8 @@ sys.path.pop()
 os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
 from django.db import connection, transaction
 
-from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \
-                                Entries_interactions, Performance, \
-                                Reason
+from Bcfg2.Reporting.models import Client, Interaction, \
+                                Performance
 
 
 def printStats(fn):
@@ -54,7 +50,8 @@ def printStats(fn):
         self.log.info("Interactions removed: %s" %
                       (start_i - Interaction.objects.count()))
         self.log.info("Interactions->Entries removed: %s" %
-                      (start_ei - Entries_interactions.objects.count()))
+                      (start_ei - 0))
+        #              (start_ei - Entries_interactions.objects.count()))
         self.log.info("Metrics removed: %s" %
                       (start_perf - Performance.objects.count()))
 
@@ -70,9 +67,6 @@ class Reports(Bcfg2.Server.Admin.Mode):
                  "\n"
                  "  Commands:\n"
                  "    init                 Initialize the database\n"
-                 "    load_stats           Load statistics data\n"
-                 "      -s|--stats         Path to statistics.xml file\n"
-                 "      -O3                Fast mode.  Duplicates data!\n"
                  "    purge                Purge records\n"
                  "      --client [n]       Client to operate on\n"
                  "      --days   [n]       Records older then n days\n"
@@ -85,6 +79,11 @@ class Reports(Bcfg2.Server.Admin.Mode):
 
     def __init__(self, setup):
         Bcfg2.Server.Admin.Mode.__init__(self, setup)
+        try:
+            import south
+        except ImportError:
+            print "Django south is required for Reporting"
+            raise SystemExit(-3)
 
     def __call__(self, args):
         Bcfg2.Server.Admin.Mode.__call__(self, args)
@@ -99,24 +98,16 @@ class Reports(Bcfg2.Server.Admin.Mode):
         elif args[0] == 'scrub':
             self.scrub()
         elif args[0] in ['init', 'update']:
+            if self.setup['verbose'] or self.setup['debug']:
+                vrb = 2
+            else:
+                vrb = 0
             try:
-                update_database()
-            except UpdaterError:
-                print("Update failed")
+                management.call_command("syncdb", verbosity=vrb)
+                management.call_command("migrate", verbosity=vrb)
+            except:
+                print("Update failed: %s" % traceback.format_exc().splitlines()[-1])
                 raise SystemExit(-1)
-        elif args[0] == 'load_stats':
-            quick = '-O3' in args
-            stats_file = None
-            i = 1
-            while i < len(args):
-                if args[i] == '-s' or args[i] == '--stats':
-                    stats_file = args[i + 1]
-                    if stats_file[0] == '-':
-                        self.errExit("Invalid statistics file: %s" % stats_file)
-                elif args[i] == '-c' or args[i] == '--clients-file':
-                    print("DeprecationWarning: %s is no longer used" % args[i])
-                i = i + 1
-            self.load_stats(stats_file, self.log.getEffectiveLevel() > logging.WARNING, quick)
         elif args[0] == 'purge':
             expired = False
             client = None
@@ -203,9 +194,9 @@ class Reports(Bcfg2.Server.Admin.Mode):
         Reason.prune_orphans()
         self.log.info("Pruned %d Reason records" % (start_count - Reason.objects.count()))
 
-        start_count = Entries.objects.count()
-        Entries.prune_orphans()
-        self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count()))
+        #start_count = Entries.objects.count()
+        #Entries.prune_orphans()
+        #self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count()))
 
     def django_command_proxy(self, command):
         '''Call a django command'''
@@ -214,37 +205,6 @@ class Reports(Bcfg2.Server.Admin.Mode):
         else:
             django.core.management.call_command(command)
 
-    def load_stats(self, stats_file=None, verb=0, quick=False):
-        '''Load statistics data into the database'''
-        location = ''
-
-        if not stats_file:
-            try:
-                stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository')
-            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
-                self.errExit("Could not read bcfg2.conf; exiting")
-        try:
-            statsdata = XML(open(stats_file).read())
-        except (IOError, XMLSyntaxError):
-            self.errExit("StatReports: Failed to parse %s" % (stats_file))
-
-        try:
-            encoding = self.cfp.get('components', 'encoding')
-        except:
-            encoding = 'UTF-8'
-
-        try:
-            load_stats(statsdata,
-                       encoding,
-                       verb,
-                       self.log,
-                       quick=quick,
-                       location=platform.node())
-        except UpdaterError:
-            self.errExit("StatReports: Database updater failed")
-        except:
-            self.errExit("failed to import stats: %s" 
-                % traceback.format_exc().splitlines()[-1])
 
     @printStats
     def purge(self, client=None, maxdate=None, state=None):
@@ -272,7 +232,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
             self.log.debug("Filtering by maxdate: %s" % maxdate)
             ipurge = ipurge.filter(timestamp__lt=maxdate)
 
-        if Bcfg2.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
             grp_limit = 100
         else:
             grp_limit = 1000
diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py
index 1eb953e2a..3686722ae 100644
--- a/src/lib/Bcfg2/Server/Admin/Syncdb.py
+++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py
@@ -1,8 +1,7 @@
 import Bcfg2.settings
 import Bcfg2.Options
 import Bcfg2.Server.Admin
-from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
-from django.core.management import setup_environ
+from django.core.management import setup_environ, call_command
 
 class Syncdb(Bcfg2.Server.Admin.Mode):
     __shorthelp__ = ("Sync the Django ORM with the configured database")
@@ -23,7 +22,13 @@ class Syncdb(Bcfg2.Server.Admin.Mode):
         Bcfg2.Server.models.load_models(cfile=self.opts['configfile'])
 
         try:
-            update_database()
-        except UpdaterError:
-            print("Update failed")
+            call_command("syncdb", interactive=False, verbosity=0)
+            self._database_available = True
+        except ImproperlyConfigured:
+            self.logger.error("Django configuration problem: %s" %
+                format_exc().splitlines()[-1])
+            raise SystemExit(-1)
+        except:
+            self.logger.error("Database update failed: %s" %
+                format_exc().splitlines()[-1])
             raise SystemExit(-1)