Reporting: Merge new reporting data

Move reporting data to a new schema
Use south for django migrations
Add bcfg2-report-collector daemon

Conflicts:
	doc/development/index.txt
	doc/server/plugins/connectors/properties.txt
	doc/server/plugins/generators/packages.txt
	setup.py
	src/lib/Bcfg2/Client/Tools/SELinux.py
	src/lib/Bcfg2/Compat.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Server/Admin/Init.py
	src/lib/Bcfg2/Server/Admin/Reports.py
	src/lib/Bcfg2/Server/BuiltinCore.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/Inotify.py
	src/lib/Bcfg2/Server/Plugin/base.py
	src/lib/Bcfg2/Server/Plugin/interfaces.py
	src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py
	src/lib/Bcfg2/Server/Plugins/FileProbes.py
	src/lib/Bcfg2/Server/Plugins/Ohai.py
	src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
	src/lib/Bcfg2/Server/Plugins/Packages/Source.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/lib/Bcfg2/Server/Plugins/Properties.py
	src/lib/Bcfg2/Server/Reports/backends.py
	src/lib/Bcfg2/Server/Reports/manage.py
	src/lib/Bcfg2/Server/Reports/nisauth.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py

1 files changed, 111 insertions, 0 deletions

diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py
new file mode 100644
index 000000000..bb2e85c21
--- /dev/null
+++ b/src/lib/Bcfg2/Reporting/Collector.py
@@ -0,0 +1,111 @@
+import atexit
+import daemon
+import lockfile
+import logging
+import time
+import traceback
+import threading
+
+import Bcfg2.Logger
+from Bcfg2.Reporting.Transport import load_transport_from_config, \
+    TransportError, TransportImportError
+from Bcfg2.Reporting.Storage import load_storage_from_config, \
+    StorageError, StorageImportError
+
+class ReportingError(Exception):
+    """Generic reporting exception"""
+    pass
+
+class ReportingCollector(object):
+    """The collecting process for reports"""
+
+    def __init__(self, setup):
+        """Setup the collector.  This may be called by the daemon or though 
+        bcfg2-admin"""
+        self.setup = setup
+        self.datastore = setup['repo']
+        self.encoding = setup['encoding']
+        self.terminate = None
+        self.context = None
+
+        if setup['debug']:
+            level = logging.DEBUG
+        elif setup['verbose']:
+            level = logging.INFO
+        else:
+            level = logging.WARNING
+
+        Bcfg2.Logger.setup_logging('bcfg2-report-collector',
+                                   to_console=logging.INFO,
+                                   to_syslog=setup['syslog'],
+                                   to_file=setup['logging'],
+                                   level=level)
+        self.logger = logging.getLogger('bcfg2-report-collector')
+
+        try:
+            self.transport = load_transport_from_config(setup)
+            self.storage = load_storage_from_config(setup)
+        except TransportError:
+            self.logger.error("Failed to load transport: %s" %
+                traceback.format_exc().splitlines()[-1])
+            raise ReportingError
+        except StorageError:
+            self.logger.error("Failed to load storage: %s" %
+                traceback.format_exc().splitlines()[-1])
+            raise ReportingError
+
+        try:
+            self.logger.debug("Validating storage %s" % 
+                self.storage.__class__.__name__)
+            self.storage.validate()
+        except:
+            self.logger.error("Storage backed %s failed to validate: %s" %
+                (self.storage.__class__.__name__, 
+                    traceback.format_exc().splitlines()[-1]))
+
+
+    def run(self):
+        """Startup the processing and go!"""
+        self.terminate = threading.Event()
+        atexit.register(self.shutdown)
+        self.context = daemon.DaemonContext()
+
+        if self.setup['daemon']:
+            self.logger.debug("Daemonizing")
+            self.context.pidfile = lockfile.FileLock(self.setup['daemon'])
+            self.context.open()
+            self.logger.info("Starting daemon")
+
+        self.transport.start_monitor(self)
+
+        while not self.terminate.isSet():
+            try:
+                interaction = self.transport.fetch()
+                if not interaction:
+                    continue
+                try:
+                    start = time.time()
+                    self.storage.import_interaction(interaction)
+                    self.logger.info("Imported interaction for %s in %ss" %
+                        (interaction.get('hostname', '<unknown>'),
+                            time.time() - start))
+                except:
+                    #TODO requeue?
+                    raise
+            except (SystemExit, KeyboardInterrupt):
+                self.logger.info("Shutting down")
+                self.shutdown()
+            except:
+                self.logger.error("Unhandled exception in main loop %s" %
+                    traceback.format_exc().splitlines()[-1])
+
+    def shutdown(self):
+        """Cleanup and go"""
+        if self.terminate:
+            # this wil be missing if called from bcfg2-admin
+            self.terminate.set()
+        if self.transport:
+            self.transport.shutdown()
+        if self.storage:
+            self.storage.shutdown()
+