Reporting: Merge new reporting data

Move reporting data to a new schema
Use south for django migrations
Add bcfg2-report-collector daemon

Conflicts:
	doc/development/index.txt
	doc/server/plugins/connectors/properties.txt
	doc/server/plugins/generators/packages.txt
	setup.py
	src/lib/Bcfg2/Client/Tools/SELinux.py
	src/lib/Bcfg2/Compat.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Server/Admin/Init.py
	src/lib/Bcfg2/Server/Admin/Reports.py
	src/lib/Bcfg2/Server/BuiltinCore.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/Inotify.py
	src/lib/Bcfg2/Server/Plugin/base.py
	src/lib/Bcfg2/Server/Plugin/interfaces.py
	src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py
	src/lib/Bcfg2/Server/Plugins/FileProbes.py
	src/lib/Bcfg2/Server/Plugins/Ohai.py
	src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
	src/lib/Bcfg2/Server/Plugins/Packages/Source.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/lib/Bcfg2/Server/Plugins/Properties.py
	src/lib/Bcfg2/Server/Reports/backends.py
	src/lib/Bcfg2/Server/Reports/manage.py
	src/lib/Bcfg2/Server/Reports/nisauth.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py

1 files changed, 230 insertions, 0 deletions

diff --git a/src/lib/Bcfg2/Reporting/migrate.py b/src/lib/Bcfg2/Reporting/migrate.py
new file mode 100644
index 000000000..d0b3c9dc4
--- /dev/null
+++ b/src/lib/Bcfg2/Reporting/migrate.py
@@ -0,0 +1,230 @@
+import logging
+import Bcfg2.Logger
+from django.core.cache import cache
+from django.db import connection, transaction, backend
+
+from Bcfg2.Reporting import models as new_models
+from Bcfg2.Server.Reports.reports import models as legacy_models
+
+logger = logging.getLogger(__name__)
+
+_our_backend = None
+
+def _quote(value):
+    """
+    Quote a string to use as a table name or column
+
+    Newer versions and various drivers require an argument
+    https://code.djangoproject.com/ticket/13630
+    """
+    global _our_backend
+    if not _our_backend:
+        try:
+            _our_backend = backend.DatabaseOperations(connection)
+        except TypeError:
+            _our_backend = backend.DatabaseOperations(connection)
+    return _our_backend.quote_name(value)
+
+
+@transaction.commit_on_success
+def _migrate_transaction(inter, entries):
+    """helper"""
+
+    logger.debug("Migrating interaction %s for %s" % 
+        (inter.id, inter.client.name))
+
+    newint = new_models.Interaction(id=inter.id,
+        client_id=inter.client_id,
+        timestamp=inter.timestamp,
+        state=inter.state,
+        repo_rev_code=inter.repo_rev_code,
+        server=inter.server,
+        good_count=inter.goodcount,
+        total_count=inter.totalcount,
+        bad_count=inter.bad_entries,
+        modified_count=inter.modified_entries,
+        extra_count=inter.extra_entries)
+
+    try:
+        newint.profile_id = inter.metadata.profile.id
+        groups = [grp.pk for grp in inter.metadata.groups.all()]
+        bundles = [bun.pk for bun in inter.metadata.bundles.all()]
+    except legacy_models.InteractionMetadata.DoesNotExist:
+        groups = []
+        bundles = []
+        unkown_profile = cache.get("PROFILE_UNKNOWN")
+        if not unkown_profile:
+            unkown_profile, created = new_models.Group.objects.get_or_create(name="<<Unknown>>")
+            cache.set("PROFILE_UNKNOWN", unkown_profile)
+        newint.profile = unkown_profile
+    newint.save()
+    if bundles:
+        newint.bundles.add(*bundles)
+    if groups:
+        newint.groups.add(*groups)
+
+    updates = dict(paths=[], packages=[], actions=[], services=[])
+    for ei in legacy_models.Entries_interactions.objects.select_related('reason')\
+            .filter(interaction=inter):
+        ent = entries[ei.entry_id]
+        name = ent.name
+        act_dict = dict(name=name, exists=ei.reason.current_exists,
+            state=ei.type)
+
+        if ent.kind == 'Action':
+            act_dict['status'] = ei.reason.status
+            if not act_dict['status']:
+                act_dict['status'] = "check"
+            act_dict['output'] = -1
+            logger.debug("Adding action %s" % name)
+            updates['actions'].append(new_models.ActionEntry.entry_get_or_create(act_dict))
+
+        elif ent.kind == 'Package':
+            act_dict['target_version'] = ei.reason.version
+            act_dict['current_version'] = ei.reason.current_version
+            logger.debug("Adding package %s %s" % 
+                (name, act_dict['target_version']))
+            updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict))
+        elif ent.kind == 'Path':
+            # these might be hard.. they aren't one to one with the old model
+            act_dict['path_type'] = 'file'
+
+            target_dict = dict(
+                owner=ei.reason.owner,
+                group=ei.reason.group,
+                perms=ei.reason.perms
+            )
+            fperm, created = new_models.FilePerms.objects.get_or_create(**target_dict)
+            act_dict['target_perms'] = fperm
+
+            current_dict = dict(
+                owner=ei.reason.current_owner,
+                group=ei.reason.current_group,
+                perms=ei.reason.current_perms
+            )
+            fperm, created = new_models.FilePerms.objects.get_or_create(**current_dict)
+            act_dict['current_perms'] = fperm
+
+            if ei.reason.to:
+                act_dict['path_type'] = 'symlink'
+                act_dict['target_path'] = ei.reason.to
+                act_dict['current_path'] = ei.reason.current_to
+                logger.debug("Adding link %s" % name)
+                updates['paths'].append(new_models.LinkEntry.entry_get_or_create(act_dict))
+                continue
+
+            act_dict['detail_type'] = new_models.PathEntry.DETAIL_UNUSED
+            if ei.reason.unpruned:
+                # this is the only other case we know what the type really is
+                act_dict['path_type'] = 'directory'
+                act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED
+                act_dict['details'] = ei.reason.unpruned
+
+            
+            if ei.reason.is_sensitive:
+                act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE
+            elif ei.reason.is_binary:
+                act_dict['detail_type'] = new_models.PathEntry.DETAIL_BINARY
+                act_dict['details'] = ei.reason.current_diff
+            elif ei.reason.current_diff:
+                act_dict['detail_type'] = new_models.PathEntry.DETAIL_DIFF
+                act_dict['details'] = ei.reason.current_diff
+            logger.debug("Adding path %s" % name)
+            updates['paths'].append(new_models.PathEntry.entry_get_or_create(act_dict))
+
+        elif ent.kind == 'Service':
+            act_dict['target_status'] = ei.reason.status
+            act_dict['current_status'] = ei.reason.current_status
+            logger.debug("Adding service %s" % name)
+            updates['services'].append(new_models.ServiceEntry.entry_get_or_create(act_dict))
+        else:
+            logger.warn("Skipping type %s" % ent.kind)
+
+    for entry_type in updates.keys():
+        i = 0
+        while(i < len(updates[entry_type])):
+            getattr(newint, entry_type).add(*updates[entry_type][i:i+100])
+            i += 100
+
+    for perf in inter.performance_items.all():
+        new_models.Performance(
+            interaction=newint,
+            metric=perf.metric,
+            value=perf.value).save()
+
+
+def _shove(old_table, new_table, columns):
+    cols = ",".join([_quote(f) for f in columns])
+    sql = "insert into %s(%s) select %s from %s" % (
+        _quote(new_table),
+        cols,
+        cols,
+        _quote(old_table))
+
+    cursor = connection.cursor()
+    cursor.execute(sql)
+    cursor.close()
+
+
+@transaction.commit_manually
+def _restructure():
+    """major restructure of reporting data"""
+
+    logger.info("Migrating clients")
+    try:
+        _shove(legacy_models.Client._meta.db_table, new_models.Client._meta.db_table,
+            ('id', 'name', 'creation', 'expiration'))
+    except:
+        logger.error("Failed to migrate clients", exc_info=1)
+        return False
+
+    logger.info("Migrating Bundles")
+    try:
+        _shove(legacy_models.Bundle._meta.db_table, new_models.Bundle._meta.db_table,
+            ('id', 'name'))
+    except:
+        logger.error("Failed to migrate bundles", exc_info=1)
+        return False
+
+    logger.info("Migrating Groups")
+    try:
+        _shove(legacy_models.Group._meta.db_table, new_models.Group._meta.db_table,
+            ('id', 'name', 'profile', 'public', 'category', 'comment'))
+    except:
+        logger.error("Failed to migrate groups", exc_info=1)
+        return False
+
+    try:
+        entries = {}
+        for ent in legacy_models.Entries.objects.all():
+            entries[ent.id] = ent
+    except:
+        logger.error("Failed to populate entries dict", exc_info=1)
+        return False
+
+    transaction.commit()
+
+    failures = []
+    int_count = legacy_models.Interaction.objects.count()
+    int_ctr = 0
+    for inter in legacy_models.Interaction.objects.select_related().all():
+        if int_ctr % 1000 == 0:
+            logger.info("Migrated %s of %s interactions" % (int_ctr, int_count))
+        try:
+            _migrate_transaction(inter, entries)
+        except:
+            logger.error("Failed to migrate interaction %s for %s" %
+                (inter.id, inter.client.name), exc_info=1)
+            failures.append(inter.id)
+        int_ctr += 1
+    if not failures:
+        logger.info("Successfully restructured reason data")
+        return True
+
+
+if __name__ == '__main__':
+    Bcfg2.Logger.setup_logging('bcfg2-report-collector',
+                                   to_console=logging.INFO,
+                                   level=logging.INFO)
+    _restructure()
+