summaryrefslogtreecommitdiffstats
path: root/src/lib/Bcfg2/Reporting/migrate.py
diff options
context:
space:
mode:
authorTim Laszlo <tim.laszlo@gmail.com>2012-10-08 10:38:02 -0500
committerTim Laszlo <tim.laszlo@gmail.com>2012-10-08 10:38:02 -0500
commit44638176067df5231bf0be30801e36863391cd1f (patch)
tree6aaba73d03f9a5532047518b9a3e8ef3e63d3f9f /src/lib/Bcfg2/Reporting/migrate.py
parent1a3ced3f45423d79e08ca7d861e8118e8618d3b2 (diff)
downloadbcfg2-44638176067df5231bf0be30801e36863391cd1f.tar.gz
bcfg2-44638176067df5231bf0be30801e36863391cd1f.tar.bz2
bcfg2-44638176067df5231bf0be30801e36863391cd1f.zip
Reporting: Merge new reporting data
Move reporting data to a new schema Use south for django migrations Add bcfg2-report-collector daemon Conflicts: doc/development/index.txt doc/server/plugins/connectors/properties.txt doc/server/plugins/generators/packages.txt setup.py src/lib/Bcfg2/Client/Tools/SELinux.py src/lib/Bcfg2/Compat.py src/lib/Bcfg2/Encryption.py src/lib/Bcfg2/Options.py src/lib/Bcfg2/Server/Admin/Init.py src/lib/Bcfg2/Server/Admin/Reports.py src/lib/Bcfg2/Server/BuiltinCore.py src/lib/Bcfg2/Server/Core.py src/lib/Bcfg2/Server/FileMonitor/Inotify.py src/lib/Bcfg2/Server/Plugin/base.py src/lib/Bcfg2/Server/Plugin/interfaces.py src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py src/lib/Bcfg2/Server/Plugins/FileProbes.py src/lib/Bcfg2/Server/Plugins/Ohai.py src/lib/Bcfg2/Server/Plugins/Packages/Collection.py src/lib/Bcfg2/Server/Plugins/Packages/Source.py src/lib/Bcfg2/Server/Plugins/Packages/Yum.py src/lib/Bcfg2/Server/Plugins/Packages/__init__.py src/lib/Bcfg2/Server/Plugins/Probes.py src/lib/Bcfg2/Server/Plugins/Properties.py src/lib/Bcfg2/Server/Reports/backends.py src/lib/Bcfg2/Server/Reports/manage.py src/lib/Bcfg2/Server/Reports/nisauth.py src/lib/Bcfg2/settings.py src/sbin/bcfg2-crypt src/sbin/bcfg2-yum-helper testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py
Diffstat (limited to 'src/lib/Bcfg2/Reporting/migrate.py')
-rw-r--r--src/lib/Bcfg2/Reporting/migrate.py230
1 files changed, 230 insertions, 0 deletions
diff --git a/src/lib/Bcfg2/Reporting/migrate.py b/src/lib/Bcfg2/Reporting/migrate.py
new file mode 100644
index 000000000..d0b3c9dc4
--- /dev/null
+++ b/src/lib/Bcfg2/Reporting/migrate.py
@@ -0,0 +1,230 @@
+import logging
+import Bcfg2.Logger
+from django.core.cache import cache
+from django.db import connection, transaction, backend
+
+from Bcfg2.Reporting import models as new_models
+from Bcfg2.Server.Reports.reports import models as legacy_models
+
+logger = logging.getLogger(__name__)
+
+_our_backend = None
+
+def _quote(value):
+ """
+ Quote a string to use as a table name or column
+
+ Newer versions and various drivers require an argument
+ https://code.djangoproject.com/ticket/13630
+ """
+ global _our_backend
+ if not _our_backend:
+ try:
+ _our_backend = backend.DatabaseOperations(connection)
+ except TypeError:
+ _our_backend = backend.DatabaseOperations(connection)
+ return _our_backend.quote_name(value)
+
+
+@transaction.commit_on_success
+def _migrate_transaction(inter, entries):
+ """helper"""
+
+ logger.debug("Migrating interaction %s for %s" %
+ (inter.id, inter.client.name))
+
+ newint = new_models.Interaction(id=inter.id,
+ client_id=inter.client_id,
+ timestamp=inter.timestamp,
+ state=inter.state,
+ repo_rev_code=inter.repo_rev_code,
+ server=inter.server,
+ good_count=inter.goodcount,
+ total_count=inter.totalcount,
+ bad_count=inter.bad_entries,
+ modified_count=inter.modified_entries,
+ extra_count=inter.extra_entries)
+
+ try:
+ newint.profile_id = inter.metadata.profile.id
+ groups = [grp.pk for grp in inter.metadata.groups.all()]
+ bundles = [bun.pk for bun in inter.metadata.bundles.all()]
+ except legacy_models.InteractionMetadata.DoesNotExist:
+ groups = []
+ bundles = []
+ unkown_profile = cache.get("PROFILE_UNKNOWN")
+ if not unkown_profile:
+ unkown_profile, created = new_models.Group.objects.get_or_create(name="<<Unknown>>")
+ cache.set("PROFILE_UNKNOWN", unkown_profile)
+ newint.profile = unkown_profile
+ newint.save()
+ if bundles:
+ newint.bundles.add(*bundles)
+ if groups:
+ newint.groups.add(*groups)
+
+ updates = dict(paths=[], packages=[], actions=[], services=[])
+ for ei in legacy_models.Entries_interactions.objects.select_related('reason')\
+ .filter(interaction=inter):
+ ent = entries[ei.entry_id]
+ name = ent.name
+ act_dict = dict(name=name, exists=ei.reason.current_exists,
+ state=ei.type)
+
+ if ent.kind == 'Action':
+ act_dict['status'] = ei.reason.status
+ if not act_dict['status']:
+ act_dict['status'] = "check"
+ act_dict['output'] = -1
+ logger.debug("Adding action %s" % name)
+ updates['actions'].append(new_models.ActionEntry.entry_get_or_create(act_dict))
+
+ elif ent.kind == 'Package':
+ act_dict['target_version'] = ei.reason.version
+ act_dict['current_version'] = ei.reason.current_version
+ logger.debug("Adding package %s %s" %
+ (name, act_dict['target_version']))
+ updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict))
+ elif ent.kind == 'Path':
+ # these might be hard.. they aren't one to one with the old model
+ act_dict['path_type'] = 'file'
+
+ target_dict = dict(
+ owner=ei.reason.owner,
+ group=ei.reason.group,
+ perms=ei.reason.perms
+ )
+ fperm, created = new_models.FilePerms.objects.get_or_create(**target_dict)
+ act_dict['target_perms'] = fperm
+
+ current_dict = dict(
+ owner=ei.reason.current_owner,
+ group=ei.reason.current_group,
+ perms=ei.reason.current_perms
+ )
+ fperm, created = new_models.FilePerms.objects.get_or_create(**current_dict)
+ act_dict['current_perms'] = fperm
+
+ if ei.reason.to:
+ act_dict['path_type'] = 'symlink'
+ act_dict['target_path'] = ei.reason.to
+ act_dict['current_path'] = ei.reason.current_to
+ logger.debug("Adding link %s" % name)
+ updates['paths'].append(new_models.LinkEntry.entry_get_or_create(act_dict))
+ continue
+
+ act_dict['detail_type'] = new_models.PathEntry.DETAIL_UNUSED
+ if ei.reason.unpruned:
+ # this is the only other case we know what the type really is
+ act_dict['path_type'] = 'directory'
+ act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED
+ act_dict['details'] = ei.reason.unpruned
+
+
+ if ei.reason.is_sensitive:
+ act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE
+ elif ei.reason.is_binary:
+ act_dict['detail_type'] = new_models.PathEntry.DETAIL_BINARY
+ act_dict['details'] = ei.reason.current_diff
+ elif ei.reason.current_diff:
+ act_dict['detail_type'] = new_models.PathEntry.DETAIL_DIFF
+ act_dict['details'] = ei.reason.current_diff
+ logger.debug("Adding path %s" % name)
+ updates['paths'].append(new_models.PathEntry.entry_get_or_create(act_dict))
+
+ elif ent.kind == 'Service':
+ act_dict['target_status'] = ei.reason.status
+ act_dict['current_status'] = ei.reason.current_status
+ logger.debug("Adding service %s" % name)
+ updates['services'].append(new_models.ServiceEntry.entry_get_or_create(act_dict))
+ else:
+ logger.warn("Skipping type %s" % ent.kind)
+
+ for entry_type in updates.keys():
+ i = 0
+ while(i < len(updates[entry_type])):
+ getattr(newint, entry_type).add(*updates[entry_type][i:i+100])
+ i += 100
+
+ for perf in inter.performance_items.all():
+ new_models.Performance(
+ interaction=newint,
+ metric=perf.metric,
+ value=perf.value).save()
+
+
+def _shove(old_table, new_table, columns):
+ cols = ",".join([_quote(f) for f in columns])
+ sql = "insert into %s(%s) select %s from %s" % (
+ _quote(new_table),
+ cols,
+ cols,
+ _quote(old_table))
+
+ cursor = connection.cursor()
+ cursor.execute(sql)
+ cursor.close()
+
+
+@transaction.commit_manually
+def _restructure():
+ """major restructure of reporting data"""
+
+ logger.info("Migrating clients")
+ try:
+ _shove(legacy_models.Client._meta.db_table, new_models.Client._meta.db_table,
+ ('id', 'name', 'creation', 'expiration'))
+ except:
+ logger.error("Failed to migrate clients", exc_info=1)
+ return False
+
+ logger.info("Migrating Bundles")
+ try:
+ _shove(legacy_models.Bundle._meta.db_table, new_models.Bundle._meta.db_table,
+ ('id', 'name'))
+ except:
+ logger.error("Failed to migrate bundles", exc_info=1)
+ return False
+
+ logger.info("Migrating Groups")
+ try:
+ _shove(legacy_models.Group._meta.db_table, new_models.Group._meta.db_table,
+ ('id', 'name', 'profile', 'public', 'category', 'comment'))
+ except:
+ logger.error("Failed to migrate groups", exc_info=1)
+ return False
+
+ try:
+ entries = {}
+ for ent in legacy_models.Entries.objects.all():
+ entries[ent.id] = ent
+ except:
+ logger.error("Failed to populate entries dict", exc_info=1)
+ return False
+
+ transaction.commit()
+
+ failures = []
+ int_count = legacy_models.Interaction.objects.count()
+ int_ctr = 0
+ for inter in legacy_models.Interaction.objects.select_related().all():
+ if int_ctr % 1000 == 0:
+ logger.info("Migrated %s of %s interactions" % (int_ctr, int_count))
+ try:
+ _migrate_transaction(inter, entries)
+ except:
+ logger.error("Failed to migrate interaction %s for %s" %
+ (inter.id, inter.client.name), exc_info=1)
+ failures.append(inter.id)
+ int_ctr += 1
+ if not failures:
+ logger.info("Successfully restructured reason data")
+ return True
+
+
+if __name__ == '__main__':
+ Bcfg2.Logger.setup_logging('bcfg2-report-collector',
+ to_console=logging.INFO,
+ level=logging.INFO)
+ _restructure()
+