From 44638176067df5231bf0be30801e36863391cd1f Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 8 Oct 2012 10:38:02 -0500 Subject: Reporting: Merge new reporting data Move reporting data to a new schema Use south for django migrations Add bcfg2-report-collector daemon Conflicts: doc/development/index.txt doc/server/plugins/connectors/properties.txt doc/server/plugins/generators/packages.txt setup.py src/lib/Bcfg2/Client/Tools/SELinux.py src/lib/Bcfg2/Compat.py src/lib/Bcfg2/Encryption.py src/lib/Bcfg2/Options.py src/lib/Bcfg2/Server/Admin/Init.py src/lib/Bcfg2/Server/Admin/Reports.py src/lib/Bcfg2/Server/BuiltinCore.py src/lib/Bcfg2/Server/Core.py src/lib/Bcfg2/Server/FileMonitor/Inotify.py src/lib/Bcfg2/Server/Plugin/base.py src/lib/Bcfg2/Server/Plugin/interfaces.py src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py src/lib/Bcfg2/Server/Plugins/FileProbes.py src/lib/Bcfg2/Server/Plugins/Ohai.py src/lib/Bcfg2/Server/Plugins/Packages/Collection.py src/lib/Bcfg2/Server/Plugins/Packages/Source.py src/lib/Bcfg2/Server/Plugins/Packages/Yum.py src/lib/Bcfg2/Server/Plugins/Packages/__init__.py src/lib/Bcfg2/Server/Plugins/Probes.py src/lib/Bcfg2/Server/Plugins/Properties.py src/lib/Bcfg2/Server/Reports/backends.py src/lib/Bcfg2/Server/Reports/manage.py src/lib/Bcfg2/Server/Reports/nisauth.py src/lib/Bcfg2/settings.py src/sbin/bcfg2-crypt src/sbin/bcfg2-yum-helper testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py --- src/lib/Bcfg2/Reporting/Collector.py | 111 ++++ src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 316 +++++++++++ src/lib/Bcfg2/Reporting/Storage/__init__.py | 32 ++ src/lib/Bcfg2/Reporting/Storage/base.py | 51 ++ .../Bcfg2/Reporting/Transport/LocalFilesystem.py | 163 ++++++ src/lib/Bcfg2/Reporting/Transport/__init__.py | 32 ++ src/lib/Bcfg2/Reporting/Transport/base.py | 45 ++ src/lib/Bcfg2/Reporting/__init__.py | 0 src/lib/Bcfg2/Reporting/migrate.py | 230 ++++++++ src/lib/Bcfg2/Reporting/migrations/0001_initial.py | 465 ++++++++++++++++ src/lib/Bcfg2/Reporting/migrations/__init__.py | 0 src/lib/Bcfg2/Reporting/models.py | 582 +++++++++++++++++++++ src/lib/Bcfg2/Reporting/templates/404.html | 8 + .../Bcfg2/Reporting/templates/base-timeview.html | 28 + src/lib/Bcfg2/Reporting/templates/base.html | 96 ++++ .../Bcfg2/Reporting/templates/clients/detail.html | 149 ++++++ .../Reporting/templates/clients/detailed-list.html | 46 ++ .../Bcfg2/Reporting/templates/clients/history.html | 20 + .../Bcfg2/Reporting/templates/clients/index.html | 35 ++ .../Bcfg2/Reporting/templates/clients/manage.html | 45 ++ .../Reporting/templates/config_items/common.html | 42 ++ .../templates/config_items/entry_status.html | 32 ++ .../templates/config_items/item-failure.html | 13 + .../Reporting/templates/config_items/item.html | 136 +++++ .../Reporting/templates/config_items/listing.html | 35 ++ .../Reporting/templates/displays/summary.html | 42 ++ .../Bcfg2/Reporting/templates/displays/timing.html | 38 ++ .../Reporting/templates/widgets/filter_bar.html | 25 + .../templates/widgets/interaction_list.inc | 38 ++ .../Reporting/templates/widgets/page_bar.html | 23 + src/lib/Bcfg2/Reporting/templatetags/__init__.py | 0 src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py | 415 +++++++++++++++ src/lib/Bcfg2/Reporting/templatetags/split.py | 8 + .../Reporting/templatetags/syntax_coloring.py | 48 ++ src/lib/Bcfg2/Reporting/urls.py | 59 +++ src/lib/Bcfg2/Reporting/utils.py | 126 +++++ src/lib/Bcfg2/Reporting/views.py | 550 +++++++++++++++++++ 37 files changed, 4084 insertions(+) create mode 100644 src/lib/Bcfg2/Reporting/Collector.py create mode 100644 src/lib/Bcfg2/Reporting/Storage/DjangoORM.py create mode 100644 src/lib/Bcfg2/Reporting/Storage/__init__.py create mode 100644 src/lib/Bcfg2/Reporting/Storage/base.py create mode 100644 src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py create mode 100644 src/lib/Bcfg2/Reporting/Transport/__init__.py create mode 100644 src/lib/Bcfg2/Reporting/Transport/base.py create mode 100644 src/lib/Bcfg2/Reporting/__init__.py create mode 100644 src/lib/Bcfg2/Reporting/migrate.py create mode 100644 src/lib/Bcfg2/Reporting/migrations/0001_initial.py create mode 100644 src/lib/Bcfg2/Reporting/migrations/__init__.py create mode 100644 src/lib/Bcfg2/Reporting/models.py create mode 100644 src/lib/Bcfg2/Reporting/templates/404.html create mode 100644 src/lib/Bcfg2/Reporting/templates/base-timeview.html create mode 100644 src/lib/Bcfg2/Reporting/templates/base.html create mode 100644 src/lib/Bcfg2/Reporting/templates/clients/detail.html create mode 100644 src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html create mode 100644 src/lib/Bcfg2/Reporting/templates/clients/history.html create mode 100644 src/lib/Bcfg2/Reporting/templates/clients/index.html create mode 100644 src/lib/Bcfg2/Reporting/templates/clients/manage.html create mode 100644 src/lib/Bcfg2/Reporting/templates/config_items/common.html create mode 100644 src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html create mode 100644 src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html create mode 100644 src/lib/Bcfg2/Reporting/templates/config_items/item.html create mode 100644 src/lib/Bcfg2/Reporting/templates/config_items/listing.html create mode 100644 src/lib/Bcfg2/Reporting/templates/displays/summary.html create mode 100644 src/lib/Bcfg2/Reporting/templates/displays/timing.html create mode 100644 src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html create mode 100644 src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc create mode 100644 src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html create mode 100644 src/lib/Bcfg2/Reporting/templatetags/__init__.py create mode 100644 src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py create mode 100644 src/lib/Bcfg2/Reporting/templatetags/split.py create mode 100644 src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py create mode 100644 src/lib/Bcfg2/Reporting/urls.py create mode 100755 src/lib/Bcfg2/Reporting/utils.py create mode 100644 src/lib/Bcfg2/Reporting/views.py (limited to 'src/lib/Bcfg2/Reporting') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py new file mode 100644 index 000000000..bb2e85c21 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -0,0 +1,111 @@ +import atexit +import daemon +import lockfile +import logging +import time +import traceback +import threading + +import Bcfg2.Logger +from Bcfg2.Reporting.Transport import load_transport_from_config, \ + TransportError, TransportImportError +from Bcfg2.Reporting.Storage import load_storage_from_config, \ + StorageError, StorageImportError + +class ReportingError(Exception): + """Generic reporting exception""" + pass + +class ReportingCollector(object): + """The collecting process for reports""" + + def __init__(self, setup): + """Setup the collector. This may be called by the daemon or though + bcfg2-admin""" + self.setup = setup + self.datastore = setup['repo'] + self.encoding = setup['encoding'] + self.terminate = None + self.context = None + + if setup['debug']: + level = logging.DEBUG + elif setup['verbose']: + level = logging.INFO + else: + level = logging.WARNING + + Bcfg2.Logger.setup_logging('bcfg2-report-collector', + to_console=logging.INFO, + to_syslog=setup['syslog'], + to_file=setup['logging'], + level=level) + self.logger = logging.getLogger('bcfg2-report-collector') + + try: + self.transport = load_transport_from_config(setup) + self.storage = load_storage_from_config(setup) + except TransportError: + self.logger.error("Failed to load transport: %s" % + traceback.format_exc().splitlines()[-1]) + raise ReportingError + except StorageError: + self.logger.error("Failed to load storage: %s" % + traceback.format_exc().splitlines()[-1]) + raise ReportingError + + try: + self.logger.debug("Validating storage %s" % + self.storage.__class__.__name__) + self.storage.validate() + except: + self.logger.error("Storage backed %s failed to validate: %s" % + (self.storage.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + + + def run(self): + """Startup the processing and go!""" + self.terminate = threading.Event() + atexit.register(self.shutdown) + self.context = daemon.DaemonContext() + + if self.setup['daemon']: + self.logger.debug("Daemonizing") + self.context.pidfile = lockfile.FileLock(self.setup['daemon']) + self.context.open() + self.logger.info("Starting daemon") + + self.transport.start_monitor(self) + + while not self.terminate.isSet(): + try: + interaction = self.transport.fetch() + if not interaction: + continue + try: + start = time.time() + self.storage.import_interaction(interaction) + self.logger.info("Imported interaction for %s in %ss" % + (interaction.get('hostname', ''), + time.time() - start)) + except: + #TODO requeue? + raise + except (SystemExit, KeyboardInterrupt): + self.logger.info("Shutting down") + self.shutdown() + except: + self.logger.error("Unhandled exception in main loop %s" % + traceback.format_exc().splitlines()[-1]) + + def shutdown(self): + """Cleanup and go""" + if self.terminate: + # this wil be missing if called from bcfg2-admin + self.terminate.set() + if self.transport: + self.transport.shutdown() + if self.storage: + self.storage.shutdown() + diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py new file mode 100644 index 000000000..17eb52f66 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -0,0 +1,316 @@ +""" +The base for the original DjangoORM (DBStats) +""" + +import os +import traceback +from lxml import etree +from datetime import datetime +from time import strptime + +os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' +from Bcfg2 import settings + +from Bcfg2.Compat import md5 +from Bcfg2.Reporting.Storage.base import StorageBase, StorageError +from Bcfg2.Server.Plugin.exceptions import PluginExecutionError +from django.core import management +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.core.cache import cache +from django.db import transaction + +#Used by GetCurrentEntry +import difflib +from Bcfg2.Compat import b64decode +from Bcfg2.Reporting.models import * + + +class DjangoORM(StorageBase): + def __init__(self, setup): + super(DjangoORM, self).__init__(setup) + self.size_limit = setup.get('reporting_size_limit') + + @transaction.commit_on_success + def _import_interaction(self, interaction): + """Real import function""" + hostname = interaction['hostname'] + stats = etree.fromstring(interaction['stats']) + metadata = interaction['metadata'] + server = metadata['server'] + + client = cache.get(hostname) + if not client: + client, created = Client.objects.get_or_create(name=hostname) + if created: + self.logger.debug("Client %s added to the db" % hostname) + cache.set(hostname, client) + + timestamp = datetime(*strptime(stats.get('time'))[0:6]) + if len(Interaction.objects.filter(client=client, timestamp=timestamp)) > 0: + self.logger.warn("Interaction for %s at %s already exists" % + (hostname, timestamp)) + return + + profile, created = Group.objects.get_or_create(name=metadata['profile']) + inter = Interaction(client=client, + timestamp=timestamp, + state=stats.get('state', default="unknown"), + repo_rev_code=stats.get('revision', + default="unknown"), + good_count=stats.get('good', default="0"), + total_count=stats.get('total', default="0"), + server=server, + profile=profile) + inter.save() + self.logger.debug("Interaction for %s at %s with INSERTED in to db" % + (client.id, timestamp)) + + #FIXME - this should be more efficient + for group_name in metadata['groups']: + group = cache.get("GROUP_" + group_name) + if not group: + group, created = Group.objects.get_or_create(name=group_name) + if created: + self.logger.debug("Added group %s" % group) + cache.set("GROUP_" + group_name, group) + + inter.groups.add(group) + for bundle_name in metadata['bundles']: + bundle = cache.get("BUNDLE_" + bundle_name) + if not bundle: + bundle, created = Bundle.objects.get_or_create(name=bundle_name) + if created: + self.logger.debug("Added bundle %s" % bundle) + cache.set("BUNDLE_" + bundle_name, bundle) + inter.bundles.add(bundle) + inter.save() + + counter_fields = {TYPE_BAD: 0, + TYPE_MODIFIED: 0, + TYPE_EXTRA: 0} + pattern = [('Bad/*', TYPE_BAD), + ('Extra/*', TYPE_EXTRA), + ('Modified/*', TYPE_MODIFIED)] + updates = dict(failures=[], paths=[], packages=[], actions=[], services=[]) + for (xpath, state) in pattern: + for entry in stats.findall(xpath): + counter_fields[state] = counter_fields[state] + 1 + + entry_type = entry.tag + name = entry.get('name') + exists = entry.get('current_exists', default="true").lower() == "true" + + # handle server failures differently + failure = entry.get('failure', '') + if failure: + act_dict = dict(name=name, entry_type=entry_type, + message=failure) + newact = FailureEntry.entry_get_or_create(act_dict) + updates['failures'].append(newact) + continue + + act_dict = dict(name=name, state=state, exists=exists) + + if entry_type == 'Action': + act_dict['status'] = entry.get('status', default="check") + act_dict['output'] = entry.get('rc', default=-1) + self.logger.debug("Adding action %s" % name) + updates['actions'].append(ActionEntry.entry_get_or_create(act_dict)) + elif entry_type == 'Package': + act_dict['target_version'] = entry.get('version', default='') + act_dict['current_version'] = entry.get('current_version', default='') + + # extra entries are a bit different. They can have Instance objects + if not act_dict['target_version']: + for instance in entry.findall("Instance"): + #TODO - this probably only works for rpms + release = instance.get('release', '') + arch = instance.get('arch', '') + act_dict['current_version'] = instance.get('version') + if release: + act_dict['current_version'] += "-" + release + if arch: + act_dict['current_version'] += "." + arch + self.logger.debug("Adding package %s %s" % (name, act_dict['current_version'])) + updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) + else: + + self.logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) + + # not implemented yet + act_dict['verification_details'] = entry.get('verification_details', '') + updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) + + elif entry_type == 'Path': + path_type = entry.get("type").lower() + act_dict['path_type'] = path_type + + target_dict = dict( + owner=entry.get('owner', default="root"), + group=entry.get('group', default="root"), + perms=entry.get('perms', default=""), + ) + fperm, created = FilePerms.objects.get_or_create(**target_dict) + act_dict['target_perms'] = fperm + + current_dict = dict( + owner=entry.get('current_owner', default=""), + group=entry.get('current_group', default=""), + perms=entry.get('current_perms', default=""), + ) + fperm, created = FilePerms.objects.get_or_create(**current_dict) + act_dict['current_perms'] = fperm + + if path_type in ('symlink', 'hardlink'): + act_dict['target_path'] = entry.get('to', default="") + act_dict['current_path'] = entry.get('current_to', default="") + self.logger.debug("Adding link %s" % name) + updates['paths'].append(LinkEntry.entry_get_or_create(act_dict)) + continue + elif path_type == 'device': + #TODO devices + self.logger.warn("device path types are not supported yet") + continue + + # TODO - vcs output + act_dict['detail_type'] = PathEntry.DETAIL_UNUSED + if path_type == 'directory' and entry.get('prune', 'false') == 'true': + unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + if unpruned_elist: + act_dict['detail_type'] = PathEntry.DETAIL_PRUNED + act_dict['details'] = "\n".join(unpruned_elist) + elif entry.get('sensitive', 'false').lower() == 'true': + act_dict['detail_type'] = PathEntry.DETAIL_SENSITIVE + else: + cdata = None + if entry.get('current_bfile', None): + act_dict['detail_type'] = PathEntry.DETAIL_BINARY + cdata = entry.get('current_bfile') + elif entry.get('current_bdiff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = b64decode(entry.get('current_bdiff')) + elif entry.get('current_diff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = entry.get('current_bdiff') + if cdata: + if len(cdata) > self.size_limit: + act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT + act_dict['details'] = md5(cdata).hexdigest() + else: + act_dict['details'] = cdata + self.logger.debug("Adding path %s" % name) + updates['paths'].append(PathEntry.entry_get_or_create(act_dict)) + + + #TODO - secontext + #TODO - acls + + elif entry_type == 'Service': + act_dict['target_status'] = entry.get('status', default='') + act_dict['current_status'] = entry.get('current_status', default='') + self.logger.debug("Adding service %s" % name) + updates['services'].append(ServiceEntry.entry_get_or_create(act_dict)) + elif entry_type == 'SELinux': + self.logger.info("SELinux not implemented yet") + else: + self.logger.error("Unknown type %s not handled by reporting yet" % entry_type) + + inter.bad_count = counter_fields[TYPE_BAD] + inter.modified_count = counter_fields[TYPE_MODIFIED] + inter.extra_count = counter_fields[TYPE_EXTRA] + inter.save() + for entry_type in updates.keys(): + getattr(inter, entry_type).add(*updates[entry_type]) + + # performance metrics + for times in stats.findall('OpStamps'): + for metric, value in list(times.items()): + Performance(interaction=inter, metric=metric, value=value).save() + + + def import_interaction(self, interaction): + """Import the data into the backend""" + + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) + + + def validate(self): + """Validate backend storage. Should be called once when loaded""" + + settings.read_config(repo=self.setup['repo']) + + # verify our database schema + try: + if self.setup['verbose'] or self.setup['debug']: + vrb = 2 + else: + vrb = 0 + management.call_command("syncdb", verbosity=vrb, interactive=False) + management.call_command("migrate", verbosity=vrb, interactive=False) + except: + self.logger.error("Failed to update database schema: %s" % \ + traceback.format_exc().splitlines()[-1]) + raise StorageError + + def GetExtra(self, client): + """Fetch extra entries for a client""" + try: + c_inst = Client.objects.get(name=client) + return [(ent.entry_type, ent.name) for ent in + c_inst.current_interaction.extra()] + except ObjectDoesNotExist: + return [] + except MultipleObjectsReturned: + self.logger.error("%s Inconsistency: Multiple entries for %s." % + (self.__class__.__name__, client)) + return [] + + def GetCurrentEntry(self, client, e_type, e_name): + """"GetCurrentEntry: Used by PullSource""" + try: + c_inst = Client.objects.get(name=client) + except ObjectDoesNotExist: + self.logger.error("Unknown client: %s" % client) + raise PluginExecutionError + except MultipleObjectsReturned: + self.logger.error("%s Inconsistency: Multiple entries for %s." % + (self.__class__.__name__, client)) + raise PluginExecutionError + try: + cls = BaseEntry.entry_from_name(e_type + "Entry") + result = cls.objects.filter(name=e_name, state=TYPE_BAD, + interaction=c_inst.current_interaction) + except ValueError: + self.logger.error("Unhandled type %s" % e_type) + raise PluginExecutionError + if not result: + raise PluginExecutionError + entry = result[0] + ret = [] + for p_entry in ('owner', 'group', 'perms'): + this_entry = getattr(entry.current_perms, p_entry) + if this_entry == '': + ret.append(getattr(entry.target_perms, p_entry)) + else: + ret.append(this_entry) + if entry.entry_type == 'Path': + if entry.is_sensitive(): + raise PluginExecutionError + elif entry.detail_type == PathEntry.DETAIL_PRUNED: + ret.append('\n'.join(entry.details)) + elif entry.is_binary(): + ret.append(b64decode(entry.details)) + elif entry.is_diff(): + ret.append('\n'.join(difflib.restore(\ + entry.details.split('\n'), 1))) + elif entry.is_too_large(): + # If len is zero the object was too large to store + raise PluginExecutionError + else: + ret.append(None) + return ret + diff --git a/src/lib/Bcfg2/Reporting/Storage/__init__.py b/src/lib/Bcfg2/Reporting/Storage/__init__.py new file mode 100644 index 000000000..85356fcfe --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/__init__.py @@ -0,0 +1,32 @@ +""" +Public storage routines +""" + +import traceback + +from Bcfg2.Reporting.Storage.base import StorageError, \ + StorageImportError + +def load_storage(storage_name, setup): + """ + Try to load the storage. Raise StorageImportError on failure + """ + try: + mod_name = "%s.%s" % (__name__, storage_name) + mod = getattr(__import__(mod_name).Reporting.Storage, storage_name) + except ImportError: + try: + mod = __import__(storage_name) + except: + raise StorageImportError("Unavailable") + try: + cls = getattr(mod, storage_name) + return cls(setup) + except: + raise StorageImportError("Storage unavailable: %s" % + traceback.format_exc().splitlines()[-1]) + +def load_storage_from_config(setup): + """Load the storage in the config... eventually""" + return load_storage('DjangoORM', setup) + diff --git a/src/lib/Bcfg2/Reporting/Storage/base.py b/src/lib/Bcfg2/Reporting/Storage/base.py new file mode 100644 index 000000000..92cc3a68b --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/base.py @@ -0,0 +1,51 @@ +""" +The base for all Storage backends +""" + +import logging + +class StorageError(Exception): + """Generic StorageError""" + pass + +class StorageImportError(StorageError): + """Raised when a storage module fails to import""" + pass + +class StorageBase(object): + """The base for all storages""" + + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + + def __init__(self, setup): + """Do something here""" + clsname = self.__class__.__name__ + self.logger = logging.getLogger(clsname) + self.logger.debug("Loading %s storage" % clsname) + self.setup = setup + self.encoding = setup['encoding'] + + def import_interaction(self, interaction): + """Import the data into the backend""" + raise NotImplementedError + + def validate(self): + """Validate backend storage. Should be called once when loaded""" + raise NotImplementedError + + def shutdown(self): + """Called at program exit""" + pass + + def Ping(self): + """Test for communication with reporting collector""" + return "Pong" + + def GetExtra(self, client): + """Return a list of extra entries for a client. Minestruct""" + raise NotImplementedError + + def GetCurrentEntry(self, client, e_type, e_name): + """Get the current status of an entry on the client""" + raise NotImplementedError + diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py new file mode 100644 index 000000000..41741ea4b --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -0,0 +1,163 @@ +""" +The local transport. Stats are pickled and written to +/store/-timestamp + +Leans on FileMonitor to detect changes +""" + +import os +import os.path +import select +import time +import traceback +import Bcfg2.Server.FileMonitor +from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError +from Bcfg2.Reporting.Transport.base import TransportBase, TransportError + +try: + import cPickle as pickle +except: + import pickle + +class LocalFilesystem(TransportBase): + def __init__(self, setup): + super(LocalFilesystem, self).__init__(setup) + + self.work_path = "%s/work" % self.data + self.logger.debug("LocalFilesystem: work path %s" % self.work_path) + self.fmon = None + self._phony_collector = None + + #setup our local paths or die + if not os.path.exists(self.work_path): + try: + os.makedirs(self.work_path) + except: + self.logger.error("%s: Unable to create storage: %s" % + (self.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + raise TransportError + + def start_monitor(self, collector): + """Start the file monitor. Most of this comes from BaseCore""" + setup = self.setup + try: + fmon = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] + except KeyError: + self.logger.error("File monitor driver %s not available; " + "forcing to default" % setup['filemonitor']) + fmon = Bcfg2.Server.FileMonitor.available['default'] + + fmdebug = setup.get('debug', False) + try: + self.fmon = fmon(debug=fmdebug) + self.logger.info("Using the %s file monitor" % self.fmon.__class__.__name__) + except IOError: + msg = "Failed to instantiate file monitor %s" % setup['filemonitor'] + self.logger.error(msg, exc_info=1) + raise TransportError(msg) + self.fmon.start() + self.fmon.AddMonitor(self.work_path, self) + + def store(self, hostname, payload): + """Store the file to disk""" + + save_file = "%s/%s-%s" % (self.work_path, hostname, time.time()) + tmp_file = "%s/.%s-%s" % (self.work_path, hostname, time.time()) + if os.path.exists(save_file): + self.logger.error("%s: Oops.. duplicate statistic in directory." % + self.__class__.__name__) + raise TransportError + + # using a tmpfile to hopefully avoid the file monitor from grabbing too + # soon + saved = open(tmp_file, 'w') + try: + saved.write(payload) + except IOError: + self.logger.error("Failed to store interaction for %s: %s" % + (hostname, traceback.format_exc().splitlines()[-1])) + os.unlink(tmp_file) + saved.close() + os.rename(tmp_file, save_file) + + def fetch(self): + """Fetch the next object""" + event = None + fmonfd = self.fmon.fileno() + if self.fmon.pending(): + event = self.fmon.get_event() + elif fmonfd: + select.select([fmonfd], [], [], self.timeout) + if self.fmon.pending(): + event = self.fmon.get_event() + else: + # pseudo.. if nothings pending sleep and loop + time.sleep(self.timeout) + + if not event or event.filename == self.work_path: + return None + + #deviate from the normal routines here we only want one event + etype = event.code2str() + self.logger.debug("Recieved event %s for %s" % (etype, event.filename)) + if os.path.basename(event.filename)[0] == '.': + return None + if etype in ('created', 'exists'): + self.logger.debug("Handling event %s" % event.filename) + payload = os.path.join(self.work_path, event.filename) + try: + payloadfd = open(payload, "r") + interaction = pickle.load(payloadfd) + payloadfd.close() + os.unlink(payload) + return interaction + except IOError: + self.logger.error("Failed to read payload: %s" % + traceback.format_exc().splitlines()[-1]) + except pickle.UnpicklingError: + self.logger.error("Failed to unpickle payload: %s" % + traceback.format_exc().splitlines()[-1]) + payloadfd.close() + raise TransportError + return None + + def shutdown(self): + """Called at program exit""" + if self.fmon: + self.fmon.shutdown() + if self._phony_collector: + self._phony_collector.shutdown() + + def rpc(self, method, *args, **kwargs): + """ + Here this is more of a dummy. Rather then start a layer + which doesn't exist or muck with files, start the collector + + This will all change when other layers are added + """ + try: + if not self._phony_collector: + self._phony_collector = ReportingCollector(self.setup) + except ReportingError: + raise TransportError + except: + self.logger.error("Failed to load collector: %s" % + traceback.format_exc().splitlines()[-1]) + raise TransportError + + if not method in self._phony_collector.storage.__class__.__rmi__ or \ + not hasattr(self._phony_collector.storage, method): + self.logger.error("Unknown method %s called on storage engine %s" % + (method, self._phony_collector.storage.__class__.__name__)) + raise TransportError + + + try: + cls_method = getattr(self._phony_collector.storage, method) + return cls_method(*args, **kwargs) + except: + self.logger.error("RPC method %s failed: %s" % + (method, traceback.format_exc().splitlines()[-1])) + raise TransportError + diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py new file mode 100644 index 000000000..ec39a1628 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -0,0 +1,32 @@ +""" +Public transport routines +""" + +import traceback + +from Bcfg2.Reporting.Transport.base import TransportError, \ + TransportImportError + +def load_transport(transport_name, setup): + """ + Try to load the transport. Raise TransportImportError on failure + """ + try: + mod_name = "%s.%s" % (__name__, transport_name) + mod = getattr(__import__(mod_name).Reporting.Transport, transport_name) + except ImportError: + try: + mod = __import__(transport_name) + except: + raise TransportImportError("Unavailable") + try: + cls = getattr(mod, transport_name) + return cls(setup) + except: + raise TransportImportError("Transport unavailable: %s" % + traceback.format_exc().splitlines()[-1]) + +def load_transport_from_config(setup): + """Load the transport in the config... eventually""" + return load_transport('LocalFilesystem', setup) + diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py new file mode 100644 index 000000000..8488d0e46 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -0,0 +1,45 @@ +""" +The base for all server -> collector Transports +""" + +import os.path +import logging + +class TransportError(Exception): + """Generic TransportError""" + pass + +class TransportImportError(TransportError): + """Raised when a transport fails to import""" + pass + +class TransportBase(object): + """The base for all transports""" + + def __init__(self, setup): + """Do something here""" + clsname = self.__class__.__name__ + self.logger = logging.getLogger(clsname) + self.logger.debug("Loading %s transport" % clsname) + self.data = os.path.join(setup['repo'], clsname.split()[-1]) + self.setup = setup + self.timeout = 2 + + def start_monitor(self, collector): + """Called to start monitoring""" + raise NotImplementedError + + def store(self, hostname, payload): + raise NotImplementedError + + def fetch(self): + raise NotImplementedError + + def shutdown(self): + """Called at program exit""" + pass + + def rpc(self, method, *args, **kwargs): + """Send a request for data to the collector""" + raise NotImplementedError + diff --git a/src/lib/Bcfg2/Reporting/__init__.py b/src/lib/Bcfg2/Reporting/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/Bcfg2/Reporting/migrate.py b/src/lib/Bcfg2/Reporting/migrate.py new file mode 100644 index 000000000..d0b3c9dc4 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrate.py @@ -0,0 +1,230 @@ +import logging +import Bcfg2.Logger +from django.core.cache import cache +from django.db import connection, transaction, backend + +from Bcfg2.Reporting import models as new_models +from Bcfg2.Server.Reports.reports import models as legacy_models + +logger = logging.getLogger(__name__) + +_our_backend = None + +def _quote(value): + """ + Quote a string to use as a table name or column + + Newer versions and various drivers require an argument + https://code.djangoproject.com/ticket/13630 + """ + global _our_backend + if not _our_backend: + try: + _our_backend = backend.DatabaseOperations(connection) + except TypeError: + _our_backend = backend.DatabaseOperations(connection) + return _our_backend.quote_name(value) + + +@transaction.commit_on_success +def _migrate_transaction(inter, entries): + """helper""" + + logger.debug("Migrating interaction %s for %s" % + (inter.id, inter.client.name)) + + newint = new_models.Interaction(id=inter.id, + client_id=inter.client_id, + timestamp=inter.timestamp, + state=inter.state, + repo_rev_code=inter.repo_rev_code, + server=inter.server, + good_count=inter.goodcount, + total_count=inter.totalcount, + bad_count=inter.bad_entries, + modified_count=inter.modified_entries, + extra_count=inter.extra_entries) + + try: + newint.profile_id = inter.metadata.profile.id + groups = [grp.pk for grp in inter.metadata.groups.all()] + bundles = [bun.pk for bun in inter.metadata.bundles.all()] + except legacy_models.InteractionMetadata.DoesNotExist: + groups = [] + bundles = [] + unkown_profile = cache.get("PROFILE_UNKNOWN") + if not unkown_profile: + unkown_profile, created = new_models.Group.objects.get_or_create(name="<>") + cache.set("PROFILE_UNKNOWN", unkown_profile) + newint.profile = unkown_profile + newint.save() + if bundles: + newint.bundles.add(*bundles) + if groups: + newint.groups.add(*groups) + + updates = dict(paths=[], packages=[], actions=[], services=[]) + for ei in legacy_models.Entries_interactions.objects.select_related('reason')\ + .filter(interaction=inter): + ent = entries[ei.entry_id] + name = ent.name + act_dict = dict(name=name, exists=ei.reason.current_exists, + state=ei.type) + + if ent.kind == 'Action': + act_dict['status'] = ei.reason.status + if not act_dict['status']: + act_dict['status'] = "check" + act_dict['output'] = -1 + logger.debug("Adding action %s" % name) + updates['actions'].append(new_models.ActionEntry.entry_get_or_create(act_dict)) + + elif ent.kind == 'Package': + act_dict['target_version'] = ei.reason.version + act_dict['current_version'] = ei.reason.current_version + logger.debug("Adding package %s %s" % + (name, act_dict['target_version'])) + updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) + elif ent.kind == 'Path': + # these might be hard.. they aren't one to one with the old model + act_dict['path_type'] = 'file' + + target_dict = dict( + owner=ei.reason.owner, + group=ei.reason.group, + perms=ei.reason.perms + ) + fperm, created = new_models.FilePerms.objects.get_or_create(**target_dict) + act_dict['target_perms'] = fperm + + current_dict = dict( + owner=ei.reason.current_owner, + group=ei.reason.current_group, + perms=ei.reason.current_perms + ) + fperm, created = new_models.FilePerms.objects.get_or_create(**current_dict) + act_dict['current_perms'] = fperm + + if ei.reason.to: + act_dict['path_type'] = 'symlink' + act_dict['target_path'] = ei.reason.to + act_dict['current_path'] = ei.reason.current_to + logger.debug("Adding link %s" % name) + updates['paths'].append(new_models.LinkEntry.entry_get_or_create(act_dict)) + continue + + act_dict['detail_type'] = new_models.PathEntry.DETAIL_UNUSED + if ei.reason.unpruned: + # this is the only other case we know what the type really is + act_dict['path_type'] = 'directory' + act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED + act_dict['details'] = ei.reason.unpruned + + + if ei.reason.is_sensitive: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE + elif ei.reason.is_binary: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_BINARY + act_dict['details'] = ei.reason.current_diff + elif ei.reason.current_diff: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_DIFF + act_dict['details'] = ei.reason.current_diff + logger.debug("Adding path %s" % name) + updates['paths'].append(new_models.PathEntry.entry_get_or_create(act_dict)) + + elif ent.kind == 'Service': + act_dict['target_status'] = ei.reason.status + act_dict['current_status'] = ei.reason.current_status + logger.debug("Adding service %s" % name) + updates['services'].append(new_models.ServiceEntry.entry_get_or_create(act_dict)) + else: + logger.warn("Skipping type %s" % ent.kind) + + for entry_type in updates.keys(): + i = 0 + while(i < len(updates[entry_type])): + getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + i += 100 + + for perf in inter.performance_items.all(): + new_models.Performance( + interaction=newint, + metric=perf.metric, + value=perf.value).save() + + +def _shove(old_table, new_table, columns): + cols = ",".join([_quote(f) for f in columns]) + sql = "insert into %s(%s) select %s from %s" % ( + _quote(new_table), + cols, + cols, + _quote(old_table)) + + cursor = connection.cursor() + cursor.execute(sql) + cursor.close() + + +@transaction.commit_manually +def _restructure(): + """major restructure of reporting data""" + + logger.info("Migrating clients") + try: + _shove(legacy_models.Client._meta.db_table, new_models.Client._meta.db_table, + ('id', 'name', 'creation', 'expiration')) + except: + logger.error("Failed to migrate clients", exc_info=1) + return False + + logger.info("Migrating Bundles") + try: + _shove(legacy_models.Bundle._meta.db_table, new_models.Bundle._meta.db_table, + ('id', 'name')) + except: + logger.error("Failed to migrate bundles", exc_info=1) + return False + + logger.info("Migrating Groups") + try: + _shove(legacy_models.Group._meta.db_table, new_models.Group._meta.db_table, + ('id', 'name', 'profile', 'public', 'category', 'comment')) + except: + logger.error("Failed to migrate groups", exc_info=1) + return False + + try: + entries = {} + for ent in legacy_models.Entries.objects.all(): + entries[ent.id] = ent + except: + logger.error("Failed to populate entries dict", exc_info=1) + return False + + transaction.commit() + + failures = [] + int_count = legacy_models.Interaction.objects.count() + int_ctr = 0 + for inter in legacy_models.Interaction.objects.select_related().all(): + if int_ctr % 1000 == 0: + logger.info("Migrated %s of %s interactions" % (int_ctr, int_count)) + try: + _migrate_transaction(inter, entries) + except: + logger.error("Failed to migrate interaction %s for %s" % + (inter.id, inter.client.name), exc_info=1) + failures.append(inter.id) + int_ctr += 1 + if not failures: + logger.info("Successfully restructured reason data") + return True + + +if __name__ == '__main__': + Bcfg2.Logger.setup_logging('bcfg2-report-collector', + to_console=logging.INFO, + level=logging.INFO) + _restructure() + diff --git a/src/lib/Bcfg2/Reporting/migrations/0001_initial.py b/src/lib/Bcfg2/Reporting/migrations/0001_initial.py new file mode 100644 index 000000000..609290edb --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0001_initial.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Client' + db.create_table('Reporting_client', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('creation', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_interaction', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='parent_client', null=True, to=orm['Reporting.Interaction'])), + ('expiration', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + )) + db.send_create_signal('Reporting', ['Client']) + + # Adding model 'Interaction' + db.create_table('Reporting_interaction', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('client', self.gf('django.db.models.fields.related.ForeignKey')(related_name='interactions', to=orm['Reporting.Client'])), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(db_index=True)), + ('state', self.gf('django.db.models.fields.CharField')(max_length=32)), + ('repo_rev_code', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('server', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('good_count', self.gf('django.db.models.fields.IntegerField')()), + ('total_count', self.gf('django.db.models.fields.IntegerField')()), + ('bad_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('modified_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('extra_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('profile', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.Group'])), + )) + db.send_create_signal('Reporting', ['Interaction']) + + # Adding unique constraint on 'Interaction', fields ['client', 'timestamp'] + db.create_unique('Reporting_interaction', ['client_id', 'timestamp']) + + # Adding M2M table for field actions on 'Interaction' + db.create_table('Reporting_interaction_actions', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('actionentry', models.ForeignKey(orm['Reporting.actionentry'], null=False)) + )) + db.create_unique('Reporting_interaction_actions', ['interaction_id', 'actionentry_id']) + + # Adding M2M table for field packages on 'Interaction' + db.create_table('Reporting_interaction_packages', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('packageentry', models.ForeignKey(orm['Reporting.packageentry'], null=False)) + )) + db.create_unique('Reporting_interaction_packages', ['interaction_id', 'packageentry_id']) + + # Adding M2M table for field paths on 'Interaction' + db.create_table('Reporting_interaction_paths', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('pathentry', models.ForeignKey(orm['Reporting.pathentry'], null=False)) + )) + db.create_unique('Reporting_interaction_paths', ['interaction_id', 'pathentry_id']) + + # Adding M2M table for field services on 'Interaction' + db.create_table('Reporting_interaction_services', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('serviceentry', models.ForeignKey(orm['Reporting.serviceentry'], null=False)) + )) + db.create_unique('Reporting_interaction_services', ['interaction_id', 'serviceentry_id']) + + # Adding M2M table for field failures on 'Interaction' + db.create_table('Reporting_interaction_failures', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('failureentry', models.ForeignKey(orm['Reporting.failureentry'], null=False)) + )) + db.create_unique('Reporting_interaction_failures', ['interaction_id', 'failureentry_id']) + + # Adding M2M table for field groups on 'Interaction' + db.create_table('Reporting_interaction_groups', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('group', models.ForeignKey(orm['Reporting.group'], null=False)) + )) + db.create_unique('Reporting_interaction_groups', ['interaction_id', 'group_id']) + + # Adding M2M table for field bundles on 'Interaction' + db.create_table('Reporting_interaction_bundles', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('bundle', models.ForeignKey(orm['Reporting.bundle'], null=False)) + )) + db.create_unique('Reporting_interaction_bundles', ['interaction_id', 'bundle_id']) + + # Adding model 'Performance' + db.create_table('Reporting_performance', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('interaction', self.gf('django.db.models.fields.related.ForeignKey')(related_name='performance_items', to=orm['Reporting.Interaction'])), + ('metric', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('value', self.gf('django.db.models.fields.DecimalField')(max_digits=32, decimal_places=16)), + )) + db.send_create_signal('Reporting', ['Performance']) + + # Adding model 'Group' + db.create_table('Reporting_group', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)), + ('profile', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('public', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('category', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + ('comment', self.gf('django.db.models.fields.TextField')(blank=True)), + )) + db.send_create_signal('Reporting', ['Group']) + + # Adding M2M table for field groups on 'Group' + db.create_table('Reporting_group_groups', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('from_group', models.ForeignKey(orm['Reporting.group'], null=False)), + ('to_group', models.ForeignKey(orm['Reporting.group'], null=False)) + )) + db.create_unique('Reporting_group_groups', ['from_group_id', 'to_group_id']) + + # Adding M2M table for field bundles on 'Group' + db.create_table('Reporting_group_bundles', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('group', models.ForeignKey(orm['Reporting.group'], null=False)), + ('bundle', models.ForeignKey(orm['Reporting.bundle'], null=False)) + )) + db.create_unique('Reporting_group_bundles', ['group_id', 'bundle_id']) + + # Adding model 'Bundle' + db.create_table('Reporting_bundle', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)), + )) + db.send_create_signal('Reporting', ['Bundle']) + + # Adding model 'FilePerms' + db.create_table('Reporting_fileperms', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('group', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('perms', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('Reporting', ['FilePerms']) + + # Adding unique constraint on 'FilePerms', fields ['owner', 'group', 'perms'] + db.create_unique('Reporting_fileperms', ['owner', 'group', 'perms']) + + # Adding model 'FileAcl' + db.create_table('Reporting_fileacl', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + )) + db.send_create_signal('Reporting', ['FileAcl']) + + # Adding model 'FailureEntry' + db.create_table('Reporting_failureentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('entry_type', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('message', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('Reporting', ['FailureEntry']) + + # Adding model 'ActionEntry' + db.create_table('Reporting_actionentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('status', self.gf('django.db.models.fields.CharField')(default='check', max_length=128)), + ('output', self.gf('django.db.models.fields.IntegerField')(default=0)), + )) + db.send_create_signal('Reporting', ['ActionEntry']) + + # Adding model 'PackageEntry' + db.create_table('Reporting_packageentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('target_version', self.gf('django.db.models.fields.CharField')(default='', max_length=1024)), + ('current_version', self.gf('django.db.models.fields.CharField')(max_length=1024)), + ('verification_details', self.gf('django.db.models.fields.TextField')(default='')), + )) + db.send_create_signal('Reporting', ['PackageEntry']) + + # Adding model 'PathEntry' + db.create_table('Reporting_pathentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('path_type', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('target_perms', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.FilePerms'])), + ('current_perms', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.FilePerms'])), + ('detail_type', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('details', self.gf('django.db.models.fields.TextField')(default='')), + )) + db.send_create_signal('Reporting', ['PathEntry']) + + # Adding M2M table for field acls on 'PathEntry' + db.create_table('Reporting_pathentry_acls', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('pathentry', models.ForeignKey(orm['Reporting.pathentry'], null=False)), + ('fileacl', models.ForeignKey(orm['Reporting.fileacl'], null=False)) + )) + db.create_unique('Reporting_pathentry_acls', ['pathentry_id', 'fileacl_id']) + + # Adding model 'LinkEntry' + db.create_table('Reporting_linkentry', ( + ('pathentry_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['Reporting.PathEntry'], unique=True, primary_key=True)), + ('target_path', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + ('current_path', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + )) + db.send_create_signal('Reporting', ['LinkEntry']) + + # Adding model 'DeviceEntry' + db.create_table('Reporting_deviceentry', ( + ('pathentry_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['Reporting.PathEntry'], unique=True, primary_key=True)), + ('device_type', self.gf('django.db.models.fields.CharField')(max_length=16)), + ('target_major', self.gf('django.db.models.fields.IntegerField')()), + ('target_minor', self.gf('django.db.models.fields.IntegerField')()), + ('current_major', self.gf('django.db.models.fields.IntegerField')()), + ('current_minor', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal('Reporting', ['DeviceEntry']) + + # Adding model 'ServiceEntry' + db.create_table('Reporting_serviceentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('target_status', self.gf('django.db.models.fields.CharField')(default='', max_length=128)), + ('current_status', self.gf('django.db.models.fields.CharField')(default='', max_length=128)), + )) + db.send_create_signal('Reporting', ['ServiceEntry']) + + + def backwards(self, orm): + # Removing unique constraint on 'FilePerms', fields ['owner', 'group', 'perms'] + db.delete_unique('Reporting_fileperms', ['owner', 'group', 'perms']) + + # Removing unique constraint on 'Interaction', fields ['client', 'timestamp'] + db.delete_unique('Reporting_interaction', ['client_id', 'timestamp']) + + # Deleting model 'Client' + db.delete_table('Reporting_client') + + # Deleting model 'Interaction' + db.delete_table('Reporting_interaction') + + # Removing M2M table for field actions on 'Interaction' + db.delete_table('Reporting_interaction_actions') + + # Removing M2M table for field packages on 'Interaction' + db.delete_table('Reporting_interaction_packages') + + # Removing M2M table for field paths on 'Interaction' + db.delete_table('Reporting_interaction_paths') + + # Removing M2M table for field services on 'Interaction' + db.delete_table('Reporting_interaction_services') + + # Removing M2M table for field failures on 'Interaction' + db.delete_table('Reporting_interaction_failures') + + # Removing M2M table for field groups on 'Interaction' + db.delete_table('Reporting_interaction_groups') + + # Removing M2M table for field bundles on 'Interaction' + db.delete_table('Reporting_interaction_bundles') + + # Deleting model 'Performance' + db.delete_table('Reporting_performance') + + # Deleting model 'Group' + db.delete_table('Reporting_group') + + # Removing M2M table for field groups on 'Group' + db.delete_table('Reporting_group_groups') + + # Removing M2M table for field bundles on 'Group' + db.delete_table('Reporting_group_bundles') + + # Deleting model 'Bundle' + db.delete_table('Reporting_bundle') + + # Deleting model 'FilePerms' + db.delete_table('Reporting_fileperms') + + # Deleting model 'FileAcl' + db.delete_table('Reporting_fileacl') + + # Deleting model 'FailureEntry' + db.delete_table('Reporting_failureentry') + + # Deleting model 'ActionEntry' + db.delete_table('Reporting_actionentry') + + # Deleting model 'PackageEntry' + db.delete_table('Reporting_packageentry') + + # Deleting model 'PathEntry' + db.delete_table('Reporting_pathentry') + + # Removing M2M table for field acls on 'PathEntry' + db.delete_table('Reporting_pathentry_acls') + + # Deleting model 'LinkEntry' + db.delete_table('Reporting_linkentry') + + # Deleting model 'DeviceEntry' + db.delete_table('Reporting_deviceentry') + + # Deleting model 'ServiceEntry' + db.delete_table('Reporting_serviceentry') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'perms'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'perms': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.Group']"}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + } + } + + complete_apps = ['Reporting'] \ No newline at end of file diff --git a/src/lib/Bcfg2/Reporting/migrations/__init__.py b/src/lib/Bcfg2/Reporting/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py new file mode 100644 index 000000000..3a540587a --- /dev/null +++ b/src/lib/Bcfg2/Reporting/models.py @@ -0,0 +1,582 @@ +"""Django models for Bcfg2 reports.""" +import sys + +from django.core.exceptions import ImproperlyConfigured +try: + from django.db import models +except ImproperlyConfigured: + e = sys.exc_info()[1] + print("Reports: unable to import django models: %s" % e) + sys.exit(1) + +from django.core.cache import cache +from datetime import datetime, timedelta + +try: + import cPickle as pickle +except: + import pickle + +KIND_CHOICES = ( + #These are the kinds of config elements + ('Package', 'Package'), + ('Path', 'directory'), + ('Path', 'file'), + ('Path', 'permissions'), + ('Path', 'symlink'), + ('Service', 'Service'), +) +TYPE_GOOD = 0 +TYPE_BAD = 1 +TYPE_MODIFIED = 2 +TYPE_EXTRA = 3 + +TYPE_CHOICES = ( + (TYPE_GOOD, 'Good'), + (TYPE_BAD, 'Bad'), + (TYPE_MODIFIED, 'Modified'), + (TYPE_EXTRA, 'Extra'), +) + + +def convert_entry_type_to_id(type_name): + """Convert a entry type to its entry id""" + for e_id, e_name in TYPE_CHOICES: + if e_name.lower() == type_name.lower(): + return e_id + return -1 + + +def hash_entry(entry_dict): + """ + Build a key for this based on its data + + entry_dict = a dict of all the data identifying this + """ + dataset = [] + for key in sorted(entry_dict.keys()): + if key in ('id', 'hash_key') or key.startswith('_'): + continue + dataset.append( (key, entry_dict[key]) ) + return hash(pickle.dumps(dataset)) + + +class Client(models.Model): + """Object representing every client we have seen stats for.""" + creation = models.DateTimeField(auto_now_add=True) + name = models.CharField(max_length=128,) + current_interaction = models.ForeignKey('Interaction', + null=True, blank=True, + related_name="parent_client") + expiration = models.DateTimeField(blank=True, null=True) + + def __str__(self): + return self.name + + +class InteractionManager(models.Manager): + """Manages interactions objects.""" + + def recent_ids(self, maxdate=None): + """ + Returns the ids of most recent interactions for clients as of a date. + + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + + """ + from django.db import connection + cursor = connection.cursor() + cfilter = "expiration is null" + + sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \ + 'as timer from Reporting_interaction' + if maxdate: + if not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + sql = sql + " where timestamp <= '%s' " % maxdate + cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) + sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \ + 'ri.client_id = x.client_id AND ri.timestamp = x.timer' + sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter + try: + cursor.execute(sql) + return [item[0] for item in cursor.fetchall()] + except: + '''FIXME - really need some error handling''' + pass + return [] + + + def recent(self, maxdate=None): + """ + Returns the most recent interactions for clients as of a date + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + + """ + if maxdate and not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + return self.filter(id__in=self.recent_ids(maxdate)) + + +class Interaction(models.Model): + """Models each reconfiguration operation interaction between client and server.""" + client = models.ForeignKey(Client, related_name="interactions") + timestamp = models.DateTimeField(db_index=True) # Timestamp for this record + state = models.CharField(max_length=32) # good/bad/modified/etc + repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + server = models.CharField(max_length=256) # Name of the server used for the interaction + good_count = models.IntegerField() # of good config-items + total_count = models.IntegerField() # of total config-items + bad_count = models.IntegerField(default=0) + modified_count = models.IntegerField(default=0) + extra_count = models.IntegerField(default=0) + + actions = models.ManyToManyField("ActionEntry") + packages = models.ManyToManyField("PackageEntry") + paths = models.ManyToManyField("PathEntry") + services = models.ManyToManyField("ServiceEntry") + failures = models.ManyToManyField("FailureEntry") + + # Formerly InteractionMetadata + profile = models.ForeignKey("Group", related_name="+") + groups = models.ManyToManyField("Group") + bundles = models.ManyToManyField("Bundle") + + objects = InteractionManager() + + def __str__(self): + return "With " + self.client.name + " @ " + self.timestamp.isoformat() + + def percentgood(self): + if not self.total_count == 0: + return (self.good_count / float(self.total_count)) * 100 + else: + return 0 + + def percentbad(self): + if not self.total_count == 0: + return ((self.total_count - self.good_count) / (float(self.total_count))) * 100 + else: + return 0 + + def isclean(self): + if (self.bad_count == 0 and self.good_count == self.total_count): + return True + else: + return False + + def isstale(self): + if (self == self.client.current_interaction): # Is Mostrecent + if(datetime.now() - self.timestamp > timedelta(hours=25)): + return True + else: + return False + else: + #Search for subsequent Interaction for this client + #Check if it happened more than 25 hrs ago. + if (self.client.interactions.filter(timestamp__gt=self.timestamp) + .order_by('timestamp')[0].timestamp - + self.timestamp > timedelta(hours=25)): + return True + else: + return False + + def save(self): + super(Interaction, self).save() # call the real save... + self.client.current_interaction = self.client.interactions.latest() + self.client.save() # save again post update + + def delete(self): + '''Override the default delete. Allows us to remove Performance items''' + pitems = list(self.performance_items.all()) + super(Interaction, self).delete() + for perf in pitems: + if perf.interaction.count() == 0: + perf.delete() + + def badcount(self): + return self.total_count - self.good_count + + def bad(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_BAD)) + return rv + + def modified(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_MODIFIED)) + return rv + + def extra(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_EXTRA)) + return rv + + class Meta: + get_latest_by = 'timestamp' + ordering = ['-timestamp'] + unique_together = ("client", "timestamp") + + +class Performance(models.Model): + """Object representing performance data for any interaction.""" + interaction = models.ForeignKey(Interaction, related_name="performance_items") + metric = models.CharField(max_length=128) + value = models.DecimalField(max_digits=32, decimal_places=16) + + def __str__(self): + return self.metric + + +class Group(models.Model): + """ + Groups extracted from interactions + + name - The group name + + TODO - Most of this is for future use + TODO - set a default group + """ + + name = models.CharField(max_length=255, unique=True) + profile = models.BooleanField(default=False) + public = models.BooleanField(default=False) + category = models.CharField(max_length=1024, blank=True) + comment = models.TextField(blank=True) + + groups = models.ManyToManyField("self", symmetrical=False) + bundles = models.ManyToManyField("Bundle") + + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + +class Bundle(models.Model): + """ + Bundles extracted from interactions + + name - The bundle name + """ + + name = models.CharField(max_length=255, unique=True) + + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + +# new interaction models +class FilePerms(models.Model): + owner = models.CharField(max_length=128) + group = models.CharField(max_length=128) + perms = models.CharField(max_length=128) + + class Meta: + unique_together = ('owner', 'group', 'perms') + + def empty(self): + """Return true if we have no real data""" + if self.owner or self.group or self.perms: + return False + else: + return True + + +class FileAcl(models.Model): + """Placeholder""" + name = models.CharField(max_length=128, db_index=True) + + +class BaseEntry(models.Model): + """ Abstract base for all entry types """ + name = models.CharField(max_length=128, db_index=True) + hash_key = models.IntegerField(editable=False, db_index=True) + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + if 'hash_key' in kwargs: + self.hash_key = kwargs['hash_key'] + del kwargs['hash_key'] + else: + self.hash_key = hash_entry(self.__dict__) + super(BaseEntry, self).save(*args, **kwargs) + + + def class_name(self): + return self.__class__.__name__ + + def short_list(self): + """todo""" + return [] + + + @classmethod + def entry_from_name(cls, name): + try: + newcls = globals()[name] + if not isinstance(newcls(), cls): + raise ValueError("%s is not an instance of %s" % (name, cls)) + return newcls + except KeyError: + raise ValueError("Invalid type %s" % name) + + + @classmethod + def entry_get_or_create(cls, act_dict): + """Helper to quickly lookup an object""" + cls_name = cls().__class__.__name__ + act_hash = hash_entry(act_dict) + + # TODO - get form cache and validate + act_key = "%s_%s" % (cls_name, act_hash) + newact = cache.get(act_key) + if newact: + return newact + + acts = cls.objects.filter(hash_key=act_hash) + if len(acts) > 0: + for act in acts: + for key in act_dict: + if act_dict[key] != getattr(act, key): + continue + #match found + newact = act + break + + # worst case, its new + if not newact: + newact = cls(**act_dict) + newact.save(hash_key=act_hash) + + cache.set(act_key, newact) + return newact + + + def is_failure(self): + return isinstance(self, FailureEntry) + + +class SuccessEntry(BaseEntry): + """Base for successful entries""" + state = models.IntegerField(choices=TYPE_CHOICES) + exists = models.BooleanField(default=True) + + ENTRY_TYPE = r"Success" + + @property + def entry_type(self): + return self.ENTRY_TYPE + + def is_extra(self): + return self.state == TYPE_EXTRA + + class Meta: + abstract = True + ordering = ('state', 'name') + + def short_list(self): + """Return a list of problems""" + rv = [] + if self.is_extra(): + rv.append("Extra") + elif not self.exists: + rv.append("Missing") + return rv + + +class FailureEntry(BaseEntry): + """Represents objects that failed to bind""" + entry_type = models.CharField(max_length=128) + message = models.TextField() + + def is_failure(self): + return True + + +class ActionEntry(SuccessEntry): + """ The new model for package information """ + status = models.CharField(max_length=128, default="check") + output = models.IntegerField(default=0) + + ENTRY_TYPE = r"Action" + #TODO - prune + + +class PackageEntry(SuccessEntry): + """ The new model for package information """ + + # if this is an extra entry trget_version will be empty + target_version = models.CharField(max_length=1024, default='') + current_version = models.CharField(max_length=1024) + verification_details = models.TextField(default="") + + ENTRY_TYPE = r"Package" + #TODO - prune + + def version_problem(self): + """Check for a version problem.""" + if not self.current_version: + return True + if self.target_version != self.current_version: + return True + elif self.target_version == 'auto': + return True + else: + return False + + def short_list(self): + """Return a list of problems""" + rv = super(PackageEntry, self).short_list() + if self.is_extra(): + return rv + if not self.version_problem() or not self.exists: + return rv + if not self.current_version: + rv.append("Missing") + else: + rv.append("Wrong version") + return rv + + +class PathEntry(SuccessEntry): + """reason why modified or bad entry did not verify, or changed.""" + + PATH_TYPES = ( + ("device", "Device"), + ("directory", "Directory"), + ("hardlink", "Hard Link"), + ("nonexistent", "Non Existent"), + ("permissions", "Permissions"), + ("symlink", "Symlink"), + ) + + DETAIL_UNUSED = 0 + DETAIL_DIFF = 1 + DETAIL_BINARY = 2 + DETAIL_SENSITIVE = 3 + DETAIL_SIZE_LIMIT = 4 + DETAIL_VCS = 5 + DETAIL_PRUNED = 6 + + DETAIL_CHOICES = ( + (DETAIL_UNUSED, 'Unused'), + (DETAIL_DIFF, 'Diff'), + (DETAIL_BINARY, 'Binary'), + (DETAIL_SENSITIVE, 'Sensitive'), + (DETAIL_SIZE_LIMIT, 'Size limit exceeded'), + (DETAIL_VCS, 'VCS output'), + (DETAIL_PRUNED, 'Pruned paths'), + ) + + path_type = models.CharField(max_length=128, choices=PATH_TYPES) + + target_perms = models.ForeignKey(FilePerms, related_name="+") + current_perms = models.ForeignKey(FilePerms, related_name="+") + + acls = models.ManyToManyField(FileAcl) + + detail_type = models.IntegerField(default=0, + choices=DETAIL_CHOICES) + details = models.TextField(default='') + + ENTRY_TYPE = r"Path" + + def perms_problem(self): + if self.current_perms.empty(): + return False + elif self.target_perms.perms != self.current_perms.perms: + return True + else: + return False + + def has_detail(self): + return self.detail_type != PathEntry.DETAIL_UNUSED + + def is_sensitive(self): + return self.detail_type == PathEntry.DETAIL_SENSITIVE + + def is_diff(self): + return self.detail_type == PathEntry.DETAIL_DIFF + + def is_sensitive(self): + return self.detail_type == PathEntry.DETAIL_SENSITIVE + + def is_binary(self): + return self.detail_type == PathEntry.DETAIL_BINARY + + def is_too_large(self): + return self.detail_type == PathEntry.DETAIL_SIZE_LIMIT + + def short_list(self): + """Return a list of problems""" + rv = super(PathEntry, self).short_list() + if self.is_extra(): + return rv + if self.perms_problem(): + rv.append("File permissions") + if self.detail_type == PathEntry.DETAIL_PRUNED: + rv.append("Directory has extra files") + elif self.detail_type != PathEntry.DETAIL_UNUSED: + rv.append("Incorrect data") + if hasattr(self, 'linkentry') and \ + self.linkentry.target_path != self.linkentry.current_path: + rv.append("Incorrect target") + return rv + + +class LinkEntry(PathEntry): + """Sym/Hard Link types""" + target_path = models.CharField(max_length=1024, blank=True) + current_path = models.CharField(max_length=1024, blank=True) + + def link_problem(self): + return self.target_path != self.current_path + + +class DeviceEntry(PathEntry): + """Device types. Best I can tell the client driver needs work here""" + DEVICE_TYPES = ( + ("block", "Block"), + ("char", "Char"), + ("fifo", "Fifo"), + ) + + device_type = models.CharField(max_length=16, choices=DEVICE_TYPES) + + target_major = models.IntegerField() + target_minor = models.IntegerField() + current_major = models.IntegerField() + current_minor = models.IntegerField() + + +class ServiceEntry(SuccessEntry): + """ The new model for package information """ + target_status = models.CharField(max_length=128, default='') + current_status = models.CharField(max_length=128, default='') + + ENTRY_TYPE = r"Service" + #TODO - prune + + def status_problem(self): + return self.target_status != self.current_status + + def short_list(self): + """Return a list of problems""" + rv = super(ServiceEntry, self).short_list() + if self.status_problem(): + rv.append("Incorrect status") + return rv + + diff --git a/src/lib/Bcfg2/Reporting/templates/404.html b/src/lib/Bcfg2/Reporting/templates/404.html new file mode 100644 index 000000000..168bd9fec --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/404.html @@ -0,0 +1,8 @@ +{% extends 'base.html' %} +{% block title %}Bcfg2 - Page not found{% endblock %} +{% block fullcontent %} +

Page not found

+

+The page or object requested could not be found. +

+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/base-timeview.html b/src/lib/Bcfg2/Reporting/templates/base-timeview.html new file mode 100644 index 000000000..9a5ef651c --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/base-timeview.html @@ -0,0 +1,28 @@ +{% extends "base.html" %} + +{% block timepiece %} + +{% if not timestamp %}Rendered at {% now "Y-m-d H:i" %} | {% else %}View as of {{ timestamp|date:"Y-m-d H:i" }} | {% endif %}{% spaceless %} + [change] +
+ + +
+{% endspaceless %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html new file mode 100644 index 000000000..6d20f86d9 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -0,0 +1,96 @@ +{% load bcfg2_tags %} + + + + + +{% block title %}Bcfg2 Reporting System{% endblock %} + + + + + + + + + + + + + +{% block extra_header_info %}{% endblock %} + + + + + + +
+
+ {% block fullcontent %} +
+

{% block pagebanner %}Page Banner{% endblock %}

+
{% block timepiece %}Rendered at {% now "Y-m-d H:i" %}{% endblock %}
+
+
+ {% block content %}{% endblock %} +
+ {% endblock %} +
+
+ {% block sidemenu %} +
    +
  • Overview
    • +
+ +
    +
  • Clients
    • +
+ +
    +
  • Entries Configured
    • +
+ +{% comment %} + TODO +
    +
  • Entry Types
    • +
+ +{% endcomment %} + + {% endblock %} +
+
+
+ + + + + diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html new file mode 100644 index 000000000..b2244bfa1 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -0,0 +1,149 @@ +{% extends "base.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Client {{client.name}}{% endblock %} + +{% block extra_header_info %} + +{% endblock %} + +{% block body_onload %}javascript:clientdetailload(){% endblock %} + +{% block pagebanner %}Client Details{% endblock %} + +{% block content %} +
+

{{client.name}}

+ [manage] + View History | Jump to  + +
+ + {% if interaction.isstale %} +
+ This node did not run within the last 24 hours — it may be out of date. +
+ {% endif %} + + + {% if interaction.server %} + + {% endif %} + + {% if interaction.repo_rev_code %} + + {% endif %} + + + {% if not interaction.isclean %} + + {% endif %} +
Timestamp{{interaction.timestamp}}
Served by{{interaction.server}}
Profile{{interaction.profile}}
Revision{{interaction.repo_rev_code}}
State{{interaction.state|capfirst}}
Managed entries{{interaction.total_count}}
Deviation{{interaction.percentbad|floatformat:"3"}}%
+ + {% for group in interaction.groups.all %} + {% if forloop.first %} +
+
+

Group membership

+
[+]
+
+ + {% endif %} + + + + {% if forloop.last %} + +
+ {% endif %} + {% endfor %} + + {% for bundle in interaction.bundles.all %} + {% if forloop.first %} +
+
+

Bundle membership

+
[+]
+
+ + {% endif %} + + + + {% if forloop.last %} + +
+ {% endif %} + {% endfor %} + + {% for entry_type, entry_list in entry_types.items %} + {% if entry_list %} +
+
+

{{ entry_type|capfirst }} Entries — {{ entry_list|length }}

+
[+]
+
+ + {% for entry in entry_list %} + + + + + {% endfor %} +
{{entry.entry_type}} + {{entry.name}}
+
+ {% endif %} + {% endfor %} + + {% if interaction.failures.all %} +
+
+

Failed entries

+
[+]
+
+ + {% for failure in interaction.failures.all %} + + + + + {% endfor %} + +
+ {% endif %} + + {% if entry_list %} +
+
+

Recent Interactions

+
+
+ {% include "widgets/interaction_list.inc" %} + +
+
+ {% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html new file mode 100644 index 000000000..06c99d899 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -0,0 +1,46 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Detailed Client Listing{% endblock %} +{% block pagebanner %}Clients - Detailed View{% endblock %} + +{% block content %} +
+ {% filter_navigator %} +{% if entry_list %} + + + + + + + + + + + + {% for entry in entry_list %} + + + + + + + + + + + {% endfor %} +
{% sort_link 'client' 'Node' %}{% sort_link 'state' 'State' %}{% sort_link '-good' 'Good' %}{% sort_link '-bad' 'Bad' %}{% sort_link '-modified' 'Modified' %}{% sort_link '-extra' 'Extra' %}{% sort_link 'timestamp' 'Last Run' %}{% sort_link 'server' 'Server' %}
{{ entry.client.name }}{{ entry.state }}{{ entry.good_count }}{{ entry.bad_count }}{{ entry.modified_count }}{{ entry.extra_count }}{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }} + {% if entry.server %} + {{ entry.server }} + {% else %} +   + {% endif %} +
+{% else %} +

No client records are available.

+{% endif %} +
+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/history.html b/src/lib/Bcfg2/Reporting/templates/clients/history.html new file mode 100644 index 000000000..01d4ec2f4 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/clients/history.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Interaction History{% endblock %} +{% block pagebanner %}Interaction history{% if client %} for {{ client.name }}{% endif %}{% endblock %} + +{% block extra_header_info %} +{% endblock %} + +{% block content %} +
+{% if entry_list %} + {% filter_navigator %} + {% include "widgets/interaction_list.inc" %} +{% else %} +

No client records are available.

+{% endif %} +
+{% page_navigator %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html new file mode 100644 index 000000000..45ba20b86 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -0,0 +1,35 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block extra_header_info %} +{% endblock%} + +{% block title %}Bcfg2 - Client Grid View{% endblock %} + +{% block pagebanner %}Clients - Grid View{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if inter_list %} + + {% for inter in inter_list %} + {% if forloop.first %}{% endif %} + + {% if forloop.last %} + + {% else %} + {% if forloop.counter|divisibleby:"4" %}{% endif %} + {% endif %} + {% endfor %} +
+ {{ inter.client.name }} +
+{% else %}

No client records are available.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html new file mode 100644 index 000000000..443ec8ccb --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} + +{% block extra_header_info %} +{% endblock%} + +{% block title %}Bcfg2 - Manage Clients{% endblock %} + +{% block pagebanner %}Clients - Manage{% endblock %} + +{% block content %} +
+ {% if message %} +
{{ message }}
+ {% endif %} +{% if clients %} + + + + + + + {% for client in clients %} + + + + + + {% endfor %} +
NodeExpirationManage
+ + + {{ client.name }}{% firstof client.expiration 'Active' %} +
+
{# here for no reason other then to validate #} + + + +
+
+
+{% else %} +

No client records are available.

+{% endif %} +
+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html new file mode 100644 index 000000000..b39957a2e --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -0,0 +1,42 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Common Problems{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}Common configuration problems{% endblock %} + +{% block content %} +
+
+ Showing items with more then {{ threshold }} entries + + +
+
+ {% for type_name, type_list in lists %} +
+
+

{{ type_name|capfirst }} entries

+
[–]
+
+ {% if type_list %} + + + {% for item in type_list %} + + + + + + + {% endfor %} +
TypeNameCountReason
{{ item.ENTRY_TYPE }}{{ item.name }}{{ item.num_entries }}{{ item.short_list|join:"," }}
+ {% else %} +

There are currently no inconsistent {{ type_name }} configuration entries.

+ {% endif %} +
+ {% endfor %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html new file mode 100644 index 000000000..e940889ab --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -0,0 +1,32 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Entry Status{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}{{ entry.entry_type }} entry {{ entry.name }} status{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if items %} +
+ + + {% for item, inters in items %} + {% for inter in inters %} + + + + + + + {% endfor %} + {% endfor %} +
NameTimestampStateReason
{{inter.client.name}}{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}{{ item.get_state_display }}({{item.pk}}) {{item.short_list|join:","}}
+
+{% else %} +

There are currently no hosts with this configuration entry.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html b/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html new file mode 100644 index 000000000..0b87fbdbd --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html @@ -0,0 +1,13 @@ +{% extends "config_items/item.html" %} +{% load syntax_coloring %} + +{% block item_details %} +
+
+

This item failed to bind on the server

+
+
+ {{ item.message|syntaxhilight:"py" }} +
+
+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html new file mode 100644 index 000000000..4c2e9c2ae --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -0,0 +1,136 @@ +{% extends "base.html" %} +{% load split %} +{% load syntax_coloring %} + + +{% block title %}Bcfg2 - Element Details{% endblock %} + + +{% block extra_header_info %} + +{% endblock%} + +{% block pagebanner %}Element Details{% endblock %} + +{% block content %} +
+

{{item.get_state_display}} {{item.entry_type}}: {{item.name}}

+
+ +
+{% block item_details %} + {% if item.is_extra %} +

This item exists on the host but is not defined in the configuration.

+ {% endif %} + + {% if not item.exists %} +
This item does not currently exist on the host but is specified to exist in the configuration.
+ {% endif %} + +{# Really need a better test here #} +{% if item.perms_problem or item.status_problem or item.linkentry.link_problem or item.version_problem %} + + + + {% if item.perms_problem %} + {% if item.current_perms.owner %} + + + {% endif %} + {% if item.current_perms.group %} + + + {% endif %} + {% if item.current_perms.perms %} + + + {% endif %} + {% endif %} + {% if item.status_problem %} + + + {% endif %} + {% if item.linkentry.link_problem %} + + + {% endif %} + {% if item.version_problem %} + + + {% endif %} +
Problem TypeExpectedFound
Owner{{item.target_perms.owner}}{{item.current_perms.owner}}
Group{{item.target_perms.group}}{{item.current_perms.group}}
Perms{{item.target_perms.perms}}{{item.current_perms.perms}}
Status{{item.target_status}}{{item.current_status}}
{{item.get_path_type_display}}{{item.linkentry.target_path}}{{item.linkentry.current_path}}
Package Version{{item.target_version|cut:"("|cut:")"}}{{item.current_version|cut:"("|cut:")"}}
+{% endif %} + + {% if item.has_detail %} +
+
+ {% if item.is_sensitive %} +

File contents unavailable, as they might contain sensitive data.

+ {% else %} +

Incorrect file contents ({{item.get_detail_type_display}})

+ {% endif %} +
+ {% if item.is_diff %} +
+ {{ item.details|syntaxhilight }} +
+ {% else %} + {{ item.details }} + {% endif %} +
+ {% endif %} + + + {% if item.reason.unpruned %} +
+
+

Extra entries found

+
+ + {% for unpruned_item in item.reason.unpruned|split %} + + {% endfor %} +
{{ unpruned_item }}
+
+ {% endif %} +{% endblock %} + + +
+
+

Occurences on {{ timestamp|date:"Y-m-d" }}

+
+ {% if associated_list %} + + {% for inter in associated_list %} + + + + {% endfor %} +
{{inter.client.name}}{{inter.timestamp}}
+ {% else %} +

Missing client list

+ {% endif %} +
+ +
+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html new file mode 100644 index 000000000..864392754 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -0,0 +1,35 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Element Listing{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}{{item_state|capfirst}} Element Listing{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if item_list %} + {% for type_name, type_data in item_list %} +
+
+

{{ type_name }} — {{ type_data|length }}

+
[–]
+
+ + + {% for entry in type_data %} + + + + + + {% endfor %} +
NameCountReason
{{entry.name}}{{entry.num_entries}}{{entry.short_list|join:","}}
+
+ {% endfor %} +{% else %} +

There are currently no inconsistent configuration entries.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html new file mode 100644 index 000000000..b9847cf96 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -0,0 +1,42 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Client Summary{% endblock %} +{% block pagebanner %}Clients - Summary{% endblock %} + +{% block body_onload %}javascript:hide_table_array(hide_tables){% endblock %} + +{% block extra_header_info %} + +{% endblock%} + +{% block content %} +
+

{{ node_count }} nodes reporting in

+
+{% if summary_data %} + {% for summary in summary_data %} +
+
+

{{ summary.nodes|length }} {{ summary.label }}

+
[+]
+
+ + + {% for node in summary.nodes|sort_interactions_by_name %} + + + + {% endfor %} +
{{ node.client.name }}
+
+ {% endfor %} +{% else %} +

No data to report on

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html new file mode 100644 index 000000000..ff775ded5 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -0,0 +1,38 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Performance Metrics{% endblock %} +{% block pagebanner %}Performance Metrics{% endblock %} + + +{% block extra_header_info %} +{% endblock%} + +{% block content %} +
+ {% if metrics %} + + + + + + + + + + + {% for metric in metrics|dictsort:"name" %} + + + {% for mitem in metric|build_metric_list %} + + {% endfor %} + + {% endfor %} +
NameParseProbeInventoryInstallConfigTotal
{{ metric.name }}{{ mitem }}
+ {% else %} +

No metric data available

+ {% endif %} +
+{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html new file mode 100644 index 000000000..759415507 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html @@ -0,0 +1,25 @@ +{% spaceless %} +
+
+{% if filters %} +{% for filter, filter_url in filters %} + {% if forloop.first %} + Active filters (click to remove): + {% endif %} + {{ filter|capfirst }}{% if not forloop.last %}, {% endif %} + {% if forloop.last %} + {% if groups %}|{% endif %} + {% endif %} +{% endfor %} +{% endif %} +{% if groups %} + + +{% endif %} +
+
+{% endspaceless %} diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc new file mode 100644 index 000000000..30ed2fd3e --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc @@ -0,0 +1,38 @@ +{% load bcfg2_tags %} +
+ + + + {% if not client %} + + {% endif %} + + + + + + + + {% for entry in entry_list %} + + + {% if not client %} + + {% endif %} + + + + + + + + {% endfor %} +
TimestampClientStateGoodBadModifiedExtraServer
{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}{{ entry.client.name }}{{ entry.state }}{{ entry.good_count }}{{ entry.bad_count }}{{ entry.modified_count }}{{ entry.extra_count }} + {% if entry.server %} + {{ entry.server }} + {% else %} +   + {% endif %} +
+
diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html new file mode 100644 index 000000000..aa0def83e --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html @@ -0,0 +1,23 @@ +{% spaceless %} +{% for page, page_url in pager %} + {% if forloop.first %} +
+ {% if prev_page %}< Prev {% endif %} + {% if first_page %}1 ... {% endif %} + {% endif %} + {% ifequal page current_page %} + {{ page }} + {% else %} + {{ page }} + {% endifequal %} + {% if forloop.last %} + {% if last_page %} ... {{ total_pages }} {% endif %} + {% if next_page %}Next > {% endif %} + |{% for limit, limit_url in page_limits %} {{ limit }}{% endfor %} +
+ {% else %} +   + {% endif %} +{% endfor %} +{% endspaceless %} + diff --git a/src/lib/Bcfg2/Reporting/templatetags/__init__.py b/src/lib/Bcfg2/Reporting/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py new file mode 100644 index 000000000..c079f4a3c --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -0,0 +1,415 @@ +import sys +from copy import copy + +from django import template +from django.conf import settings +from django.core.urlresolvers import resolve, reverse, \ + Resolver404, NoReverseMatch +from django.template.loader import get_template, \ + get_template_from_string,TemplateDoesNotExist +from django.utils.encoding import smart_unicode, smart_str +from django.utils.safestring import mark_safe +from datetime import datetime, timedelta +from Bcfg2.Reporting.utils import filter_list +from Bcfg2.Reporting.models import Group + +register = template.Library() + +__PAGE_NAV_LIMITS__ = (10, 25, 50, 100) + + +@register.inclusion_tag('widgets/page_bar.html', takes_context=True) +def page_navigator(context): + """ + Creates paginated links. + + Expects the context to be a RequestContext and + views.prepare_paginated_list() to have populated page information. + """ + fragment = dict() + try: + path = context['request'].META['PATH_INFO'] + total_pages = int(context['total_pages']) + records_per_page = int(context['records_per_page']) + except KeyError: + return fragment + except ValueError: + return fragment + + if total_pages < 2: + return {} + + try: + view, args, kwargs = resolve(path) + current_page = int(kwargs.get('page_number', 1)) + fragment['current_page'] = current_page + fragment['page_number'] = current_page + fragment['total_pages'] = total_pages + fragment['records_per_page'] = records_per_page + if current_page > 1: + kwargs['page_number'] = current_page - 1 + fragment['prev_page'] = reverse(view, args=args, kwargs=kwargs) + if current_page < total_pages: + kwargs['page_number'] = current_page + 1 + fragment['next_page'] = reverse(view, args=args, kwargs=kwargs) + + view_range = 5 + if total_pages > view_range: + pager_start = current_page - 2 + pager_end = current_page + 2 + if pager_start < 1: + pager_end += (1 - pager_start) + pager_start = 1 + if pager_end > total_pages: + pager_start -= (pager_end - total_pages) + pager_end = total_pages + else: + pager_start = 1 + pager_end = total_pages + + if pager_start > 1: + kwargs['page_number'] = 1 + fragment['first_page'] = reverse(view, args=args, kwargs=kwargs) + if pager_end < total_pages: + kwargs['page_number'] = total_pages + fragment['last_page'] = reverse(view, args=args, kwargs=kwargs) + + pager = [] + for page in range(pager_start, int(pager_end) + 1): + kwargs['page_number'] = page + pager.append((page, reverse(view, args=args, kwargs=kwargs))) + + kwargs['page_number'] = 1 + page_limits = [] + for limit in __PAGE_NAV_LIMITS__: + kwargs['page_limit'] = limit + page_limits.append((limit, + reverse(view, args=args, kwargs=kwargs))) + # resolver doesn't like this + del kwargs['page_number'] + del kwargs['page_limit'] + page_limits.append(('all', + reverse(view, args=args, kwargs=kwargs) + "|all")) + + fragment['pager'] = pager + fragment['page_limits'] = page_limits + + except Resolver404: + path = "404" + except NoReverseMatch: + nr = sys.exc_info()[1] + path = "NoReverseMatch: %s" % nr + except ValueError: + path = "ValueError" + #FIXME - Handle these + + fragment['path'] = path + return fragment + + +@register.inclusion_tag('widgets/filter_bar.html', takes_context=True) +def filter_navigator(context): + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + filters.append((filter, + reverse(view, args=args, kwargs=myargs))) + filters.sort(lambda x, y: cmp(x[0], y[0])) + + myargs = kwargs.copy() + selected=True + if 'group' in myargs: + del myargs['group'] + selected=False + groups = [('---', reverse(view, args=args, kwargs=myargs), selected)] + for group in Group.objects.values('name'): + myargs['group'] = group['name'] + groups.append((group['name'], reverse(view, args=args, kwargs=myargs), + group['name'] == kwargs.get('group', ''))) + + return {'filters': filters, 'groups': groups} + except (Resolver404, NoReverseMatch, ValueError, KeyError): + pass + return dict() + + +def _subtract_or_na(mdict, x, y): + """ + Shortcut for build_metric_list + """ + try: + return round(mdict[x] - mdict[y], 4) + except: + return "n/a" + + +@register.filter +def build_metric_list(mdict): + """ + Create a list of metric table entries + + Moving this here to simplify the view. + Should really handle the case where these are missing... + """ + td_list = [] + # parse + td_list.append(_subtract_or_na(mdict, 'config_parse', 'config_download')) + #probe + td_list.append(_subtract_or_na(mdict, 'probe_upload', 'start')) + #inventory + td_list.append(_subtract_or_na(mdict, 'inventory', 'initialization')) + #install + td_list.append(_subtract_or_na(mdict, 'install', 'inventory')) + #cfg download & parse + td_list.append(_subtract_or_na(mdict, 'config_parse', 'probe_upload')) + #total + td_list.append(_subtract_or_na(mdict, 'finished', 'start')) + return td_list + + +@register.filter +def isstale(timestamp, entry_max=None): + """ + Check for a stale timestamp + + Compares two timestamps and returns True if the + difference is greater then 24 hours. + """ + if not entry_max: + entry_max = datetime.now() + return entry_max - timestamp > timedelta(hours=24) + + +@register.filter +def sort_interactions_by_name(value): + """ + Sort an interaction list by client name + """ + inters = list(value) + inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) + return inters + + +class AddUrlFilter(template.Node): + def __init__(self, filter_name, filter_value): + self.filter_name = filter_name + self.filter_value = filter_value + self.fallback_view = 'Bcfg2.Reporting.views.render_history_view' + + def render(self, context): + link = '#' + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + filter_value = self.filter_value.resolve(context, True) + if filter_value: + filter_name = smart_str(self.filter_name) + filter_value = smart_unicode(filter_value) + kwargs[filter_name] = filter_value + # These two don't make sense + if filter_name == 'server' and 'hostname' in kwargs: + del kwargs['hostname'] + elif filter_name == 'hostname' and 'server' in kwargs: + del kwargs['server'] + try: + link = reverse(view, args=args, kwargs=kwargs) + except NoReverseMatch: + link = reverse(self.fallback_view, args=None, + kwargs={filter_name: filter_value}) + except NoReverseMatch: + rm = sys.exc_info()[1] + raise rm + except (Resolver404, ValueError): + pass + return link + + +@register.tag +def add_url_filter(parser, token): + """ + Return a url with the filter added to the current view. + + Takes a new filter and resolves the current view with the new filter + applied. Resolves to Bcfg2.Reporting.views.client_history + by default. + + {% add_url_filter server=interaction.server %} + """ + try: + tag_name, filter_pair = token.split_contents() + filter_name, filter_value = filter_pair.split('=', 1) + filter_name = filter_name.strip() + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + if not filter_name or not filter_value: + raise template.TemplateSyntaxError("argument should be a filter=value pair") + + return AddUrlFilter(filter_name, filter_value) + + +class MediaTag(template.Node): + def __init__(self, filter_value): + self.filter_value = filter_value + + def render(self, context): + base = context['MEDIA_URL'] + try: + request = context['request'] + try: + base = request.environ['bcfg2.media_url'] + except: + if request.path != request.META['PATH_INFO']: + offset = request.path.find(request.META['PATH_INFO']) + if offset > 0: + base = "%s/%s" % (request.path[:offset], \ + context['MEDIA_URL'].strip('/')) + except: + pass + return "%s/%s" % (base, self.filter_value) + + +@register.tag +def to_media_url(parser, token): + """ + Return a url relative to the media_url. + + {% to_media_url /bcfg2.css %} + """ + try: + filter_value = token.split_contents()[1] + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + + return MediaTag(filter_value) + +@register.filter +def determine_client_state(entry): + """ + Determine client state. + + This is used to determine whether a client is reporting clean or + dirty. If the client is reporting dirty, this will figure out just + _how_ dirty and adjust the color accordingly. + """ + if entry.state == 'clean': + return "clean-lineitem" + + bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) + if bad_percentage < 33: + thisdirty = "slightly-dirty-lineitem" + elif bad_percentage < 66: + thisdirty = "dirty-lineitem" + else: + thisdirty = "very-dirty-lineitem" + return thisdirty + + +@register.tag(name='qs') +def do_qs(parser, token): + """ + qs tag + + accepts a name value pair and inserts or replaces it in the query string + """ + try: + tag, name, value = token.split_contents() + except ValueError: + raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ + % token.contents.split()[0] + return QsNode(name, value) + +class QsNode(template.Node): + def __init__(self, name, value): + self.name = template.Variable(name) + self.value = template.Variable(value) + + def render(self, context): + try: + name = self.name.resolve(context) + value = self.value.resolve(context) + request = context['request'] + qs = copy(request.GET) + qs[name] = value + return "?%s" % qs.urlencode() + except template.VariableDoesNotExist: + return '' + except KeyError: + if settings.TEMPLATE_DEBUG: + raise Exception, "'qs' tag requires context['request']" + return '' + except: + return '' + + +@register.tag +def sort_link(parser, token): + ''' + Create a sort anchor tag. Reverse it if active. + + {% sort_link sort_key text %} + ''' + try: + tag, sort_key, text = token.split_contents() + except ValueError: + raise template.TemplateSyntaxError("%r tag requires at least four arguments" \ + % token.split_contents()[0]) + + return SortLinkNode(sort_key, text) + +class SortLinkNode(template.Node): + __TMPL__ = "{% load bcfg2_tags %}{{ text }}" + + def __init__(self, sort_key, text): + self.sort_key = template.Variable(sort_key) + self.text = template.Variable(text) + + def render(self, context): + try: + try: + sort = context['request'].GET['sort'] + except KeyError: + #fall back on this + sort = context.get('sort', '') + sort_key = self.sort_key.resolve(context) + text = self.text.resolve(context) + + # add arrows + try: + sort_base = sort_key.lstrip('-') + if sort[0] == '-' and sort[1:] == sort_base: + text = text + '▼' + sort_key = sort_base + elif sort_base == sort: + text = text + '▲' + sort_key = '-' + sort_base + except IndexError: + pass + + context.push() + context['key'] = sort_key + context['text'] = mark_safe(text) + output = get_template_from_string(self.__TMPL__).render(context) + context.pop() + return output + except: + if settings.DEBUG: + raise + raise + return '' + diff --git a/src/lib/Bcfg2/Reporting/templatetags/split.py b/src/lib/Bcfg2/Reporting/templatetags/split.py new file mode 100644 index 000000000..a9b4f0371 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/split.py @@ -0,0 +1,8 @@ +from django import template +register = template.Library() + + +@register.filter +def split(s): + """split by newlines""" + return s.split('\n') diff --git a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py new file mode 100644 index 000000000..2712d6395 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py @@ -0,0 +1,48 @@ +import sys +from django import template +from django.utils.encoding import smart_unicode +from django.utils.html import conditional_escape +from django.utils.safestring import mark_safe + +from Bcfg2.Compat import u_str + +register = template.Library() + +# pylint: disable=E0611 +try: + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter + colorize = True +except: + colorize = False +# pylint: enable=E0611 + + +@register.filter +def syntaxhilight(value, arg="diff", autoescape=None): + """ + Returns a syntax-hilighted version of Code; + requires code/language arguments + """ + + if autoescape: + # Seems to cause a double escape + #value = conditional_escape(value) + arg = conditional_escape(arg) + + if colorize: + try: + output = u_str('') + + lexer = get_lexer_by_name(arg) + output += highlight(value, lexer, HtmlFormatter()) + return mark_safe(output) + except: + return value + else: + return mark_safe(u_str('
Tip: Install pygments ' + 'for highlighting
%s
') % value) +syntaxhilight.needs_autoescape = True diff --git a/src/lib/Bcfg2/Reporting/urls.py b/src/lib/Bcfg2/Reporting/urls.py new file mode 100644 index 000000000..4dd343905 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/urls.py @@ -0,0 +1,59 @@ +from django.conf.urls.defaults import * +from django.core.urlresolvers import reverse, NoReverseMatch +from django.http import HttpResponsePermanentRedirect +from Bcfg2.Reporting.utils import filteredUrls, paginatedUrls, timeviewUrls + +def newRoot(request): + try: + grid_view = reverse('reports_grid_view') + except NoReverseMatch: + grid_view = '/grid' + return HttpResponsePermanentRedirect(grid_view) + +urlpatterns = patterns('Bcfg2.Reporting', + (r'^$', newRoot), + + url(r'^manage/?$', 'views.client_manage', name='reports_client_manage'), + url(r'^client/(?P[^/]+)/(?P\d+)/?$', 'views.client_detail', name='reports_client_detail_pk'), + url(r'^client/(?P[^/]+)/?$', 'views.client_detail', name='reports_client_detail'), + url(r'^element/(?P\w+)/(?P\d+)/(?P\d+)?/?$', 'views.config_item', name='reports_item'), + url(r'^element/(?P\w+)/(?P\d+)/?$', 'views.config_item', name='reports_item'), + url(r'^entry/(?P\w+)/(?P\w+)/?$', 'views.entry_status', name='reports_entry'), +) + +urlpatterns += patterns('Bcfg2.Reporting', + *timeviewUrls( + (r'^summary/?$', 'views.display_summary', None, 'reports_summary'), + (r'^timing/?$', 'views.display_timing', None, 'reports_timing'), + (r'^common/(?P\d+)/?$', 'views.common_problems', None, 'reports_common_problems'), + (r'^common/?$', 'views.common_problems', None, 'reports_common_problems'), +)) + +urlpatterns += patterns('Bcfg2.Reporting', + *filteredUrls(*timeviewUrls( + (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), + (r'^detailed/?$', + 'views.client_detailed_list', None, 'reports_detailed_list'), + (r'^elements/(?P\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), +))) + +urlpatterns += patterns('Bcfg2.Reporting', + *paginatedUrls( *filteredUrls( + (r'^history/?$', + 'views.render_history_view', None, 'reports_history'), + (r'^history/(?P[^/|]+)/?$', + 'views.render_history_view', None, 'reports_client_history'), +))) + + # Uncomment this for admin: + #(r'^admin/', include('django.contrib.admin.urls')), + + +## Uncomment this section if using authentication +#urlpatterns += patterns('', +# (r'^login/$', 'django.contrib.auth.views.login', +# {'template_name': 'auth/login.html'}), +# (r'^logout/$', 'django.contrib.auth.views.logout', +# {'template_name': 'auth/logout.html'}) +# ) + diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py new file mode 100755 index 000000000..c47763e39 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -0,0 +1,126 @@ +"""Helper functions for reports""" +from django.conf.urls.defaults import * +import re + +"""List of filters provided by filteredUrls""" +filter_list = ('server', 'state', 'group') + + +class BatchFetch(object): + """Fetch Django objects in smaller batches to save memory""" + + def __init__(self, obj, step=10000): + self.count = 0 + self.block_count = 0 + self.obj = obj + self.data = None + self.step = step + self.max = obj.count() + + def __iter__(self): + return self + + def next(self): + """Provide compatibility with python < 3.0""" + return self.__next__() + + def __next__(self): + """Return the next object from our array and fetch from the + database when needed""" + if self.block_count + self.count - self.step == self.max: + raise StopIteration + if self.block_count == 0 or self.count == self.step: + # Without list() this turns into LIMIT 1 OFFSET x queries + self.data = list(self.obj.all()[self.block_count: \ + (self.block_count + self.step)]) + self.block_count += self.step + self.count = 0 + self.count += 1 + return self.data[self.count - 1] + + +def generateUrls(fn): + """ + Parse url tuples and send to functions. + + Decorator for url generators. Handles url tuple parsing + before the actual function is called. + """ + def url_gen(*urls): + results = [] + for url_tuple in urls: + if isinstance(url_tuple, (list, tuple)): + results += fn(*url_tuple) + else: + raise ValueError("Unable to handle compiled urls") + return results + return url_gen + + +@generateUrls +def paginatedUrls(pattern, view, kwargs=None, name=None): + """ + Takes a group of url tuples and adds paginated urls. + + Extends a url tuple to include paginated urls. + Currently doesn't handle url() compiled patterns. + + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + results += [(pattern + "/(?P\d+)" + tail, view, kwargs)] + results += [(pattern + "/(?P\d+)\|(?P\d+)" + + tail, view, kwargs)] + if not kwargs: + kwargs = dict() + kwargs['page_limit'] = 0 + results += [(pattern + "/?\|(?Pall)" + tail, view, kwargs)] + return results + + +@generateUrls +def filteredUrls(pattern, view, kwargs=None, name=None): + """ + Takes a url and adds filtered urls. + + Extends a url tuple to include filtered view urls. Currently doesn't + handle url() compiled patterns. + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + for filter in ('/state/(?P\w+)', + '/group/(?P[\w\-\.]+)', + '/group/(?P[\w\-\.]+)/(?P[A-Za-z]+)', + '/server/(?P[\w\-\.]+)', + '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): + results += [(pattern + filter + tail, view, kwargs)] + return results + + +@generateUrls +def timeviewUrls(pattern, view, kwargs=None, name=None): + """ + Takes a url and adds timeview urls + + Extends a url tuple to include filtered view urls. Currently doesn't + handle url() compiled patterns. + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + for filter in ('/(?P\d{4})-(?P\d{2})-(?P\d{2})/' + \ + '(?P\d\d)-(?P\d\d)', + '/(?P\d{4})-(?P\d{2})-(?P\d{2})'): + results += [(pattern + filter + tail, view, kwargs)] + return results diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py new file mode 100644 index 000000000..58774831f --- /dev/null +++ b/src/lib/Bcfg2/Reporting/views.py @@ -0,0 +1,550 @@ +""" +Report views + +Functions to handle all of the reporting views. +""" +from datetime import datetime, timedelta +import sys +from time import strptime + +from django.template import Context, RequestContext +from django.http import \ + HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 +from django.shortcuts import render_to_response, get_object_or_404 +from django.core.urlresolvers import \ + resolve, reverse, Resolver404, NoReverseMatch +from django.db import connection, DatabaseError +from django.db.models import Q, Count + +from Bcfg2.Reporting.models import * + + +__SORT_FIELDS__ = ( 'client', 'state', 'good', 'bad', 'modified', 'extra', \ + 'timestamp', 'server' ) + +class PaginationError(Exception): + """This error is raised when pagination cannot be completed.""" + pass + + +def _in_bulk(model, ids): + """ + Short cut to fetch in bulk and trap database errors. sqlite will raise + a "too many SQL variables" exception if this list is too long. Try using + django and fetch manually if an error occurs + + returns a dict of this form { id: } + """ + + try: + return model.objects.in_bulk(ids) + except DatabaseError: + pass + + # if objects.in_bulk fails so will obejcts.filter(pk__in=ids) + bulk_dict = {} + [bulk_dict.__setitem__(i.id, i) \ + for i in model.objects.all() if i.id in ids] + return bulk_dict + + +def server_error(request): + """ + 500 error handler. + + For now always return the debug response. Mailing isn't appropriate here. + + """ + from django.views import debug + return debug.technical_500_response(request, *sys.exc_info()) + + +def timeview(fn): + """ + Setup a timeview view + + Handles backend posts from the calendar and converts date pieces + into a 'timestamp' parameter + + """ + def _handle_timeview(request, **kwargs): + """Send any posts back.""" + if request.method == 'POST' and request.POST.get('op', '') == 'timeview': + cal_date = request.POST['cal_date'] + try: + fmt = "%Y/%m/%d" + if cal_date.find(' ') > -1: + fmt += " %H:%M" + timestamp = datetime(*strptime(cal_date, fmt)[0:6]) + view, args, kw = resolve(request.META['PATH_INFO']) + kw['year'] = "%0.4d" % timestamp.year + kw['month'] = "%02.d" % timestamp.month + kw['day'] = "%02.d" % timestamp.day + if cal_date.find(' ') > -1: + kw['hour'] = timestamp.hour + kw['minute'] = timestamp.minute + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) + except KeyError: + pass + except: + pass + # FIXME - Handle this + + """Extract timestamp from args.""" + timestamp = None + try: + timestamp = datetime(int(kwargs.pop('year')), + int(kwargs.pop('month')), + int(kwargs.pop('day')), int(kwargs.pop('hour', 0)), + int(kwargs.pop('minute', 0)), 0) + kwargs['timestamp'] = timestamp + except KeyError: + pass + except: + raise + return fn(request, **kwargs) + + return _handle_timeview + + +def _handle_filters(query, **kwargs): + """ + Applies standard filters to a query object + + Returns an updated query object + + query - query object to filter + + server -- Filter interactions by server + state -- Filter interactions by state + group -- Filter interactions by group + + """ + if 'state' in kwargs and kwargs['state']: + query = query.filter(state__exact=kwargs['state']) + if 'server' in kwargs and kwargs['server']: + query = query.filter(server__exact=kwargs['server']) + + if 'group' in kwargs and kwargs['group']: + group = get_object_or_404(Group, name=kwargs['group']) + query = query.filter(metadata__groups__id=group.pk) + return query + + +def config_item(request, pk, entry_type, interaction=None): + """ + Display a single entry. + + Displays information about a single entry. + + """ + try: + cls = BaseEntry.entry_from_name(entry_type) + except ValueError: + # TODO - handle this + raise + item = get_object_or_404(cls, pk=pk) + + # TODO - timestamp + if interaction: + try: + inter = Interaction.objects.get(pk=interaction) + except Interaction.DoesNotExist: + raise Http404("Not a valid interaction") + timestamp = inter.timestamp + else: + timestamp = datetime.now() + + ts_start = timestamp.replace(hour=1, minute=0, second=0, microsecond=0) + ts_end = ts_start + timedelta(days=1) + associated_list = item.interaction_set.select_related('client').filter(\ + timestamp__gte=ts_start, timestamp__lt=ts_end) + + if item.is_failure(): + template = 'config_items/item-failure.html' + else: + template = 'config_items/item.html' + return render_to_response(template, + {'item': item, + 'associated_list': associated_list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def config_item_list(request, item_state, timestamp=None, **kwargs): + """Render a listing of affected elements""" + state = convert_entry_type_to_id(item_state.lower()) + if state < 0: + raise Http404 + + current_clients = Interaction.objects.recent(timestamp) + current_clients = [q['id'] for q in _handle_filters(current_clients, **kwargs).values('id')] + + lists = [] + for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: + ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ + .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') + if len(ldata) > 0: + # Property doesn't render properly.. + lists.append((etype.ENTRY_TYPE, ldata)) + + return render_to_response('config_items/listing.html', + {'item_list': lists, + 'item_state': item_state, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def entry_status(request, entry_type, pk, timestamp=None, **kwargs): + """Render a listing of affected elements by type and name""" + try: + cls = BaseEntry.entry_from_name(entry_type) + except ValueError: + # TODO - handle this + raise + item = get_object_or_404(cls, pk=pk) + + current_clients = Interaction.objects.recent(timestamp) + current_clients = [i['pk'] for i in _handle_filters(current_clients, **kwargs).values('pk')] + + # There is no good way to do this... + items = [] + for it in cls.objects.filter(interaction__in=current_clients, name=item.name).distinct("id").select_related(): + items.append((it, it.interaction_set.filter(pk__in=current_clients).order_by('client__name').select_related('client'))) + + return render_to_response('config_items/entry_status.html', + {'entry': item, + 'items': items, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def common_problems(request, timestamp=None, threshold=None): + """Mine config entries""" + + if request.method == 'POST': + try: + threshold = int(request.POST['threshold']) + view, args, kw = resolve(request.META['PATH_INFO']) + kw['threshold'] = threshold + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) + except: + pass + + try: + threshold = int(threshold) + except: + threshold = 10 + + current_clients = Interaction.objects.recent_ids(timestamp) + lists = [] + for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: + ldata = etype.objects.exclude(state=TYPE_GOOD).filter( + interaction__in=current_clients).annotate(num_entries=Count('id')).filter(num_entries__gte=threshold)\ + .order_by('-num_entries', 'name') + if len(ldata) > 0: + # Property doesn't render properly.. + lists.append((etype.ENTRY_TYPE, ldata)) + + return render_to_response('config_items/common.html', + {'lists': lists, + 'timestamp': timestamp, + 'threshold': threshold}, + context_instance=RequestContext(request)) + + +@timeview +def client_index(request, timestamp=None, **kwargs): + """ + Render a grid view of active clients. + + Keyword parameters: + timestamp -- datetime object to render from + + """ + list = _handle_filters(Interaction.objects.recent(timestamp), **kwargs).\ + select_related().order_by("client__name").all() + + return render_to_response('clients/index.html', + {'inter_list': list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def client_detailed_list(request, timestamp=None, **kwargs): + """ + Provides a more detailed list view of the clients. Allows for extra + filters to be passed in. + + """ + + try: + sort = request.GET['sort'] + if sort[0] == '-': + sort_key = sort[1:] + else: + sort_key = sort + if not sort_key in __SORT_FIELDS__: + raise ValueError + + if sort_key == "client": + kwargs['orderby'] = "%s__name" % sort + elif sort_key == "good": + kwargs['orderby'] = "%scount" % sort + elif sort_key in ["bad", "modified", "extra"]: + kwargs['orderby'] = "%s_entries" % sort + else: + kwargs['orderby'] = sort + kwargs['sort'] = sort + except (ValueError, KeyError): + kwargs['orderby'] = "client__name" + kwargs['sort'] = "client" + + kwargs['interaction_base'] = Interaction.objects.recent(timestamp).select_related() + kwargs['page_limit'] = 0 + return render_history_view(request, 'clients/detailed-list.html', **kwargs) + + +def client_detail(request, hostname=None, pk=None): + context = dict() + client = get_object_or_404(Client, name=hostname) + if(pk == None): + inter = client.current_interaction + maxdate = None + else: + inter = client.interactions.get(pk=pk) + maxdate = inter.timestamp + + etypes = { TYPE_BAD: 'bad', TYPE_MODIFIED: 'modified', TYPE_EXTRA: 'extra' } + edict = dict() + for label in etypes.values(): + edict[label] = [] + for ekind in ('actions', 'packages', 'paths', 'services'): + for ent in getattr(inter, ekind).all(): + edict[etypes[ent.state]].append(ent) + context['entry_types'] = edict + + context['interaction']=inter + return render_history_view(request, 'clients/detail.html', page_limit=5, + client=client, maxdate=maxdate, context=context) + + +def client_manage(request): + """Manage client expiration""" + message = '' + if request.method == 'POST': + try: + client_name = request.POST.get('client_name', None) + client_action = request.POST.get('client_action', None) + client = Client.objects.get(name=client_name) + if client_action == 'expire': + client.expiration = datetime.now() + client.save() + message = "Expiration for %s set to %s." % \ + (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) + elif client_action == 'unexpire': + client.expiration = None + client.save() + message = "%s is now active." % client_name + else: + message = "Missing action" + except Client.DoesNotExist: + if not client_name: + client_name = "" + message = "Couldn't find client \"%s\"" % client_name + + return render_to_response('clients/manage.html', + {'clients': Client.objects.order_by('name').all(), 'message': message}, + context_instance=RequestContext(request)) + + +@timeview +def display_summary(request, timestamp=None): + """ + Display a summary of the bcfg2 world + """ + recent_data = Interaction.objects.recent(timestamp) \ + .select_related() + node_count = len(recent_data) + if not timestamp: + timestamp = datetime.now() + + collected_data = dict(clean=[], + bad=[], + modified=[], + extra=[], + stale=[]) + for node in recent_data: + if timestamp - node.timestamp > timedelta(hours=24): + collected_data['stale'].append(node) + # If stale check for uptime + if node.bad_count > 0: + collected_data['bad'].append(node) + else: + collected_data['clean'].append(node) + if node.modified_count > 0: + collected_data['modified'].append(node) + if node.extra_count > 0: + collected_data['extra'].append(node) + + # label, header_text, node_list + summary_data = [] + get_dict = lambda name, label: {'name': name, + 'nodes': collected_data[name], + 'label': label} + if len(collected_data['clean']) > 0: + summary_data.append(get_dict('clean', + 'nodes are clean.')) + if len(collected_data['bad']) > 0: + summary_data.append(get_dict('bad', + 'nodes are bad.')) + if len(collected_data['modified']) > 0: + summary_data.append(get_dict('modified', + 'nodes were modified.')) + if len(collected_data['extra']) > 0: + summary_data.append(get_dict('extra', + 'nodes have extra configurations.')) + if len(collected_data['stale']) > 0: + summary_data.append(get_dict('stale', + 'nodes did not run within the last 24 hours.')) + + return render_to_response('displays/summary.html', + {'summary_data': summary_data, 'node_count': node_count, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def display_timing(request, timestamp=None): + perfs = Performance.objects.filter(interaction__in=Interaction.objects.recent_ids(timestamp))\ + .select_related('interaction__client') + + mdict = dict() + for perf in perfs: + client = perf.interaction.client.name + if client not in mdict: + mdict[client] = { 'name': client } + mdict[client][perf.metric] = perf.value + + return render_to_response('displays/timing.html', + {'metrics': list(mdict.values()), + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +def render_history_view(request, template='clients/history.html', **kwargs): + """ + Provides a detailed history of a clients interactions. + + Renders a detailed history of a clients interactions. Allows for various + filters and settings. Automatically sets pagination data into the context. + + Keyword arguments: + interaction_base -- Interaction QuerySet to build on + (default Interaction.objects) + context -- Additional context data to render with + page_number -- Page to display (default 1) + page_limit -- Number of results per page, if 0 show all (default 25) + client -- Client object to render + hostname -- Client hostname to lookup and render. Returns a 404 if + not found + server -- Filter interactions by server + state -- Filter interactions by state + group -- Filter interactions by group + entry_max -- Most recent interaction to display + orderby -- Sort results using this field + + """ + + context = kwargs.get('context', dict()) + max_results = int(kwargs.get('page_limit', 25)) + page = int(kwargs.get('page_number', 1)) + + client = kwargs.get('client', None) + if not client and 'hostname' in kwargs: + client = get_object_or_404(Client, name=kwargs['hostname']) + if client: + context['client'] = client + + entry_max = kwargs.get('maxdate', None) + context['entry_max'] = entry_max + + # Either filter by client or limit by clients + iquery = kwargs.get('interaction_base', Interaction.objects) + if client: + iquery = iquery.filter(client__exact=client) + iquery = iquery.select_related('client') + + if 'orderby' in kwargs and kwargs['orderby']: + iquery = iquery.order_by(kwargs['orderby']) + if 'sort' in kwargs: + context['sort'] = kwargs['sort'] + + iquery = _handle_filters(iquery, **kwargs) + + if entry_max: + iquery = iquery.filter(timestamp__lte=entry_max) + + if max_results < 0: + max_results = 1 + entry_list = [] + if max_results > 0: + try: + rec_start, rec_end = prepare_paginated_list(request, + context, + iquery, + page, + max_results) + except PaginationError: + page_error = sys.exc_info()[1] + if isinstance(page_error[0], HttpResponse): + return page_error[0] + return HttpResponseServerError(page_error) + context['entry_list'] = iquery.all()[rec_start:rec_end] + else: + context['entry_list'] = iquery.all() + + return render_to_response(template, context, + context_instance=RequestContext(request)) + + +def prepare_paginated_list(request, context, paged_list, page=1, max_results=25): + """ + Prepare context and slice an object for pagination. + """ + if max_results < 1: + raise PaginationError("Max results less then 1") + if paged_list == None: + raise PaginationError("Invalid object") + + try: + nitems = paged_list.count() + except TypeError: + nitems = len(paged_list) + + rec_start = (page - 1) * int(max_results) + try: + total_pages = (nitems / int(max_results)) + 1 + except: + total_pages = 1 + if page > total_pages: + # If we passed beyond the end send back + try: + view, args, kwargs = resolve(request.META['PATH_INFO']) + kwargs['page_number'] = total_pages + raise PaginationError(HttpResponseRedirect(reverse(view, + kwargs=kwargs))) + except (Resolver404, NoReverseMatch, ValueError): + raise "Accessing beyond last page. Unable to resolve redirect." + + context['total_pages'] = total_pages + context['records_per_page'] = max_results + return (rec_start, rec_start + int(max_results)) -- cgit v1.2.3-1-g7c22