diff options
Diffstat (limited to 'src/lib/Bcfg2/Reporting')
17 files changed, 770 insertions, 231 deletions
diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 8e8e8b1fa..f05a25732 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -4,7 +4,6 @@ import atexit import daemon import logging import time -import traceback import threading from lockfile import LockFailed, LockTimeout @@ -16,11 +15,11 @@ except ImportError: # pylint: enable=E0611 import Bcfg2.Logger -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError, TransportImportError +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Reporting.Transport.DirectStore import DirectStore -from Bcfg2.Reporting.Storage import load_storage_from_config, \ - StorageError, StorageImportError +from Bcfg2.Reporting.Storage.base import StorageError + class ReportingError(Exception): @@ -31,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -39,59 +38,71 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - traceback.format_exc().splitlines()[-1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" - - def __init__(self, setup): + options = [Bcfg2.Options.Common.reporting_storage, + Bcfg2.Options.Common.reporting_transport, + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] + + def __init__(self): """Setup the collector. This may be called by the daemon or though bcfg2-admin""" - self.setup = setup - self.datastore = setup['repo'] - self.encoding = setup['encoding'] self.terminate = None self.context = None self.children = [] self.cleanup_threshold = 25 - if setup['debug']: + self.semaphore = None + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + + if Bcfg2.Options.setup.debug: level = logging.DEBUG - elif setup['verbose']: + elif Bcfg2.Options.setup.verbose: level = logging.INFO else: level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - to_syslog=setup['syslog'], - to_file=setup['logging'], - level=level) + Bcfg2.Logger.setup_logging() self.logger = logging.getLogger('bcfg2-report-collector') try: - self.transport = load_transport_from_config(setup) - self.storage = load_storage_from_config(setup) + self.transport = Bcfg2.Options.setup.reporting_transport() + self.storage = Bcfg2.Options.setup.reporting_storage() except TransportError: self.logger.error("Failed to load transport: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError except StorageError: self.logger.error("Failed to load storage: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError if isinstance(self.transport, DirectStore): @@ -102,12 +113,12 @@ class ReportingCollector(object): try: self.logger.debug("Validating storage %s" % - self.storage.__class__.__name__) + self.storage.__class__.__name__) self.storage.validate() except: self.logger.error("Storage backend %s failed to validate: %s" % - (self.storage.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (self.storage.__class__.__name__, + sys.exc_info()[1])) def run(self): """Startup the processing and go!""" @@ -116,10 +127,10 @@ class ReportingCollector(object): self.context = daemon.DaemonContext(detach_process=True) iter = 0 - if self.setup['daemon']: + if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") - self.context.pidfile = TimeoutPIDLockFile(self.setup['daemon'], - acquire_timeout=5) + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) # Attempt to ensure lockfile is able to be created and not stale try: self.context.pidfile.acquire() @@ -136,7 +147,7 @@ class ReportingCollector(object): else: self.logger.error("Failed to daemonize: " "Failed to acquire lock on %s" % - self.setup['daemon']) + Bcfg2.Options.setup.daemon) self.shutdown() return else: @@ -152,8 +163,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - - store_thread = ReportingStoreThread(interaction, self.storage) + if self.semaphore: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) @@ -167,7 +180,7 @@ class ReportingCollector(object): self.shutdown() except: self.logger.error("Unhandled exception in main loop %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) def shutdown(self): """Cleanup and go""" diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py new file mode 100755 index 000000000..ebd0db58f --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python +"""Query reporting system for client status.""" + +import sys +import argparse +import datetime +import Bcfg2.DBSettings + + +def print_entries(interaction, etype): + items = getattr(interaction, etype)() + for item in items: + print("%-70s %s" % (item.entry_type + ":" + item.name, etype)) + + +class _FlagsFilterMixin(object): + """ Mixin that allows to filter the interactions based on the + only_important and/or the dry_run flag """ + + options = [ + Bcfg2.Options.BooleanOption( + "-n", "--no-dry-run", + help="Do not consider interactions created with the --dry-run " + "flag"), + Bcfg2.Options.BooleanOption( + "-i", "--no-only-important", + help="Do not consider interactions created with the " + "--only-important flag")] + + def get_interaction(self, client, setup): + if not setup.no_dry_run and not setup.no_only_important: + return client.current_interaction + + filter = {} + if setup.no_dry_run: + filter['dry_run'] = False + if setup.no_only_important: + filter['only_important'] = False + + from Bcfg2.Reporting.models import Interaction + return Interaction.objects.filter(client=client, **filter).latest() + + +class _SingleHostCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 + """ Base class for bcfg2-reports modes that take a single host as + a positional argument """ + options = [Bcfg2.Options.PositionalArgument("host")] + + def get_client(self, setup): + from Bcfg2.Reporting.models import Client + try: + return Client.objects.select_related().get(name=setup.host) + except Client.DoesNotExist: + print("No such host: %s" % setup.host) + raise SystemExit(2) + + +class Show(_SingleHostCmd, _FlagsFilterMixin): + """ Show bad, extra, modified, or all entries from a given host """ + + options = _SingleHostCmd.options + _FlagsFilterMixin.options + [ + Bcfg2.Options.BooleanOption( + "-b", "--bad", help="Show bad entries from HOST"), + Bcfg2.Options.BooleanOption( + "-e", "--extra", help="Show extra entries from HOST"), + Bcfg2.Options.BooleanOption( + "-m", "--modified", help="Show modified entries from HOST")] + + def run(self, setup): + client = self.get_client(setup) + show_all = not setup.bad and not setup.extra and not setup.modified + interaction = self.get_interaction(client, setup) + if setup.bad or show_all: + print_entries(interaction, "bad") + if setup.modified or show_all: + print_entries(interaction, "modified") + if setup.extra or show_all: + print_entries(interaction, "extra") + + +class Total(_SingleHostCmd, _FlagsFilterMixin): + """ Show total number of managed and good entries from HOST """ + + options = _SingleHostCmd.options + _FlagsFilterMixin.options + + def run(self, setup): + client = self.get_client(setup) + interaction = self.get_interaction(client, setup) + managed = interaction.total_count + good = interaction.good_count + print("Total managed entries: %d (good: %d)" % (managed, good)) + + +class Expire(_SingleHostCmd): + """ Toggle the expired/unexpired state of HOST """ + + def run(self, setup): + client = self.get_client(setup) + if client.expiration is None: + client.expiration = datetime.datetime.now() + print("%s expired." % client.name) + else: + client.expiration = None + print("%s un-expired." % client.name) + client.save() + + +class _ClientSelectCmd(Bcfg2.Options.Subcommand, _FlagsFilterMixin): + """ Base class for subcommands that display lists of clients """ + options = _FlagsFilterMixin.options + [ + Bcfg2.Options.Option("--fields", metavar="FIELD,FIELD,...", + help="Only display the listed fields", + type=Bcfg2.Options.Types.comma_list, + default=['name', 'time', 'state'])] + + def get_clients(self): + from Bcfg2.Reporting.models import Client + return Client.objects.exclude(current_interaction__isnull=True) + + def _print_fields(self, setup, fields, client, fmt, extra=None): + """ Prints the fields specified in fields of client, max_name + specifies the column width of the name column. """ + fdata = [] + if extra is None: + extra = dict() + interaction = self.get_interaction(client, setup) + for field in fields: + if field == 'time': + fdata.append(str(interaction.timestamp)) + elif field == 'state': + if interaction.isclean(): + fdata.append("clean") + else: + fdata.append("dirty") + elif field == 'total': + fdata.append(interaction.total_count) + elif field == 'good': + fdata.append(interaction.good_count) + elif field == 'modified': + fdata.append(interaction.modified_count) + elif field == 'extra': + fdata.append(interaction.extra_count) + elif field == 'bad': + fdata.append(interaction.bad_count) + elif field == 'stale': + fdata.append(interaction.isstale()) + else: + try: + fdata.append(getattr(client, field)) + except AttributeError: + fdata.append(extra.get(field, "N/A")) + + print(fmt % tuple(fdata)) + + def display(self, setup, result, fields, extra=None): + if 'name' not in fields: + fields.insert(0, "name") + if not result: + print("No match found") + return + if extra is None: + extra = dict() + max_name = max(len(c.name) for c in result) + ffmt = [] + for field in fields: + if field == "name": + ffmt.append("%%-%ds" % max_name) + elif field == "time": + ffmt.append("%-19s") + else: + ffmt.append("%%-%ds" % len(field)) + fmt = " ".join(ffmt) + print(fmt % tuple(f.title() for f in fields)) + for client in result: + if not client.expiration: + self._print_fields(setup, fields, client, fmt, + extra=extra.get(client, None)) + + +class Clients(_ClientSelectCmd): + """ Query hosts """ + options = _ClientSelectCmd.options + [ + Bcfg2.Options.BooleanOption( + "-c", "--clean", help="Show only clean hosts"), + Bcfg2.Options.BooleanOption( + "-d", "--dirty", help="Show only dirty hosts"), + Bcfg2.Options.BooleanOption( + "--stale", + help="Show hosts that haven't run in the last 24 hours")] + + def run(self, setup): + result = [] + show_all = not setup.stale and not setup.clean and not setup.dirty + for client in self.get_clients(): + interaction = self.get_interaction(client, setup) + if (show_all or + (setup.stale and interaction.isstale()) or + (setup.clean and interaction.isclean()) or + (setup.dirty and not interaction.isclean())): + result.append(client) + + self.display(setup, result, setup.fields) + + +class Entries(_ClientSelectCmd): + """ Query hosts by entries """ + options = _ClientSelectCmd.options + [ + Bcfg2.Options.BooleanOption( + "--badentry", + help="Show hosts that have bad entries that match"), + Bcfg2.Options.BooleanOption( + "--modifiedentry", + help="Show hosts that have modified entries that match"), + Bcfg2.Options.BooleanOption( + "--extraentry", + help="Show hosts that have extra entries that match"), + Bcfg2.Options.PathOption( + "--file", type=argparse.FileType('r'), + help="Read TYPE:NAME pairs from the specified file instead of " + "from the command line"), + Bcfg2.Options.PositionalArgument( + "entries", metavar="TYPE:NAME", nargs="*")] + + def _hosts_by_entry_type(self, setup, clients, etype, entryspec): + result = [] + for entry in entryspec: + for client in clients: + interaction = self.get_interaction(client, setup) + items = getattr(interaction, etype)() + for item in items: + if (item.entry_type == entry[0] and + item.name == entry[1]): + result.append(client) + return result + + def run(self, setup): + result = [] + if setup.file: + try: + entries = [l.strip().split(":") for l in setup.file] + except IOError: + err = sys.exc_info()[1] + print("Cannot read entries from %s: %s" % (setup.file.name, + err)) + return 2 + else: + entries = [a.split(":") for a in setup.entries] + + clients = self.get_clients() + if setup.badentry: + result = self._hosts_by_entry_type(setup, clients, "bad", entries) + elif setup.modifiedentry: + result = self._hosts_by_entry_type(setup, clients, "modified", + entries) + elif setup.extraentry: + result = self._hosts_by_entry_type(setup, clients, "extra", + entries) + + self.display(setup, result, setup.fields) + + +class Entry(_ClientSelectCmd): + """ Show the status of a single entry on all hosts """ + + options = _ClientSelectCmd.options + [ + Bcfg2.Options.PositionalArgument( + "entry", metavar="TYPE:NAME", nargs=1)] + + def run(self, setup): + from Bcfg2.Reporting.models import BaseEntry + result = [] + fields = setup.fields + if 'state' in fields: + fields.remove('state') + fields.append("entry state") + + etype, ename = setup.entry[0].split(":") + try: + entry_cls = BaseEntry.entry_from_type(etype) + except ValueError: + print("Unhandled/unknown type %s" % etype) + return 2 + + # TODO: batch fetch this. sqlite could break + extra = dict() + for client in self.get_clients(): + ents = entry_cls.objects.filter( + name=ename, + interaction=self.get_interaction(client, setup)) + if len(ents) == 0: + continue + extra[client] = {"entry state": ents[0].get_state_display(), + "reason": ents[0]} + result.append(client) + + self.display(setup, result, fields, extra=extra) + + +class CLI(Bcfg2.Options.CommandRegistry): + """ CLI class for bcfg2-reports """ + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + self.register_commands(globals().values()) + parser = Bcfg2.Options.get_parser( + description="Query the Bcfg2 reporting subsystem", + components=[self]) + parser.add_options(self.subcommand_options) + parser.parse() + + def run(self): + """ Run bcfg2-reports """ + return self.runcommand() diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 98226dc4e..c9aa169bf 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -2,15 +2,12 @@ The base for the original DjangoORM (DBStats) """ -import os -import traceback from lxml import etree from datetime import datetime +import traceback from time import strptime - -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' -from Bcfg2 import settings - +import Bcfg2.Options +import Bcfg2.DBSettings from Bcfg2.Compat import md5 from Bcfg2.Reporting.Storage.base import StorageBase, StorageError from Bcfg2.Server.Plugin.exceptions import PluginExecutionError @@ -28,9 +25,13 @@ from Bcfg2.Reporting.Compat import transaction class DjangoORM(StorageBase): - def __init__(self, setup): - super(DjangoORM, self).__init__(setup) - self.size_limit = setup.get('reporting_file_limit') + options = StorageBase.options + [ + Bcfg2.Options.Common.repository, + Bcfg2.Options.Option( + cf=('reporting', 'file_limit'), + type=Bcfg2.Options.Types.size, + help='Reporting file size limit', + default=1024 * 1024)] def _import_default(self, entry, state, entrytype=None, defaults=None, mapping=None, boolean=None, xforms=None): @@ -108,8 +109,8 @@ class DjangoORM(StorageBase): # extra entries are a bit different. They can have Instance # objects if not act_dict['target_version']: - for instance in entry.findall("Instance"): - # FIXME - this probably only works for rpms + instance = entry.find("Instance") + if instance: release = instance.get('release', '') arch = instance.get('arch', '') act_dict['current_version'] = instance.get('version') @@ -117,9 +118,8 @@ class DjangoORM(StorageBase): act_dict['current_version'] += "-" + release if arch: act_dict['current_version'] += "." + arch - self.logger.debug("Adding package %s %s" % - (name, act_dict['current_version'])) - return PackageEntry.entry_get_or_create(act_dict) + self.logger.debug("Adding extra package %s %s" % + (name, act_dict['current_version'])) else: self.logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) @@ -127,7 +127,7 @@ class DjangoORM(StorageBase): # not implemented yet act_dict['verification_details'] = \ entry.get('verification_details', '') - return PackageEntry.entry_get_or_create(act_dict) + return PackageEntry.entry_get_or_create(act_dict) def _import_Path(self, entry, state): name = entry.get('name') @@ -185,7 +185,7 @@ class DjangoORM(StorageBase): act_dict['detail_type'] = PathEntry.DETAIL_DIFF cdata = entry.get('current_bdiff') if cdata: - if len(cdata) > self.size_limit: + if len(cdata) > Bcfg2.Options.setup.file_limit: act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT act_dict['details'] = md5(cdata).hexdigest() else: @@ -284,6 +284,14 @@ class DjangoORM(StorageBase): Group.objects.get_or_create(name=metadata['profile']) else: profile = None + + flags = {'dry_run': False, 'only_important': False} + for flag in stats.findall('./Flags/Flag'): + value = flag.get('value', default='false').lower() == 'true' + name = flag.get('name') + if name in flags: + flags[name] = value + inter = Interaction(client=client, timestamp=timestamp, state=stats.get('state', default="unknown"), @@ -292,7 +300,8 @@ class DjangoORM(StorageBase): good_count=stats.get('good', default="0"), total_count=stats.get('total', default="0"), server=server, - profile=profile) + profile=profile, + **flags) inter.save() self.logger.debug("Interaction for %s at %s with INSERTED in to db" % (client.id, timestamp)) @@ -365,7 +374,6 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" - try: try: self._import_interaction(interaction) @@ -380,23 +388,21 @@ class DjangoORM(StorageBase): def validate(self): """Validate backend storage. Should be called once when loaded""" - - settings.read_config(repo=self.setup['repo']) - # verify our database schema try: - if self.setup['debug']: + if Bcfg2.Options.setup.debug: vrb = 2 - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: - self.logger.error("Failed to update database schema: %s" % \ - traceback.format_exc().splitlines()[-1]) - raise StorageError + msg = "Failed to update database schema: %s" % sys.exc_info()[1] + self.logger.error(msg) + raise StorageError(msg) def GetExtra(self, client): """Fetch extra entries for a client""" diff --git a/src/lib/Bcfg2/Reporting/Storage/__init__.py b/src/lib/Bcfg2/Reporting/Storage/__init__.py index 85356fcfe..953104d4b 100644 --- a/src/lib/Bcfg2/Reporting/Storage/__init__.py +++ b/src/lib/Bcfg2/Reporting/Storage/__init__.py @@ -1,32 +1,3 @@ """ Public storage routines """ - -import traceback - -from Bcfg2.Reporting.Storage.base import StorageError, \ - StorageImportError - -def load_storage(storage_name, setup): - """ - Try to load the storage. Raise StorageImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, storage_name) - mod = getattr(__import__(mod_name).Reporting.Storage, storage_name) - except ImportError: - try: - mod = __import__(storage_name) - except: - raise StorageImportError("Unavailable") - try: - cls = getattr(mod, storage_name) - return cls(setup) - except: - raise StorageImportError("Storage unavailable: %s" % - traceback.format_exc().splitlines()[-1]) - -def load_storage_from_config(setup): - """Load the storage in the config... eventually""" - return load_storage('DjangoORM', setup) - diff --git a/src/lib/Bcfg2/Reporting/Storage/base.py b/src/lib/Bcfg2/Reporting/Storage/base.py index 92cc3a68b..771f755a1 100644 --- a/src/lib/Bcfg2/Reporting/Storage/base.py +++ b/src/lib/Bcfg2/Reporting/Storage/base.py @@ -2,28 +2,25 @@ The base for all Storage backends """ -import logging +import logging + class StorageError(Exception): """Generic StorageError""" pass -class StorageImportError(StorageError): - """Raised when a storage module fails to import""" - pass - class StorageBase(object): """The base for all storages""" + options = [] + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] - def __init__(self, setup): + def __init__(self): """Do something here""" clsname = self.__class__.__name__ self.logger = logging.getLogger(clsname) self.logger.debug("Loading %s storage" % clsname) - self.setup = setup - self.encoding = setup['encoding'] def import_interaction(self, interaction): """Import the data into the backend""" @@ -48,4 +45,3 @@ class StorageBase(object): def GetCurrentEntry(self, client, e_type, e_name): """Get the current status of an entry on the client""" raise NotImplementedError - diff --git a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py index 79d1b5aba..b9d17212e 100644 --- a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py +++ b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py @@ -5,18 +5,20 @@ import os import sys import time import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError -from Bcfg2.Reporting.Storage import load_storage_from_config from Bcfg2.Compat import Queue, Full, Empty, cPickle class DirectStore(TransportBase, threading.Thread): - def __init__(self, setup): - TransportBase.__init__(self, setup) + options = TransportBase.options + [Bcfg2.Options.Common.reporting_storage] + + def __init__(self): + TransportBase.__init__(self) threading.Thread.__init__(self) self.save_file = os.path.join(self.data, ".saved") - self.storage = load_storage_from_config(setup) + self.storage = Bcfg2.Options.setup.reporting_storage() self.storage.validate() self.queue = Queue(100000) @@ -30,10 +32,9 @@ class DirectStore(TransportBase, threading.Thread): def store(self, hostname, metadata, stats): try: - self.queue.put_nowait(dict( - hostname=hostname, - metadata=metadata, - stats=stats)) + self.queue.put_nowait(dict(hostname=hostname, + metadata=metadata, + stats=stats)) except Full: self.logger.warning("Reporting: Queue is full, " "dropping statistics") diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index c7d5c512a..189967cb0 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -9,6 +9,7 @@ import os import select import time import traceback +import Bcfg2.Options import Bcfg2.Server.FileMonitor from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError from Bcfg2.Reporting.Transport.base import TransportBase, TransportError @@ -16,8 +17,10 @@ from Bcfg2.Compat import cPickle class LocalFilesystem(TransportBase): - def __init__(self, setup): - super(LocalFilesystem, self).__init__(setup) + options = TransportBase.options + [Bcfg2.Options.Common.filemonitor] + + def __init__(self): + super(LocalFilesystem, self).__init__() self.work_path = "%s/work" % self.data self.debug_log("LocalFilesystem: work path %s" % self.work_path) @@ -42,24 +45,16 @@ class LocalFilesystem(TransportBase): def start_monitor(self, collector): """Start the file monitor. Most of this comes from BaseCore""" - setup = self.setup - try: - fmon = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] - except KeyError: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % setup['filemonitor']) - fmon = Bcfg2.Server.FileMonitor.available['default'] - if self.debug_flag: - self.fmon.set_debug(self.debug_flag) try: - self.fmon = fmon(debug=self.debug_flag) - self.logger.info("Using the %s file monitor" % - self.fmon.__class__.__name__) + self.fmon = Bcfg2.Server.FileMonitor.get_fam() except IOError: - msg = "Failed to instantiate file monitor %s" % \ - setup['filemonitor'] + msg = "Failed to instantiate fam driver %s" % \ + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise TransportError(msg) + + if self.debug_flag: + self.fmon.set_debug(self.debug_flag) self.fmon.start() self.fmon.AddMonitor(self.work_path, self) @@ -154,7 +149,7 @@ class LocalFilesystem(TransportBase): """ try: if not self._phony_collector: - self._phony_collector = ReportingCollector(self.setup) + self._phony_collector = ReportingCollector() except ReportingError: raise TransportError except: @@ -176,4 +171,3 @@ class LocalFilesystem(TransportBase): self.logger.error("RPC method %s failed: %s" % (method, traceback.format_exc().splitlines()[-1])) raise TransportError - diff --git a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py index 22d9af57e..7427c2e1d 100644 --- a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py +++ b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py @@ -9,9 +9,9 @@ import signal import platform import traceback import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError from Bcfg2.Compat import cPickle -from Bcfg2.Options import Option try: import redis @@ -34,9 +34,19 @@ class RedisTransport(TransportBase): STATS_KEY = 'bcfg2_statistics' COMMAND_KEY = 'bcfg2_command' - def __init__(self, setup): - super(RedisTransport, self).__init__(setup) - self._redis = None + options = TransportBase.options + [ + Bcfg2.Options.Option( + cf=('reporting', 'redis_host'), dest="reporting_redis_host", + default='127.0.0.1', help='Reporting Redis host'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_port'), dest="reporting_redis_port", + default=6379, type=int, help='Reporting Redis port'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_db'), dest="reporting_redis_db", + default=0, type=int, help='Reporting Redis DB')] + + def __init__(self): + super(RedisTransport, self).__init__() self._commands = None self.logger.error("Warning: RedisTransport is experimental") @@ -45,36 +55,15 @@ class RedisTransport(TransportBase): self.logger.error("redis python module is not available") raise TransportError - setup.update(dict( - reporting_redis_host=Option( - 'Redis Host', - default='127.0.0.1', - cf=('reporting', 'redis_host')), - reporting_redis_port=Option( - 'Redis Port', - default=6379, - cf=('reporting', 'redis_port')), - reporting_redis_db=Option( - 'Redis DB', - default=0, - cf=('reporting', 'redis_db')), - )) - setup.reparse() - - self._redis_host = setup.get('reporting_redis_host', '127.0.0.1') - try: - self._redis_port = int(setup.get('reporting_redis_port', 6379)) - except ValueError: - self.logger.error("Redis port must be an integer") - raise TransportError - self._redis_db = setup.get('reporting_redis_db', 0) - self._redis = redis.Redis(host=self._redis_host, - port=self._redis_port, db=self._redis_db) + self._redis = redis.Redis( + host=Bcfg2.Options.setup.reporting_redis_host, + port=Bcfg2.Options.setup.reporting_redis_port, + db=Bcfg2.Options.setup.reporting_redis_db) def start_monitor(self, collector): """Start the monitor. Eventaully start the command thread""" - self._commands = threading.Thread(target=self.monitor_thread, + self._commands = threading.Thread(target=self.monitor_thread, args=(self._redis, collector)) self._commands.start() @@ -129,7 +118,7 @@ class RedisTransport(TransportBase): channel = "%s%s" % (platform.node(), int(time.time())) pubsub.subscribe(channel) - self._redis.rpush(RedisTransport.COMMAND_KEY, + self._redis.rpush(RedisTransport.COMMAND_KEY, cPickle.dumps(RedisMessage(channel, method, args, kwargs))) resp = pubsub.listen() @@ -160,7 +149,7 @@ class RedisTransport(TransportBase): continue message = cPickle.loads(payload[1]) if not isinstance(message, RedisMessage): - self.logger.error("Message \"%s\" is not a RedisMessage" % + self.logger.error("Message \"%s\" is not a RedisMessage" % message) if not message.method in collector.storage.__class__.__rmi__ or\ @@ -192,5 +181,3 @@ class RedisTransport(TransportBase): self.logger.error("Unhandled exception in command thread: %s" % traceback.format_exc().splitlines()[-1]) self.logger.info("Command thread shutdown") - - diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 73bdd0b3a..04b574ed7 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -1,35 +1,3 @@ """ Public transport routines """ - -import sys -from Bcfg2.Reporting.Transport.base import TransportError, \ - TransportImportError - - -def load_transport(transport_name, setup): - """ - Try to load the transport. Raise TransportImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, transport_name) - mod = getattr(__import__(mod_name).Reporting.Transport, transport_name) - except ImportError: - try: - mod = __import__(transport_name) - except: - raise TransportImportError("Error importing transport %s: %s" % - (transport_name, sys.exc_info()[1])) - try: - return getattr(mod, transport_name)(setup) - except: - raise TransportImportError("Error instantiating transport %s: %s" % - (transport_name, sys.exc_info()[1])) - - -def load_transport_from_config(setup): - """Load the transport in the config... eventually""" - try: - return load_transport(setup['reporting_transport'], setup) - except KeyError: - raise TransportImportError('Transport missing in config') diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index 530011e47..9a0a4262f 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -4,7 +4,8 @@ The base for all server -> collector Transports import os import sys -from Bcfg2.Server.Plugin import Debuggable +import Bcfg2.Options +from Bcfg2.Logger import Debuggable class TransportError(Exception): @@ -12,20 +13,18 @@ class TransportError(Exception): pass -class TransportImportError(TransportError): - """Raised when a transport fails to import""" - pass - - class TransportBase(Debuggable): """The base for all transports""" - def __init__(self, setup): + options = Debuggable.options + + def __init__(self): """Do something here""" clsname = self.__class__.__name__ Debuggable.__init__(self, name=clsname) self.debug_log("Loading %s transport" % clsname) - self.data = os.path.join(setup['repo'], 'Reporting', clsname) + self.data = os.path.join(Bcfg2.Options.setup.repository, 'Reporting', + clsname) if not os.path.exists(self.data): self.logger.info("%s does not exist, creating" % self.data) try: @@ -34,7 +33,6 @@ class TransportBase(Debuggable): self.logger.warning("Could not create %s: %s" % (self.data, sys.exc_info()[1])) self.logger.warning("The transport may not function properly") - self.setup = setup self.timeout = 2 def start_monitor(self, collector): diff --git a/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py b/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py index 668094cf5..37cdd146c 100644 --- a/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py +++ b/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py @@ -3,8 +3,7 @@ import datetime from south.db import db from south.v2 import SchemaMigration from django.db import models - -from Bcfg2 import settings +from django.conf import settings class Migration(SchemaMigration): diff --git a/src/lib/Bcfg2/Reporting/migrations/0007_add_flag_fields_interaction.py b/src/lib/Bcfg2/Reporting/migrations/0007_add_flag_fields_interaction.py new file mode 100644 index 000000000..491ecb845 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0007_add_flag_fields_interaction.py @@ -0,0 +1,298 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Interaction.dry_run' + db.add_column('Reporting_interaction', 'dry_run', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Interaction.only_important' + db.add_column('Reporting_interaction', 'only_important', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Interaction.dry_run' + db.delete_column('Reporting_interaction', 'dry_run') + + # Deleting field 'Interaction.only_important' + db.delete_column('Reporting_interaction', 'only_important') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'mode'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'dry_run': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'only_important': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'posixgroups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXGroupEntry']", 'symmetrical': 'False'}), + 'posixusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXUserEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': "orm['Reporting.Group']"}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'sebooleans': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEBooleanEntry']", 'symmetrical': 'False'}), + 'sefcontexts': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEFcontextEntry']", 'symmetrical': 'False'}), + 'seinterfaces': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEInterfaceEntry']", 'symmetrical': 'False'}), + 'selogins': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SELoginEntry']", 'symmetrical': 'False'}), + 'semodules': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEModuleEntry']", 'symmetrical': 'False'}), + 'senodes': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SENodeEntry']", 'symmetrical': 'False'}), + 'sepermissives': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPermissiveEntry']", 'symmetrical': 'False'}), + 'seports': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPortEntry']", 'symmetrical': 'False'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'seusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEUserEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.posixgroupentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXGroupEntry'}, + 'current_gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.posixuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXUserEntry'}, + 'current_gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_group': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'current_home': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_shell': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'group': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'home': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'shell': ('django.db.models.fields.CharField', [], {'default': "'/bin/bash'", 'max_length': '1024'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + 'Reporting.sebooleanentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEBooleanEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'value': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'Reporting.sefcontextentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEFcontextEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'filetype': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seinterfaceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEInterfaceEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seloginentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SELoginEntry'}, + 'current_selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.semoduleentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEModuleEntry'}, + 'current_disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.senodeentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SENodeEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'proto': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.sepermissiveentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPermissiveEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seportentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPortEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + }, + 'Reporting.seuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEUserEntry'}, + 'current_prefix': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'current_roles': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'prefix': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['Reporting']
\ No newline at end of file diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 29f08e787..8e2c644fb 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models, backend, connection + from django.db import models, backend, connections except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -12,6 +12,7 @@ except ImproperlyConfigured: from django.core.cache import cache from datetime import datetime, timedelta from Bcfg2.Compat import cPickle +from Bcfg2.DBSettings import get_db_label TYPE_GOOD = 0 @@ -61,7 +62,8 @@ def _quote(value): global _our_backend if not _our_backend: try: - _our_backend = backend.DatabaseOperations(connection) + _our_backend = backend.DatabaseOperations( + connections[get_db_label('Reporting')]) except TypeError: _our_backend = backend.DatabaseOperations() return _our_backend.quote_name(value) @@ -91,8 +93,8 @@ class InteractionManager(models.Manager): maxdate -- datetime object. Most recent date to pull. (default None) """ - from django.db import connection - cursor = connection.cursor() + from django.db import connections + cursor = connections[get_db_label('Reporting')].cursor() cfilter = "expiration is null" sql = 'select ri.id, x.client_id from ' + \ @@ -142,6 +144,8 @@ class Interaction(models.Model): bad_count = models.IntegerField(default=0) modified_count = models.IntegerField(default=0) extra_count = models.IntegerField(default=0) + dry_run = models.BooleanField(default=False) + only_important = models.BooleanField(default=False) actions = models.ManyToManyField("ActionEntry") packages = models.ManyToManyField("PackageEntry") @@ -381,7 +385,7 @@ class BaseEntry(models.Model): @classmethod def entry_from_type(cls, etype): - for entry_cls in ENTRY_CLASSES: + for entry_cls in ENTRY_TYPES: if etype == entry_cls.ENTRY_TYPE: return entry_cls else: diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7589f404e..8b197231c 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.6</span> + <span>Bcfg2 Version 1.4.0pre1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 33c78a5f0..6a314bd88 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -32,7 +32,7 @@ This is needed for Django versions less than 1.5 <td class='right_column_narrow'>{{ entry.bad_count }}</td> <td class='right_column_narrow'>{{ entry.modified_count }}</td> <td class='right_column_narrow'>{{ entry.extra_count }}</td> - <td class='right_column'><span {% if entry.timestamp|isstale:entry_max %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> + <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 0ee5cd0d6..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -200,19 +200,6 @@ def build_metric_list(mdict): @register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - - -@register.filter def sort_interactions_by_name(value): """ Sort an interaction list by client name @@ -329,7 +316,11 @@ def determine_client_state(entry): dirty. If the client is reporting dirty, this will figure out just _how_ dirty and adjust the color accordingly. """ + if entry.isstale(): + return "stale-lineitem" if entry.state == 'clean': + if entry.extra_count > 0: + return "extra-lineitem" return "clean-lineitem" bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index c7c2a503f..0b8ed65cc 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -13,7 +13,7 @@ from django.http import \ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection, DatabaseError +from django.db import DatabaseError from django.db.models import Q, Count from Bcfg2.Reporting.models import * |