diff options
61 files changed, 2683 insertions, 1414 deletions
@@ -140,6 +140,11 @@ setup(cmdclass=cmdclass, "Bcfg2.Client", "Bcfg2.Client.Tools", "Bcfg2.Client.Tools.POSIX", + "Bcfg2.Reporting", + "Bcfg2.Reporting.Storage", + "Bcfg2.Reporting.Transport", + "Bcfg2.Reporting.migrations", + "Bcfg2.Reporting.templatetags", 'Bcfg2.Server', "Bcfg2.Server.Admin", "Bcfg2.Server.FileMonitor", @@ -152,17 +157,14 @@ setup(cmdclass=cmdclass, "Bcfg2.Server.Plugins.Cfg", "Bcfg2.Server.Reports", "Bcfg2.Server.Reports.reports", - "Bcfg2.Server.Reports.reports.templatetags", - "Bcfg2.Server.SchemaUpdater", "Bcfg2.Server.Snapshots", ], install_requires=inst_reqs, tests_require=['mock', 'nose', 'sqlalchemy'], package_dir={'': 'src/lib', }, - package_data={'Bcfg2.Server.Reports.reports': ['fixtures/*.xml', - 'templates/*.html', - 'templates/*/*.html', - 'templates/*/*.inc']}, + package_data={'Bcfg2.Reporting': [ 'templates/*.html', + 'templates/*/*.html', + 'templates/*/*.inc']}, scripts=glob('src/sbin/*'), data_files=[('share/bcfg2/schemas', glob('schemas/*.xsd')), diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 4fda79dfb..0b4b1b047 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -333,6 +333,23 @@ def get_bool(val): else: raise ValueError +def get_size(value): + if value == -1: + return value + mat = re.match("(\d+)([KkMmGg])?", value) + if not mat: + raise ValueError + rvalue = int(mat.group(1)) + mult = mat.group(2).lower() + if mult == 'k': + return rvalue * 1024 + elif mult == 'm': + return rvalue * 1024 * 1024 + elif mult == 'g': + return rvalue * 1024 * 1024 * 1024 + else: + return rvalue + def get_gid(val): """ This takes a group name or gid and returns the corresponding @@ -607,6 +624,12 @@ DJANGO_WEB_PREFIX = \ default=None, cf=('statistics', 'web_prefix'),) +# Reporting options +REPORTING_FILE_LIMIT = \ + Option('Reporting file size limit', + default=get_size('512m'), + cf=('reporting', 'file_limit'), + cook=get_size,) # Client options CLIENT_KEY = \ @@ -1135,6 +1158,8 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) +REPORTING_COMMON_OPTIONS = dict(reporting_file_limit=REPORTING_FILE_LIMIT) + class OptionParser(OptionSet): """ diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py new file mode 100644 index 000000000..bb2e85c21 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -0,0 +1,111 @@ +import atexit +import daemon +import lockfile +import logging +import time +import traceback +import threading + +import Bcfg2.Logger +from Bcfg2.Reporting.Transport import load_transport_from_config, \ + TransportError, TransportImportError +from Bcfg2.Reporting.Storage import load_storage_from_config, \ + StorageError, StorageImportError + +class ReportingError(Exception): + """Generic reporting exception""" + pass + +class ReportingCollector(object): + """The collecting process for reports""" + + def __init__(self, setup): + """Setup the collector. This may be called by the daemon or though + bcfg2-admin""" + self.setup = setup + self.datastore = setup['repo'] + self.encoding = setup['encoding'] + self.terminate = None + self.context = None + + if setup['debug']: + level = logging.DEBUG + elif setup['verbose']: + level = logging.INFO + else: + level = logging.WARNING + + Bcfg2.Logger.setup_logging('bcfg2-report-collector', + to_console=logging.INFO, + to_syslog=setup['syslog'], + to_file=setup['logging'], + level=level) + self.logger = logging.getLogger('bcfg2-report-collector') + + try: + self.transport = load_transport_from_config(setup) + self.storage = load_storage_from_config(setup) + except TransportError: + self.logger.error("Failed to load transport: %s" % + traceback.format_exc().splitlines()[-1]) + raise ReportingError + except StorageError: + self.logger.error("Failed to load storage: %s" % + traceback.format_exc().splitlines()[-1]) + raise ReportingError + + try: + self.logger.debug("Validating storage %s" % + self.storage.__class__.__name__) + self.storage.validate() + except: + self.logger.error("Storage backed %s failed to validate: %s" % + (self.storage.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + + + def run(self): + """Startup the processing and go!""" + self.terminate = threading.Event() + atexit.register(self.shutdown) + self.context = daemon.DaemonContext() + + if self.setup['daemon']: + self.logger.debug("Daemonizing") + self.context.pidfile = lockfile.FileLock(self.setup['daemon']) + self.context.open() + self.logger.info("Starting daemon") + + self.transport.start_monitor(self) + + while not self.terminate.isSet(): + try: + interaction = self.transport.fetch() + if not interaction: + continue + try: + start = time.time() + self.storage.import_interaction(interaction) + self.logger.info("Imported interaction for %s in %ss" % + (interaction.get('hostname', '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + raise + except (SystemExit, KeyboardInterrupt): + self.logger.info("Shutting down") + self.shutdown() + except: + self.logger.error("Unhandled exception in main loop %s" % + traceback.format_exc().splitlines()[-1]) + + def shutdown(self): + """Cleanup and go""" + if self.terminate: + # this wil be missing if called from bcfg2-admin + self.terminate.set() + if self.transport: + self.transport.shutdown() + if self.storage: + self.storage.shutdown() + diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py new file mode 100644 index 000000000..17eb52f66 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -0,0 +1,316 @@ +""" +The base for the original DjangoORM (DBStats) +""" + +import os +import traceback +from lxml import etree +from datetime import datetime +from time import strptime + +os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' +from Bcfg2 import settings + +from Bcfg2.Compat import md5 +from Bcfg2.Reporting.Storage.base import StorageBase, StorageError +from Bcfg2.Server.Plugin.exceptions import PluginExecutionError +from django.core import management +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.core.cache import cache +from django.db import transaction + +#Used by GetCurrentEntry +import difflib +from Bcfg2.Compat import b64decode +from Bcfg2.Reporting.models import * + + +class DjangoORM(StorageBase): + def __init__(self, setup): + super(DjangoORM, self).__init__(setup) + self.size_limit = setup.get('reporting_size_limit') + + @transaction.commit_on_success + def _import_interaction(self, interaction): + """Real import function""" + hostname = interaction['hostname'] + stats = etree.fromstring(interaction['stats']) + metadata = interaction['metadata'] + server = metadata['server'] + + client = cache.get(hostname) + if not client: + client, created = Client.objects.get_or_create(name=hostname) + if created: + self.logger.debug("Client %s added to the db" % hostname) + cache.set(hostname, client) + + timestamp = datetime(*strptime(stats.get('time'))[0:6]) + if len(Interaction.objects.filter(client=client, timestamp=timestamp)) > 0: + self.logger.warn("Interaction for %s at %s already exists" % + (hostname, timestamp)) + return + + profile, created = Group.objects.get_or_create(name=metadata['profile']) + inter = Interaction(client=client, + timestamp=timestamp, + state=stats.get('state', default="unknown"), + repo_rev_code=stats.get('revision', + default="unknown"), + good_count=stats.get('good', default="0"), + total_count=stats.get('total', default="0"), + server=server, + profile=profile) + inter.save() + self.logger.debug("Interaction for %s at %s with INSERTED in to db" % + (client.id, timestamp)) + + #FIXME - this should be more efficient + for group_name in metadata['groups']: + group = cache.get("GROUP_" + group_name) + if not group: + group, created = Group.objects.get_or_create(name=group_name) + if created: + self.logger.debug("Added group %s" % group) + cache.set("GROUP_" + group_name, group) + + inter.groups.add(group) + for bundle_name in metadata['bundles']: + bundle = cache.get("BUNDLE_" + bundle_name) + if not bundle: + bundle, created = Bundle.objects.get_or_create(name=bundle_name) + if created: + self.logger.debug("Added bundle %s" % bundle) + cache.set("BUNDLE_" + bundle_name, bundle) + inter.bundles.add(bundle) + inter.save() + + counter_fields = {TYPE_BAD: 0, + TYPE_MODIFIED: 0, + TYPE_EXTRA: 0} + pattern = [('Bad/*', TYPE_BAD), + ('Extra/*', TYPE_EXTRA), + ('Modified/*', TYPE_MODIFIED)] + updates = dict(failures=[], paths=[], packages=[], actions=[], services=[]) + for (xpath, state) in pattern: + for entry in stats.findall(xpath): + counter_fields[state] = counter_fields[state] + 1 + + entry_type = entry.tag + name = entry.get('name') + exists = entry.get('current_exists', default="true").lower() == "true" + + # handle server failures differently + failure = entry.get('failure', '') + if failure: + act_dict = dict(name=name, entry_type=entry_type, + message=failure) + newact = FailureEntry.entry_get_or_create(act_dict) + updates['failures'].append(newact) + continue + + act_dict = dict(name=name, state=state, exists=exists) + + if entry_type == 'Action': + act_dict['status'] = entry.get('status', default="check") + act_dict['output'] = entry.get('rc', default=-1) + self.logger.debug("Adding action %s" % name) + updates['actions'].append(ActionEntry.entry_get_or_create(act_dict)) + elif entry_type == 'Package': + act_dict['target_version'] = entry.get('version', default='') + act_dict['current_version'] = entry.get('current_version', default='') + + # extra entries are a bit different. They can have Instance objects + if not act_dict['target_version']: + for instance in entry.findall("Instance"): + #TODO - this probably only works for rpms + release = instance.get('release', '') + arch = instance.get('arch', '') + act_dict['current_version'] = instance.get('version') + if release: + act_dict['current_version'] += "-" + release + if arch: + act_dict['current_version'] += "." + arch + self.logger.debug("Adding package %s %s" % (name, act_dict['current_version'])) + updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) + else: + + self.logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) + + # not implemented yet + act_dict['verification_details'] = entry.get('verification_details', '') + updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) + + elif entry_type == 'Path': + path_type = entry.get("type").lower() + act_dict['path_type'] = path_type + + target_dict = dict( + owner=entry.get('owner', default="root"), + group=entry.get('group', default="root"), + perms=entry.get('perms', default=""), + ) + fperm, created = FilePerms.objects.get_or_create(**target_dict) + act_dict['target_perms'] = fperm + + current_dict = dict( + owner=entry.get('current_owner', default=""), + group=entry.get('current_group', default=""), + perms=entry.get('current_perms', default=""), + ) + fperm, created = FilePerms.objects.get_or_create(**current_dict) + act_dict['current_perms'] = fperm + + if path_type in ('symlink', 'hardlink'): + act_dict['target_path'] = entry.get('to', default="") + act_dict['current_path'] = entry.get('current_to', default="") + self.logger.debug("Adding link %s" % name) + updates['paths'].append(LinkEntry.entry_get_or_create(act_dict)) + continue + elif path_type == 'device': + #TODO devices + self.logger.warn("device path types are not supported yet") + continue + + # TODO - vcs output + act_dict['detail_type'] = PathEntry.DETAIL_UNUSED + if path_type == 'directory' and entry.get('prune', 'false') == 'true': + unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + if unpruned_elist: + act_dict['detail_type'] = PathEntry.DETAIL_PRUNED + act_dict['details'] = "\n".join(unpruned_elist) + elif entry.get('sensitive', 'false').lower() == 'true': + act_dict['detail_type'] = PathEntry.DETAIL_SENSITIVE + else: + cdata = None + if entry.get('current_bfile', None): + act_dict['detail_type'] = PathEntry.DETAIL_BINARY + cdata = entry.get('current_bfile') + elif entry.get('current_bdiff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = b64decode(entry.get('current_bdiff')) + elif entry.get('current_diff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = entry.get('current_bdiff') + if cdata: + if len(cdata) > self.size_limit: + act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT + act_dict['details'] = md5(cdata).hexdigest() + else: + act_dict['details'] = cdata + self.logger.debug("Adding path %s" % name) + updates['paths'].append(PathEntry.entry_get_or_create(act_dict)) + + + #TODO - secontext + #TODO - acls + + elif entry_type == 'Service': + act_dict['target_status'] = entry.get('status', default='') + act_dict['current_status'] = entry.get('current_status', default='') + self.logger.debug("Adding service %s" % name) + updates['services'].append(ServiceEntry.entry_get_or_create(act_dict)) + elif entry_type == 'SELinux': + self.logger.info("SELinux not implemented yet") + else: + self.logger.error("Unknown type %s not handled by reporting yet" % entry_type) + + inter.bad_count = counter_fields[TYPE_BAD] + inter.modified_count = counter_fields[TYPE_MODIFIED] + inter.extra_count = counter_fields[TYPE_EXTRA] + inter.save() + for entry_type in updates.keys(): + getattr(inter, entry_type).add(*updates[entry_type]) + + # performance metrics + for times in stats.findall('OpStamps'): + for metric, value in list(times.items()): + Performance(interaction=inter, metric=metric, value=value).save() + + + def import_interaction(self, interaction): + """Import the data into the backend""" + + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) + + + def validate(self): + """Validate backend storage. Should be called once when loaded""" + + settings.read_config(repo=self.setup['repo']) + + # verify our database schema + try: + if self.setup['verbose'] or self.setup['debug']: + vrb = 2 + else: + vrb = 0 + management.call_command("syncdb", verbosity=vrb, interactive=False) + management.call_command("migrate", verbosity=vrb, interactive=False) + except: + self.logger.error("Failed to update database schema: %s" % \ + traceback.format_exc().splitlines()[-1]) + raise StorageError + + def GetExtra(self, client): + """Fetch extra entries for a client""" + try: + c_inst = Client.objects.get(name=client) + return [(ent.entry_type, ent.name) for ent in + c_inst.current_interaction.extra()] + except ObjectDoesNotExist: + return [] + except MultipleObjectsReturned: + self.logger.error("%s Inconsistency: Multiple entries for %s." % + (self.__class__.__name__, client)) + return [] + + def GetCurrentEntry(self, client, e_type, e_name): + """"GetCurrentEntry: Used by PullSource""" + try: + c_inst = Client.objects.get(name=client) + except ObjectDoesNotExist: + self.logger.error("Unknown client: %s" % client) + raise PluginExecutionError + except MultipleObjectsReturned: + self.logger.error("%s Inconsistency: Multiple entries for %s." % + (self.__class__.__name__, client)) + raise PluginExecutionError + try: + cls = BaseEntry.entry_from_name(e_type + "Entry") + result = cls.objects.filter(name=e_name, state=TYPE_BAD, + interaction=c_inst.current_interaction) + except ValueError: + self.logger.error("Unhandled type %s" % e_type) + raise PluginExecutionError + if not result: + raise PluginExecutionError + entry = result[0] + ret = [] + for p_entry in ('owner', 'group', 'perms'): + this_entry = getattr(entry.current_perms, p_entry) + if this_entry == '': + ret.append(getattr(entry.target_perms, p_entry)) + else: + ret.append(this_entry) + if entry.entry_type == 'Path': + if entry.is_sensitive(): + raise PluginExecutionError + elif entry.detail_type == PathEntry.DETAIL_PRUNED: + ret.append('\n'.join(entry.details)) + elif entry.is_binary(): + ret.append(b64decode(entry.details)) + elif entry.is_diff(): + ret.append('\n'.join(difflib.restore(\ + entry.details.split('\n'), 1))) + elif entry.is_too_large(): + # If len is zero the object was too large to store + raise PluginExecutionError + else: + ret.append(None) + return ret + diff --git a/src/lib/Bcfg2/Reporting/Storage/__init__.py b/src/lib/Bcfg2/Reporting/Storage/__init__.py new file mode 100644 index 000000000..85356fcfe --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/__init__.py @@ -0,0 +1,32 @@ +""" +Public storage routines +""" + +import traceback + +from Bcfg2.Reporting.Storage.base import StorageError, \ + StorageImportError + +def load_storage(storage_name, setup): + """ + Try to load the storage. Raise StorageImportError on failure + """ + try: + mod_name = "%s.%s" % (__name__, storage_name) + mod = getattr(__import__(mod_name).Reporting.Storage, storage_name) + except ImportError: + try: + mod = __import__(storage_name) + except: + raise StorageImportError("Unavailable") + try: + cls = getattr(mod, storage_name) + return cls(setup) + except: + raise StorageImportError("Storage unavailable: %s" % + traceback.format_exc().splitlines()[-1]) + +def load_storage_from_config(setup): + """Load the storage in the config... eventually""" + return load_storage('DjangoORM', setup) + diff --git a/src/lib/Bcfg2/Reporting/Storage/base.py b/src/lib/Bcfg2/Reporting/Storage/base.py new file mode 100644 index 000000000..92cc3a68b --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Storage/base.py @@ -0,0 +1,51 @@ +""" +The base for all Storage backends +""" + +import logging + +class StorageError(Exception): + """Generic StorageError""" + pass + +class StorageImportError(StorageError): + """Raised when a storage module fails to import""" + pass + +class StorageBase(object): + """The base for all storages""" + + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + + def __init__(self, setup): + """Do something here""" + clsname = self.__class__.__name__ + self.logger = logging.getLogger(clsname) + self.logger.debug("Loading %s storage" % clsname) + self.setup = setup + self.encoding = setup['encoding'] + + def import_interaction(self, interaction): + """Import the data into the backend""" + raise NotImplementedError + + def validate(self): + """Validate backend storage. Should be called once when loaded""" + raise NotImplementedError + + def shutdown(self): + """Called at program exit""" + pass + + def Ping(self): + """Test for communication with reporting collector""" + return "Pong" + + def GetExtra(self, client): + """Return a list of extra entries for a client. Minestruct""" + raise NotImplementedError + + def GetCurrentEntry(self, client, e_type, e_name): + """Get the current status of an entry on the client""" + raise NotImplementedError + diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py new file mode 100644 index 000000000..41741ea4b --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -0,0 +1,163 @@ +""" +The local transport. Stats are pickled and written to +<repo>/store/<hostname>-timestamp + +Leans on FileMonitor to detect changes +""" + +import os +import os.path +import select +import time +import traceback +import Bcfg2.Server.FileMonitor +from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError +from Bcfg2.Reporting.Transport.base import TransportBase, TransportError + +try: + import cPickle as pickle +except: + import pickle + +class LocalFilesystem(TransportBase): + def __init__(self, setup): + super(LocalFilesystem, self).__init__(setup) + + self.work_path = "%s/work" % self.data + self.logger.debug("LocalFilesystem: work path %s" % self.work_path) + self.fmon = None + self._phony_collector = None + + #setup our local paths or die + if not os.path.exists(self.work_path): + try: + os.makedirs(self.work_path) + except: + self.logger.error("%s: Unable to create storage: %s" % + (self.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + raise TransportError + + def start_monitor(self, collector): + """Start the file monitor. Most of this comes from BaseCore""" + setup = self.setup + try: + fmon = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] + except KeyError: + self.logger.error("File monitor driver %s not available; " + "forcing to default" % setup['filemonitor']) + fmon = Bcfg2.Server.FileMonitor.available['default'] + + fmdebug = setup.get('debug', False) + try: + self.fmon = fmon(debug=fmdebug) + self.logger.info("Using the %s file monitor" % self.fmon.__class__.__name__) + except IOError: + msg = "Failed to instantiate file monitor %s" % setup['filemonitor'] + self.logger.error(msg, exc_info=1) + raise TransportError(msg) + self.fmon.start() + self.fmon.AddMonitor(self.work_path, self) + + def store(self, hostname, payload): + """Store the file to disk""" + + save_file = "%s/%s-%s" % (self.work_path, hostname, time.time()) + tmp_file = "%s/.%s-%s" % (self.work_path, hostname, time.time()) + if os.path.exists(save_file): + self.logger.error("%s: Oops.. duplicate statistic in directory." % + self.__class__.__name__) + raise TransportError + + # using a tmpfile to hopefully avoid the file monitor from grabbing too + # soon + saved = open(tmp_file, 'w') + try: + saved.write(payload) + except IOError: + self.logger.error("Failed to store interaction for %s: %s" % + (hostname, traceback.format_exc().splitlines()[-1])) + os.unlink(tmp_file) + saved.close() + os.rename(tmp_file, save_file) + + def fetch(self): + """Fetch the next object""" + event = None + fmonfd = self.fmon.fileno() + if self.fmon.pending(): + event = self.fmon.get_event() + elif fmonfd: + select.select([fmonfd], [], [], self.timeout) + if self.fmon.pending(): + event = self.fmon.get_event() + else: + # pseudo.. if nothings pending sleep and loop + time.sleep(self.timeout) + + if not event or event.filename == self.work_path: + return None + + #deviate from the normal routines here we only want one event + etype = event.code2str() + self.logger.debug("Recieved event %s for %s" % (etype, event.filename)) + if os.path.basename(event.filename)[0] == '.': + return None + if etype in ('created', 'exists'): + self.logger.debug("Handling event %s" % event.filename) + payload = os.path.join(self.work_path, event.filename) + try: + payloadfd = open(payload, "r") + interaction = pickle.load(payloadfd) + payloadfd.close() + os.unlink(payload) + return interaction + except IOError: + self.logger.error("Failed to read payload: %s" % + traceback.format_exc().splitlines()[-1]) + except pickle.UnpicklingError: + self.logger.error("Failed to unpickle payload: %s" % + traceback.format_exc().splitlines()[-1]) + payloadfd.close() + raise TransportError + return None + + def shutdown(self): + """Called at program exit""" + if self.fmon: + self.fmon.shutdown() + if self._phony_collector: + self._phony_collector.shutdown() + + def rpc(self, method, *args, **kwargs): + """ + Here this is more of a dummy. Rather then start a layer + which doesn't exist or muck with files, start the collector + + This will all change when other layers are added + """ + try: + if not self._phony_collector: + self._phony_collector = ReportingCollector(self.setup) + except ReportingError: + raise TransportError + except: + self.logger.error("Failed to load collector: %s" % + traceback.format_exc().splitlines()[-1]) + raise TransportError + + if not method in self._phony_collector.storage.__class__.__rmi__ or \ + not hasattr(self._phony_collector.storage, method): + self.logger.error("Unknown method %s called on storage engine %s" % + (method, self._phony_collector.storage.__class__.__name__)) + raise TransportError + + + try: + cls_method = getattr(self._phony_collector.storage, method) + return cls_method(*args, **kwargs) + except: + self.logger.error("RPC method %s failed: %s" % + (method, traceback.format_exc().splitlines()[-1])) + raise TransportError + diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py new file mode 100644 index 000000000..ec39a1628 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -0,0 +1,32 @@ +""" +Public transport routines +""" + +import traceback + +from Bcfg2.Reporting.Transport.base import TransportError, \ + TransportImportError + +def load_transport(transport_name, setup): + """ + Try to load the transport. Raise TransportImportError on failure + """ + try: + mod_name = "%s.%s" % (__name__, transport_name) + mod = getattr(__import__(mod_name).Reporting.Transport, transport_name) + except ImportError: + try: + mod = __import__(transport_name) + except: + raise TransportImportError("Unavailable") + try: + cls = getattr(mod, transport_name) + return cls(setup) + except: + raise TransportImportError("Transport unavailable: %s" % + traceback.format_exc().splitlines()[-1]) + +def load_transport_from_config(setup): + """Load the transport in the config... eventually""" + return load_transport('LocalFilesystem', setup) + diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py new file mode 100644 index 000000000..8488d0e46 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -0,0 +1,45 @@ +""" +The base for all server -> collector Transports +""" + +import os.path +import logging + +class TransportError(Exception): + """Generic TransportError""" + pass + +class TransportImportError(TransportError): + """Raised when a transport fails to import""" + pass + +class TransportBase(object): + """The base for all transports""" + + def __init__(self, setup): + """Do something here""" + clsname = self.__class__.__name__ + self.logger = logging.getLogger(clsname) + self.logger.debug("Loading %s transport" % clsname) + self.data = os.path.join(setup['repo'], clsname.split()[-1]) + self.setup = setup + self.timeout = 2 + + def start_monitor(self, collector): + """Called to start monitoring""" + raise NotImplementedError + + def store(self, hostname, payload): + raise NotImplementedError + + def fetch(self): + raise NotImplementedError + + def shutdown(self): + """Called at program exit""" + pass + + def rpc(self, method, *args, **kwargs): + """Send a request for data to the collector""" + raise NotImplementedError + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Reporting/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py +++ b/src/lib/Bcfg2/Reporting/__init__.py diff --git a/src/lib/Bcfg2/Reporting/migrate.py b/src/lib/Bcfg2/Reporting/migrate.py new file mode 100644 index 000000000..d0b3c9dc4 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrate.py @@ -0,0 +1,230 @@ +import logging +import Bcfg2.Logger +from django.core.cache import cache +from django.db import connection, transaction, backend + +from Bcfg2.Reporting import models as new_models +from Bcfg2.Server.Reports.reports import models as legacy_models + +logger = logging.getLogger(__name__) + +_our_backend = None + +def _quote(value): + """ + Quote a string to use as a table name or column + + Newer versions and various drivers require an argument + https://code.djangoproject.com/ticket/13630 + """ + global _our_backend + if not _our_backend: + try: + _our_backend = backend.DatabaseOperations(connection) + except TypeError: + _our_backend = backend.DatabaseOperations(connection) + return _our_backend.quote_name(value) + + +@transaction.commit_on_success +def _migrate_transaction(inter, entries): + """helper""" + + logger.debug("Migrating interaction %s for %s" % + (inter.id, inter.client.name)) + + newint = new_models.Interaction(id=inter.id, + client_id=inter.client_id, + timestamp=inter.timestamp, + state=inter.state, + repo_rev_code=inter.repo_rev_code, + server=inter.server, + good_count=inter.goodcount, + total_count=inter.totalcount, + bad_count=inter.bad_entries, + modified_count=inter.modified_entries, + extra_count=inter.extra_entries) + + try: + newint.profile_id = inter.metadata.profile.id + groups = [grp.pk for grp in inter.metadata.groups.all()] + bundles = [bun.pk for bun in inter.metadata.bundles.all()] + except legacy_models.InteractionMetadata.DoesNotExist: + groups = [] + bundles = [] + unkown_profile = cache.get("PROFILE_UNKNOWN") + if not unkown_profile: + unkown_profile, created = new_models.Group.objects.get_or_create(name="<<Unknown>>") + cache.set("PROFILE_UNKNOWN", unkown_profile) + newint.profile = unkown_profile + newint.save() + if bundles: + newint.bundles.add(*bundles) + if groups: + newint.groups.add(*groups) + + updates = dict(paths=[], packages=[], actions=[], services=[]) + for ei in legacy_models.Entries_interactions.objects.select_related('reason')\ + .filter(interaction=inter): + ent = entries[ei.entry_id] + name = ent.name + act_dict = dict(name=name, exists=ei.reason.current_exists, + state=ei.type) + + if ent.kind == 'Action': + act_dict['status'] = ei.reason.status + if not act_dict['status']: + act_dict['status'] = "check" + act_dict['output'] = -1 + logger.debug("Adding action %s" % name) + updates['actions'].append(new_models.ActionEntry.entry_get_or_create(act_dict)) + + elif ent.kind == 'Package': + act_dict['target_version'] = ei.reason.version + act_dict['current_version'] = ei.reason.current_version + logger.debug("Adding package %s %s" % + (name, act_dict['target_version'])) + updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) + elif ent.kind == 'Path': + # these might be hard.. they aren't one to one with the old model + act_dict['path_type'] = 'file' + + target_dict = dict( + owner=ei.reason.owner, + group=ei.reason.group, + perms=ei.reason.perms + ) + fperm, created = new_models.FilePerms.objects.get_or_create(**target_dict) + act_dict['target_perms'] = fperm + + current_dict = dict( + owner=ei.reason.current_owner, + group=ei.reason.current_group, + perms=ei.reason.current_perms + ) + fperm, created = new_models.FilePerms.objects.get_or_create(**current_dict) + act_dict['current_perms'] = fperm + + if ei.reason.to: + act_dict['path_type'] = 'symlink' + act_dict['target_path'] = ei.reason.to + act_dict['current_path'] = ei.reason.current_to + logger.debug("Adding link %s" % name) + updates['paths'].append(new_models.LinkEntry.entry_get_or_create(act_dict)) + continue + + act_dict['detail_type'] = new_models.PathEntry.DETAIL_UNUSED + if ei.reason.unpruned: + # this is the only other case we know what the type really is + act_dict['path_type'] = 'directory' + act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED + act_dict['details'] = ei.reason.unpruned + + + if ei.reason.is_sensitive: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE + elif ei.reason.is_binary: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_BINARY + act_dict['details'] = ei.reason.current_diff + elif ei.reason.current_diff: + act_dict['detail_type'] = new_models.PathEntry.DETAIL_DIFF + act_dict['details'] = ei.reason.current_diff + logger.debug("Adding path %s" % name) + updates['paths'].append(new_models.PathEntry.entry_get_or_create(act_dict)) + + elif ent.kind == 'Service': + act_dict['target_status'] = ei.reason.status + act_dict['current_status'] = ei.reason.current_status + logger.debug("Adding service %s" % name) + updates['services'].append(new_models.ServiceEntry.entry_get_or_create(act_dict)) + else: + logger.warn("Skipping type %s" % ent.kind) + + for entry_type in updates.keys(): + i = 0 + while(i < len(updates[entry_type])): + getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + i += 100 + + for perf in inter.performance_items.all(): + new_models.Performance( + interaction=newint, + metric=perf.metric, + value=perf.value).save() + + +def _shove(old_table, new_table, columns): + cols = ",".join([_quote(f) for f in columns]) + sql = "insert into %s(%s) select %s from %s" % ( + _quote(new_table), + cols, + cols, + _quote(old_table)) + + cursor = connection.cursor() + cursor.execute(sql) + cursor.close() + + +@transaction.commit_manually +def _restructure(): + """major restructure of reporting data""" + + logger.info("Migrating clients") + try: + _shove(legacy_models.Client._meta.db_table, new_models.Client._meta.db_table, + ('id', 'name', 'creation', 'expiration')) + except: + logger.error("Failed to migrate clients", exc_info=1) + return False + + logger.info("Migrating Bundles") + try: + _shove(legacy_models.Bundle._meta.db_table, new_models.Bundle._meta.db_table, + ('id', 'name')) + except: + logger.error("Failed to migrate bundles", exc_info=1) + return False + + logger.info("Migrating Groups") + try: + _shove(legacy_models.Group._meta.db_table, new_models.Group._meta.db_table, + ('id', 'name', 'profile', 'public', 'category', 'comment')) + except: + logger.error("Failed to migrate groups", exc_info=1) + return False + + try: + entries = {} + for ent in legacy_models.Entries.objects.all(): + entries[ent.id] = ent + except: + logger.error("Failed to populate entries dict", exc_info=1) + return False + + transaction.commit() + + failures = [] + int_count = legacy_models.Interaction.objects.count() + int_ctr = 0 + for inter in legacy_models.Interaction.objects.select_related().all(): + if int_ctr % 1000 == 0: + logger.info("Migrated %s of %s interactions" % (int_ctr, int_count)) + try: + _migrate_transaction(inter, entries) + except: + logger.error("Failed to migrate interaction %s for %s" % + (inter.id, inter.client.name), exc_info=1) + failures.append(inter.id) + int_ctr += 1 + if not failures: + logger.info("Successfully restructured reason data") + return True + + +if __name__ == '__main__': + Bcfg2.Logger.setup_logging('bcfg2-report-collector', + to_console=logging.INFO, + level=logging.INFO) + _restructure() + diff --git a/src/lib/Bcfg2/Reporting/migrations/0001_initial.py b/src/lib/Bcfg2/Reporting/migrations/0001_initial.py new file mode 100644 index 000000000..609290edb --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0001_initial.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Client' + db.create_table('Reporting_client', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('creation', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_interaction', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='parent_client', null=True, to=orm['Reporting.Interaction'])), + ('expiration', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + )) + db.send_create_signal('Reporting', ['Client']) + + # Adding model 'Interaction' + db.create_table('Reporting_interaction', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('client', self.gf('django.db.models.fields.related.ForeignKey')(related_name='interactions', to=orm['Reporting.Client'])), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(db_index=True)), + ('state', self.gf('django.db.models.fields.CharField')(max_length=32)), + ('repo_rev_code', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('server', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('good_count', self.gf('django.db.models.fields.IntegerField')()), + ('total_count', self.gf('django.db.models.fields.IntegerField')()), + ('bad_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('modified_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('extra_count', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('profile', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.Group'])), + )) + db.send_create_signal('Reporting', ['Interaction']) + + # Adding unique constraint on 'Interaction', fields ['client', 'timestamp'] + db.create_unique('Reporting_interaction', ['client_id', 'timestamp']) + + # Adding M2M table for field actions on 'Interaction' + db.create_table('Reporting_interaction_actions', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('actionentry', models.ForeignKey(orm['Reporting.actionentry'], null=False)) + )) + db.create_unique('Reporting_interaction_actions', ['interaction_id', 'actionentry_id']) + + # Adding M2M table for field packages on 'Interaction' + db.create_table('Reporting_interaction_packages', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('packageentry', models.ForeignKey(orm['Reporting.packageentry'], null=False)) + )) + db.create_unique('Reporting_interaction_packages', ['interaction_id', 'packageentry_id']) + + # Adding M2M table for field paths on 'Interaction' + db.create_table('Reporting_interaction_paths', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('pathentry', models.ForeignKey(orm['Reporting.pathentry'], null=False)) + )) + db.create_unique('Reporting_interaction_paths', ['interaction_id', 'pathentry_id']) + + # Adding M2M table for field services on 'Interaction' + db.create_table('Reporting_interaction_services', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('serviceentry', models.ForeignKey(orm['Reporting.serviceentry'], null=False)) + )) + db.create_unique('Reporting_interaction_services', ['interaction_id', 'serviceentry_id']) + + # Adding M2M table for field failures on 'Interaction' + db.create_table('Reporting_interaction_failures', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('failureentry', models.ForeignKey(orm['Reporting.failureentry'], null=False)) + )) + db.create_unique('Reporting_interaction_failures', ['interaction_id', 'failureentry_id']) + + # Adding M2M table for field groups on 'Interaction' + db.create_table('Reporting_interaction_groups', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('group', models.ForeignKey(orm['Reporting.group'], null=False)) + )) + db.create_unique('Reporting_interaction_groups', ['interaction_id', 'group_id']) + + # Adding M2M table for field bundles on 'Interaction' + db.create_table('Reporting_interaction_bundles', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('bundle', models.ForeignKey(orm['Reporting.bundle'], null=False)) + )) + db.create_unique('Reporting_interaction_bundles', ['interaction_id', 'bundle_id']) + + # Adding model 'Performance' + db.create_table('Reporting_performance', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('interaction', self.gf('django.db.models.fields.related.ForeignKey')(related_name='performance_items', to=orm['Reporting.Interaction'])), + ('metric', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('value', self.gf('django.db.models.fields.DecimalField')(max_digits=32, decimal_places=16)), + )) + db.send_create_signal('Reporting', ['Performance']) + + # Adding model 'Group' + db.create_table('Reporting_group', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)), + ('profile', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('public', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('category', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + ('comment', self.gf('django.db.models.fields.TextField')(blank=True)), + )) + db.send_create_signal('Reporting', ['Group']) + + # Adding M2M table for field groups on 'Group' + db.create_table('Reporting_group_groups', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('from_group', models.ForeignKey(orm['Reporting.group'], null=False)), + ('to_group', models.ForeignKey(orm['Reporting.group'], null=False)) + )) + db.create_unique('Reporting_group_groups', ['from_group_id', 'to_group_id']) + + # Adding M2M table for field bundles on 'Group' + db.create_table('Reporting_group_bundles', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('group', models.ForeignKey(orm['Reporting.group'], null=False)), + ('bundle', models.ForeignKey(orm['Reporting.bundle'], null=False)) + )) + db.create_unique('Reporting_group_bundles', ['group_id', 'bundle_id']) + + # Adding model 'Bundle' + db.create_table('Reporting_bundle', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)), + )) + db.send_create_signal('Reporting', ['Bundle']) + + # Adding model 'FilePerms' + db.create_table('Reporting_fileperms', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('group', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('perms', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('Reporting', ['FilePerms']) + + # Adding unique constraint on 'FilePerms', fields ['owner', 'group', 'perms'] + db.create_unique('Reporting_fileperms', ['owner', 'group', 'perms']) + + # Adding model 'FileAcl' + db.create_table('Reporting_fileacl', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + )) + db.send_create_signal('Reporting', ['FileAcl']) + + # Adding model 'FailureEntry' + db.create_table('Reporting_failureentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('entry_type', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('message', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('Reporting', ['FailureEntry']) + + # Adding model 'ActionEntry' + db.create_table('Reporting_actionentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('status', self.gf('django.db.models.fields.CharField')(default='check', max_length=128)), + ('output', self.gf('django.db.models.fields.IntegerField')(default=0)), + )) + db.send_create_signal('Reporting', ['ActionEntry']) + + # Adding model 'PackageEntry' + db.create_table('Reporting_packageentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('target_version', self.gf('django.db.models.fields.CharField')(default='', max_length=1024)), + ('current_version', self.gf('django.db.models.fields.CharField')(max_length=1024)), + ('verification_details', self.gf('django.db.models.fields.TextField')(default='')), + )) + db.send_create_signal('Reporting', ['PackageEntry']) + + # Adding model 'PathEntry' + db.create_table('Reporting_pathentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('path_type', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('target_perms', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.FilePerms'])), + ('current_perms', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['Reporting.FilePerms'])), + ('detail_type', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('details', self.gf('django.db.models.fields.TextField')(default='')), + )) + db.send_create_signal('Reporting', ['PathEntry']) + + # Adding M2M table for field acls on 'PathEntry' + db.create_table('Reporting_pathentry_acls', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('pathentry', models.ForeignKey(orm['Reporting.pathentry'], null=False)), + ('fileacl', models.ForeignKey(orm['Reporting.fileacl'], null=False)) + )) + db.create_unique('Reporting_pathentry_acls', ['pathentry_id', 'fileacl_id']) + + # Adding model 'LinkEntry' + db.create_table('Reporting_linkentry', ( + ('pathentry_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['Reporting.PathEntry'], unique=True, primary_key=True)), + ('target_path', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + ('current_path', self.gf('django.db.models.fields.CharField')(max_length=1024, blank=True)), + )) + db.send_create_signal('Reporting', ['LinkEntry']) + + # Adding model 'DeviceEntry' + db.create_table('Reporting_deviceentry', ( + ('pathentry_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['Reporting.PathEntry'], unique=True, primary_key=True)), + ('device_type', self.gf('django.db.models.fields.CharField')(max_length=16)), + ('target_major', self.gf('django.db.models.fields.IntegerField')()), + ('target_minor', self.gf('django.db.models.fields.IntegerField')()), + ('current_major', self.gf('django.db.models.fields.IntegerField')()), + ('current_minor', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal('Reporting', ['DeviceEntry']) + + # Adding model 'ServiceEntry' + db.create_table('Reporting_serviceentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.IntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('target_status', self.gf('django.db.models.fields.CharField')(default='', max_length=128)), + ('current_status', self.gf('django.db.models.fields.CharField')(default='', max_length=128)), + )) + db.send_create_signal('Reporting', ['ServiceEntry']) + + + def backwards(self, orm): + # Removing unique constraint on 'FilePerms', fields ['owner', 'group', 'perms'] + db.delete_unique('Reporting_fileperms', ['owner', 'group', 'perms']) + + # Removing unique constraint on 'Interaction', fields ['client', 'timestamp'] + db.delete_unique('Reporting_interaction', ['client_id', 'timestamp']) + + # Deleting model 'Client' + db.delete_table('Reporting_client') + + # Deleting model 'Interaction' + db.delete_table('Reporting_interaction') + + # Removing M2M table for field actions on 'Interaction' + db.delete_table('Reporting_interaction_actions') + + # Removing M2M table for field packages on 'Interaction' + db.delete_table('Reporting_interaction_packages') + + # Removing M2M table for field paths on 'Interaction' + db.delete_table('Reporting_interaction_paths') + + # Removing M2M table for field services on 'Interaction' + db.delete_table('Reporting_interaction_services') + + # Removing M2M table for field failures on 'Interaction' + db.delete_table('Reporting_interaction_failures') + + # Removing M2M table for field groups on 'Interaction' + db.delete_table('Reporting_interaction_groups') + + # Removing M2M table for field bundles on 'Interaction' + db.delete_table('Reporting_interaction_bundles') + + # Deleting model 'Performance' + db.delete_table('Reporting_performance') + + # Deleting model 'Group' + db.delete_table('Reporting_group') + + # Removing M2M table for field groups on 'Group' + db.delete_table('Reporting_group_groups') + + # Removing M2M table for field bundles on 'Group' + db.delete_table('Reporting_group_bundles') + + # Deleting model 'Bundle' + db.delete_table('Reporting_bundle') + + # Deleting model 'FilePerms' + db.delete_table('Reporting_fileperms') + + # Deleting model 'FileAcl' + db.delete_table('Reporting_fileacl') + + # Deleting model 'FailureEntry' + db.delete_table('Reporting_failureentry') + + # Deleting model 'ActionEntry' + db.delete_table('Reporting_actionentry') + + # Deleting model 'PackageEntry' + db.delete_table('Reporting_packageentry') + + # Deleting model 'PathEntry' + db.delete_table('Reporting_pathentry') + + # Removing M2M table for field acls on 'PathEntry' + db.delete_table('Reporting_pathentry_acls') + + # Deleting model 'LinkEntry' + db.delete_table('Reporting_linkentry') + + # Deleting model 'DeviceEntry' + db.delete_table('Reporting_deviceentry') + + # Deleting model 'ServiceEntry' + db.delete_table('Reporting_serviceentry') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'perms'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'perms': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.Group']"}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + } + } + + complete_apps = ['Reporting']
\ No newline at end of file diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py b/src/lib/Bcfg2/Reporting/migrations/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py +++ b/src/lib/Bcfg2/Reporting/migrations/__init__.py diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py new file mode 100644 index 000000000..3a540587a --- /dev/null +++ b/src/lib/Bcfg2/Reporting/models.py @@ -0,0 +1,582 @@ +"""Django models for Bcfg2 reports.""" +import sys + +from django.core.exceptions import ImproperlyConfigured +try: + from django.db import models +except ImproperlyConfigured: + e = sys.exc_info()[1] + print("Reports: unable to import django models: %s" % e) + sys.exit(1) + +from django.core.cache import cache +from datetime import datetime, timedelta + +try: + import cPickle as pickle +except: + import pickle + +KIND_CHOICES = ( + #These are the kinds of config elements + ('Package', 'Package'), + ('Path', 'directory'), + ('Path', 'file'), + ('Path', 'permissions'), + ('Path', 'symlink'), + ('Service', 'Service'), +) +TYPE_GOOD = 0 +TYPE_BAD = 1 +TYPE_MODIFIED = 2 +TYPE_EXTRA = 3 + +TYPE_CHOICES = ( + (TYPE_GOOD, 'Good'), + (TYPE_BAD, 'Bad'), + (TYPE_MODIFIED, 'Modified'), + (TYPE_EXTRA, 'Extra'), +) + + +def convert_entry_type_to_id(type_name): + """Convert a entry type to its entry id""" + for e_id, e_name in TYPE_CHOICES: + if e_name.lower() == type_name.lower(): + return e_id + return -1 + + +def hash_entry(entry_dict): + """ + Build a key for this based on its data + + entry_dict = a dict of all the data identifying this + """ + dataset = [] + for key in sorted(entry_dict.keys()): + if key in ('id', 'hash_key') or key.startswith('_'): + continue + dataset.append( (key, entry_dict[key]) ) + return hash(pickle.dumps(dataset)) + + +class Client(models.Model): + """Object representing every client we have seen stats for.""" + creation = models.DateTimeField(auto_now_add=True) + name = models.CharField(max_length=128,) + current_interaction = models.ForeignKey('Interaction', + null=True, blank=True, + related_name="parent_client") + expiration = models.DateTimeField(blank=True, null=True) + + def __str__(self): + return self.name + + +class InteractionManager(models.Manager): + """Manages interactions objects.""" + + def recent_ids(self, maxdate=None): + """ + Returns the ids of most recent interactions for clients as of a date. + + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + + """ + from django.db import connection + cursor = connection.cursor() + cfilter = "expiration is null" + + sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \ + 'as timer from Reporting_interaction' + if maxdate: + if not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + sql = sql + " where timestamp <= '%s' " % maxdate + cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) + sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \ + 'ri.client_id = x.client_id AND ri.timestamp = x.timer' + sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter + try: + cursor.execute(sql) + return [item[0] for item in cursor.fetchall()] + except: + '''FIXME - really need some error handling''' + pass + return [] + + + def recent(self, maxdate=None): + """ + Returns the most recent interactions for clients as of a date + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + + """ + if maxdate and not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + return self.filter(id__in=self.recent_ids(maxdate)) + + +class Interaction(models.Model): + """Models each reconfiguration operation interaction between client and server.""" + client = models.ForeignKey(Client, related_name="interactions") + timestamp = models.DateTimeField(db_index=True) # Timestamp for this record + state = models.CharField(max_length=32) # good/bad/modified/etc + repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + server = models.CharField(max_length=256) # Name of the server used for the interaction + good_count = models.IntegerField() # of good config-items + total_count = models.IntegerField() # of total config-items + bad_count = models.IntegerField(default=0) + modified_count = models.IntegerField(default=0) + extra_count = models.IntegerField(default=0) + + actions = models.ManyToManyField("ActionEntry") + packages = models.ManyToManyField("PackageEntry") + paths = models.ManyToManyField("PathEntry") + services = models.ManyToManyField("ServiceEntry") + failures = models.ManyToManyField("FailureEntry") + + # Formerly InteractionMetadata + profile = models.ForeignKey("Group", related_name="+") + groups = models.ManyToManyField("Group") + bundles = models.ManyToManyField("Bundle") + + objects = InteractionManager() + + def __str__(self): + return "With " + self.client.name + " @ " + self.timestamp.isoformat() + + def percentgood(self): + if not self.total_count == 0: + return (self.good_count / float(self.total_count)) * 100 + else: + return 0 + + def percentbad(self): + if not self.total_count == 0: + return ((self.total_count - self.good_count) / (float(self.total_count))) * 100 + else: + return 0 + + def isclean(self): + if (self.bad_count == 0 and self.good_count == self.total_count): + return True + else: + return False + + def isstale(self): + if (self == self.client.current_interaction): # Is Mostrecent + if(datetime.now() - self.timestamp > timedelta(hours=25)): + return True + else: + return False + else: + #Search for subsequent Interaction for this client + #Check if it happened more than 25 hrs ago. + if (self.client.interactions.filter(timestamp__gt=self.timestamp) + .order_by('timestamp')[0].timestamp - + self.timestamp > timedelta(hours=25)): + return True + else: + return False + + def save(self): + super(Interaction, self).save() # call the real save... + self.client.current_interaction = self.client.interactions.latest() + self.client.save() # save again post update + + def delete(self): + '''Override the default delete. Allows us to remove Performance items''' + pitems = list(self.performance_items.all()) + super(Interaction, self).delete() + for perf in pitems: + if perf.interaction.count() == 0: + perf.delete() + + def badcount(self): + return self.total_count - self.good_count + + def bad(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_BAD)) + return rv + + def modified(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_MODIFIED)) + return rv + + def extra(self): + rv = [] + for entry in ('actions', 'packages', 'paths', 'services'): + rv.extend(getattr(self, entry).filter(state=TYPE_EXTRA)) + return rv + + class Meta: + get_latest_by = 'timestamp' + ordering = ['-timestamp'] + unique_together = ("client", "timestamp") + + +class Performance(models.Model): + """Object representing performance data for any interaction.""" + interaction = models.ForeignKey(Interaction, related_name="performance_items") + metric = models.CharField(max_length=128) + value = models.DecimalField(max_digits=32, decimal_places=16) + + def __str__(self): + return self.metric + + +class Group(models.Model): + """ + Groups extracted from interactions + + name - The group name + + TODO - Most of this is for future use + TODO - set a default group + """ + + name = models.CharField(max_length=255, unique=True) + profile = models.BooleanField(default=False) + public = models.BooleanField(default=False) + category = models.CharField(max_length=1024, blank=True) + comment = models.TextField(blank=True) + + groups = models.ManyToManyField("self", symmetrical=False) + bundles = models.ManyToManyField("Bundle") + + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + +class Bundle(models.Model): + """ + Bundles extracted from interactions + + name - The bundle name + """ + + name = models.CharField(max_length=255, unique=True) + + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + +# new interaction models +class FilePerms(models.Model): + owner = models.CharField(max_length=128) + group = models.CharField(max_length=128) + perms = models.CharField(max_length=128) + + class Meta: + unique_together = ('owner', 'group', 'perms') + + def empty(self): + """Return true if we have no real data""" + if self.owner or self.group or self.perms: + return False + else: + return True + + +class FileAcl(models.Model): + """Placeholder""" + name = models.CharField(max_length=128, db_index=True) + + +class BaseEntry(models.Model): + """ Abstract base for all entry types """ + name = models.CharField(max_length=128, db_index=True) + hash_key = models.IntegerField(editable=False, db_index=True) + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + if 'hash_key' in kwargs: + self.hash_key = kwargs['hash_key'] + del kwargs['hash_key'] + else: + self.hash_key = hash_entry(self.__dict__) + super(BaseEntry, self).save(*args, **kwargs) + + + def class_name(self): + return self.__class__.__name__ + + def short_list(self): + """todo""" + return [] + + + @classmethod + def entry_from_name(cls, name): + try: + newcls = globals()[name] + if not isinstance(newcls(), cls): + raise ValueError("%s is not an instance of %s" % (name, cls)) + return newcls + except KeyError: + raise ValueError("Invalid type %s" % name) + + + @classmethod + def entry_get_or_create(cls, act_dict): + """Helper to quickly lookup an object""" + cls_name = cls().__class__.__name__ + act_hash = hash_entry(act_dict) + + # TODO - get form cache and validate + act_key = "%s_%s" % (cls_name, act_hash) + newact = cache.get(act_key) + if newact: + return newact + + acts = cls.objects.filter(hash_key=act_hash) + if len(acts) > 0: + for act in acts: + for key in act_dict: + if act_dict[key] != getattr(act, key): + continue + #match found + newact = act + break + + # worst case, its new + if not newact: + newact = cls(**act_dict) + newact.save(hash_key=act_hash) + + cache.set(act_key, newact) + return newact + + + def is_failure(self): + return isinstance(self, FailureEntry) + + +class SuccessEntry(BaseEntry): + """Base for successful entries""" + state = models.IntegerField(choices=TYPE_CHOICES) + exists = models.BooleanField(default=True) + + ENTRY_TYPE = r"Success" + + @property + def entry_type(self): + return self.ENTRY_TYPE + + def is_extra(self): + return self.state == TYPE_EXTRA + + class Meta: + abstract = True + ordering = ('state', 'name') + + def short_list(self): + """Return a list of problems""" + rv = [] + if self.is_extra(): + rv.append("Extra") + elif not self.exists: + rv.append("Missing") + return rv + + +class FailureEntry(BaseEntry): + """Represents objects that failed to bind""" + entry_type = models.CharField(max_length=128) + message = models.TextField() + + def is_failure(self): + return True + + +class ActionEntry(SuccessEntry): + """ The new model for package information """ + status = models.CharField(max_length=128, default="check") + output = models.IntegerField(default=0) + + ENTRY_TYPE = r"Action" + #TODO - prune + + +class PackageEntry(SuccessEntry): + """ The new model for package information """ + + # if this is an extra entry trget_version will be empty + target_version = models.CharField(max_length=1024, default='') + current_version = models.CharField(max_length=1024) + verification_details = models.TextField(default="") + + ENTRY_TYPE = r"Package" + #TODO - prune + + def version_problem(self): + """Check for a version problem.""" + if not self.current_version: + return True + if self.target_version != self.current_version: + return True + elif self.target_version == 'auto': + return True + else: + return False + + def short_list(self): + """Return a list of problems""" + rv = super(PackageEntry, self).short_list() + if self.is_extra(): + return rv + if not self.version_problem() or not self.exists: + return rv + if not self.current_version: + rv.append("Missing") + else: + rv.append("Wrong version") + return rv + + +class PathEntry(SuccessEntry): + """reason why modified or bad entry did not verify, or changed.""" + + PATH_TYPES = ( + ("device", "Device"), + ("directory", "Directory"), + ("hardlink", "Hard Link"), + ("nonexistent", "Non Existent"), + ("permissions", "Permissions"), + ("symlink", "Symlink"), + ) + + DETAIL_UNUSED = 0 + DETAIL_DIFF = 1 + DETAIL_BINARY = 2 + DETAIL_SENSITIVE = 3 + DETAIL_SIZE_LIMIT = 4 + DETAIL_VCS = 5 + DETAIL_PRUNED = 6 + + DETAIL_CHOICES = ( + (DETAIL_UNUSED, 'Unused'), + (DETAIL_DIFF, 'Diff'), + (DETAIL_BINARY, 'Binary'), + (DETAIL_SENSITIVE, 'Sensitive'), + (DETAIL_SIZE_LIMIT, 'Size limit exceeded'), + (DETAIL_VCS, 'VCS output'), + (DETAIL_PRUNED, 'Pruned paths'), + ) + + path_type = models.CharField(max_length=128, choices=PATH_TYPES) + + target_perms = models.ForeignKey(FilePerms, related_name="+") + current_perms = models.ForeignKey(FilePerms, related_name="+") + + acls = models.ManyToManyField(FileAcl) + + detail_type = models.IntegerField(default=0, + choices=DETAIL_CHOICES) + details = models.TextField(default='') + + ENTRY_TYPE = r"Path" + + def perms_problem(self): + if self.current_perms.empty(): + return False + elif self.target_perms.perms != self.current_perms.perms: + return True + else: + return False + + def has_detail(self): + return self.detail_type != PathEntry.DETAIL_UNUSED + + def is_sensitive(self): + return self.detail_type == PathEntry.DETAIL_SENSITIVE + + def is_diff(self): + return self.detail_type == PathEntry.DETAIL_DIFF + + def is_sensitive(self): + return self.detail_type == PathEntry.DETAIL_SENSITIVE + + def is_binary(self): + return self.detail_type == PathEntry.DETAIL_BINARY + + def is_too_large(self): + return self.detail_type == PathEntry.DETAIL_SIZE_LIMIT + + def short_list(self): + """Return a list of problems""" + rv = super(PathEntry, self).short_list() + if self.is_extra(): + return rv + if self.perms_problem(): + rv.append("File permissions") + if self.detail_type == PathEntry.DETAIL_PRUNED: + rv.append("Directory has extra files") + elif self.detail_type != PathEntry.DETAIL_UNUSED: + rv.append("Incorrect data") + if hasattr(self, 'linkentry') and \ + self.linkentry.target_path != self.linkentry.current_path: + rv.append("Incorrect target") + return rv + + +class LinkEntry(PathEntry): + """Sym/Hard Link types""" + target_path = models.CharField(max_length=1024, blank=True) + current_path = models.CharField(max_length=1024, blank=True) + + def link_problem(self): + return self.target_path != self.current_path + + +class DeviceEntry(PathEntry): + """Device types. Best I can tell the client driver needs work here""" + DEVICE_TYPES = ( + ("block", "Block"), + ("char", "Char"), + ("fifo", "Fifo"), + ) + + device_type = models.CharField(max_length=16, choices=DEVICE_TYPES) + + target_major = models.IntegerField() + target_minor = models.IntegerField() + current_major = models.IntegerField() + current_minor = models.IntegerField() + + +class ServiceEntry(SuccessEntry): + """ The new model for package information """ + target_status = models.CharField(max_length=128, default='') + current_status = models.CharField(max_length=128, default='') + + ENTRY_TYPE = r"Service" + #TODO - prune + + def status_problem(self): + return self.target_status != self.current_status + + def short_list(self): + """Return a list of problems""" + rv = super(ServiceEntry, self).short_list() + if self.status_problem(): + rv.append("Incorrect status") + return rv + + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/404.html b/src/lib/Bcfg2/Reporting/templates/404.html index 168bd9fec..168bd9fec 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/404.html +++ b/src/lib/Bcfg2/Reporting/templates/404.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html b/src/lib/Bcfg2/Reporting/templates/base-timeview.html index 9a5ef651c..9a5ef651c 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html +++ b/src/lib/Bcfg2/Reporting/templates/base-timeview.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 6d20f86d9..6d20f86d9 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 9b86b609f..b2244bfa1 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -35,7 +35,7 @@ span.history_links a { <select id="quick" name="quick" onchange="javascript:pageJump('quick');"> <option value="" selected="selected">--- Time ---</option> {% for i in client.interactions.all|slice:":25" %} - <option value="{% url reports_client_detail_pk hostname=client.name, pk=i.id %}">{{i.timestamp}}</option> + <option value="{% url reports_client_detail_pk hostname=client.name, pk=i.id %}">{{i.timestamp|date:"c"}}</option> {% endfor %} </select></span> </div> @@ -50,64 +50,66 @@ span.history_links a { {% if interaction.server %} <tr><td>Served by</td><td>{{interaction.server}}</td></tr> {% endif %} - {% if interaction.metadata %} - <tr><td>Profile</td><td>{{interaction.metadata.profile}}</td></tr> - {% endif %} + <tr><td>Profile</td><td>{{interaction.profile}}</td></tr> {% if interaction.repo_rev_code %} <tr><td>Revision</td><td>{{interaction.repo_rev_code}}</td></tr> {% endif %} <tr><td>State</td><td class='{{interaction.state}}-lineitem'>{{interaction.state|capfirst}}</td></tr> - <tr><td>Managed entries</td><td>{{interaction.totalcount}}</td></tr> + <tr><td>Managed entries</td><td>{{interaction.total_count}}</td></tr> {% if not interaction.isclean %} <tr><td>Deviation</td><td>{{interaction.percentbad|floatformat:"3"}}%</td></tr> {% endif %} </table> - {% if interaction.metadata.groups.count %} + {% for group in interaction.groups.all %} + {% if forloop.first %} <div class='entry_list'> <div class='entry_list_head' onclick='javascript:toggleMe("groups_table");'> <h3>Group membership</h3> <div class='entry_expand_tab' id='plusminus_groups_table'>[+]</div> </div> <table id='groups_table' class='entry_list' style='display: none'> - {% for group in interaction.metadata.groups.all %} + {% endif %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{group}}</td> </tr> - {% endfor %} + {% if forloop.last %} </table> </div> {% endif %} + {% endfor %} - {% if interaction.metadata.bundles.count %} + {% for bundle in interaction.bundles.all %} + {% if forloop.first %} <div class='entry_list'> <div class='entry_list_head' onclick='javascript:toggleMe("bundles_table");'> <h3>Bundle membership</h3> <div class='entry_expand_tab' id='plusminus_bundless_table'>[+]</div> </div> <table id='bundles_table' class='entry_list' style='display: none'> - {% for bundle in interaction.metadata.bundles.all %} + {% endif %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{bundle}}</td> </tr> - {% endfor %} + {% if forloop.last %} </table> </div> {% endif %} + {% endfor %} - {% for type, ei_list in ei_lists %} - {% if ei_list %} + {% for entry_type, entry_list in entry_types.items %} + {% if entry_list %} <div class='entry_list'> - <div class='entry_list_head {{type}}-lineitem' onclick='javascript:toggleMe("{{type}}_table");'> - <h3>{{ type|capfirst }} Entries — {{ ei_list|length }}</h3> - <div class='entry_expand_tab' id='plusminus_{{type}}_table'>[+]</div> + <div class='entry_list_head {{entry_type}}-lineitem' onclick='javascript:toggleMe("{{entry_type}}_table");'> + <h3>{{ entry_type|capfirst }} Entries — {{ entry_list|length }}</h3> + <div class='entry_expand_tab' id='plusminus_{{entry_type}}_table'>[+]</div> </div> - <table id='{{type}}_table' class='entry_list'> - {% for ei in ei_list %} + <table id='{{entry_type}}_table' class='entry_list'> + {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='entry_list_type'>{{ei.entry.kind}}</td> - <td><a href="{% url reports_item type ei.id %}"> - {{ei.entry.name}}</a></td> + <td class='entry_list_type'>{{entry.entry_type}}</td> + <td><a href="{% url reports_item entry.class_name entry.pk interaction.pk %}"> + {{entry.name}}</a></td> </tr> {% endfor %} </table> @@ -115,6 +117,24 @@ span.history_links a { {% endif %} {% endfor %} + {% if interaction.failures.all %} + <div class='entry_list'> + <div class='entry_list_head' onclick='javascript:toggleMe("failures_table");'> + <h3>Failed entries</h3> + <div class='entry_expand_tab' id='plusminus_failuress_table'>[+]</div> + </div> + <table id='failures_table' class='entry_list' style='display: none'> + {% for failure in interaction.failures.all %} + <tr class='{% cycle listview,listview_alt %}'> + <td class='entry_list_type'>{{failure.entry_type}}</td> + <td><a href="{% url reports_item failure.class_name failure.pk interaction.pk %}"> + {{failure.name}}</a></td> + </tr> + {% endfor %} + </table> + </div> + {% endif %} + {% if entry_list %} <div class="entry_list recent_history_wrapper"> <div class="entry_list_head" style="border-bottom: 2px solid #98DBCC;"> diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 9be59e7d2..06c99d899 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -21,13 +21,13 @@ </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='left_column'><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> + <td class='left_column'><a href='{% url Bcfg2.Reporting.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> - <td class='right_column_narrow'>{{ entry.goodcount }}</td> - <td class='right_column_narrow'>{{ entry.bad_entry_count }}</td> - <td class='right_column_narrow'>{{ entry.modified_entry_count }}</td> - <td class='right_column_narrow'>{{ entry.extra_entry_count }}</td> + <td class='right_column_narrow'>{{ entry.good_count }}</td> + <td class='right_column_narrow'>{{ entry.bad_count }}</td> + <td class='right_column_narrow'>{{ entry.modified_count }}</td> + <td class='right_column_narrow'>{{ entry.extra_count }}</td> <td class='right_column'><span {% if entry.timestamp|isstale:entry_max %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> <td class='right_column_wide'> {% if entry.server %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html b/src/lib/Bcfg2/Reporting/templates/clients/history.html index 01d4ec2f4..01d4ec2f4 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/history.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index 45ba20b86..45ba20b86 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 443ec8ccb..443ec8ccb 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index d6ad303fc..b39957a2e 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -25,12 +25,12 @@ {% if type_list %} <table id='table_{{ type_name }}' class='entry_list'> <tr style='text-align: left'><th>Type</th><th>Name</th><th>Count</th><th>Reason</th></tr> - {% for entry, reason, interaction in type_list %} + {% for item in type_list %} <tr class='{% cycle listview,listview_alt %}'> - <td>{{ entry.kind }}</td> - <td><a href="{% url reports_entry eid=entry.pk %}">{{ entry.name }}</a></td> - <td>{{ interaction|length }}</td> - <td><a href="{% url reports_item type=type_name pk=interaction.0 %}">{{ reason.short_list|join:"," }}</a></td> + <td>{{ item.ENTRY_TYPE }}</td> + <td><a href="{% url reports_entry item.class_name, item.pk %}">{{ item.name }}</a></td> + <td>{{ item.num_entries }}</td> + <td><a href="{% url reports_item item.ENTRY_TYPE, item.pk %}">{{ item.short_list|join:"," }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html new file mode 100644 index 000000000..e940889ab --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -0,0 +1,32 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Entry Status{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}{{ entry.entry_type }} entry {{ entry.name }} status{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if items %} + <div class='entry_list'> + <table class='entry_list'> + <tr style='text-align: left' ><th>Name</th><th>Timestamp</th><th>State</th><th>Reason</th></tr> + {% for item, inters in items %} + {% for inter in inters %} + <tr class='{% cycle listview,listview_alt %}'> + <td><a href='{% url reports_client_detail hostname=inter.client.name %}'>{{inter.client.name}}</a></td> + <td><a href='{% url reports_client_detail_pk hostname=inter.client.name, pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> + <td>{{ item.get_state_display }}</td> + <td style='white-space: nowrap'><a href="{% url reports_item entry_type=item.class_name pk=item.pk %}">({{item.pk}}) {{item.short_list|join:","}}</a></td> + </tr> + {% endfor %} + {% endfor %} + </table> + </div> +{% else %} + <p>There are currently no hosts with this configuration entry.</p> +{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html b/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html new file mode 100644 index 000000000..0b87fbdbd --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item-failure.html @@ -0,0 +1,13 @@ +{% extends "config_items/item.html" %} +{% load syntax_coloring %} + +{% block item_details %} +<div class='entry_list'> + <div class='entry_list_head'> + <h3>This item failed to bind on the server</h3> + </div> + <div class='diff_wrapper'> + {{ item.message|syntaxhilight:"py" }} + </div> +</div> +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index cadc178a7..4c2e9c2ae 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -31,63 +31,68 @@ div.entry_list h3 { {% block content %} <div class='detail_header'> - <h3>{{mod_or_bad|capfirst}} {{item.entry.kind}}: {{item.entry.name}}</h3> + <h3>{{item.get_state_display}} {{item.entry_type}}: {{item.name}}</h3> </div> <div class="information_wrapper"> - - {% if isextra %} +{% block item_details %} + {% if item.is_extra %} <p>This item exists on the host but is not defined in the configuration.</p> {% endif %} - {% if not item.reason.current_exists %} + {% if not item.exists %} <div class="warning">This item does not currently exist on the host but is specified to exist in the configuration.</div> {% endif %} - {% if item.reason.current_owner or item.reason.current_group or item.reason.current_perms or item.reason.current_status or item.reason.current_status or item.reason.current_to or item.reason.current_version %} +{# Really need a better test here #} +{% if item.perms_problem or item.status_problem or item.linkentry.link_problem or item.version_problem %} <table class='entry_list'> <tr id='table_list_header'> <td style='text-align: right;'>Problem Type</td><td>Expected</td><td style='border-bottom: 1px solid #98DBCC;'>Found</td></tr> - {% if item.reason.current_owner %} - <tr><td style='text-align: right'><b>Owner</b></td><td>{{item.reason.owner}}</td> - <td>{{item.reason.current_owner}}</td></tr> - {% endif %} - {% if item.reason.current_group %} - <tr><td style='text-align: right'><b>Group</b></td><td>{{item.reason.group}}</td> - <td>{{item.reason.current_group}}</td></tr> - {% endif %} - {% if item.reason.current_perms %} - <tr><td style='text-align: right'><b>Permissions</b></td><td>{{item.reason.perms}}</td> - <td>{{item.reason.current_perms}}</td></tr> - {% endif %} - {% if item.reason.current_status %} - <tr><td style='text-align: right'><b>Status</b></td><td>{{item.reason.status}}</td> - <td>{{item.reason.current_status}}</td></tr> - {% endif %} - {% if item.reason.current_to %} - <tr><td style='text-align: right'><b>Symlink Target</b></td><td>{{item.reason.to}}</td> - <td>{{item.reason.current_to}}</td></tr> - {% endif %} - {% if item.reason.current_version %} - <tr><td style='text-align: right'><b>Package Version</b></td><td>{{item.reason.version|cut:"("|cut:")"}}</td> - <td>{{item.reason.current_version|cut:"("|cut:")"}}</td></tr> - {% endif %} - </table> + {% if item.perms_problem %} + {% if item.current_perms.owner %} + <tr><td style='text-align: right'><b>Owner</b></td><td>{{item.target_perms.owner}}</td> + <td>{{item.current_perms.owner}}</td></tr> + {% endif %} + {% if item.current_perms.group %} + <tr><td style='text-align: right'><b>Group</b></td><td>{{item.target_perms.group}}</td> + <td>{{item.current_perms.group}}</td></tr> + {% endif %} + {% if item.current_perms.perms %} + <tr><td style='text-align: right'><b>Perms</b></td><td>{{item.target_perms.perms}}</td> + <td>{{item.current_perms.perms}}</td></tr> + {% endif %} + {% endif %} + {% if item.status_problem %} + <tr><td style='text-align: right'><b>Status</b></td><td>{{item.target_status}}</td> + <td>{{item.current_status}}</td></tr> {% endif %} + {% if item.linkentry.link_problem %} + <tr><td style='text-align: right'><b>{{item.get_path_type_display}}</b></td><td>{{item.linkentry.target_path}}</td> + <td>{{item.linkentry.current_path}}</td></tr> + {% endif %} + {% if item.version_problem %} + <tr><td style='text-align: right'><b>Package Version</b></td><td>{{item.target_version|cut:"("|cut:")"}}</td> + <td>{{item.current_version|cut:"("|cut:")"}}</td></tr> + {% endif %} + </table> +{% endif %} - {% if item.reason.current_diff or item.reason.is_sensitive %} + {% if item.has_detail %} <div class='entry_list'> <div class='entry_list_head'> - {% if item.reason.is_sensitive %} + {% if item.is_sensitive %} <h3>File contents unavailable, as they might contain sensitive data.</h3> {% else %} - <h3>Incorrect file contents</h3> + <h3>Incorrect file contents ({{item.get_detail_type_display}})</h3> {% endif %} </div> - {% if not item.reason.is_sensitive %} + {% if item.is_diff %} <div class='diff_wrapper'> - {{ item.reason.current_diff|syntaxhilight }} + {{ item.details|syntaxhilight }} </div> + {% else %} + {{ item.details }} {% endif %} </div> {% endif %} @@ -105,6 +110,7 @@ div.entry_list h3 { </table> </div> {% endif %} +{% endblock %} <div class='entry_list'> diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 0a92e7fc0..864392754 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -6,7 +6,7 @@ {% block extra_header_info %} {% endblock%} -{% block pagebanner %}{{mod_or_bad|capfirst}} Element Listing{% endblock %} +{% block pagebanner %}{{item_state|capfirst}} Element Listing{% endblock %} {% block content %} {% filter_navigator %} @@ -18,12 +18,12 @@ <div class='entry_expand_tab' id='plusminus_table_{{ type_name }}'>[–]</div> </div> <table id='table_{{ type_name }}' class='entry_list'> - <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> - {% for entry, reason, eis in type_data %} + <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> + {% for entry in type_data %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_entry eid=entry.pk %}">{{entry.name}}</a></td> - <td>{{ eis|length }}</td> - <td><a href="{% url reports_item type=mod_or_bad,pk=eis.0 %}">{{ reason.short_list|join:"," }}</a></td> + <td><a href="{% url reports_entry entry.class_name entry.pk %}">{{entry.name}}</a></td> + <td>{{entry.num_entries}}</td> + <td><a href="{% url reports_item entry.class_name entry.pk %}">{{entry.short_list|join:","}}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index b9847cf96..b9847cf96 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index ff775ded5..ff775ded5 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html index 759415507..759415507 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html +++ b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc index 6fe7e6547..30ed2fd3e 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc +++ b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc @@ -21,10 +21,10 @@ {% endif %} <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> - <td class='right_column_narrow'>{{ entry.goodcount }}</td> - <td class='right_column_narrow'>{{ entry.bad_entry_count }}</td> - <td class='right_column_narrow'>{{ entry.modified_entry_count }}</td> - <td class='right_column_narrow'>{{ entry.extra_entry_count }}</td> + <td class='right_column_narrow'>{{ entry.good_count }}</td> + <td class='right_column_narrow'>{{ entry.bad_count }}</td> + <td class='right_column_narrow'>{{ entry.modified_count }}</td> + <td class='right_column_narrow'>{{ entry.extra_count }}</td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html index aa0def83e..aa0def83e 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html +++ b/src/lib/Bcfg2/Reporting/templates/widgets/page_bar.html diff --git a/src/lib/Bcfg2/Reporting/templatetags/__init__.py b/src/lib/Bcfg2/Reporting/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/__init__.py diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 736d6448a..c079f4a3c 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -10,8 +10,8 @@ from django.template.loader import get_template, \ from django.utils.encoding import smart_unicode, smart_str from django.utils.safestring import mark_safe from datetime import datetime, timedelta -from Bcfg2.Server.Reports.utils import filter_list -from Bcfg2.Server.Reports.reports.models import Group +from Bcfg2.Reporting.utils import filter_list +from Bcfg2.Reporting.models import Group register = template.Library() @@ -208,7 +208,7 @@ class AddUrlFilter(template.Node): def __init__(self, filter_name, filter_value): self.filter_name = filter_name self.filter_value = filter_value - self.fallback_view = 'Bcfg2.Server.Reports.reports.views.render_history_view' + self.fallback_view = 'Bcfg2.Reporting.views.render_history_view' def render(self, context): link = '#' @@ -244,7 +244,7 @@ def add_url_filter(parser, token): Return a url with the filter added to the current view. Takes a new filter and resolves the current view with the new filter - applied. Resolves to Bcfg2.Server.Reports.reports.views.client_history + applied. Resolves to Bcfg2.Reporting.views.client_history by default. {% add_url_filter server=interaction.server %} @@ -310,7 +310,7 @@ def determine_client_state(entry): if entry.state == 'clean': return "clean-lineitem" - bad_percentage = 100 * (float(entry.badcount()) / entry.totalcount) + bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) if bad_percentage < 33: thisdirty = "slightly-dirty-lineitem" elif bad_percentage < 66: diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py b/src/lib/Bcfg2/Reporting/templatetags/split.py index a9b4f0371..a9b4f0371 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py +++ b/src/lib/Bcfg2/Reporting/templatetags/split.py diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py index bd379b98d..2712d6395 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py @@ -27,7 +27,8 @@ def syntaxhilight(value, arg="diff", autoescape=None): """ if autoescape: - value = conditional_escape(value) + # Seems to cause a double escape + #value = conditional_escape(value) arg = conditional_escape(arg) if colorize: diff --git a/src/lib/Bcfg2/Server/Reports/reports/urls.py b/src/lib/Bcfg2/Reporting/urls.py index 1cfe725c2..4dd343905 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/urls.py +++ b/src/lib/Bcfg2/Reporting/urls.py @@ -1,7 +1,7 @@ from django.conf.urls.defaults import * from django.core.urlresolvers import reverse, NoReverseMatch from django.http import HttpResponsePermanentRedirect -from Bcfg2.Server.Reports.utils import filteredUrls, paginatedUrls, timeviewUrls +from Bcfg2.Reporting.utils import filteredUrls, paginatedUrls, timeviewUrls def newRoot(request): try: @@ -10,17 +10,18 @@ def newRoot(request): grid_view = '/grid' return HttpResponsePermanentRedirect(grid_view) -urlpatterns = patterns('Bcfg2.Server.Reports.reports', +urlpatterns = patterns('Bcfg2.Reporting', (r'^$', newRoot), url(r'^manage/?$', 'views.client_manage', name='reports_client_manage'), url(r'^client/(?P<hostname>[^/]+)/(?P<pk>\d+)/?$', 'views.client_detail', name='reports_client_detail_pk'), url(r'^client/(?P<hostname>[^/]+)/?$', 'views.client_detail', name='reports_client_detail'), - url(r'^elements/(?P<type>\w+)/(?P<pk>\d+)/?$', 'views.config_item', name='reports_item'), - url(r'^entry/(?P<eid>\w+)/?$', 'views.entry_status', name='reports_entry'), + url(r'^element/(?P<entry_type>\w+)/(?P<pk>\d+)/(?P<interaction>\d+)?/?$', 'views.config_item', name='reports_item'), + url(r'^element/(?P<entry_type>\w+)/(?P<pk>\d+)/?$', 'views.config_item', name='reports_item'), + url(r'^entry/(?P<entry_type>\w+)/(?P<pk>\w+)/?$', 'views.entry_status', name='reports_entry'), ) -urlpatterns += patterns('Bcfg2.Server.Reports.reports', +urlpatterns += patterns('Bcfg2.Reporting', *timeviewUrls( (r'^summary/?$', 'views.display_summary', None, 'reports_summary'), (r'^timing/?$', 'views.display_timing', None, 'reports_timing'), @@ -28,15 +29,15 @@ urlpatterns += patterns('Bcfg2.Server.Reports.reports', (r'^common/?$', 'views.common_problems', None, 'reports_common_problems'), )) -urlpatterns += patterns('Bcfg2.Server.Reports.reports', +urlpatterns += patterns('Bcfg2.Reporting', *filteredUrls(*timeviewUrls( (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), (r'^detailed/?$', 'views.client_detailed_list', None, 'reports_detailed_list'), - (r'^elements/(?P<type>\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), + (r'^elements/(?P<item_state>\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), ))) -urlpatterns += patterns('Bcfg2.Server.Reports.reports', +urlpatterns += patterns('Bcfg2.Reporting', *paginatedUrls( *filteredUrls( (r'^history/?$', 'views.render_history_view', None, 'reports_history'), diff --git a/src/lib/Bcfg2/Server/Reports/utils.py b/src/lib/Bcfg2/Reporting/utils.py index c47763e39..c47763e39 100755 --- a/src/lib/Bcfg2/Server/Reports/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py diff --git a/src/lib/Bcfg2/Server/Reports/reports/views.py b/src/lib/Bcfg2/Reporting/views.py index ca9e5f1f9..58774831f 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -14,9 +14,9 @@ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch from django.db import connection, DatabaseError -from django.db.models import Q +from django.db.models import Q, Count -from Bcfg2.Server.Reports.reports.models import * +from Bcfg2.Reporting.models import * __SORT_FIELDS__ = ( 'client', 'state', 'good', 'bad', 'modified', 'extra', \ @@ -133,105 +133,92 @@ def _handle_filters(query, **kwargs): return query -def config_item(request, pk, type="bad"): +def config_item(request, pk, entry_type, interaction=None): """ Display a single entry. - Dispalys information about a single entry. + Displays information about a single entry. """ - item = get_object_or_404(Entries_interactions, id=pk) - timestamp = item.interaction.timestamp - time_start = item.interaction.timestamp.replace(hour=0, - minute=0, - second=0, - microsecond=0) - time_end = time_start + timedelta(days=1) - - todays_data = Interaction.objects.filter(timestamp__gte=time_start, - timestamp__lt=time_end) - shared_entries = Entries_interactions.objects.filter(entry=item.entry, - reason=item.reason, - type=item.type, - interaction__in=[x['id']\ - for x in todays_data.values('id')]) - - associated_list = Interaction.objects.filter(id__in=[x['interaction']\ - for x in shared_entries.values('interaction')])\ - .order_by('client__name', 'timestamp').select_related().all() - - return render_to_response('config_items/item.html', + try: + cls = BaseEntry.entry_from_name(entry_type) + except ValueError: + # TODO - handle this + raise + item = get_object_or_404(cls, pk=pk) + + # TODO - timestamp + if interaction: + try: + inter = Interaction.objects.get(pk=interaction) + except Interaction.DoesNotExist: + raise Http404("Not a valid interaction") + timestamp = inter.timestamp + else: + timestamp = datetime.now() + + ts_start = timestamp.replace(hour=1, minute=0, second=0, microsecond=0) + ts_end = ts_start + timedelta(days=1) + associated_list = item.interaction_set.select_related('client').filter(\ + timestamp__gte=ts_start, timestamp__lt=ts_end) + + if item.is_failure(): + template = 'config_items/item-failure.html' + else: + template = 'config_items/item.html' + return render_to_response(template, {'item': item, - 'isextra': item.type == TYPE_EXTRA, - 'mod_or_bad': type, 'associated_list': associated_list, 'timestamp': timestamp}, context_instance=RequestContext(request)) @timeview -def config_item_list(request, type, timestamp=None, **kwargs): +def config_item_list(request, item_state, timestamp=None, **kwargs): """Render a listing of affected elements""" - mod_or_bad = type.lower() - type = convert_entry_type_to_id(type) - if type < 0: + state = convert_entry_type_to_id(item_state.lower()) + if state < 0: raise Http404 - current_clients = Interaction.objects.interaction_per_client(timestamp) + current_clients = Interaction.objects.recent(timestamp) current_clients = [q['id'] for q in _handle_filters(current_clients, **kwargs).values('id')] - ldata = list(Entries_interactions.objects.filter( - interaction__in=current_clients, type=type).values()) - entry_ids = set([x['entry_id'] for x in ldata]) - reason_ids = set([x['reason_id'] for x in ldata]) - - entries = _in_bulk(Entries, entry_ids) - reasons = _in_bulk(Reason, reason_ids) - - kind_list = {} - [kind_list.__setitem__(kind, {}) for kind in set([e.kind for e in entries.values()])] - for x in ldata: - kind = entries[x['entry_id']].kind - data_key = (x['entry_id'], x['reason_id']) - try: - kind_list[kind][data_key].append(x['id']) - except KeyError: - kind_list[kind][data_key] = [x['id']] - lists = [] - for kind in kind_list.keys(): - lists.append((kind, [(entries[e[0][0]], reasons[e[0][1]], e[1]) - for e in sorted(kind_list[kind].iteritems(), key=lambda x: entries[x[0][0]].name)])) + for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: + ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ + .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') + if len(ldata) > 0: + # Property doesn't render properly.. + lists.append((etype.ENTRY_TYPE, ldata)) return render_to_response('config_items/listing.html', {'item_list': lists, - 'mod_or_bad': mod_or_bad, + 'item_state': item_state, 'timestamp': timestamp}, context_instance=RequestContext(request)) @timeview -def entry_status(request, eid, timestamp=None, **kwargs): - """Render a listing of affected elements""" - entry = get_object_or_404(Entries, pk=eid) - - current_clients = Interaction.objects.interaction_per_client(timestamp) - inters = {} - [inters.__setitem__(i.id, i) \ - for i in _handle_filters(current_clients, **kwargs).select_related('client')] - - eis = Entries_interactions.objects.filter( - interaction__in=inters.keys(), entry=entry) - - reasons = _in_bulk(Reason, set([x.reason_id for x in eis])) - - item_data = [] - for ei in eis: - item_data.append((ei, inters[ei.interaction_id], reasons[ei.reason_id])) - +def entry_status(request, entry_type, pk, timestamp=None, **kwargs): + """Render a listing of affected elements by type and name""" + try: + cls = BaseEntry.entry_from_name(entry_type) + except ValueError: + # TODO - handle this + raise + item = get_object_or_404(cls, pk=pk) + + current_clients = Interaction.objects.recent(timestamp) + current_clients = [i['pk'] for i in _handle_filters(current_clients, **kwargs).values('pk')] + + # There is no good way to do this... + items = [] + for it in cls.objects.filter(interaction__in=current_clients, name=item.name).distinct("id").select_related(): + items.append((it, it.interaction_set.filter(pk__in=current_clients).order_by('client__name').select_related('client'))) + return render_to_response('config_items/entry_status.html', - {'entry': entry, - 'item_data': item_data, + {'entry': item, + 'items': items, 'timestamp': timestamp}, context_instance=RequestContext(request)) @@ -256,33 +243,15 @@ def common_problems(request, timestamp=None, threshold=None): except: threshold = 10 - c_intr = Interaction.objects.get_interaction_per_client_ids(timestamp) - data_list = {} - [data_list.__setitem__(t_id, {}) \ - for t_id, t_label in TYPE_CHOICES if t_id != TYPE_GOOD] - ldata = list(Entries_interactions.objects.filter( - interaction__in=c_intr).exclude(type=TYPE_GOOD).values()) - - entry_ids = set([x['entry_id'] for x in ldata]) - reason_ids = set([x['reason_id'] for x in ldata]) - for x in ldata: - type = x['type'] - data_key = (x['entry_id'], x['reason_id']) - try: - data_list[type][data_key].append(x['id']) - except KeyError: - data_list[type][data_key] = [x['id']] - - entries = _in_bulk(Entries, entry_ids) - reasons = _in_bulk(Reason, reason_ids) - + current_clients = Interaction.objects.recent_ids(timestamp) lists = [] - for type, type_name in TYPE_CHOICES: - if type == TYPE_GOOD: - continue - lists.append([type_name.lower(), [(entries[e[0][0]], reasons[e[0][1]], e[1]) - for e in sorted(data_list[type].items(), key=lambda x: len(x[1]), reverse=True) - if len(e[1]) > threshold]]) + for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: + ldata = etype.objects.exclude(state=TYPE_GOOD).filter( + interaction__in=current_clients).annotate(num_entries=Count('id')).filter(num_entries__gte=threshold)\ + .order_by('-num_entries', 'name') + if len(ldata) > 0: + # Property doesn't render properly.. + lists.append((etype.ENTRY_TYPE, ldata)) return render_to_response('config_items/common.html', {'lists': lists, @@ -300,7 +269,7 @@ def client_index(request, timestamp=None, **kwargs): timestamp -- datetime object to render from """ - list = _handle_filters(Interaction.objects.interaction_per_client(timestamp), **kwargs).\ + list = _handle_filters(Interaction.objects.recent(timestamp), **kwargs).\ select_related().order_by("client__name").all() return render_to_response('clients/index.html', @@ -339,7 +308,7 @@ def client_detailed_list(request, timestamp=None, **kwargs): kwargs['orderby'] = "client__name" kwargs['sort'] = "client" - kwargs['interaction_base'] = Interaction.objects.interaction_per_client(timestamp).select_related() + kwargs['interaction_base'] = Interaction.objects.recent(timestamp).select_related() kwargs['page_limit'] = 0 return render_history_view(request, 'clients/detailed-list.html', **kwargs) @@ -354,15 +323,14 @@ def client_detail(request, hostname=None, pk=None): inter = client.interactions.get(pk=pk) maxdate = inter.timestamp - ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry').order_by('entry__kind', 'entry__name') - #ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry') - #ei = sorted(Entries_interactions.objects.filter(interaction=inter).select_related('entry'), - # key=lambda x: (x.entry.kind, x.entry.name)) - context['ei_lists'] = ( - ('bad', [x for x in ei if x.type == TYPE_BAD]), - ('modified', [x for x in ei if x.type == TYPE_MODIFIED]), - ('extra', [x for x in ei if x.type == TYPE_EXTRA]) - ) + etypes = { TYPE_BAD: 'bad', TYPE_MODIFIED: 'modified', TYPE_EXTRA: 'extra' } + edict = dict() + for label in etypes.values(): + edict[label] = [] + for ekind in ('actions', 'packages', 'paths', 'services'): + for ent in getattr(inter, ekind).all(): + edict[etypes[ent.state]].append(ent) + context['entry_types'] = edict context['interaction']=inter return render_history_view(request, 'clients/detail.html', page_limit=5, @@ -403,8 +371,8 @@ def display_summary(request, timestamp=None): """ Display a summary of the bcfg2 world """ - recent_data = Interaction.objects.interaction_per_client(timestamp) \ - .select_related().all() + recent_data = Interaction.objects.recent(timestamp) \ + .select_related() node_count = len(recent_data) if not timestamp: timestamp = datetime.now() @@ -418,13 +386,13 @@ def display_summary(request, timestamp=None): if timestamp - node.timestamp > timedelta(hours=24): collected_data['stale'].append(node) # If stale check for uptime - if node.bad_entry_count() > 0: + if node.bad_count > 0: collected_data['bad'].append(node) else: collected_data['clean'].append(node) - if node.modified_entry_count() > 0: + if node.modified_count > 0: collected_data['modified'].append(node) - if node.extra_entry_count() > 0: + if node.extra_count > 0: collected_data['extra'].append(node) # label, header_text, node_list @@ -456,17 +424,16 @@ def display_summary(request, timestamp=None): @timeview def display_timing(request, timestamp=None): + perfs = Performance.objects.filter(interaction__in=Interaction.objects.recent_ids(timestamp))\ + .select_related('interaction__client') + mdict = dict() - inters = Interaction.objects.interaction_per_client(timestamp).select_related().all() - [mdict.__setitem__(inter, {'name': inter.client.name}) \ - for inter in inters] - for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): - for i in metric.interaction.all(): - try: - mdict[i][metric.metric] = metric.value - except KeyError: - #In the unlikely event two interactions share a metric, ignore it - pass + for perf in perfs: + client = perf.interaction.client.name + if client not in mdict: + mdict[client] = { 'name': client } + mdict[client][perf.metric] = perf.value + return render_to_response('displays/timing.html', {'metrics': list(mdict.values()), 'timestamp': timestamp}, @@ -514,7 +481,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): iquery = kwargs.get('interaction_base', Interaction.objects) if client: iquery = iquery.filter(client__exact=client) - iquery = iquery.select_related() + iquery = iquery.select_related('client') if 'orderby' in kwargs and kwargs['orderby']: iquery = iquery.order_by(kwargs['orderby']) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 63a0092d5..815d34e97 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -8,19 +8,16 @@ import pickle import platform import sys import traceback -from lxml.etree import XML, XMLSyntaxError -from Bcfg2.Compat import ConfigParser, md5 +from Bcfg2.Compat import md5 -import Bcfg2.settings +from Bcfg2 import settings # Load django and reports stuff _after_ we know we can load settings -import django.core.management -from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError -from Bcfg2.Server.Reports.utils import * +from django.core import management +from Bcfg2.Reporting.utils import * -project_directory = os.path.dirname(Bcfg2.settings.__file__) +project_directory = os.path.dirname(settings.__file__) project_name = os.path.basename(project_directory) sys.path.append(os.path.join(project_directory, '..')) project_module = __import__(project_name, '', '', ['']) @@ -30,9 +27,8 @@ sys.path.pop() os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name from django.db import connection, transaction -from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ - Entries_interactions, Performance, \ - Reason +from Bcfg2.Reporting.models import Client, Interaction, \ + Performance def printStats(fn): @@ -54,7 +50,8 @@ def printStats(fn): self.log.info("Interactions removed: %s" % (start_i - Interaction.objects.count())) self.log.info("Interactions->Entries removed: %s" % - (start_ei - Entries_interactions.objects.count())) + (start_ei - 0)) + # (start_ei - Entries_interactions.objects.count())) self.log.info("Metrics removed: %s" % (start_perf - Performance.objects.count())) @@ -70,9 +67,6 @@ class Reports(Bcfg2.Server.Admin.Mode): "\n" " Commands:\n" " init Initialize the database\n" - " load_stats Load statistics data\n" - " -s|--stats Path to statistics.xml file\n" - " -O3 Fast mode. Duplicates data!\n" " purge Purge records\n" " --client [n] Client to operate on\n" " --days [n] Records older then n days\n" @@ -85,6 +79,11 @@ class Reports(Bcfg2.Server.Admin.Mode): def __init__(self, setup): Bcfg2.Server.Admin.Mode.__init__(self, setup) + try: + import south + except ImportError: + print "Django south is required for Reporting" + raise SystemExit(-3) def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) @@ -99,24 +98,16 @@ class Reports(Bcfg2.Server.Admin.Mode): elif args[0] == 'scrub': self.scrub() elif args[0] in ['init', 'update']: + if self.setup['verbose'] or self.setup['debug']: + vrb = 2 + else: + vrb = 0 try: - update_database() - except UpdaterError: - print("Update failed") + management.call_command("syncdb", verbosity=vrb) + management.call_command("migrate", verbosity=vrb) + except: + print("Update failed: %s" % traceback.format_exc().splitlines()[-1]) raise SystemExit(-1) - elif args[0] == 'load_stats': - quick = '-O3' in args - stats_file = None - i = 1 - while i < len(args): - if args[i] == '-s' or args[i] == '--stats': - stats_file = args[i + 1] - if stats_file[0] == '-': - self.errExit("Invalid statistics file: %s" % stats_file) - elif args[i] == '-c' or args[i] == '--clients-file': - print("DeprecationWarning: %s is no longer used" % args[i]) - i = i + 1 - self.load_stats(stats_file, self.log.getEffectiveLevel() > logging.WARNING, quick) elif args[0] == 'purge': expired = False client = None @@ -203,9 +194,9 @@ class Reports(Bcfg2.Server.Admin.Mode): Reason.prune_orphans() self.log.info("Pruned %d Reason records" % (start_count - Reason.objects.count())) - start_count = Entries.objects.count() - Entries.prune_orphans() - self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) + #start_count = Entries.objects.count() + #Entries.prune_orphans() + #self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) def django_command_proxy(self, command): '''Call a django command''' @@ -214,37 +205,6 @@ class Reports(Bcfg2.Server.Admin.Mode): else: django.core.management.call_command(command) - def load_stats(self, stats_file=None, verb=0, quick=False): - '''Load statistics data into the database''' - location = '' - - if not stats_file: - try: - stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.errExit("Could not read bcfg2.conf; exiting") - try: - statsdata = XML(open(stats_file).read()) - except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s" % (stats_file)) - - try: - encoding = self.cfp.get('components', 'encoding') - except: - encoding = 'UTF-8' - - try: - load_stats(statsdata, - encoding, - verb, - self.log, - quick=quick, - location=platform.node()) - except UpdaterError: - self.errExit("StatReports: Database updater failed") - except: - self.errExit("failed to import stats: %s" - % traceback.format_exc().splitlines()[-1]) @printStats def purge(self, client=None, maxdate=None, state=None): @@ -272,7 +232,7 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) - if Bcfg2.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': + if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': grp_limit = 100 else: grp_limit = 1000 diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 1eb953e2a..3686722ae 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -1,8 +1,7 @@ import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin -from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError -from django.core.management import setup_environ +from django.core.management import setup_environ, call_command class Syncdb(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("Sync the Django ORM with the configured database") @@ -23,7 +22,13 @@ class Syncdb(Bcfg2.Server.Admin.Mode): Bcfg2.Server.models.load_models(cfile=self.opts['configfile']) try: - update_database() - except UpdaterError: - print("Update failed") + call_command("syncdb", interactive=False, verbosity=0) + self._database_available = True + except ImproperlyConfigured: + self.logger.error("Django configuration problem: %s" % + format_exc().splitlines()[-1]) + raise SystemExit(-1) + except: + self.logger.error("Database update failed: %s" % + format_exc().splitlines()[-1]) raise SystemExit(-1) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 14b9d9d0a..ae1c578fa 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -148,25 +148,18 @@ class BaseCore(object): Bcfg2.settings.read_config(repo=self.datastore) self._database_available = False - # verify our database schema - try: - from Bcfg2.Server.SchemaUpdater import update_database, \ - UpdaterError + if Bcfg2.settings.HAS_DJANGO: + from django.core.exceptions import ImproperlyConfigured + from django.core import management try: - update_database() + management.call_command("syncdb", interactive=False, verbosity=0) self._database_available = True - except UpdaterError: - err = sys.exc_info()[1] - self.logger.error("Failed to update database schema: %s" % err) - except ImportError: - # assume django is not installed - pass - except Exception: - inst = sys.exc_info()[1] - self.logger.error("Failed to update database schema") - self.logger.error(str(inst)) - self.logger.error(str(type(inst))) - raise CoreInitError + except ImproperlyConfigured: + self.logger.error("Django configuration problem: %s" % + format_exc().splitlines()[-1]) + except: + self.logger.error("Database update failed: %s" % + format_exc().splitlines()[-1]) if '' in setup['plugins']: setup['plugins'].remove('') diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 12965c040..23f5424d0 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -7,6 +7,7 @@ from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ from Bcfg2.Server.FileMonitor import Event, FileMonitor + class GaminEvent(Event): """ This class provides an event analogous to diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 31c3d79b0..01f590b06 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -21,6 +21,8 @@ except ImportError: LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py new file mode 100644 index 000000000..883b95ba4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -0,0 +1,105 @@ +import time +import platform +import traceback +from lxml import etree + +from Bcfg2.Reporting.Transport import load_transport_from_config, \ + TransportError, TransportImportError + +try: + import cPickle as pickle +except: + import pickle + +from Bcfg2.Options import REPORTING_COMMON_OPTIONS +from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \ + PluginExecutionError + +def _rpc_call(method): + def _real_rpc_call(self, *args, **kwargs): + """Wrapper for calls to the reporting collector""" + + try: + return self.transport.rpc(method, *args, **kwargs) + except TransportError: + # this is needed for Admin.Pull + raise PluginExecutionError + return _real_rpc_call + +class Reporting(Statistics, PullSource): + + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + + CLIENT_METADATA_FILEDS = ('profile', 'bundles', 'aliases', 'addresses', + 'groups', 'categories', 'uuid', 'version') + + def __init__(self, core, datastore): + Statistics.__init__(self, core, datastore) + PullSource.__init__(self) + self.core = core + self.experimental = True + + self.whoami = platform.node() + self.transport = None + + core.setup.update(REPORTING_COMMON_OPTIONS) + core.setup.reparse() + self.logger.error("File limit: %s" % core.setup['reporting_file_limit']) + + try: + self.transport = load_transport_from_config(core.setup) + except TransportError: + self.logger.error("%s: Failed to load transport: %s" % + (self.name, traceback.format_exc().splitlines()[-1])) + raise PluginInitError + + + def process_statistics(self, client, xdata): + stats = xdata.find("Statistics") + stats.set('time', time.asctime(time.localtime())) + + cdata = { 'server': self.whoami } + for field in self.CLIENT_METADATA_FILEDS: + try: + value = getattr(client, field) + except AttributeError: + continue + if value: + if isinstance(value, set): + value = [v for v in value] + cdata[field] = value + + try: + interaction_data = pickle.dumps({ 'hostname': client.hostname, + 'metadata': cdata, 'stats': + etree.tostring(stats, xml_declaration=False).decode('UTF-8') }) + except: + self.logger.error("%s: Failed to build interaction object: %s" % + (self.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + + # try 3 times to store the data + for i in [1, 2, 3]: + try: + self.transport.store(client.hostname, interaction_data) + self.logger.debug("%s: Queued statistics data for %s" % + (self.__class__.__name__, client.hostname)) + return + except TransportError: + continue + except: + self.logger.error("%s: Attempt %s: Failed to add statistic %s" % + (self.__class__.__name__, i, + traceback.format_exc().splitlines()[-1])) + self.logger.error("%s: Retry limit reached for %s" % + (self.__class__.__name__, client.hostname)) + + def shutdown(self): + super(Reporting, self).shutdown() + if self.transport: + self.transport.shutdown() + + Ping = _rpc_call('Ping') + GetExtra = _rpc_call('GetExtra') + GetCurrentEntry = _rpc_call('GetCurrentEntry') + diff --git a/src/lib/Bcfg2/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py deleted file mode 100644 index 9f07c104f..000000000 --- a/src/lib/Bcfg2/Server/Reports/backends.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -from django.contrib.auth.models import User -from nisauth import * - - -class NISBackend(object): - - def authenticate(self, username=None, password=None): - try: - print("start nis authenticate") - n = nisauth(username, password) - temp_pass = User.objects.make_random_password(100) - nis_user = dict(username=username, - ) - - user_session_obj = dict(email=username, - first_name=None, - last_name=None, - uid=n.uid) - user, created = User.objects.get_or_create(username=username) - - return user - - except NISAUTHError: - e = sys.exc_info()[1] - print(e) - return None - - def get_user(self, user_id): - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - e = sys.exc_info()[1] - print(e) - return None diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py deleted file mode 100755 index ace07a75d..000000000 --- a/src/lib/Bcfg2/Server/Reports/importscript.py +++ /dev/null @@ -1,335 +0,0 @@ -#! /usr/bin/env python -""" -Imports statistics.xml and clients.xml files in to database backend for -new statistics engine -""" - -import os -import sys -import traceback -try: - import Bcfg2.settings -except Exception: - e = sys.exc_info()[1] - sys.stderr.write("Failed to load configuration settings. %s\n" % e) - sys.exit(1) - -project_directory = os.path.dirname(Bcfg2.settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from Bcfg2.Server.Reports.reports.models import * -from lxml.etree import XML, XMLSyntaxError -from getopt import getopt, GetoptError -from datetime import datetime -from time import strptime -from django.db import connection, transaction -from Bcfg2.Server.Plugins.Metadata import ClientMetadata -import logging -import Bcfg2.Logger -import platform - -# Compatibility import -from Bcfg2.Compat import ConfigParser, b64decode - - -def build_reason_kwargs(r_ent, encoding, logger): - binary_file = False - sensitive_file = False - unpruned_entries = '' - if r_ent.get('sensitive') in ['true', 'True']: - sensitive_file = True - rc_diff = '' - elif r_ent.get('current_bfile', False): - binary_file = True - rc_diff = r_ent.get('current_bfile') - if len(rc_diff) > 1024 * 1024: - rc_diff = '' - elif len(rc_diff) == 0: - # No point in flagging binary if we have no data - binary_file = False - elif r_ent.get('current_bdiff', False): - rc_diff = b64decode(r_ent.get('current_bdiff')) - elif r_ent.get('current_diff', False): - rc_diff = r_ent.get('current_diff') - else: - rc_diff = '' - # detect unmanaged entries in pruned directories - if r_ent.get('prune', 'false') == 'true' and r_ent.get('qtest'): - unpruned_elist = [e.get('path') for e in r_ent.findall('Prune')] - unpruned_entries = "\n".join(unpruned_elist) - if not binary_file: - try: - rc_diff = rc_diff.decode(encoding) - except: - logger.error("Reason isn't %s encoded, cannot decode it" % encoding) - rc_diff = '' - return dict(owner=r_ent.get('owner', default=""), - current_owner=r_ent.get('current_owner', default=""), - group=r_ent.get('group', default=""), - current_group=r_ent.get('current_group', default=""), - perms=r_ent.get('perms', default=""), - current_perms=r_ent.get('current_perms', default=""), - status=r_ent.get('status', default=""), - current_status=r_ent.get('current_status', default=""), - to=r_ent.get('to', default=""), - current_to=r_ent.get('current_to', default=""), - version=r_ent.get('version', default=""), - current_version=r_ent.get('current_version', default=""), - current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", - current_diff=rc_diff, - is_binary=binary_file, - is_sensitive=sensitive_file, - unpruned=unpruned_entries) - -def _fetch_reason(elem, kargs, logger): - try: - rr = None - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - rr = Reason(**kargs) - rr.save() - logger.debug("Created reason: %s" % rr.id) - except Exception: - ex = sys.exc_info()[1] - logger.error("Failed to create reason for %s: %s" % (elem.get('name'), ex)) - rr = Reason(current_exists=elem.get('current_exists', - default="True").capitalize() == "True") - rr.save() - return rr - - -def load_stats(sdata, encoding, vlevel, logger, quick=False, location=''): - for node in sdata.findall('Node'): - name = node.get('name') - for statistics in node.findall('Statistics'): - try: - load_stat(name, statistics, encoding, vlevel, logger, quick, location) - except: - logger.error("Failed to create interaction for %s: %s" % - (name, traceback.format_exc().splitlines()[-1])) - -@transaction.commit_on_success -def load_stat(cobj, statistics, encoding, vlevel, logger, quick, location): - if isinstance(cobj, ClientMetadata): - client_name = cobj.hostname - else: - client_name = cobj - client, created = Client.objects.get_or_create(name=client_name) - if created and vlevel > 0: - logger.info("Client %s added to db" % client_name) - - timestamp = datetime(*strptime(statistics.get('time'))[0:6]) - ilist = Interaction.objects.filter(client=client, - timestamp=timestamp) - if ilist: - current_interaction = ilist[0] - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s already exists" % \ - (client.id, timestamp, current_interaction.id)) - return - else: - newint = Interaction(client=client, - timestamp=timestamp, - state=statistics.get('state', - default="unknown"), - repo_rev_code=statistics.get('revision', - default="unknown"), - goodcount=statistics.get('good', - default="0"), - totalcount=statistics.get('total', - default="0"), - server=location) - newint.save() - current_interaction = newint - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (client.id, - timestamp, current_interaction.id)) - - if isinstance(cobj, ClientMetadata): - try: - imeta = InteractionMetadata(interaction=current_interaction) - profile, created = Group.objects.get_or_create(name=cobj.profile) - imeta.profile = profile - imeta.save() # save here for m2m - - #FIXME - this should be more efficient - group_set = [] - for group_name in cobj.groups: - group, created = Group.objects.get_or_create(name=group_name) - if created: - logger.debug("Added group %s" % group) - imeta.groups.add(group) - for bundle_name in cobj.bundles: - bundle, created = Bundle.objects.get_or_create(name=bundle_name) - if created: - logger.debug("Added bundle %s" % bundle) - imeta.bundles.add(bundle) - imeta.save() - except: - logger.error("Failed to save interaction metadata for %s: %s" % - (client_name, traceback.format_exc().splitlines()[-1])) - - - entries_cache = {} - [entries_cache.__setitem__((e.kind, e.name), e) \ - for e in Entries.objects.all()] - counter_fields = {TYPE_BAD: 0, - TYPE_MODIFIED: 0, - TYPE_EXTRA: 0} - pattern = [('Bad/*', TYPE_BAD), - ('Extra/*', TYPE_EXTRA), - ('Modified/*', TYPE_MODIFIED)] - for (xpath, type) in pattern: - for x in statistics.findall(xpath): - counter_fields[type] = counter_fields[type] + 1 - rr = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) - - try: - entry = entries_cache[(x.tag, x.get('name'))] - except KeyError: - entry, created = Entries.objects.get_or_create(\ - name=x.get('name'), kind=x.tag) - - Entries_interactions(entry=entry, reason=rr, - interaction=current_interaction, - type=type).save() - if vlevel > 0: - logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) - - # add good entries - good_reason = None - for x in statistics.findall('Good/*'): - if good_reason == None: - # Do this once. Really need to fix Reasons... - good_reason = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) - try: - entry = entries_cache[(x.tag, x.get('name'))] - except KeyError: - entry, created = Entries.objects.get_or_create(\ - name=x.get('name'), kind=x.tag) - Entries_interactions(entry=entry, reason=good_reason, - interaction=current_interaction, - type=TYPE_GOOD).save() - if vlevel > 0: - logger.info("%s interaction created with reason id %s and entry %s" % (xpath, good_reason.id, entry.id)) - - # Update interaction counters - current_interaction.bad_entries = counter_fields[TYPE_BAD] - current_interaction.modified_entries = counter_fields[TYPE_MODIFIED] - current_interaction.extra_entries = counter_fields[TYPE_EXTRA] - current_interaction.save() - - mperfs = [] - for times in statistics.findall('OpStamps'): - for metric, value in list(times.items()): - mmatch = [] - if not quick: - mmatch = Performance.objects.filter(metric=metric, value=value) - - if mmatch: - mperf = mmatch[0] - else: - mperf = Performance(metric=metric, value=value) - mperf.save() - mperfs.append(mperf) - current_interaction.performance_items.add(*mperfs) - - -if __name__ == '__main__': - from sys import argv - verb = 0 - cpath = "/etc/bcfg2.conf" - clientpath = False - statpath = False - syslog = False - - try: - opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", - "verbose", - "updates", - "debug", - "clients=", - "stats=", - "config=", - "syslog"]) - except GetoptError: - mesg = sys.exc_info()[1] - # print help information and exit: - print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-s statistics-file]" % (mesg)) - raise SystemExit(2) - - for o, a in opts: - if o in ("-h", "--help"): - print("Usage:\nimportscript.py [-h] [-v] -s <statistics-file> \n") - print("h : help; this message") - print("v : verbose; print messages on record insertion/skip") - print("u : updates; print status messages as items inserted semi-verbose") - print("d : debug; print most SQL used to manipulate database") - print("C : path to bcfg2.conf config file.") - print("s : statistics.xml file") - print("S : syslog; output to syslog") - raise SystemExit - if o in ["-C", "--config"]: - cpath = a - - if o in ("-v", "--verbose"): - verb = 1 - if o in ("-u", "--updates"): - verb = 2 - if o in ("-d", "--debug"): - verb = 3 - if o in ("-c", "--clients"): - print("DeprecationWarning: %s is no longer used" % o) - - if o in ("-s", "--stats"): - statpath = a - if o in ("-S", "--syslog"): - syslog = True - - logger = logging.getLogger('importscript.py') - logging.getLogger().setLevel(logging.INFO) - Bcfg2.Logger.setup_logging('importscript.py', - True, - syslog, level=logging.INFO) - - cf = ConfigParser.ConfigParser() - cf.read([cpath]) - - if not statpath: - try: - statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print("Could not read bcfg2.conf; exiting") - raise SystemExit(1) - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s" % (statpath)) - raise SystemExit(1) - - try: - encoding = cf.get('components', 'encoding') - except: - encoding = 'UTF-8' - - q = '-O3' in sys.argv - - # don't load this at the top. causes a circular import error - from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError - # Be sure the database is ready for new schema - try: - update_database() - except UpdaterError: - raise SystemExit(1) - load_stats(statsdata, - encoding, - verb, - logger, - quick=q, - location=platform.node()) diff --git a/src/lib/Bcfg2/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py deleted file mode 100755 index 1c8fb03f6..000000000 --- a/src/lib/Bcfg2/Server/Reports/manage.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -try: - import Bcfg2.settings -except ImportError: - import sys - sys.stderr.write("Error: Can't find the Bcfg2.settings module. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) - sys.exit(1) - -if __name__ == "__main__": - execute_manager(Bcfg2.settings) diff --git a/src/lib/Bcfg2/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py deleted file mode 100644 index dd1f2f742..000000000 --- a/src/lib/Bcfg2/Server/Reports/nisauth.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Checks with NIS to see if the current user is in the support group""" - -import crypt -import nis -from Bcfg2.settings import AUTHORIZED_GROUP # pylint: disable=E0611 - - -class NISAUTHError(Exception): - """NISAUTHError is raised when somehting goes boom.""" - pass - - -class nisauth(object): - group_test = False - samAcctName = None - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None # this will be a list - mail = None - extensionAttribute1 = None # badgenumber - badge_no = None - uid = None - - def __init__(self, login, passwd=None): - """get user profile from NIS""" - try: - p = nis.match(login, 'passwd.byname').split(":") - print(p) - except: - raise NISAUTHError('username') - # check user password using crypt and 2 character salt from passwd file - if p[1] == crypt.crypt(passwd, p[1][:2]): - # check to see if user is in valid support groups - # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, - 'group.byname').split(':')[-1].split(','): - raise NISAUTHError('group') - self.uid = p[2] - print(self.uid) - else: - raise NISAUTHError('password') diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html deleted file mode 100644 index 5f7579eb9..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Entry Status{% endblock %} - -{% block extra_header_info %} -{% endblock%} - -{% block pagebanner %}{{ entry.kind }} entry {{ entry.name }} status{% endblock %} - -{% block content %} -{% filter_navigator %} -{% if item_data %} - <div class='entry_list'> - <table class='entry_list'> - <tr style='text-align: left' ><th>Name</th><th>Timestamp</th><th>State</th><th>Reason</th></tr> - {% for ei, inter, reason in item_data %} - <tr class='{% cycle listview,listview_alt %}'> - <td><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=inter.client.name, pk=inter.id %}'>{{ inter.client.name }}</a></td> - <td style='white-space: nowrap'><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=inter.client.name, pk=inter.id %}'>{{ inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</a></td> - <td>{{ ei.get_type_display }}</td> - <td style='white-space: nowrap'><a href="{% url reports_item type=ei.get_type_display pk=ei.pk %}">{{ reason.short_list|join:"," }}</a></td> - </tr> - {% endfor %} - </table> - </div> -{% else %} - <p>There are currently no hosts with this configuration entry.</p> -{% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py new file mode 100644 index 000000000..b377806ab --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -0,0 +1,155 @@ +import Bcfg2.settings + +from django.db import connection +import django.core.management +import sys +import logging +import traceback +from Bcfg2.Server.models import InternalDatabaseVersion +logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + + +# all update function should go here +def _merge_database_table_entries(): + cursor = connection.cursor() + insert_cursor = connection.cursor() + find_cursor = connection.cursor() + cursor.execute(""" + Select name, kind from reports_bad + union + select name, kind from reports_modified + union + select name, kind from reports_extra + """) + # this fetch could be better done + entries_map = {} + for row in cursor.fetchall(): + insert_cursor.execute("insert into reports_entries (name, kind) \ + values (%s, %s)", (row[0], row[1])) + entries_map[(row[0], row[1])] = insert_cursor.lastrowid + + cursor.execute(""" + Select name, kind, reason_id, interaction_id, 1 from reports_bad + inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id + union + Select name, kind, reason_id, interaction_id, 2 from reports_modified + inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id + union + Select name, kind, reason_id, interaction_id, 3 from reports_extra + inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id + """) + for row in cursor.fetchall(): + key = (row[0], row[1]) + if entries_map.get(key, None): + entry_id = entries_map[key] + else: + find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) + rowe = find_cursor.fetchone() + entry_id = rowe[0] + insert_cursor.execute("insert into reports_entries_interactions \ + (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + + +def _interactions_constraint_or_idx(): + '''sqlite doesn't support alter tables.. or constraints''' + cursor = connection.cursor() + try: + cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + except: + cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + + +def _populate_interaction_entry_counts(): + '''Populate up the type totals for the interaction table''' + cursor = connection.cursor() + count_field = {1: 'bad_entries', + 2: 'modified_entries', + 3: 'extra_entries'} + + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + + "from reports_entries_interactions where type = %s group by interaction_id" % type) + updates = [] + for row in cursor.fetchall(): + updates.append(row) + try: + cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + except Exception: + e = sys.exc_info()[1] + print(e) + cursor.close() + + +# be sure to test your upgrade query before reflecting the change in the models +# the list of function and sql command to do should go here +_fixes = [_merge_database_table_entries, + # this will remove unused tables + "drop table reports_bad;", + "drop table reports_bad_interactions;", + "drop table reports_extra;", + "drop table reports_extra_interactions;", + "drop table reports_modified;", + "drop table reports_modified_interactions;", + "drop table reports_repository;", + "drop table reports_metadata;", + "alter table reports_interaction add server varchar(256) not null default 'N/A';", + # fix revision data type to support $VCS hashes + "alter table reports_interaction add repo_rev_code varchar(64) default '';", + # Performance enhancements for large sites + 'alter table reports_interaction add column bad_entries integer not null default -1;', + 'alter table reports_interaction add column modified_entries integer not null default -1;', + 'alter table reports_interaction add column extra_entries integer not null default -1;', + _populate_interaction_entry_counts, + _interactions_constraint_or_idx, + 'alter table reports_reason add is_binary bool NOT NULL default False;', + 'alter table reports_reason add is_sensitive bool NOT NULL default False;', +] + +# this will calculate the last possible version of the database +lastversion = len(_fixes) + + +def rollupdate(current_version): + """ function responsible to coordinates all the updates + need current_version as integer + """ + ret = None + if current_version < lastversion: + for i in range(current_version, lastversion): + try: + if type(_fixes[i]) == str: + connection.cursor().execute(_fixes[i]) + else: + _fixes[i]() + except: + logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) + # since array start at 0 but version start at 1 we add 1 to the normal count + ret = InternalDatabaseVersion.objects.create(version=i + 1) + return ret + else: + return None + + +def update_database(): + ''' methode to search where we are in the revision of the database models and update them ''' + try: + logger.debug("Running upgrade of models to the new one") + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + know_version = InternalDatabaseVersion.objects.order_by('-version') + if not know_version: + logger.debug("No version, creating initial version") + know_version = InternalDatabaseVersion.objects.create(version=lastversion) + else: + know_version = know_version[0] + logger.debug("Presently at %s" % know_version) + if know_version.version > 13000: + # SchemaUpdater stuff + return + elif know_version.version < lastversion: + new_version = rollupdate(know_version.version) + if new_version: + logger.debug("upgraded to %s" % new_version) + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) diff --git a/src/lib/Bcfg2/Server/Reports/urls.py b/src/lib/Bcfg2/Server/Reports/urls.py deleted file mode 100644 index d7ff1eee5..000000000 --- a/src/lib/Bcfg2/Server/Reports/urls.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.conf.urls.defaults import * -from django.http import HttpResponsePermanentRedirect - -handler500 = 'Bcfg2.Server.Reports.reports.views.server_error' - -urlpatterns = patterns('', - (r'^', include('Bcfg2.Server.Reports.reports.urls')) -) - -#urlpatterns += patterns("django.views", -# url(r"media/(?P<path>.*)$", "static.serve", { -# "document_root": '/Users/tlaszlo/svn/bcfg2/reports/site_media/', -# }) -#) diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py deleted file mode 100644 index ff4c24328..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -1_0_x.py - -This file should contain updates relevant to the 1.0.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import UnsupportedUpdate - -def updates(): - return UnsupportedUpdate("1.0", 10) - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py deleted file mode 100644 index 0d28786fd..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -1_1_x.py - -This file should contain updates relevant to the 1.1.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -from django.db import connection -import sys -import Bcfg2.settings -from Bcfg2.Server.Reports.reports.models import \ - TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA - -@updatercallable -def _interactions_constraint_or_idx(): - """sqlite doesn't support alter tables.. or constraints""" - cursor = connection.cursor() - try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') - except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - - -@updatercallable -def _populate_interaction_entry_counts(): - '''Populate up the type totals for the interaction table''' - cursor = connection.cursor() - count_field = {TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries'} - - for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) - updates = [] - for row in cursor.fetchall(): - updates.append(row) - try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception: - e = sys.exc_info()[1] - print(e) - cursor.close() - - -def updates(): - fixes = Updater("1.1") - fixes.override_base_version(12) # Do not do this in new code - - fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;') - fixes.add(_populate_interaction_entry_counts()) - fixes.add(_interactions_constraint_or_idx()) - fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py deleted file mode 100644 index 024965bd5..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -1_2_x.py - -This file should contain updates relevant to the 1.2.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -def updates(): - fixes = Updater("1.2") - fixes.override_base_version(18) # Do not do this in new code - fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py deleted file mode 100644 index 4fc57c653..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -1_3_0.py - -This file should contain updates relevant to the 1.3.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater, UpdaterError -from Bcfg2.Server.SchemaUpdater.Routines import AddColumns, \ - RemoveColumns, RebuildTable, DropTable - -from Bcfg2.Server.Reports.reports.models import Reason, Interaction - - -def updates(): - fixes = Updater("1.3") - fixes.add(RemoveColumns(Interaction, 'client_version')) - fixes.add(AddColumns(Reason)) - fixes.add(RebuildTable(Reason, [ - 'owner', 'current_owner', - 'group', 'current_group', - 'perms', 'current_perms', - 'status', 'current_status', - 'to', 'current_to'])) - fixes.add(DropTable('reports_ping')) - - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py deleted file mode 100644 index 4fcf0e6bf..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py +++ /dev/null @@ -1,279 +0,0 @@ -import logging -import traceback -from django.db.models.fields import NOT_PROVIDED -from django.db import connection, DatabaseError, backend, models -from django.core.management.color import no_style -from django.core.management.sql import sql_create -import django.core.management - -import Bcfg2.settings - -logger = logging.getLogger(__name__) - -def _quote(value): - """ - Quote a string to use as a table name or column - """ - return backend.DatabaseOperations().quote_name(value) - - -def _rebuild_sqlite_table(model): - """Sqlite doesn't support most alter table statments. This streamlines the - rebuild process""" - try: - cursor = connection.cursor() - table_name = model._meta.db_table - - # Build create staement from django - model._meta.db_table = "%s_temp" % table_name - sql, references = connection.creation.sql_create_model(model, no_style()) - columns = ",".join([_quote(f.column) \ - for f in model._meta.fields]) - - # Create a temp table - [cursor.execute(s) for s in sql] - - # Fill the table - tbl_name = _quote(table_name) - tmp_tbl_name = _quote(model._meta.db_table) - # Reset this - model._meta.db_table = table_name - cursor.execute("insert into %s(%s) select %s from %s;" % ( - tmp_tbl_name, - columns, - columns, - tbl_name)) - cursor.execute("drop table %s" % tbl_name) - - # Call syncdb to create the table again - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - # syncdb closes our cursor - cursor = connection.cursor() - # Repopulate - cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, - columns, - columns, - tmp_tbl_name)) - cursor.execute('DROP TABLE %s;' % tmp_tbl_name) - except DatabaseError: - logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1) - raise UpdaterRoutineException - - -class UpdaterRoutineException(Exception): - pass - - -class UpdaterRoutine(object): - """Base for routines.""" - def __init__(self): - pass - - def __str__(self): - return __name__ - - def run(self): - """Called to execute the action""" - raise UpdaterRoutineException - - - -class AddColumns(UpdaterRoutine): - """ - Routine to add new columns to an existing model - """ - def __init__(self, model): - self.model = model - self.model_name = model.__name__ - - def __str__(self): - return "Add new columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - desc = {} - for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table): - desc[d[0]] = d - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for field in self.model._meta.fields: - if field.column in desc: - continue - logger.debug("Column %s does not exist yet" % field.column) - if field.default == NOT_PROVIDED: - logger.error("Cannot add a column with out a default value") - raise UpdaterRoutineException - - sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % ( - _quote(self.model._meta.db_table), - _quote(field.column), field.db_type(), ) - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - sql += _quote(field.default) - sql_values = () - else: - sql += '%s' - sql_values = (field.default, ) - try: - cursor.execute(sql, sql_values) - logger.debug("Added column %s to %s" % - (field.column, self.model._meta.db_table)) - except DatabaseError: - logger.error("Unable to add column %s" % field.column) - raise UpdaterRoutineException - - -class RebuildTable(UpdaterRoutine): - """ - Rebuild the table for an existing model. Use this if field types have changed. - """ - def __init__(self, model, columns): - self.model = model - self.model_name = model.__name__ - - if type(columns) == str: - self.columns = [columns] - elif type(columns) in (tuple, list): - self.columns = columns - else: - logger.error("Columns must be a str, tuple, or list") - raise UpdaterRoutineException - - - def __str__(self): - return "Rebuild columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - """ Sqlite is a special case. Altering columns is not supported. """ - _rebuild_sqlite_table(self.model) - return - - if db_engine == 'django.db.backends.mysql': - modify_cmd = 'MODIFY ' - else: - modify_cmd = 'ALTER COLUMN ' - - col_strings = [] - for column in self.columns: - col_strings.append("%s %s %s" % ( \ - modify_cmd, - _quote(column), - self.model._meta.get_field(column).db_type() - )) - - try: - cursor.execute('ALTER TABLE %s %s' % - (_quote(self.model._meta.db_table), ", ".join(col_strings))) - except DatabaseError: - logger.debug("Failed modify table %s" % self.model._meta.db_table) - raise UpdaterRoutineException - - - -class RemoveColumns(RebuildTable): - """ - Routine to remove columns from an existing model - """ - def __init__(self, model, columns): - super(RemoveColumns, self).__init__(model, columns) - - - def __str__(self): - return "Remove columns from model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - columns = [d[0] for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table)] - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for column in self.columns: - if column not in columns: - logger.warning("Cannot drop column %s: does not exist" % column) - continue - - logger.debug("Dropping column %s" % column) - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - _rebuild_sqlite_table(self.model) - else: - sql = "alter table %s drop column %s" % \ - (_quote(self.model._meta.db_table), _quote(column), ) - try: - cursor.execute(sql) - except DatabaseError: - logger.debug("Failed to drop column %s from %s" % - (column, self.model._meta.db_table)) - raise UpdaterRoutineException - - -class DropTable(UpdaterRoutine): - """ - Drop a table - """ - def __init__(self, table_name): - self.table_name = table_name - - def __str__(self): - return "Drop table %s" % self.table_name - - def run(self): - try: - cursor = connection.cursor() - cursor.execute('DROP TABLE %s' % _quote(self.table_name)) - except DatabaseError: - logger.error("Failed to drop table: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterRoutineException - - -class UpdaterCallable(UpdaterRoutine): - """Helper for routines. Basically delays execution""" - def __init__(self, fn): - self.fn = fn - self.args = [] - self.kwargs = {} - - def __call__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - return self - - def __str__(self): - return self.fn.__name__ - - def run(self): - self.fn(*self.args, **self.kwargs) - -def updatercallable(fn): - """Decorator for UpdaterCallable. Use for any function passed - into the fixes list""" - return UpdaterCallable(fn) - - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py deleted file mode 100644 index e7a3191bc..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py +++ /dev/null @@ -1,239 +0,0 @@ -from django.db import connection, DatabaseError -from django.core.exceptions import ImproperlyConfigured -import django.core.management -import logging -import re -import sys -import traceback - -from Bcfg2.Compat import CmpMixin, walk_packages -from Bcfg2.Server.models import InternalDatabaseVersion -from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \ - UpdaterRoutine -from Bcfg2.Server.SchemaUpdater import Changes - -logger = logging.getLogger(__name__) - -class UpdaterError(Exception): - pass - - -class SchemaTooOldError(UpdaterError): - pass - - -def _release_to_version(release): - """ - Build a release base for a version - - Expects a string of the form 00.00 - - returns an integer of the form MMmm00 - """ - regex = re.compile("^(\d+)\.(\d+)$") - m = regex.match(release) - if not m: - logger.error("Invalid release string: %s" % release) - raise TypeError - return int("%02d%02d00" % (int(m.group(1)), int(m.group(2)))) - - -class Updater(CmpMixin): - """Database updater to standardize updates""" - - def __init__(self, release): - CmpMixin.__init__(self) - - self._cursor = None - self._release = release - try: - self._base_version = _release_to_version(release) - except: - err = "Invalid release string: %s" % release - logger.error(err) - raise UpdaterError(err) - - self._fixes = [] - self._version = -1 - - def __cmp__(self, other): - return self._base_version - other._base_version - - @property - def release(self): - return self._release - - @property - def version(self): - if self._version < 0: - try: - iv = InternalDatabaseVersion.objects.latest() - self._version = iv.version - except InternalDatabaseVersion.DoesNotExist: - raise UpdaterError("No database version stored internally") - return self._version - - @property - def cursor(self): - if not self._cursor: - self._cursor = connection.cursor() - return self._cursor - - @property - def target_version(self): - if(len(self._fixes) == 0): - return self._base_version - else: - return self._base_version + len(self._fixes) - 1 - - - def add(self, update): - if type(update) == str or isinstance(update, UpdaterRoutine): - self._fixes.append(update) - else: - raise TypeError - - - def override_base_version(self, version): - """Override our starting point for old releases. New code should - not use this method""" - self._base_version = int(version) - - - @staticmethod - def get_current_version(): - """Queries the db for the latest version. Returns 0 for a - fresh install""" - - if "call_command" in dir(django.core.management): - django.core.management.call_command("syncdb", interactive=False, - verbosity=0) - else: - msg = "Unable to call syndb routine" - logger.warning(msg) - raise UpdaterError(msg) - - try: - iv = InternalDatabaseVersion.objects.latest() - version = iv.version - except InternalDatabaseVersion.DoesNotExist: - version = 0 - - return version - - - def syncdb(self): - """Function to do the syncronisation for the models""" - - self._version = Updater.get_current_version() - self._cursor = None - - - def increment(self): - """Increment schema version in the database""" - if self._version < self._base_version: - self._version = self._base_version - else: - self._version += 1 - InternalDatabaseVersion.objects.create(version=self._version) - - def apply(self): - """Apply pending schema changes""" - - if self.version >= self.target_version: - logger.debug("No updates for release %s" % self._release) - return - - logger.debug("Applying updates for release %s" % self._release) - - if self.version < self._base_version: - start = 0 - else: - start = self.version - self._base_version + 1 - - try: - for fix in self._fixes[start:]: - if type(fix) == str: - self.cursor.execute(fix) - elif isinstance(fix, UpdaterRoutine): - fix.run() - else: - logger.error("Invalid schema change at %s" % \ - self._version + 1) - self.increment() - logger.debug("Applied schema change number %s: %s" % \ - (self.version, fix)) - logger.info("Applied schema changes for release %s" % self._release) - except: - msg = "Failed to perform db update %s (%s): %s" % \ - (self._version + 1, fix, - traceback.format_exc().splitlines()[-1]) - logger.error(msg) - raise UpdaterError(msg) - - -class UnsupportedUpdate(Updater): - """Handle an unsupported update""" - - def __init__(self, release, version): - super(UnsupportedUpdate, self).__init__(release) - self._base_version = version - - def apply(self): - """Raise an exception if we're too old""" - - if self.version < self.target_version: - logger.error("Upgrade from release %s unsupported" % self._release) - raise SchemaTooOldError - - -def update_database(): - """method to search where we are in the revision - of the database models and update them""" - try: - logger.debug("Verifying database schema") - - updaters = [] - for loader, submodule, ispkg in walk_packages(path=Changes.__path__): - if ispkg: - continue - try: - updates = getattr( - __import__("%s.%s" % (Changes.__name__, submodule), - globals(), locals(), ['*']), - "updates") - updaters.append(updates()) - except ImportError: - logger.error("Failed to import %s" % submodule) - except AttributeError: - logger.warning("Module %s does not have an updates function" % - submodule) - except: - msg = "Failed to build updater for %s" % submodule - logger.error(msg, exc_info=1) - raise UpdaterError(msg) - - current_version = Updater.get_current_version() - logger.debug("Database version at %s" % current_version) - - updaters.sort() - if current_version > 0: - [u.apply() for u in updaters] - logger.debug("Database version at %s" % - Updater.get_current_version()) - else: - target = updaters[-1].target_version - InternalDatabaseVersion.objects.create(version=target) - logger.info("A new database was created") - - except UpdaterError: - raise - except ImproperlyConfigured: - logger.error("Django is not properly configured: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterError - except: - logger.error("Error while updating the database") - for x in traceback.format_exc().splitlines(): - logger.error(x) - raise UpdaterError diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index cfb515136..395bb97d6 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -10,6 +10,13 @@ try: except ImportError: HAS_DJANGO = False +# required for reporting +try: + import south + has_south = True +except: + has_south = False + DATABASES = dict() # Django < 1.2 compat @@ -91,6 +98,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): TIME_ZONE = setup['time_zone'] DEBUG = setup['django_debug'] + DEBUG = True TEMPLATE_DEBUG = DEBUG if DEBUG: print("Warning: Setting web_debug to True causes extraordinary memory " @@ -101,6 +109,14 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): else: MEDIA_URL = '/site_media' + if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: + CACHE_BACKEND = 'locmem:///' + else: + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + } # initialize settings from /etc/bcfg2-web.conf or /etc/bcfg2.conf, or # set up basic defaults. this lets manage.py work in all cases @@ -123,9 +139,13 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', - 'Bcfg2.Server.Reports.reports', - 'Bcfg2.Server' + 'Bcfg2.Server', ) +if has_south: + INSTALLED_APPS = INSTALLED_APPS + ( + 'south', + 'Bcfg2.Reporting', + ) # Imported from Bcfg2.Server.Reports MEDIA_ROOT = '' @@ -158,7 +178,7 @@ MIDDLEWARE_CLASSES = ( ) # TODO - move this to a higher root and dynamically import -ROOT_URLCONF = 'Bcfg2.Server.Reports.urls' +ROOT_URLCONF = 'Bcfg2.Reporting.urls' # TODO - this isn't usable # Authentication Settings diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector new file mode 100755 index 000000000..cba5be2b3 --- /dev/null +++ b/src/sbin/bcfg2-report-collector @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import sys +import logging +import Bcfg2.Logger +import Bcfg2.Options +from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError + +logger = logging.getLogger('bcfg2-report-collector') + +if __name__ == '__main__': + optinfo = dict( + daemon=Bcfg2.Options.DAEMON, + repo=Bcfg2.Options.SERVER_REPOSITORY, + filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, + web_configfile=Bcfg2.Options.WEB_CFILE, + ) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) + setup.parse(sys.argv[1:]) + + # run collector + try: + collector = ReportingCollector(setup) + collector.run() + except ReportingError: + msg = sys.exc_info()[1] + logger.error(msg) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) + sys.exit(0) diff --git a/tools/export.py b/tools/export.py index df867418d..3b52b35bb 100755 --- a/tools/export.py +++ b/tools/export.py @@ -238,7 +238,7 @@ E.G. 1.2.0pre1 is a valid version. 'Release: 0.0%s\n' % version_info['build'], dryrun=options.dryrun) # update the version in reports - find_and_replace('src/lib/Bcfg2/Server/Reports/reports/templates/base.html', + find_and_replace('src/lib/Bcfg2/Reporting/templates/base.html', 'Bcfg2 Version', ' <span>Bcfg2 Version %s</span>\n' % version, dryrun=options.dryrun) |