From 44638176067df5231bf0be30801e36863391cd1f Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 8 Oct 2012 10:38:02 -0500 Subject: Reporting: Merge new reporting data Move reporting data to a new schema Use south for django migrations Add bcfg2-report-collector daemon Conflicts: doc/development/index.txt doc/server/plugins/connectors/properties.txt doc/server/plugins/generators/packages.txt setup.py src/lib/Bcfg2/Client/Tools/SELinux.py src/lib/Bcfg2/Compat.py src/lib/Bcfg2/Encryption.py src/lib/Bcfg2/Options.py src/lib/Bcfg2/Server/Admin/Init.py src/lib/Bcfg2/Server/Admin/Reports.py src/lib/Bcfg2/Server/BuiltinCore.py src/lib/Bcfg2/Server/Core.py src/lib/Bcfg2/Server/FileMonitor/Inotify.py src/lib/Bcfg2/Server/Plugin/base.py src/lib/Bcfg2/Server/Plugin/interfaces.py src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py src/lib/Bcfg2/Server/Plugins/FileProbes.py src/lib/Bcfg2/Server/Plugins/Ohai.py src/lib/Bcfg2/Server/Plugins/Packages/Collection.py src/lib/Bcfg2/Server/Plugins/Packages/Source.py src/lib/Bcfg2/Server/Plugins/Packages/Yum.py src/lib/Bcfg2/Server/Plugins/Packages/__init__.py src/lib/Bcfg2/Server/Plugins/Probes.py src/lib/Bcfg2/Server/Plugins/Properties.py src/lib/Bcfg2/Server/Reports/backends.py src/lib/Bcfg2/Server/Reports/manage.py src/lib/Bcfg2/Server/Reports/nisauth.py src/lib/Bcfg2/settings.py src/sbin/bcfg2-crypt src/sbin/bcfg2-yum-helper testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py --- src/lib/Bcfg2/Server/Admin/Reports.py | 92 +--- src/lib/Bcfg2/Server/Admin/Syncdb.py | 15 +- src/lib/Bcfg2/Server/Core.py | 27 +- src/lib/Bcfg2/Server/FileMonitor/Gamin.py | 1 + .../Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 2 + src/lib/Bcfg2/Server/Plugins/Reporting.py | 105 ++++ src/lib/Bcfg2/Server/Reports/backends.py | 35 -- src/lib/Bcfg2/Server/Reports/importscript.py | 335 ------------ src/lib/Bcfg2/Server/Reports/manage.py | 11 - src/lib/Bcfg2/Server/Reports/nisauth.py | 44 -- .../Server/Reports/reports/templates/404.html | 8 - .../Reports/reports/templates/base-timeview.html | 28 - .../Server/Reports/reports/templates/base.html | 96 ---- .../Reports/reports/templates/clients/detail.html | 129 ----- .../reports/templates/clients/detailed-list.html | 46 -- .../Reports/reports/templates/clients/history.html | 20 - .../Reports/reports/templates/clients/index.html | 35 -- .../Reports/reports/templates/clients/manage.html | 45 -- .../reports/templates/config_items/common.html | 42 -- .../templates/config_items/entry_status.html | 30 -- .../reports/templates/config_items/item.html | 130 ----- .../reports/templates/config_items/listing.html | 35 -- .../reports/templates/displays/summary.html | 42 -- .../Reports/reports/templates/displays/timing.html | 38 -- .../reports/templates/widgets/filter_bar.html | 25 - .../reports/templates/widgets/interaction_list.inc | 38 -- .../reports/templates/widgets/page_bar.html | 23 - .../Reports/reports/templatetags/__init__.py | 0 .../Reports/reports/templatetags/bcfg2_tags.py | 415 --------------- .../Server/Reports/reports/templatetags/split.py | 8 - .../reports/templatetags/syntax_coloring.py | 47 -- src/lib/Bcfg2/Server/Reports/reports/urls.py | 58 -- src/lib/Bcfg2/Server/Reports/reports/views.py | 583 --------------------- src/lib/Bcfg2/Server/Reports/updatefix.py | 155 ++++++ src/lib/Bcfg2/Server/Reports/urls.py | 14 - src/lib/Bcfg2/Server/Reports/utils.py | 126 ----- .../Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py | 11 - .../Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py | 59 --- .../Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py | 15 - .../Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py | 27 - .../Bcfg2/Server/SchemaUpdater/Changes/__init__.py | 0 src/lib/Bcfg2/Server/SchemaUpdater/Routines.py | 279 ---------- src/lib/Bcfg2/Server/SchemaUpdater/__init__.py | 239 --------- 43 files changed, 309 insertions(+), 3204 deletions(-) create mode 100644 src/lib/Bcfg2/Server/Plugins/Reporting.py delete mode 100644 src/lib/Bcfg2/Server/Reports/backends.py delete mode 100755 src/lib/Bcfg2/Server/Reports/importscript.py delete mode 100755 src/lib/Bcfg2/Server/Reports/manage.py delete mode 100644 src/lib/Bcfg2/Server/Reports/nisauth.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/404.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/base.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/urls.py delete mode 100644 src/lib/Bcfg2/Server/Reports/reports/views.py create mode 100644 src/lib/Bcfg2/Server/Reports/updatefix.py delete mode 100644 src/lib/Bcfg2/Server/Reports/urls.py delete mode 100755 src/lib/Bcfg2/Server/Reports/utils.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/Routines.py delete mode 100644 src/lib/Bcfg2/Server/SchemaUpdater/__init__.py (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 63a0092d5..815d34e97 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -8,19 +8,16 @@ import pickle import platform import sys import traceback -from lxml.etree import XML, XMLSyntaxError -from Bcfg2.Compat import ConfigParser, md5 +from Bcfg2.Compat import md5 -import Bcfg2.settings +from Bcfg2 import settings # Load django and reports stuff _after_ we know we can load settings -import django.core.management -from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError -from Bcfg2.Server.Reports.utils import * +from django.core import management +from Bcfg2.Reporting.utils import * -project_directory = os.path.dirname(Bcfg2.settings.__file__) +project_directory = os.path.dirname(settings.__file__) project_name = os.path.basename(project_directory) sys.path.append(os.path.join(project_directory, '..')) project_module = __import__(project_name, '', '', ['']) @@ -30,9 +27,8 @@ sys.path.pop() os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name from django.db import connection, transaction -from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ - Entries_interactions, Performance, \ - Reason +from Bcfg2.Reporting.models import Client, Interaction, \ + Performance def printStats(fn): @@ -54,7 +50,8 @@ def printStats(fn): self.log.info("Interactions removed: %s" % (start_i - Interaction.objects.count())) self.log.info("Interactions->Entries removed: %s" % - (start_ei - Entries_interactions.objects.count())) + (start_ei - 0)) + # (start_ei - Entries_interactions.objects.count())) self.log.info("Metrics removed: %s" % (start_perf - Performance.objects.count())) @@ -70,9 +67,6 @@ class Reports(Bcfg2.Server.Admin.Mode): "\n" " Commands:\n" " init Initialize the database\n" - " load_stats Load statistics data\n" - " -s|--stats Path to statistics.xml file\n" - " -O3 Fast mode. Duplicates data!\n" " purge Purge records\n" " --client [n] Client to operate on\n" " --days [n] Records older then n days\n" @@ -85,6 +79,11 @@ class Reports(Bcfg2.Server.Admin.Mode): def __init__(self, setup): Bcfg2.Server.Admin.Mode.__init__(self, setup) + try: + import south + except ImportError: + print "Django south is required for Reporting" + raise SystemExit(-3) def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) @@ -99,24 +98,16 @@ class Reports(Bcfg2.Server.Admin.Mode): elif args[0] == 'scrub': self.scrub() elif args[0] in ['init', 'update']: + if self.setup['verbose'] or self.setup['debug']: + vrb = 2 + else: + vrb = 0 try: - update_database() - except UpdaterError: - print("Update failed") + management.call_command("syncdb", verbosity=vrb) + management.call_command("migrate", verbosity=vrb) + except: + print("Update failed: %s" % traceback.format_exc().splitlines()[-1]) raise SystemExit(-1) - elif args[0] == 'load_stats': - quick = '-O3' in args - stats_file = None - i = 1 - while i < len(args): - if args[i] == '-s' or args[i] == '--stats': - stats_file = args[i + 1] - if stats_file[0] == '-': - self.errExit("Invalid statistics file: %s" % stats_file) - elif args[i] == '-c' or args[i] == '--clients-file': - print("DeprecationWarning: %s is no longer used" % args[i]) - i = i + 1 - self.load_stats(stats_file, self.log.getEffectiveLevel() > logging.WARNING, quick) elif args[0] == 'purge': expired = False client = None @@ -203,9 +194,9 @@ class Reports(Bcfg2.Server.Admin.Mode): Reason.prune_orphans() self.log.info("Pruned %d Reason records" % (start_count - Reason.objects.count())) - start_count = Entries.objects.count() - Entries.prune_orphans() - self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) + #start_count = Entries.objects.count() + #Entries.prune_orphans() + #self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) def django_command_proxy(self, command): '''Call a django command''' @@ -214,37 +205,6 @@ class Reports(Bcfg2.Server.Admin.Mode): else: django.core.management.call_command(command) - def load_stats(self, stats_file=None, verb=0, quick=False): - '''Load statistics data into the database''' - location = '' - - if not stats_file: - try: - stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.errExit("Could not read bcfg2.conf; exiting") - try: - statsdata = XML(open(stats_file).read()) - except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s" % (stats_file)) - - try: - encoding = self.cfp.get('components', 'encoding') - except: - encoding = 'UTF-8' - - try: - load_stats(statsdata, - encoding, - verb, - self.log, - quick=quick, - location=platform.node()) - except UpdaterError: - self.errExit("StatReports: Database updater failed") - except: - self.errExit("failed to import stats: %s" - % traceback.format_exc().splitlines()[-1]) @printStats def purge(self, client=None, maxdate=None, state=None): @@ -272,7 +232,7 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) - if Bcfg2.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': + if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': grp_limit = 100 else: grp_limit = 1000 diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 1eb953e2a..3686722ae 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -1,8 +1,7 @@ import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin -from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError -from django.core.management import setup_environ +from django.core.management import setup_environ, call_command class Syncdb(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("Sync the Django ORM with the configured database") @@ -23,7 +22,13 @@ class Syncdb(Bcfg2.Server.Admin.Mode): Bcfg2.Server.models.load_models(cfile=self.opts['configfile']) try: - update_database() - except UpdaterError: - print("Update failed") + call_command("syncdb", interactive=False, verbosity=0) + self._database_available = True + except ImproperlyConfigured: + self.logger.error("Django configuration problem: %s" % + format_exc().splitlines()[-1]) + raise SystemExit(-1) + except: + self.logger.error("Database update failed: %s" % + format_exc().splitlines()[-1]) raise SystemExit(-1) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 14b9d9d0a..ae1c578fa 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -148,25 +148,18 @@ class BaseCore(object): Bcfg2.settings.read_config(repo=self.datastore) self._database_available = False - # verify our database schema - try: - from Bcfg2.Server.SchemaUpdater import update_database, \ - UpdaterError + if Bcfg2.settings.HAS_DJANGO: + from django.core.exceptions import ImproperlyConfigured + from django.core import management try: - update_database() + management.call_command("syncdb", interactive=False, verbosity=0) self._database_available = True - except UpdaterError: - err = sys.exc_info()[1] - self.logger.error("Failed to update database schema: %s" % err) - except ImportError: - # assume django is not installed - pass - except Exception: - inst = sys.exc_info()[1] - self.logger.error("Failed to update database schema") - self.logger.error(str(inst)) - self.logger.error(str(type(inst))) - raise CoreInitError + except ImproperlyConfigured: + self.logger.error("Django configuration problem: %s" % + format_exc().splitlines()[-1]) + except: + self.logger.error("Database update failed: %s" % + format_exc().splitlines()[-1]) if '' in setup['plugins']: setup['plugins'].remove('') diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 12965c040..23f5424d0 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -7,6 +7,7 @@ from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ from Bcfg2.Server.FileMonitor import Event, FileMonitor + class GaminEvent(Event): """ This class provides an event analogous to diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 31c3d79b0..01f590b06 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -21,6 +21,8 @@ except ImportError: LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py new file mode 100644 index 000000000..883b95ba4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -0,0 +1,105 @@ +import time +import platform +import traceback +from lxml import etree + +from Bcfg2.Reporting.Transport import load_transport_from_config, \ + TransportError, TransportImportError + +try: + import cPickle as pickle +except: + import pickle + +from Bcfg2.Options import REPORTING_COMMON_OPTIONS +from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \ + PluginExecutionError + +def _rpc_call(method): + def _real_rpc_call(self, *args, **kwargs): + """Wrapper for calls to the reporting collector""" + + try: + return self.transport.rpc(method, *args, **kwargs) + except TransportError: + # this is needed for Admin.Pull + raise PluginExecutionError + return _real_rpc_call + +class Reporting(Statistics, PullSource): + + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + + CLIENT_METADATA_FILEDS = ('profile', 'bundles', 'aliases', 'addresses', + 'groups', 'categories', 'uuid', 'version') + + def __init__(self, core, datastore): + Statistics.__init__(self, core, datastore) + PullSource.__init__(self) + self.core = core + self.experimental = True + + self.whoami = platform.node() + self.transport = None + + core.setup.update(REPORTING_COMMON_OPTIONS) + core.setup.reparse() + self.logger.error("File limit: %s" % core.setup['reporting_file_limit']) + + try: + self.transport = load_transport_from_config(core.setup) + except TransportError: + self.logger.error("%s: Failed to load transport: %s" % + (self.name, traceback.format_exc().splitlines()[-1])) + raise PluginInitError + + + def process_statistics(self, client, xdata): + stats = xdata.find("Statistics") + stats.set('time', time.asctime(time.localtime())) + + cdata = { 'server': self.whoami } + for field in self.CLIENT_METADATA_FILEDS: + try: + value = getattr(client, field) + except AttributeError: + continue + if value: + if isinstance(value, set): + value = [v for v in value] + cdata[field] = value + + try: + interaction_data = pickle.dumps({ 'hostname': client.hostname, + 'metadata': cdata, 'stats': + etree.tostring(stats, xml_declaration=False).decode('UTF-8') }) + except: + self.logger.error("%s: Failed to build interaction object: %s" % + (self.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + + # try 3 times to store the data + for i in [1, 2, 3]: + try: + self.transport.store(client.hostname, interaction_data) + self.logger.debug("%s: Queued statistics data for %s" % + (self.__class__.__name__, client.hostname)) + return + except TransportError: + continue + except: + self.logger.error("%s: Attempt %s: Failed to add statistic %s" % + (self.__class__.__name__, i, + traceback.format_exc().splitlines()[-1])) + self.logger.error("%s: Retry limit reached for %s" % + (self.__class__.__name__, client.hostname)) + + def shutdown(self): + super(Reporting, self).shutdown() + if self.transport: + self.transport.shutdown() + + Ping = _rpc_call('Ping') + GetExtra = _rpc_call('GetExtra') + GetCurrentEntry = _rpc_call('GetCurrentEntry') + diff --git a/src/lib/Bcfg2/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py deleted file mode 100644 index 9f07c104f..000000000 --- a/src/lib/Bcfg2/Server/Reports/backends.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -from django.contrib.auth.models import User -from nisauth import * - - -class NISBackend(object): - - def authenticate(self, username=None, password=None): - try: - print("start nis authenticate") - n = nisauth(username, password) - temp_pass = User.objects.make_random_password(100) - nis_user = dict(username=username, - ) - - user_session_obj = dict(email=username, - first_name=None, - last_name=None, - uid=n.uid) - user, created = User.objects.get_or_create(username=username) - - return user - - except NISAUTHError: - e = sys.exc_info()[1] - print(e) - return None - - def get_user(self, user_id): - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - e = sys.exc_info()[1] - print(e) - return None diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py deleted file mode 100755 index ace07a75d..000000000 --- a/src/lib/Bcfg2/Server/Reports/importscript.py +++ /dev/null @@ -1,335 +0,0 @@ -#! /usr/bin/env python -""" -Imports statistics.xml and clients.xml files in to database backend for -new statistics engine -""" - -import os -import sys -import traceback -try: - import Bcfg2.settings -except Exception: - e = sys.exc_info()[1] - sys.stderr.write("Failed to load configuration settings. %s\n" % e) - sys.exit(1) - -project_directory = os.path.dirname(Bcfg2.settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from Bcfg2.Server.Reports.reports.models import * -from lxml.etree import XML, XMLSyntaxError -from getopt import getopt, GetoptError -from datetime import datetime -from time import strptime -from django.db import connection, transaction -from Bcfg2.Server.Plugins.Metadata import ClientMetadata -import logging -import Bcfg2.Logger -import platform - -# Compatibility import -from Bcfg2.Compat import ConfigParser, b64decode - - -def build_reason_kwargs(r_ent, encoding, logger): - binary_file = False - sensitive_file = False - unpruned_entries = '' - if r_ent.get('sensitive') in ['true', 'True']: - sensitive_file = True - rc_diff = '' - elif r_ent.get('current_bfile', False): - binary_file = True - rc_diff = r_ent.get('current_bfile') - if len(rc_diff) > 1024 * 1024: - rc_diff = '' - elif len(rc_diff) == 0: - # No point in flagging binary if we have no data - binary_file = False - elif r_ent.get('current_bdiff', False): - rc_diff = b64decode(r_ent.get('current_bdiff')) - elif r_ent.get('current_diff', False): - rc_diff = r_ent.get('current_diff') - else: - rc_diff = '' - # detect unmanaged entries in pruned directories - if r_ent.get('prune', 'false') == 'true' and r_ent.get('qtest'): - unpruned_elist = [e.get('path') for e in r_ent.findall('Prune')] - unpruned_entries = "\n".join(unpruned_elist) - if not binary_file: - try: - rc_diff = rc_diff.decode(encoding) - except: - logger.error("Reason isn't %s encoded, cannot decode it" % encoding) - rc_diff = '' - return dict(owner=r_ent.get('owner', default=""), - current_owner=r_ent.get('current_owner', default=""), - group=r_ent.get('group', default=""), - current_group=r_ent.get('current_group', default=""), - perms=r_ent.get('perms', default=""), - current_perms=r_ent.get('current_perms', default=""), - status=r_ent.get('status', default=""), - current_status=r_ent.get('current_status', default=""), - to=r_ent.get('to', default=""), - current_to=r_ent.get('current_to', default=""), - version=r_ent.get('version', default=""), - current_version=r_ent.get('current_version', default=""), - current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", - current_diff=rc_diff, - is_binary=binary_file, - is_sensitive=sensitive_file, - unpruned=unpruned_entries) - -def _fetch_reason(elem, kargs, logger): - try: - rr = None - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - rr = Reason(**kargs) - rr.save() - logger.debug("Created reason: %s" % rr.id) - except Exception: - ex = sys.exc_info()[1] - logger.error("Failed to create reason for %s: %s" % (elem.get('name'), ex)) - rr = Reason(current_exists=elem.get('current_exists', - default="True").capitalize() == "True") - rr.save() - return rr - - -def load_stats(sdata, encoding, vlevel, logger, quick=False, location=''): - for node in sdata.findall('Node'): - name = node.get('name') - for statistics in node.findall('Statistics'): - try: - load_stat(name, statistics, encoding, vlevel, logger, quick, location) - except: - logger.error("Failed to create interaction for %s: %s" % - (name, traceback.format_exc().splitlines()[-1])) - -@transaction.commit_on_success -def load_stat(cobj, statistics, encoding, vlevel, logger, quick, location): - if isinstance(cobj, ClientMetadata): - client_name = cobj.hostname - else: - client_name = cobj - client, created = Client.objects.get_or_create(name=client_name) - if created and vlevel > 0: - logger.info("Client %s added to db" % client_name) - - timestamp = datetime(*strptime(statistics.get('time'))[0:6]) - ilist = Interaction.objects.filter(client=client, - timestamp=timestamp) - if ilist: - current_interaction = ilist[0] - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s already exists" % \ - (client.id, timestamp, current_interaction.id)) - return - else: - newint = Interaction(client=client, - timestamp=timestamp, - state=statistics.get('state', - default="unknown"), - repo_rev_code=statistics.get('revision', - default="unknown"), - goodcount=statistics.get('good', - default="0"), - totalcount=statistics.get('total', - default="0"), - server=location) - newint.save() - current_interaction = newint - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (client.id, - timestamp, current_interaction.id)) - - if isinstance(cobj, ClientMetadata): - try: - imeta = InteractionMetadata(interaction=current_interaction) - profile, created = Group.objects.get_or_create(name=cobj.profile) - imeta.profile = profile - imeta.save() # save here for m2m - - #FIXME - this should be more efficient - group_set = [] - for group_name in cobj.groups: - group, created = Group.objects.get_or_create(name=group_name) - if created: - logger.debug("Added group %s" % group) - imeta.groups.add(group) - for bundle_name in cobj.bundles: - bundle, created = Bundle.objects.get_or_create(name=bundle_name) - if created: - logger.debug("Added bundle %s" % bundle) - imeta.bundles.add(bundle) - imeta.save() - except: - logger.error("Failed to save interaction metadata for %s: %s" % - (client_name, traceback.format_exc().splitlines()[-1])) - - - entries_cache = {} - [entries_cache.__setitem__((e.kind, e.name), e) \ - for e in Entries.objects.all()] - counter_fields = {TYPE_BAD: 0, - TYPE_MODIFIED: 0, - TYPE_EXTRA: 0} - pattern = [('Bad/*', TYPE_BAD), - ('Extra/*', TYPE_EXTRA), - ('Modified/*', TYPE_MODIFIED)] - for (xpath, type) in pattern: - for x in statistics.findall(xpath): - counter_fields[type] = counter_fields[type] + 1 - rr = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) - - try: - entry = entries_cache[(x.tag, x.get('name'))] - except KeyError: - entry, created = Entries.objects.get_or_create(\ - name=x.get('name'), kind=x.tag) - - Entries_interactions(entry=entry, reason=rr, - interaction=current_interaction, - type=type).save() - if vlevel > 0: - logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) - - # add good entries - good_reason = None - for x in statistics.findall('Good/*'): - if good_reason == None: - # Do this once. Really need to fix Reasons... - good_reason = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) - try: - entry = entries_cache[(x.tag, x.get('name'))] - except KeyError: - entry, created = Entries.objects.get_or_create(\ - name=x.get('name'), kind=x.tag) - Entries_interactions(entry=entry, reason=good_reason, - interaction=current_interaction, - type=TYPE_GOOD).save() - if vlevel > 0: - logger.info("%s interaction created with reason id %s and entry %s" % (xpath, good_reason.id, entry.id)) - - # Update interaction counters - current_interaction.bad_entries = counter_fields[TYPE_BAD] - current_interaction.modified_entries = counter_fields[TYPE_MODIFIED] - current_interaction.extra_entries = counter_fields[TYPE_EXTRA] - current_interaction.save() - - mperfs = [] - for times in statistics.findall('OpStamps'): - for metric, value in list(times.items()): - mmatch = [] - if not quick: - mmatch = Performance.objects.filter(metric=metric, value=value) - - if mmatch: - mperf = mmatch[0] - else: - mperf = Performance(metric=metric, value=value) - mperf.save() - mperfs.append(mperf) - current_interaction.performance_items.add(*mperfs) - - -if __name__ == '__main__': - from sys import argv - verb = 0 - cpath = "/etc/bcfg2.conf" - clientpath = False - statpath = False - syslog = False - - try: - opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", - "verbose", - "updates", - "debug", - "clients=", - "stats=", - "config=", - "syslog"]) - except GetoptError: - mesg = sys.exc_info()[1] - # print help information and exit: - print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-s statistics-file]" % (mesg)) - raise SystemExit(2) - - for o, a in opts: - if o in ("-h", "--help"): - print("Usage:\nimportscript.py [-h] [-v] -s \n") - print("h : help; this message") - print("v : verbose; print messages on record insertion/skip") - print("u : updates; print status messages as items inserted semi-verbose") - print("d : debug; print most SQL used to manipulate database") - print("C : path to bcfg2.conf config file.") - print("s : statistics.xml file") - print("S : syslog; output to syslog") - raise SystemExit - if o in ["-C", "--config"]: - cpath = a - - if o in ("-v", "--verbose"): - verb = 1 - if o in ("-u", "--updates"): - verb = 2 - if o in ("-d", "--debug"): - verb = 3 - if o in ("-c", "--clients"): - print("DeprecationWarning: %s is no longer used" % o) - - if o in ("-s", "--stats"): - statpath = a - if o in ("-S", "--syslog"): - syslog = True - - logger = logging.getLogger('importscript.py') - logging.getLogger().setLevel(logging.INFO) - Bcfg2.Logger.setup_logging('importscript.py', - True, - syslog, level=logging.INFO) - - cf = ConfigParser.ConfigParser() - cf.read([cpath]) - - if not statpath: - try: - statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print("Could not read bcfg2.conf; exiting") - raise SystemExit(1) - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s" % (statpath)) - raise SystemExit(1) - - try: - encoding = cf.get('components', 'encoding') - except: - encoding = 'UTF-8' - - q = '-O3' in sys.argv - - # don't load this at the top. causes a circular import error - from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError - # Be sure the database is ready for new schema - try: - update_database() - except UpdaterError: - raise SystemExit(1) - load_stats(statsdata, - encoding, - verb, - logger, - quick=q, - location=platform.node()) diff --git a/src/lib/Bcfg2/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py deleted file mode 100755 index 1c8fb03f6..000000000 --- a/src/lib/Bcfg2/Server/Reports/manage.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -try: - import Bcfg2.settings -except ImportError: - import sys - sys.stderr.write("Error: Can't find the Bcfg2.settings module. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) - sys.exit(1) - -if __name__ == "__main__": - execute_manager(Bcfg2.settings) diff --git a/src/lib/Bcfg2/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py deleted file mode 100644 index dd1f2f742..000000000 --- a/src/lib/Bcfg2/Server/Reports/nisauth.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Checks with NIS to see if the current user is in the support group""" - -import crypt -import nis -from Bcfg2.settings import AUTHORIZED_GROUP # pylint: disable=E0611 - - -class NISAUTHError(Exception): - """NISAUTHError is raised when somehting goes boom.""" - pass - - -class nisauth(object): - group_test = False - samAcctName = None - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None # this will be a list - mail = None - extensionAttribute1 = None # badgenumber - badge_no = None - uid = None - - def __init__(self, login, passwd=None): - """get user profile from NIS""" - try: - p = nis.match(login, 'passwd.byname').split(":") - print(p) - except: - raise NISAUTHError('username') - # check user password using crypt and 2 character salt from passwd file - if p[1] == crypt.crypt(passwd, p[1][:2]): - # check to see if user is in valid support groups - # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, - 'group.byname').split(':')[-1].split(','): - raise NISAUTHError('group') - self.uid = p[2] - print(self.uid) - else: - raise NISAUTHError('password') diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/404.html b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html deleted file mode 100644 index 168bd9fec..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/404.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'base.html' %} -{% block title %}Bcfg2 - Page not found{% endblock %} -{% block fullcontent %} -

Page not found

-

-The page or object requested could not be found. -

-{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html deleted file mode 100644 index 9a5ef651c..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html +++ /dev/null @@ -1,28 +0,0 @@ -{% extends "base.html" %} - -{% block timepiece %} - -{% if not timestamp %}Rendered at {% now "Y-m-d H:i" %} | {% else %}View as of {{ timestamp|date:"Y-m-d H:i" }} | {% endif %}{% spaceless %} - [change] -
- - -
-{% endspaceless %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html deleted file mode 100644 index 6d20f86d9..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/base.html +++ /dev/null @@ -1,96 +0,0 @@ -{% load bcfg2_tags %} - - - - - -{% block title %}Bcfg2 Reporting System{% endblock %} - - - - - - - - - - - - - -{% block extra_header_info %}{% endblock %} - - - - - - -
-
- {% block fullcontent %} -
-

{% block pagebanner %}Page Banner{% endblock %}

-
{% block timepiece %}Rendered at {% now "Y-m-d H:i" %}{% endblock %}
-
-
- {% block content %}{% endblock %} -
- {% endblock %} -
-
- {% block sidemenu %} -
    -
  • Overview
    • -
- -
    -
  • Clients
    • -
- -
    -
  • Entries Configured
    • -
- -{% comment %} - TODO -
    -
  • Entry Types
    • -
- -{% endcomment %} - - {% endblock %} -
-
-
- - - - - diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html deleted file mode 100644 index 9b86b609f..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html +++ /dev/null @@ -1,129 +0,0 @@ -{% extends "base.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Client {{client.name}}{% endblock %} - -{% block extra_header_info %} - -{% endblock %} - -{% block body_onload %}javascript:clientdetailload(){% endblock %} - -{% block pagebanner %}Client Details{% endblock %} - -{% block content %} -
-

{{client.name}}

- [manage] - View History | Jump to  - -
- - {% if interaction.isstale %} -
- This node did not run within the last 24 hours — it may be out of date. -
- {% endif %} - - - {% if interaction.server %} - - {% endif %} - {% if interaction.metadata %} - - {% endif %} - {% if interaction.repo_rev_code %} - - {% endif %} - - - {% if not interaction.isclean %} - - {% endif %} -
Timestamp{{interaction.timestamp}}
Served by{{interaction.server}}
Profile{{interaction.metadata.profile}}
Revision{{interaction.repo_rev_code}}
State{{interaction.state|capfirst}}
Managed entries{{interaction.totalcount}}
Deviation{{interaction.percentbad|floatformat:"3"}}%
- - {% if interaction.metadata.groups.count %} -
-
-

Group membership

-
[+]
-
- - {% for group in interaction.metadata.groups.all %} - - - - {% endfor %} - -
- {% endif %} - - {% if interaction.metadata.bundles.count %} -
-
-

Bundle membership

-
[+]
-
- - {% for bundle in interaction.metadata.bundles.all %} - - - - {% endfor %} - -
- {% endif %} - - {% for type, ei_list in ei_lists %} - {% if ei_list %} -
-
-

{{ type|capfirst }} Entries — {{ ei_list|length }}

-
[+]
-
- - {% for ei in ei_list %} - - - - - {% endfor %} -
{{ei.entry.kind}} - {{ei.entry.name}}
-
- {% endif %} - {% endfor %} - - {% if entry_list %} -
-
-

Recent Interactions

-
-
- {% include "widgets/interaction_list.inc" %} -
more...
-
-
- {% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html deleted file mode 100644 index 9be59e7d2..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html +++ /dev/null @@ -1,46 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Detailed Client Listing{% endblock %} -{% block pagebanner %}Clients - Detailed View{% endblock %} - -{% block content %} -
- {% filter_navigator %} -{% if entry_list %} - - - - - - - - - - - - {% for entry in entry_list %} - - - - - - - - - - - {% endfor %} -
{% sort_link 'client' 'Node' %}{% sort_link 'state' 'State' %}{% sort_link '-good' 'Good' %}{% sort_link '-bad' 'Bad' %}{% sort_link '-modified' 'Modified' %}{% sort_link '-extra' 'Extra' %}{% sort_link 'timestamp' 'Last Run' %}{% sort_link 'server' 'Server' %}
{{ entry.client.name }}{{ entry.state }}{{ entry.goodcount }}{{ entry.bad_entry_count }}{{ entry.modified_entry_count }}{{ entry.extra_entry_count }}{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }} - {% if entry.server %} - {{ entry.server }} - {% else %} -   - {% endif %} -
-{% else %} -

No client records are available.

-{% endif %} -
-{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html deleted file mode 100644 index 01d4ec2f4..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "base.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Interaction History{% endblock %} -{% block pagebanner %}Interaction history{% if client %} for {{ client.name }}{% endif %}{% endblock %} - -{% block extra_header_info %} -{% endblock %} - -{% block content %} -
-{% if entry_list %} - {% filter_navigator %} - {% include "widgets/interaction_list.inc" %} -{% else %} -

No client records are available.

-{% endif %} -
-{% page_navigator %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html deleted file mode 100644 index 45ba20b86..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block extra_header_info %} -{% endblock%} - -{% block title %}Bcfg2 - Client Grid View{% endblock %} - -{% block pagebanner %}Clients - Grid View{% endblock %} - -{% block content %} -{% filter_navigator %} -{% if inter_list %} - - {% for inter in inter_list %} - {% if forloop.first %}{% endif %} - - {% if forloop.last %} - - {% else %} - {% if forloop.counter|divisibleby:"4" %}{% endif %} - {% endif %} - {% endfor %} -
- {{ inter.client.name }} -
-{% else %}

No client records are available.

-{% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html deleted file mode 100644 index 443ec8ccb..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "base.html" %} - -{% block extra_header_info %} -{% endblock%} - -{% block title %}Bcfg2 - Manage Clients{% endblock %} - -{% block pagebanner %}Clients - Manage{% endblock %} - -{% block content %} -
- {% if message %} -
{{ message }}
- {% endif %} -{% if clients %} - - - - - - - {% for client in clients %} - - - - - - {% endfor %} -
NodeExpirationManage
- - - {{ client.name }}{% firstof client.expiration 'Active' %} -
-
{# here for no reason other then to validate #} - - - -
-
-
-{% else %} -

No client records are available.

-{% endif %} -
-{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html deleted file mode 100644 index d6ad303fc..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Common Problems{% endblock %} - -{% block extra_header_info %} -{% endblock%} - -{% block pagebanner %}Common configuration problems{% endblock %} - -{% block content %} -
-
- Showing items with more then {{ threshold }} entries - - -
-
- {% for type_name, type_list in lists %} -
-
-

{{ type_name|capfirst }} entries

-
[–]
-
- {% if type_list %} - - - {% for entry, reason, interaction in type_list %} - - - - - - - {% endfor %} -
TypeNameCountReason
{{ entry.kind }}{{ entry.name }}{{ interaction|length }}{{ reason.short_list|join:"," }}
- {% else %} -

There are currently no inconsistent {{ type_name }} configuration entries.

- {% endif %} -
- {% endfor %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html deleted file mode 100644 index 5f7579eb9..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Entry Status{% endblock %} - -{% block extra_header_info %} -{% endblock%} - -{% block pagebanner %}{{ entry.kind }} entry {{ entry.name }} status{% endblock %} - -{% block content %} -{% filter_navigator %} -{% if item_data %} -
- - - {% for ei, inter, reason in item_data %} - - - - - - - {% endfor %} -
NameTimestampStateReason
{{ inter.client.name }}{{ inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}{{ ei.get_type_display }}{{ reason.short_list|join:"," }}
-
-{% else %} -

There are currently no hosts with this configuration entry.

-{% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html deleted file mode 100644 index cadc178a7..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html +++ /dev/null @@ -1,130 +0,0 @@ -{% extends "base.html" %} -{% load split %} -{% load syntax_coloring %} - - -{% block title %}Bcfg2 - Element Details{% endblock %} - - -{% block extra_header_info %} - -{% endblock%} - -{% block pagebanner %}Element Details{% endblock %} - -{% block content %} -
-

{{mod_or_bad|capfirst}} {{item.entry.kind}}: {{item.entry.name}}

-
- -
- - {% if isextra %} -

This item exists on the host but is not defined in the configuration.

- {% endif %} - - {% if not item.reason.current_exists %} -
This item does not currently exist on the host but is specified to exist in the configuration.
- {% endif %} - - {% if item.reason.current_owner or item.reason.current_group or item.reason.current_perms or item.reason.current_status or item.reason.current_status or item.reason.current_to or item.reason.current_version %} - - - - {% if item.reason.current_owner %} - - - {% endif %} - {% if item.reason.current_group %} - - - {% endif %} - {% if item.reason.current_perms %} - - - {% endif %} - {% if item.reason.current_status %} - - - {% endif %} - {% if item.reason.current_to %} - - - {% endif %} - {% if item.reason.current_version %} - - - {% endif %} -
Problem TypeExpectedFound
Owner{{item.reason.owner}}{{item.reason.current_owner}}
Group{{item.reason.group}}{{item.reason.current_group}}
Permissions{{item.reason.perms}}{{item.reason.current_perms}}
Status{{item.reason.status}}{{item.reason.current_status}}
Symlink Target{{item.reason.to}}{{item.reason.current_to}}
Package Version{{item.reason.version|cut:"("|cut:")"}}{{item.reason.current_version|cut:"("|cut:")"}}
- {% endif %} - - {% if item.reason.current_diff or item.reason.is_sensitive %} -
-
- {% if item.reason.is_sensitive %} -

File contents unavailable, as they might contain sensitive data.

- {% else %} -

Incorrect file contents

- {% endif %} -
- {% if not item.reason.is_sensitive %} -
- {{ item.reason.current_diff|syntaxhilight }} -
- {% endif %} -
- {% endif %} - - - {% if item.reason.unpruned %} -
-
-

Extra entries found

-
- - {% for unpruned_item in item.reason.unpruned|split %} - - {% endfor %} -
{{ unpruned_item }}
-
- {% endif %} - - -
-
-

Occurences on {{ timestamp|date:"Y-m-d" }}

-
- {% if associated_list %} - - {% for inter in associated_list %} - - - - {% endfor %} -
{{inter.client.name}}{{inter.timestamp}}
- {% else %} -

Missing client list

- {% endif %} -
- -
-{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html deleted file mode 100644 index 0a92e7fc0..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Element Listing{% endblock %} - -{% block extra_header_info %} -{% endblock%} - -{% block pagebanner %}{{mod_or_bad|capfirst}} Element Listing{% endblock %} - -{% block content %} -{% filter_navigator %} -{% if item_list %} - {% for type_name, type_data in item_list %} -
-
-

{{ type_name }} — {{ type_data|length }}

-
[–]
-
- - - {% for entry, reason, eis in type_data %} - - - - - - {% endfor %} -
NameCountReason
{{entry.name}}{{ eis|length }}{{ reason.short_list|join:"," }}
-
- {% endfor %} -{% else %} -

There are currently no inconsistent configuration entries.

-{% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html deleted file mode 100644 index b9847cf96..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Client Summary{% endblock %} -{% block pagebanner %}Clients - Summary{% endblock %} - -{% block body_onload %}javascript:hide_table_array(hide_tables){% endblock %} - -{% block extra_header_info %} - -{% endblock%} - -{% block content %} -
-

{{ node_count }} nodes reporting in

-
-{% if summary_data %} - {% for summary in summary_data %} -
-
-

{{ summary.nodes|length }} {{ summary.label }}

-
[+]
-
- - - {% for node in summary.nodes|sort_interactions_by_name %} - - - - {% endfor %} -
{{ node.client.name }}
-
- {% endfor %} -{% else %} -

No data to report on

-{% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html deleted file mode 100644 index ff775ded5..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html +++ /dev/null @@ -1,38 +0,0 @@ -{% extends "base-timeview.html" %} -{% load bcfg2_tags %} - -{% block title %}Bcfg2 - Performance Metrics{% endblock %} -{% block pagebanner %}Performance Metrics{% endblock %} - - -{% block extra_header_info %} -{% endblock%} - -{% block content %} -
- {% if metrics %} - - - - - - - - - - - {% for metric in metrics|dictsort:"name" %} - - - {% for mitem in metric|build_metric_list %} - - {% endfor %} - - {% endfor %} -
NameParseProbeInventoryInstallConfigTotal
{{ metric.name }}{{ mitem }}
- {% else %} -

No metric data available

- {% endif %} -
-{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html deleted file mode 100644 index 759415507..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html +++ /dev/null @@ -1,25 +0,0 @@ -{% spaceless %} -
-
-{% if filters %} -{% for filter, filter_url in filters %} - {% if forloop.first %} - Active filters (click to remove): - {% endif %} - {{ filter|capfirst }}{% if not forloop.last %}, {% endif %} - {% if forloop.last %} - {% if groups %}|{% endif %} - {% endif %} -{% endfor %} -{% endif %} -{% if groups %} - - -{% endif %} -
-
-{% endspaceless %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc deleted file mode 100644 index 6fe7e6547..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc +++ /dev/null @@ -1,38 +0,0 @@ -{% load bcfg2_tags %} -
- - - - {% if not client %} - - {% endif %} - - - - - - - - {% for entry in entry_list %} - - - {% if not client %} - - {% endif %} - - - - - - - - {% endfor %} -
TimestampClientStateGoodBadModifiedExtraServer
{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}{{ entry.client.name }}{{ entry.state }}{{ entry.goodcount }}{{ entry.bad_entry_count }}{{ entry.modified_entry_count }}{{ entry.extra_entry_count }} - {% if entry.server %} - {{ entry.server }} - {% else %} -   - {% endif %} -
-
diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html deleted file mode 100644 index aa0def83e..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html +++ /dev/null @@ -1,23 +0,0 @@ -{% spaceless %} -{% for page, page_url in pager %} - {% if forloop.first %} -
- {% if prev_page %}< Prev {% endif %} - {% if first_page %}1 ... {% endif %} - {% endif %} - {% ifequal page current_page %} - {{ page }} - {% else %} - {{ page }} - {% endifequal %} - {% if forloop.last %} - {% if last_page %} ... {{ total_pages }} {% endif %} - {% if next_page %}Next > {% endif %} - |{% for limit, limit_url in page_limits %} {{ limit }}{% endfor %} -
- {% else %} -   - {% endif %} -{% endfor %} -{% endspaceless %} - diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py deleted file mode 100644 index 736d6448a..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py +++ /dev/null @@ -1,415 +0,0 @@ -import sys -from copy import copy - -from django import template -from django.conf import settings -from django.core.urlresolvers import resolve, reverse, \ - Resolver404, NoReverseMatch -from django.template.loader import get_template, \ - get_template_from_string,TemplateDoesNotExist -from django.utils.encoding import smart_unicode, smart_str -from django.utils.safestring import mark_safe -from datetime import datetime, timedelta -from Bcfg2.Server.Reports.utils import filter_list -from Bcfg2.Server.Reports.reports.models import Group - -register = template.Library() - -__PAGE_NAV_LIMITS__ = (10, 25, 50, 100) - - -@register.inclusion_tag('widgets/page_bar.html', takes_context=True) -def page_navigator(context): - """ - Creates paginated links. - - Expects the context to be a RequestContext and - views.prepare_paginated_list() to have populated page information. - """ - fragment = dict() - try: - path = context['request'].META['PATH_INFO'] - total_pages = int(context['total_pages']) - records_per_page = int(context['records_per_page']) - except KeyError: - return fragment - except ValueError: - return fragment - - if total_pages < 2: - return {} - - try: - view, args, kwargs = resolve(path) - current_page = int(kwargs.get('page_number', 1)) - fragment['current_page'] = current_page - fragment['page_number'] = current_page - fragment['total_pages'] = total_pages - fragment['records_per_page'] = records_per_page - if current_page > 1: - kwargs['page_number'] = current_page - 1 - fragment['prev_page'] = reverse(view, args=args, kwargs=kwargs) - if current_page < total_pages: - kwargs['page_number'] = current_page + 1 - fragment['next_page'] = reverse(view, args=args, kwargs=kwargs) - - view_range = 5 - if total_pages > view_range: - pager_start = current_page - 2 - pager_end = current_page + 2 - if pager_start < 1: - pager_end += (1 - pager_start) - pager_start = 1 - if pager_end > total_pages: - pager_start -= (pager_end - total_pages) - pager_end = total_pages - else: - pager_start = 1 - pager_end = total_pages - - if pager_start > 1: - kwargs['page_number'] = 1 - fragment['first_page'] = reverse(view, args=args, kwargs=kwargs) - if pager_end < total_pages: - kwargs['page_number'] = total_pages - fragment['last_page'] = reverse(view, args=args, kwargs=kwargs) - - pager = [] - for page in range(pager_start, int(pager_end) + 1): - kwargs['page_number'] = page - pager.append((page, reverse(view, args=args, kwargs=kwargs))) - - kwargs['page_number'] = 1 - page_limits = [] - for limit in __PAGE_NAV_LIMITS__: - kwargs['page_limit'] = limit - page_limits.append((limit, - reverse(view, args=args, kwargs=kwargs))) - # resolver doesn't like this - del kwargs['page_number'] - del kwargs['page_limit'] - page_limits.append(('all', - reverse(view, args=args, kwargs=kwargs) + "|all")) - - fragment['pager'] = pager - fragment['page_limits'] = page_limits - - except Resolver404: - path = "404" - except NoReverseMatch: - nr = sys.exc_info()[1] - path = "NoReverseMatch: %s" % nr - except ValueError: - path = "ValueError" - #FIXME - Handle these - - fragment['path'] = path - return fragment - - -@register.inclusion_tag('widgets/filter_bar.html', takes_context=True) -def filter_navigator(context): - try: - path = context['request'].META['PATH_INFO'] - view, args, kwargs = resolve(path) - - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] - filters.append((filter, - reverse(view, args=args, kwargs=myargs))) - filters.sort(lambda x, y: cmp(x[0], y[0])) - - myargs = kwargs.copy() - selected=True - if 'group' in myargs: - del myargs['group'] - selected=False - groups = [('---', reverse(view, args=args, kwargs=myargs), selected)] - for group in Group.objects.values('name'): - myargs['group'] = group['name'] - groups.append((group['name'], reverse(view, args=args, kwargs=myargs), - group['name'] == kwargs.get('group', ''))) - - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() - - -def _subtract_or_na(mdict, x, y): - """ - Shortcut for build_metric_list - """ - try: - return round(mdict[x] - mdict[y], 4) - except: - return "n/a" - - -@register.filter -def build_metric_list(mdict): - """ - Create a list of metric table entries - - Moving this here to simplify the view. - Should really handle the case where these are missing... - """ - td_list = [] - # parse - td_list.append(_subtract_or_na(mdict, 'config_parse', 'config_download')) - #probe - td_list.append(_subtract_or_na(mdict, 'probe_upload', 'start')) - #inventory - td_list.append(_subtract_or_na(mdict, 'inventory', 'initialization')) - #install - td_list.append(_subtract_or_na(mdict, 'install', 'inventory')) - #cfg download & parse - td_list.append(_subtract_or_na(mdict, 'config_parse', 'probe_upload')) - #total - td_list.append(_subtract_or_na(mdict, 'finished', 'start')) - return td_list - - -@register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - - -@register.filter -def sort_interactions_by_name(value): - """ - Sort an interaction list by client name - """ - inters = list(value) - inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) - return inters - - -class AddUrlFilter(template.Node): - def __init__(self, filter_name, filter_value): - self.filter_name = filter_name - self.filter_value = filter_value - self.fallback_view = 'Bcfg2.Server.Reports.reports.views.render_history_view' - - def render(self, context): - link = '#' - try: - path = context['request'].META['PATH_INFO'] - view, args, kwargs = resolve(path) - filter_value = self.filter_value.resolve(context, True) - if filter_value: - filter_name = smart_str(self.filter_name) - filter_value = smart_unicode(filter_value) - kwargs[filter_name] = filter_value - # These two don't make sense - if filter_name == 'server' and 'hostname' in kwargs: - del kwargs['hostname'] - elif filter_name == 'hostname' and 'server' in kwargs: - del kwargs['server'] - try: - link = reverse(view, args=args, kwargs=kwargs) - except NoReverseMatch: - link = reverse(self.fallback_view, args=None, - kwargs={filter_name: filter_value}) - except NoReverseMatch: - rm = sys.exc_info()[1] - raise rm - except (Resolver404, ValueError): - pass - return link - - -@register.tag -def add_url_filter(parser, token): - """ - Return a url with the filter added to the current view. - - Takes a new filter and resolves the current view with the new filter - applied. Resolves to Bcfg2.Server.Reports.reports.views.client_history - by default. - - {% add_url_filter server=interaction.server %} - """ - try: - tag_name, filter_pair = token.split_contents() - filter_name, filter_value = filter_pair.split('=', 1) - filter_name = filter_name.strip() - filter_value = parser.compile_filter(filter_value) - except ValueError: - raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) - if not filter_name or not filter_value: - raise template.TemplateSyntaxError("argument should be a filter=value pair") - - return AddUrlFilter(filter_name, filter_value) - - -class MediaTag(template.Node): - def __init__(self, filter_value): - self.filter_value = filter_value - - def render(self, context): - base = context['MEDIA_URL'] - try: - request = context['request'] - try: - base = request.environ['bcfg2.media_url'] - except: - if request.path != request.META['PATH_INFO']: - offset = request.path.find(request.META['PATH_INFO']) - if offset > 0: - base = "%s/%s" % (request.path[:offset], \ - context['MEDIA_URL'].strip('/')) - except: - pass - return "%s/%s" % (base, self.filter_value) - - -@register.tag -def to_media_url(parser, token): - """ - Return a url relative to the media_url. - - {% to_media_url /bcfg2.css %} - """ - try: - filter_value = token.split_contents()[1] - filter_value = parser.compile_filter(filter_value) - except ValueError: - raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) - - return MediaTag(filter_value) - -@register.filter -def determine_client_state(entry): - """ - Determine client state. - - This is used to determine whether a client is reporting clean or - dirty. If the client is reporting dirty, this will figure out just - _how_ dirty and adjust the color accordingly. - """ - if entry.state == 'clean': - return "clean-lineitem" - - bad_percentage = 100 * (float(entry.badcount()) / entry.totalcount) - if bad_percentage < 33: - thisdirty = "slightly-dirty-lineitem" - elif bad_percentage < 66: - thisdirty = "dirty-lineitem" - else: - thisdirty = "very-dirty-lineitem" - return thisdirty - - -@register.tag(name='qs') -def do_qs(parser, token): - """ - qs tag - - accepts a name value pair and inserts or replaces it in the query string - """ - try: - tag, name, value = token.split_contents() - except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ - % token.contents.split()[0] - return QsNode(name, value) - -class QsNode(template.Node): - def __init__(self, name, value): - self.name = template.Variable(name) - self.value = template.Variable(value) - - def render(self, context): - try: - name = self.name.resolve(context) - value = self.value.resolve(context) - request = context['request'] - qs = copy(request.GET) - qs[name] = value - return "?%s" % qs.urlencode() - except template.VariableDoesNotExist: - return '' - except KeyError: - if settings.TEMPLATE_DEBUG: - raise Exception, "'qs' tag requires context['request']" - return '' - except: - return '' - - -@register.tag -def sort_link(parser, token): - ''' - Create a sort anchor tag. Reverse it if active. - - {% sort_link sort_key text %} - ''' - try: - tag, sort_key, text = token.split_contents() - except ValueError: - raise template.TemplateSyntaxError("%r tag requires at least four arguments" \ - % token.split_contents()[0]) - - return SortLinkNode(sort_key, text) - -class SortLinkNode(template.Node): - __TMPL__ = "{% load bcfg2_tags %}{{ text }}" - - def __init__(self, sort_key, text): - self.sort_key = template.Variable(sort_key) - self.text = template.Variable(text) - - def render(self, context): - try: - try: - sort = context['request'].GET['sort'] - except KeyError: - #fall back on this - sort = context.get('sort', '') - sort_key = self.sort_key.resolve(context) - text = self.text.resolve(context) - - # add arrows - try: - sort_base = sort_key.lstrip('-') - if sort[0] == '-' and sort[1:] == sort_base: - text = text + '▼' - sort_key = sort_base - elif sort_base == sort: - text = text + '▲' - sort_key = '-' + sort_base - except IndexError: - pass - - context.push() - context['key'] = sort_key - context['text'] = mark_safe(text) - output = get_template_from_string(self.__TMPL__).render(context) - context.pop() - return output - except: - if settings.DEBUG: - raise - raise - return '' - diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py deleted file mode 100644 index a9b4f0371..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py +++ /dev/null @@ -1,8 +0,0 @@ -from django import template -register = template.Library() - - -@register.filter -def split(s): - """split by newlines""" - return s.split('\n') diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py deleted file mode 100644 index bd379b98d..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py +++ /dev/null @@ -1,47 +0,0 @@ -import sys -from django import template -from django.utils.encoding import smart_unicode -from django.utils.html import conditional_escape -from django.utils.safestring import mark_safe - -from Bcfg2.Compat import u_str - -register = template.Library() - -# pylint: disable=E0611 -try: - from pygments import highlight - from pygments.lexers import get_lexer_by_name - from pygments.formatters import HtmlFormatter - colorize = True -except: - colorize = False -# pylint: enable=E0611 - - -@register.filter -def syntaxhilight(value, arg="diff", autoescape=None): - """ - Returns a syntax-hilighted version of Code; - requires code/language arguments - """ - - if autoescape: - value = conditional_escape(value) - arg = conditional_escape(arg) - - if colorize: - try: - output = u_str('') - - lexer = get_lexer_by_name(arg) - output += highlight(value, lexer, HtmlFormatter()) - return mark_safe(output) - except: - return value - else: - return mark_safe(u_str('
Tip: Install pygments ' - 'for highlighting
%s
') % value) -syntaxhilight.needs_autoescape = True diff --git a/src/lib/Bcfg2/Server/Reports/reports/urls.py b/src/lib/Bcfg2/Server/Reports/reports/urls.py deleted file mode 100644 index 1cfe725c2..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/urls.py +++ /dev/null @@ -1,58 +0,0 @@ -from django.conf.urls.defaults import * -from django.core.urlresolvers import reverse, NoReverseMatch -from django.http import HttpResponsePermanentRedirect -from Bcfg2.Server.Reports.utils import filteredUrls, paginatedUrls, timeviewUrls - -def newRoot(request): - try: - grid_view = reverse('reports_grid_view') - except NoReverseMatch: - grid_view = '/grid' - return HttpResponsePermanentRedirect(grid_view) - -urlpatterns = patterns('Bcfg2.Server.Reports.reports', - (r'^$', newRoot), - - url(r'^manage/?$', 'views.client_manage', name='reports_client_manage'), - url(r'^client/(?P[^/]+)/(?P\d+)/?$', 'views.client_detail', name='reports_client_detail_pk'), - url(r'^client/(?P[^/]+)/?$', 'views.client_detail', name='reports_client_detail'), - url(r'^elements/(?P\w+)/(?P\d+)/?$', 'views.config_item', name='reports_item'), - url(r'^entry/(?P\w+)/?$', 'views.entry_status', name='reports_entry'), -) - -urlpatterns += patterns('Bcfg2.Server.Reports.reports', - *timeviewUrls( - (r'^summary/?$', 'views.display_summary', None, 'reports_summary'), - (r'^timing/?$', 'views.display_timing', None, 'reports_timing'), - (r'^common/(?P\d+)/?$', 'views.common_problems', None, 'reports_common_problems'), - (r'^common/?$', 'views.common_problems', None, 'reports_common_problems'), -)) - -urlpatterns += patterns('Bcfg2.Server.Reports.reports', - *filteredUrls(*timeviewUrls( - (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), - (r'^detailed/?$', - 'views.client_detailed_list', None, 'reports_detailed_list'), - (r'^elements/(?P\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), -))) - -urlpatterns += patterns('Bcfg2.Server.Reports.reports', - *paginatedUrls( *filteredUrls( - (r'^history/?$', - 'views.render_history_view', None, 'reports_history'), - (r'^history/(?P[^/|]+)/?$', - 'views.render_history_view', None, 'reports_client_history'), -))) - - # Uncomment this for admin: - #(r'^admin/', include('django.contrib.admin.urls')), - - -## Uncomment this section if using authentication -#urlpatterns += patterns('', -# (r'^login/$', 'django.contrib.auth.views.login', -# {'template_name': 'auth/login.html'}), -# (r'^logout/$', 'django.contrib.auth.views.logout', -# {'template_name': 'auth/logout.html'}) -# ) - diff --git a/src/lib/Bcfg2/Server/Reports/reports/views.py b/src/lib/Bcfg2/Server/Reports/reports/views.py deleted file mode 100644 index ca9e5f1f9..000000000 --- a/src/lib/Bcfg2/Server/Reports/reports/views.py +++ /dev/null @@ -1,583 +0,0 @@ -""" -Report views - -Functions to handle all of the reporting views. -""" -from datetime import datetime, timedelta -import sys -from time import strptime - -from django.template import Context, RequestContext -from django.http import \ - HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 -from django.shortcuts import render_to_response, get_object_or_404 -from django.core.urlresolvers import \ - resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection, DatabaseError -from django.db.models import Q - -from Bcfg2.Server.Reports.reports.models import * - - -__SORT_FIELDS__ = ( 'client', 'state', 'good', 'bad', 'modified', 'extra', \ - 'timestamp', 'server' ) - -class PaginationError(Exception): - """This error is raised when pagination cannot be completed.""" - pass - - -def _in_bulk(model, ids): - """ - Short cut to fetch in bulk and trap database errors. sqlite will raise - a "too many SQL variables" exception if this list is too long. Try using - django and fetch manually if an error occurs - - returns a dict of this form { id: } - """ - - try: - return model.objects.in_bulk(ids) - except DatabaseError: - pass - - # if objects.in_bulk fails so will obejcts.filter(pk__in=ids) - bulk_dict = {} - [bulk_dict.__setitem__(i.id, i) \ - for i in model.objects.all() if i.id in ids] - return bulk_dict - - -def server_error(request): - """ - 500 error handler. - - For now always return the debug response. Mailing isn't appropriate here. - - """ - from django.views import debug - return debug.technical_500_response(request, *sys.exc_info()) - - -def timeview(fn): - """ - Setup a timeview view - - Handles backend posts from the calendar and converts date pieces - into a 'timestamp' parameter - - """ - def _handle_timeview(request, **kwargs): - """Send any posts back.""" - if request.method == 'POST' and request.POST.get('op', '') == 'timeview': - cal_date = request.POST['cal_date'] - try: - fmt = "%Y/%m/%d" - if cal_date.find(' ') > -1: - fmt += " %H:%M" - timestamp = datetime(*strptime(cal_date, fmt)[0:6]) - view, args, kw = resolve(request.META['PATH_INFO']) - kw['year'] = "%0.4d" % timestamp.year - kw['month'] = "%02.d" % timestamp.month - kw['day'] = "%02.d" % timestamp.day - if cal_date.find(' ') > -1: - kw['hour'] = timestamp.hour - kw['minute'] = timestamp.minute - return HttpResponseRedirect(reverse(view, - args=args, - kwargs=kw)) - except KeyError: - pass - except: - pass - # FIXME - Handle this - - """Extract timestamp from args.""" - timestamp = None - try: - timestamp = datetime(int(kwargs.pop('year')), - int(kwargs.pop('month')), - int(kwargs.pop('day')), int(kwargs.pop('hour', 0)), - int(kwargs.pop('minute', 0)), 0) - kwargs['timestamp'] = timestamp - except KeyError: - pass - except: - raise - return fn(request, **kwargs) - - return _handle_timeview - - -def _handle_filters(query, **kwargs): - """ - Applies standard filters to a query object - - Returns an updated query object - - query - query object to filter - - server -- Filter interactions by server - state -- Filter interactions by state - group -- Filter interactions by group - - """ - if 'state' in kwargs and kwargs['state']: - query = query.filter(state__exact=kwargs['state']) - if 'server' in kwargs and kwargs['server']: - query = query.filter(server__exact=kwargs['server']) - - if 'group' in kwargs and kwargs['group']: - group = get_object_or_404(Group, name=kwargs['group']) - query = query.filter(metadata__groups__id=group.pk) - return query - - -def config_item(request, pk, type="bad"): - """ - Display a single entry. - - Dispalys information about a single entry. - - """ - item = get_object_or_404(Entries_interactions, id=pk) - timestamp = item.interaction.timestamp - time_start = item.interaction.timestamp.replace(hour=0, - minute=0, - second=0, - microsecond=0) - time_end = time_start + timedelta(days=1) - - todays_data = Interaction.objects.filter(timestamp__gte=time_start, - timestamp__lt=time_end) - shared_entries = Entries_interactions.objects.filter(entry=item.entry, - reason=item.reason, - type=item.type, - interaction__in=[x['id']\ - for x in todays_data.values('id')]) - - associated_list = Interaction.objects.filter(id__in=[x['interaction']\ - for x in shared_entries.values('interaction')])\ - .order_by('client__name', 'timestamp').select_related().all() - - return render_to_response('config_items/item.html', - {'item': item, - 'isextra': item.type == TYPE_EXTRA, - 'mod_or_bad': type, - 'associated_list': associated_list, - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -@timeview -def config_item_list(request, type, timestamp=None, **kwargs): - """Render a listing of affected elements""" - mod_or_bad = type.lower() - type = convert_entry_type_to_id(type) - if type < 0: - raise Http404 - - current_clients = Interaction.objects.interaction_per_client(timestamp) - current_clients = [q['id'] for q in _handle_filters(current_clients, **kwargs).values('id')] - - ldata = list(Entries_interactions.objects.filter( - interaction__in=current_clients, type=type).values()) - entry_ids = set([x['entry_id'] for x in ldata]) - reason_ids = set([x['reason_id'] for x in ldata]) - - entries = _in_bulk(Entries, entry_ids) - reasons = _in_bulk(Reason, reason_ids) - - kind_list = {} - [kind_list.__setitem__(kind, {}) for kind in set([e.kind for e in entries.values()])] - for x in ldata: - kind = entries[x['entry_id']].kind - data_key = (x['entry_id'], x['reason_id']) - try: - kind_list[kind][data_key].append(x['id']) - except KeyError: - kind_list[kind][data_key] = [x['id']] - - lists = [] - for kind in kind_list.keys(): - lists.append((kind, [(entries[e[0][0]], reasons[e[0][1]], e[1]) - for e in sorted(kind_list[kind].iteritems(), key=lambda x: entries[x[0][0]].name)])) - - return render_to_response('config_items/listing.html', - {'item_list': lists, - 'mod_or_bad': mod_or_bad, - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -@timeview -def entry_status(request, eid, timestamp=None, **kwargs): - """Render a listing of affected elements""" - entry = get_object_or_404(Entries, pk=eid) - - current_clients = Interaction.objects.interaction_per_client(timestamp) - inters = {} - [inters.__setitem__(i.id, i) \ - for i in _handle_filters(current_clients, **kwargs).select_related('client')] - - eis = Entries_interactions.objects.filter( - interaction__in=inters.keys(), entry=entry) - - reasons = _in_bulk(Reason, set([x.reason_id for x in eis])) - - item_data = [] - for ei in eis: - item_data.append((ei, inters[ei.interaction_id], reasons[ei.reason_id])) - - return render_to_response('config_items/entry_status.html', - {'entry': entry, - 'item_data': item_data, - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -@timeview -def common_problems(request, timestamp=None, threshold=None): - """Mine config entries""" - - if request.method == 'POST': - try: - threshold = int(request.POST['threshold']) - view, args, kw = resolve(request.META['PATH_INFO']) - kw['threshold'] = threshold - return HttpResponseRedirect(reverse(view, - args=args, - kwargs=kw)) - except: - pass - - try: - threshold = int(threshold) - except: - threshold = 10 - - c_intr = Interaction.objects.get_interaction_per_client_ids(timestamp) - data_list = {} - [data_list.__setitem__(t_id, {}) \ - for t_id, t_label in TYPE_CHOICES if t_id != TYPE_GOOD] - ldata = list(Entries_interactions.objects.filter( - interaction__in=c_intr).exclude(type=TYPE_GOOD).values()) - - entry_ids = set([x['entry_id'] for x in ldata]) - reason_ids = set([x['reason_id'] for x in ldata]) - for x in ldata: - type = x['type'] - data_key = (x['entry_id'], x['reason_id']) - try: - data_list[type][data_key].append(x['id']) - except KeyError: - data_list[type][data_key] = [x['id']] - - entries = _in_bulk(Entries, entry_ids) - reasons = _in_bulk(Reason, reason_ids) - - lists = [] - for type, type_name in TYPE_CHOICES: - if type == TYPE_GOOD: - continue - lists.append([type_name.lower(), [(entries[e[0][0]], reasons[e[0][1]], e[1]) - for e in sorted(data_list[type].items(), key=lambda x: len(x[1]), reverse=True) - if len(e[1]) > threshold]]) - - return render_to_response('config_items/common.html', - {'lists': lists, - 'timestamp': timestamp, - 'threshold': threshold}, - context_instance=RequestContext(request)) - - -@timeview -def client_index(request, timestamp=None, **kwargs): - """ - Render a grid view of active clients. - - Keyword parameters: - timestamp -- datetime object to render from - - """ - list = _handle_filters(Interaction.objects.interaction_per_client(timestamp), **kwargs).\ - select_related().order_by("client__name").all() - - return render_to_response('clients/index.html', - {'inter_list': list, - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -@timeview -def client_detailed_list(request, timestamp=None, **kwargs): - """ - Provides a more detailed list view of the clients. Allows for extra - filters to be passed in. - - """ - - try: - sort = request.GET['sort'] - if sort[0] == '-': - sort_key = sort[1:] - else: - sort_key = sort - if not sort_key in __SORT_FIELDS__: - raise ValueError - - if sort_key == "client": - kwargs['orderby'] = "%s__name" % sort - elif sort_key == "good": - kwargs['orderby'] = "%scount" % sort - elif sort_key in ["bad", "modified", "extra"]: - kwargs['orderby'] = "%s_entries" % sort - else: - kwargs['orderby'] = sort - kwargs['sort'] = sort - except (ValueError, KeyError): - kwargs['orderby'] = "client__name" - kwargs['sort'] = "client" - - kwargs['interaction_base'] = Interaction.objects.interaction_per_client(timestamp).select_related() - kwargs['page_limit'] = 0 - return render_history_view(request, 'clients/detailed-list.html', **kwargs) - - -def client_detail(request, hostname=None, pk=None): - context = dict() - client = get_object_or_404(Client, name=hostname) - if(pk == None): - inter = client.current_interaction - maxdate = None - else: - inter = client.interactions.get(pk=pk) - maxdate = inter.timestamp - - ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry').order_by('entry__kind', 'entry__name') - #ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry') - #ei = sorted(Entries_interactions.objects.filter(interaction=inter).select_related('entry'), - # key=lambda x: (x.entry.kind, x.entry.name)) - context['ei_lists'] = ( - ('bad', [x for x in ei if x.type == TYPE_BAD]), - ('modified', [x for x in ei if x.type == TYPE_MODIFIED]), - ('extra', [x for x in ei if x.type == TYPE_EXTRA]) - ) - - context['interaction']=inter - return render_history_view(request, 'clients/detail.html', page_limit=5, - client=client, maxdate=maxdate, context=context) - - -def client_manage(request): - """Manage client expiration""" - message = '' - if request.method == 'POST': - try: - client_name = request.POST.get('client_name', None) - client_action = request.POST.get('client_action', None) - client = Client.objects.get(name=client_name) - if client_action == 'expire': - client.expiration = datetime.now() - client.save() - message = "Expiration for %s set to %s." % \ - (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) - elif client_action == 'unexpire': - client.expiration = None - client.save() - message = "%s is now active." % client_name - else: - message = "Missing action" - except Client.DoesNotExist: - if not client_name: - client_name = "" - message = "Couldn't find client \"%s\"" % client_name - - return render_to_response('clients/manage.html', - {'clients': Client.objects.order_by('name').all(), 'message': message}, - context_instance=RequestContext(request)) - - -@timeview -def display_summary(request, timestamp=None): - """ - Display a summary of the bcfg2 world - """ - recent_data = Interaction.objects.interaction_per_client(timestamp) \ - .select_related().all() - node_count = len(recent_data) - if not timestamp: - timestamp = datetime.now() - - collected_data = dict(clean=[], - bad=[], - modified=[], - extra=[], - stale=[]) - for node in recent_data: - if timestamp - node.timestamp > timedelta(hours=24): - collected_data['stale'].append(node) - # If stale check for uptime - if node.bad_entry_count() > 0: - collected_data['bad'].append(node) - else: - collected_data['clean'].append(node) - if node.modified_entry_count() > 0: - collected_data['modified'].append(node) - if node.extra_entry_count() > 0: - collected_data['extra'].append(node) - - # label, header_text, node_list - summary_data = [] - get_dict = lambda name, label: {'name': name, - 'nodes': collected_data[name], - 'label': label} - if len(collected_data['clean']) > 0: - summary_data.append(get_dict('clean', - 'nodes are clean.')) - if len(collected_data['bad']) > 0: - summary_data.append(get_dict('bad', - 'nodes are bad.')) - if len(collected_data['modified']) > 0: - summary_data.append(get_dict('modified', - 'nodes were modified.')) - if len(collected_data['extra']) > 0: - summary_data.append(get_dict('extra', - 'nodes have extra configurations.')) - if len(collected_data['stale']) > 0: - summary_data.append(get_dict('stale', - 'nodes did not run within the last 24 hours.')) - - return render_to_response('displays/summary.html', - {'summary_data': summary_data, 'node_count': node_count, - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -@timeview -def display_timing(request, timestamp=None): - mdict = dict() - inters = Interaction.objects.interaction_per_client(timestamp).select_related().all() - [mdict.__setitem__(inter, {'name': inter.client.name}) \ - for inter in inters] - for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): - for i in metric.interaction.all(): - try: - mdict[i][metric.metric] = metric.value - except KeyError: - #In the unlikely event two interactions share a metric, ignore it - pass - return render_to_response('displays/timing.html', - {'metrics': list(mdict.values()), - 'timestamp': timestamp}, - context_instance=RequestContext(request)) - - -def render_history_view(request, template='clients/history.html', **kwargs): - """ - Provides a detailed history of a clients interactions. - - Renders a detailed history of a clients interactions. Allows for various - filters and settings. Automatically sets pagination data into the context. - - Keyword arguments: - interaction_base -- Interaction QuerySet to build on - (default Interaction.objects) - context -- Additional context data to render with - page_number -- Page to display (default 1) - page_limit -- Number of results per page, if 0 show all (default 25) - client -- Client object to render - hostname -- Client hostname to lookup and render. Returns a 404 if - not found - server -- Filter interactions by server - state -- Filter interactions by state - group -- Filter interactions by group - entry_max -- Most recent interaction to display - orderby -- Sort results using this field - - """ - - context = kwargs.get('context', dict()) - max_results = int(kwargs.get('page_limit', 25)) - page = int(kwargs.get('page_number', 1)) - - client = kwargs.get('client', None) - if not client and 'hostname' in kwargs: - client = get_object_or_404(Client, name=kwargs['hostname']) - if client: - context['client'] = client - - entry_max = kwargs.get('maxdate', None) - context['entry_max'] = entry_max - - # Either filter by client or limit by clients - iquery = kwargs.get('interaction_base', Interaction.objects) - if client: - iquery = iquery.filter(client__exact=client) - iquery = iquery.select_related() - - if 'orderby' in kwargs and kwargs['orderby']: - iquery = iquery.order_by(kwargs['orderby']) - if 'sort' in kwargs: - context['sort'] = kwargs['sort'] - - iquery = _handle_filters(iquery, **kwargs) - - if entry_max: - iquery = iquery.filter(timestamp__lte=entry_max) - - if max_results < 0: - max_results = 1 - entry_list = [] - if max_results > 0: - try: - rec_start, rec_end = prepare_paginated_list(request, - context, - iquery, - page, - max_results) - except PaginationError: - page_error = sys.exc_info()[1] - if isinstance(page_error[0], HttpResponse): - return page_error[0] - return HttpResponseServerError(page_error) - context['entry_list'] = iquery.all()[rec_start:rec_end] - else: - context['entry_list'] = iquery.all() - - return render_to_response(template, context, - context_instance=RequestContext(request)) - - -def prepare_paginated_list(request, context, paged_list, page=1, max_results=25): - """ - Prepare context and slice an object for pagination. - """ - if max_results < 1: - raise PaginationError("Max results less then 1") - if paged_list == None: - raise PaginationError("Invalid object") - - try: - nitems = paged_list.count() - except TypeError: - nitems = len(paged_list) - - rec_start = (page - 1) * int(max_results) - try: - total_pages = (nitems / int(max_results)) + 1 - except: - total_pages = 1 - if page > total_pages: - # If we passed beyond the end send back - try: - view, args, kwargs = resolve(request.META['PATH_INFO']) - kwargs['page_number'] = total_pages - raise PaginationError(HttpResponseRedirect(reverse(view, - kwargs=kwargs))) - except (Resolver404, NoReverseMatch, ValueError): - raise "Accessing beyond last page. Unable to resolve redirect." - - context['total_pages'] = total_pages - context['records_per_page'] = max_results - return (rec_start, rec_start + int(max_results)) diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py new file mode 100644 index 000000000..b377806ab --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -0,0 +1,155 @@ +import Bcfg2.settings + +from django.db import connection +import django.core.management +import sys +import logging +import traceback +from Bcfg2.Server.models import InternalDatabaseVersion +logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + + +# all update function should go here +def _merge_database_table_entries(): + cursor = connection.cursor() + insert_cursor = connection.cursor() + find_cursor = connection.cursor() + cursor.execute(""" + Select name, kind from reports_bad + union + select name, kind from reports_modified + union + select name, kind from reports_extra + """) + # this fetch could be better done + entries_map = {} + for row in cursor.fetchall(): + insert_cursor.execute("insert into reports_entries (name, kind) \ + values (%s, %s)", (row[0], row[1])) + entries_map[(row[0], row[1])] = insert_cursor.lastrowid + + cursor.execute(""" + Select name, kind, reason_id, interaction_id, 1 from reports_bad + inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id + union + Select name, kind, reason_id, interaction_id, 2 from reports_modified + inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id + union + Select name, kind, reason_id, interaction_id, 3 from reports_extra + inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id + """) + for row in cursor.fetchall(): + key = (row[0], row[1]) + if entries_map.get(key, None): + entry_id = entries_map[key] + else: + find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) + rowe = find_cursor.fetchone() + entry_id = rowe[0] + insert_cursor.execute("insert into reports_entries_interactions \ + (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + + +def _interactions_constraint_or_idx(): + '''sqlite doesn't support alter tables.. or constraints''' + cursor = connection.cursor() + try: + cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + except: + cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + + +def _populate_interaction_entry_counts(): + '''Populate up the type totals for the interaction table''' + cursor = connection.cursor() + count_field = {1: 'bad_entries', + 2: 'modified_entries', + 3: 'extra_entries'} + + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + + "from reports_entries_interactions where type = %s group by interaction_id" % type) + updates = [] + for row in cursor.fetchall(): + updates.append(row) + try: + cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + except Exception: + e = sys.exc_info()[1] + print(e) + cursor.close() + + +# be sure to test your upgrade query before reflecting the change in the models +# the list of function and sql command to do should go here +_fixes = [_merge_database_table_entries, + # this will remove unused tables + "drop table reports_bad;", + "drop table reports_bad_interactions;", + "drop table reports_extra;", + "drop table reports_extra_interactions;", + "drop table reports_modified;", + "drop table reports_modified_interactions;", + "drop table reports_repository;", + "drop table reports_metadata;", + "alter table reports_interaction add server varchar(256) not null default 'N/A';", + # fix revision data type to support $VCS hashes + "alter table reports_interaction add repo_rev_code varchar(64) default '';", + # Performance enhancements for large sites + 'alter table reports_interaction add column bad_entries integer not null default -1;', + 'alter table reports_interaction add column modified_entries integer not null default -1;', + 'alter table reports_interaction add column extra_entries integer not null default -1;', + _populate_interaction_entry_counts, + _interactions_constraint_or_idx, + 'alter table reports_reason add is_binary bool NOT NULL default False;', + 'alter table reports_reason add is_sensitive bool NOT NULL default False;', +] + +# this will calculate the last possible version of the database +lastversion = len(_fixes) + + +def rollupdate(current_version): + """ function responsible to coordinates all the updates + need current_version as integer + """ + ret = None + if current_version < lastversion: + for i in range(current_version, lastversion): + try: + if type(_fixes[i]) == str: + connection.cursor().execute(_fixes[i]) + else: + _fixes[i]() + except: + logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) + # since array start at 0 but version start at 1 we add 1 to the normal count + ret = InternalDatabaseVersion.objects.create(version=i + 1) + return ret + else: + return None + + +def update_database(): + ''' methode to search where we are in the revision of the database models and update them ''' + try: + logger.debug("Running upgrade of models to the new one") + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + know_version = InternalDatabaseVersion.objects.order_by('-version') + if not know_version: + logger.debug("No version, creating initial version") + know_version = InternalDatabaseVersion.objects.create(version=lastversion) + else: + know_version = know_version[0] + logger.debug("Presently at %s" % know_version) + if know_version.version > 13000: + # SchemaUpdater stuff + return + elif know_version.version < lastversion: + new_version = rollupdate(know_version.version) + if new_version: + logger.debug("upgraded to %s" % new_version) + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) diff --git a/src/lib/Bcfg2/Server/Reports/urls.py b/src/lib/Bcfg2/Server/Reports/urls.py deleted file mode 100644 index d7ff1eee5..000000000 --- a/src/lib/Bcfg2/Server/Reports/urls.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.conf.urls.defaults import * -from django.http import HttpResponsePermanentRedirect - -handler500 = 'Bcfg2.Server.Reports.reports.views.server_error' - -urlpatterns = patterns('', - (r'^', include('Bcfg2.Server.Reports.reports.urls')) -) - -#urlpatterns += patterns("django.views", -# url(r"media/(?P.*)$", "static.serve", { -# "document_root": '/Users/tlaszlo/svn/bcfg2/reports/site_media/', -# }) -#) diff --git a/src/lib/Bcfg2/Server/Reports/utils.py b/src/lib/Bcfg2/Server/Reports/utils.py deleted file mode 100755 index c47763e39..000000000 --- a/src/lib/Bcfg2/Server/Reports/utils.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Helper functions for reports""" -from django.conf.urls.defaults import * -import re - -"""List of filters provided by filteredUrls""" -filter_list = ('server', 'state', 'group') - - -class BatchFetch(object): - """Fetch Django objects in smaller batches to save memory""" - - def __init__(self, obj, step=10000): - self.count = 0 - self.block_count = 0 - self.obj = obj - self.data = None - self.step = step - self.max = obj.count() - - def __iter__(self): - return self - - def next(self): - """Provide compatibility with python < 3.0""" - return self.__next__() - - def __next__(self): - """Return the next object from our array and fetch from the - database when needed""" - if self.block_count + self.count - self.step == self.max: - raise StopIteration - if self.block_count == 0 or self.count == self.step: - # Without list() this turns into LIMIT 1 OFFSET x queries - self.data = list(self.obj.all()[self.block_count: \ - (self.block_count + self.step)]) - self.block_count += self.step - self.count = 0 - self.count += 1 - return self.data[self.count - 1] - - -def generateUrls(fn): - """ - Parse url tuples and send to functions. - - Decorator for url generators. Handles url tuple parsing - before the actual function is called. - """ - def url_gen(*urls): - results = [] - for url_tuple in urls: - if isinstance(url_tuple, (list, tuple)): - results += fn(*url_tuple) - else: - raise ValueError("Unable to handle compiled urls") - return results - return url_gen - - -@generateUrls -def paginatedUrls(pattern, view, kwargs=None, name=None): - """ - Takes a group of url tuples and adds paginated urls. - - Extends a url tuple to include paginated urls. - Currently doesn't handle url() compiled patterns. - - """ - results = [(pattern, view, kwargs, name)] - tail = '' - mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) - if mtail: - tail = mtail.group(1) - pattern = pattern[:len(pattern) - len(tail)] - results += [(pattern + "/(?P\d+)" + tail, view, kwargs)] - results += [(pattern + "/(?P\d+)\|(?P\d+)" + - tail, view, kwargs)] - if not kwargs: - kwargs = dict() - kwargs['page_limit'] = 0 - results += [(pattern + "/?\|(?Pall)" + tail, view, kwargs)] - return results - - -@generateUrls -def filteredUrls(pattern, view, kwargs=None, name=None): - """ - Takes a url and adds filtered urls. - - Extends a url tuple to include filtered view urls. Currently doesn't - handle url() compiled patterns. - """ - results = [(pattern, view, kwargs, name)] - tail = '' - mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) - if mtail: - tail = mtail.group(1) - pattern = pattern[:len(pattern) - len(tail)] - for filter in ('/state/(?P\w+)', - '/group/(?P[\w\-\.]+)', - '/group/(?P[\w\-\.]+)/(?P[A-Za-z]+)', - '/server/(?P[\w\-\.]+)', - '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): - results += [(pattern + filter + tail, view, kwargs)] - return results - - -@generateUrls -def timeviewUrls(pattern, view, kwargs=None, name=None): - """ - Takes a url and adds timeview urls - - Extends a url tuple to include filtered view urls. Currently doesn't - handle url() compiled patterns. - """ - results = [(pattern, view, kwargs, name)] - tail = '' - mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) - if mtail: - tail = mtail.group(1) - pattern = pattern[:len(pattern) - len(tail)] - for filter in ('/(?P\d{4})-(?P\d{2})-(?P\d{2})/' + \ - '(?P\d\d)-(?P\d\d)', - '/(?P\d{4})-(?P\d{2})-(?P\d{2})'): - results += [(pattern + filter + tail, view, kwargs)] - return results diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py deleted file mode 100644 index ff4c24328..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -1_0_x.py - -This file should contain updates relevant to the 1.0.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import UnsupportedUpdate - -def updates(): - return UnsupportedUpdate("1.0", 10) - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py deleted file mode 100644 index 0d28786fd..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -1_1_x.py - -This file should contain updates relevant to the 1.1.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -from django.db import connection -import sys -import Bcfg2.settings -from Bcfg2.Server.Reports.reports.models import \ - TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA - -@updatercallable -def _interactions_constraint_or_idx(): - """sqlite doesn't support alter tables.. or constraints""" - cursor = connection.cursor() - try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') - except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - - -@updatercallable -def _populate_interaction_entry_counts(): - '''Populate up the type totals for the interaction table''' - cursor = connection.cursor() - count_field = {TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries'} - - for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) - updates = [] - for row in cursor.fetchall(): - updates.append(row) - try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception: - e = sys.exc_info()[1] - print(e) - cursor.close() - - -def updates(): - fixes = Updater("1.1") - fixes.override_base_version(12) # Do not do this in new code - - fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;') - fixes.add(_populate_interaction_entry_counts()) - fixes.add(_interactions_constraint_or_idx()) - fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py deleted file mode 100644 index 024965bd5..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -1_2_x.py - -This file should contain updates relevant to the 1.2.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -def updates(): - fixes = Updater("1.2") - fixes.override_base_version(18) # Do not do this in new code - fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py deleted file mode 100644 index 4fc57c653..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -1_3_0.py - -This file should contain updates relevant to the 1.3.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater, UpdaterError -from Bcfg2.Server.SchemaUpdater.Routines import AddColumns, \ - RemoveColumns, RebuildTable, DropTable - -from Bcfg2.Server.Reports.reports.models import Reason, Interaction - - -def updates(): - fixes = Updater("1.3") - fixes.add(RemoveColumns(Interaction, 'client_version')) - fixes.add(AddColumns(Reason)) - fixes.add(RebuildTable(Reason, [ - 'owner', 'current_owner', - 'group', 'current_group', - 'perms', 'current_perms', - 'status', 'current_status', - 'to', 'current_to'])) - fixes.add(DropTable('reports_ping')) - - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py deleted file mode 100644 index 4fcf0e6bf..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py +++ /dev/null @@ -1,279 +0,0 @@ -import logging -import traceback -from django.db.models.fields import NOT_PROVIDED -from django.db import connection, DatabaseError, backend, models -from django.core.management.color import no_style -from django.core.management.sql import sql_create -import django.core.management - -import Bcfg2.settings - -logger = logging.getLogger(__name__) - -def _quote(value): - """ - Quote a string to use as a table name or column - """ - return backend.DatabaseOperations().quote_name(value) - - -def _rebuild_sqlite_table(model): - """Sqlite doesn't support most alter table statments. This streamlines the - rebuild process""" - try: - cursor = connection.cursor() - table_name = model._meta.db_table - - # Build create staement from django - model._meta.db_table = "%s_temp" % table_name - sql, references = connection.creation.sql_create_model(model, no_style()) - columns = ",".join([_quote(f.column) \ - for f in model._meta.fields]) - - # Create a temp table - [cursor.execute(s) for s in sql] - - # Fill the table - tbl_name = _quote(table_name) - tmp_tbl_name = _quote(model._meta.db_table) - # Reset this - model._meta.db_table = table_name - cursor.execute("insert into %s(%s) select %s from %s;" % ( - tmp_tbl_name, - columns, - columns, - tbl_name)) - cursor.execute("drop table %s" % tbl_name) - - # Call syncdb to create the table again - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - # syncdb closes our cursor - cursor = connection.cursor() - # Repopulate - cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, - columns, - columns, - tmp_tbl_name)) - cursor.execute('DROP TABLE %s;' % tmp_tbl_name) - except DatabaseError: - logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1) - raise UpdaterRoutineException - - -class UpdaterRoutineException(Exception): - pass - - -class UpdaterRoutine(object): - """Base for routines.""" - def __init__(self): - pass - - def __str__(self): - return __name__ - - def run(self): - """Called to execute the action""" - raise UpdaterRoutineException - - - -class AddColumns(UpdaterRoutine): - """ - Routine to add new columns to an existing model - """ - def __init__(self, model): - self.model = model - self.model_name = model.__name__ - - def __str__(self): - return "Add new columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - desc = {} - for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table): - desc[d[0]] = d - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for field in self.model._meta.fields: - if field.column in desc: - continue - logger.debug("Column %s does not exist yet" % field.column) - if field.default == NOT_PROVIDED: - logger.error("Cannot add a column with out a default value") - raise UpdaterRoutineException - - sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % ( - _quote(self.model._meta.db_table), - _quote(field.column), field.db_type(), ) - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - sql += _quote(field.default) - sql_values = () - else: - sql += '%s' - sql_values = (field.default, ) - try: - cursor.execute(sql, sql_values) - logger.debug("Added column %s to %s" % - (field.column, self.model._meta.db_table)) - except DatabaseError: - logger.error("Unable to add column %s" % field.column) - raise UpdaterRoutineException - - -class RebuildTable(UpdaterRoutine): - """ - Rebuild the table for an existing model. Use this if field types have changed. - """ - def __init__(self, model, columns): - self.model = model - self.model_name = model.__name__ - - if type(columns) == str: - self.columns = [columns] - elif type(columns) in (tuple, list): - self.columns = columns - else: - logger.error("Columns must be a str, tuple, or list") - raise UpdaterRoutineException - - - def __str__(self): - return "Rebuild columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - """ Sqlite is a special case. Altering columns is not supported. """ - _rebuild_sqlite_table(self.model) - return - - if db_engine == 'django.db.backends.mysql': - modify_cmd = 'MODIFY ' - else: - modify_cmd = 'ALTER COLUMN ' - - col_strings = [] - for column in self.columns: - col_strings.append("%s %s %s" % ( \ - modify_cmd, - _quote(column), - self.model._meta.get_field(column).db_type() - )) - - try: - cursor.execute('ALTER TABLE %s %s' % - (_quote(self.model._meta.db_table), ", ".join(col_strings))) - except DatabaseError: - logger.debug("Failed modify table %s" % self.model._meta.db_table) - raise UpdaterRoutineException - - - -class RemoveColumns(RebuildTable): - """ - Routine to remove columns from an existing model - """ - def __init__(self, model, columns): - super(RemoveColumns, self).__init__(model, columns) - - - def __str__(self): - return "Remove columns from model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - columns = [d[0] for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table)] - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for column in self.columns: - if column not in columns: - logger.warning("Cannot drop column %s: does not exist" % column) - continue - - logger.debug("Dropping column %s" % column) - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - _rebuild_sqlite_table(self.model) - else: - sql = "alter table %s drop column %s" % \ - (_quote(self.model._meta.db_table), _quote(column), ) - try: - cursor.execute(sql) - except DatabaseError: - logger.debug("Failed to drop column %s from %s" % - (column, self.model._meta.db_table)) - raise UpdaterRoutineException - - -class DropTable(UpdaterRoutine): - """ - Drop a table - """ - def __init__(self, table_name): - self.table_name = table_name - - def __str__(self): - return "Drop table %s" % self.table_name - - def run(self): - try: - cursor = connection.cursor() - cursor.execute('DROP TABLE %s' % _quote(self.table_name)) - except DatabaseError: - logger.error("Failed to drop table: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterRoutineException - - -class UpdaterCallable(UpdaterRoutine): - """Helper for routines. Basically delays execution""" - def __init__(self, fn): - self.fn = fn - self.args = [] - self.kwargs = {} - - def __call__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - return self - - def __str__(self): - return self.fn.__name__ - - def run(self): - self.fn(*self.args, **self.kwargs) - -def updatercallable(fn): - """Decorator for UpdaterCallable. Use for any function passed - into the fixes list""" - return UpdaterCallable(fn) - - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py deleted file mode 100644 index e7a3191bc..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py +++ /dev/null @@ -1,239 +0,0 @@ -from django.db import connection, DatabaseError -from django.core.exceptions import ImproperlyConfigured -import django.core.management -import logging -import re -import sys -import traceback - -from Bcfg2.Compat import CmpMixin, walk_packages -from Bcfg2.Server.models import InternalDatabaseVersion -from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \ - UpdaterRoutine -from Bcfg2.Server.SchemaUpdater import Changes - -logger = logging.getLogger(__name__) - -class UpdaterError(Exception): - pass - - -class SchemaTooOldError(UpdaterError): - pass - - -def _release_to_version(release): - """ - Build a release base for a version - - Expects a string of the form 00.00 - - returns an integer of the form MMmm00 - """ - regex = re.compile("^(\d+)\.(\d+)$") - m = regex.match(release) - if not m: - logger.error("Invalid release string: %s" % release) - raise TypeError - return int("%02d%02d00" % (int(m.group(1)), int(m.group(2)))) - - -class Updater(CmpMixin): - """Database updater to standardize updates""" - - def __init__(self, release): - CmpMixin.__init__(self) - - self._cursor = None - self._release = release - try: - self._base_version = _release_to_version(release) - except: - err = "Invalid release string: %s" % release - logger.error(err) - raise UpdaterError(err) - - self._fixes = [] - self._version = -1 - - def __cmp__(self, other): - return self._base_version - other._base_version - - @property - def release(self): - return self._release - - @property - def version(self): - if self._version < 0: - try: - iv = InternalDatabaseVersion.objects.latest() - self._version = iv.version - except InternalDatabaseVersion.DoesNotExist: - raise UpdaterError("No database version stored internally") - return self._version - - @property - def cursor(self): - if not self._cursor: - self._cursor = connection.cursor() - return self._cursor - - @property - def target_version(self): - if(len(self._fixes) == 0): - return self._base_version - else: - return self._base_version + len(self._fixes) - 1 - - - def add(self, update): - if type(update) == str or isinstance(update, UpdaterRoutine): - self._fixes.append(update) - else: - raise TypeError - - - def override_base_version(self, version): - """Override our starting point for old releases. New code should - not use this method""" - self._base_version = int(version) - - - @staticmethod - def get_current_version(): - """Queries the db for the latest version. Returns 0 for a - fresh install""" - - if "call_command" in dir(django.core.management): - django.core.management.call_command("syncdb", interactive=False, - verbosity=0) - else: - msg = "Unable to call syndb routine" - logger.warning(msg) - raise UpdaterError(msg) - - try: - iv = InternalDatabaseVersion.objects.latest() - version = iv.version - except InternalDatabaseVersion.DoesNotExist: - version = 0 - - return version - - - def syncdb(self): - """Function to do the syncronisation for the models""" - - self._version = Updater.get_current_version() - self._cursor = None - - - def increment(self): - """Increment schema version in the database""" - if self._version < self._base_version: - self._version = self._base_version - else: - self._version += 1 - InternalDatabaseVersion.objects.create(version=self._version) - - def apply(self): - """Apply pending schema changes""" - - if self.version >= self.target_version: - logger.debug("No updates for release %s" % self._release) - return - - logger.debug("Applying updates for release %s" % self._release) - - if self.version < self._base_version: - start = 0 - else: - start = self.version - self._base_version + 1 - - try: - for fix in self._fixes[start:]: - if type(fix) == str: - self.cursor.execute(fix) - elif isinstance(fix, UpdaterRoutine): - fix.run() - else: - logger.error("Invalid schema change at %s" % \ - self._version + 1) - self.increment() - logger.debug("Applied schema change number %s: %s" % \ - (self.version, fix)) - logger.info("Applied schema changes for release %s" % self._release) - except: - msg = "Failed to perform db update %s (%s): %s" % \ - (self._version + 1, fix, - traceback.format_exc().splitlines()[-1]) - logger.error(msg) - raise UpdaterError(msg) - - -class UnsupportedUpdate(Updater): - """Handle an unsupported update""" - - def __init__(self, release, version): - super(UnsupportedUpdate, self).__init__(release) - self._base_version = version - - def apply(self): - """Raise an exception if we're too old""" - - if self.version < self.target_version: - logger.error("Upgrade from release %s unsupported" % self._release) - raise SchemaTooOldError - - -def update_database(): - """method to search where we are in the revision - of the database models and update them""" - try: - logger.debug("Verifying database schema") - - updaters = [] - for loader, submodule, ispkg in walk_packages(path=Changes.__path__): - if ispkg: - continue - try: - updates = getattr( - __import__("%s.%s" % (Changes.__name__, submodule), - globals(), locals(), ['*']), - "updates") - updaters.append(updates()) - except ImportError: - logger.error("Failed to import %s" % submodule) - except AttributeError: - logger.warning("Module %s does not have an updates function" % - submodule) - except: - msg = "Failed to build updater for %s" % submodule - logger.error(msg, exc_info=1) - raise UpdaterError(msg) - - current_version = Updater.get_current_version() - logger.debug("Database version at %s" % current_version) - - updaters.sort() - if current_version > 0: - [u.apply() for u in updaters] - logger.debug("Database version at %s" % - Updater.get_current_version()) - else: - target = updaters[-1].target_version - InternalDatabaseVersion.objects.create(version=target) - logger.info("A new database was created") - - except UpdaterError: - raise - except ImproperlyConfigured: - logger.error("Django is not properly configured: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterError - except: - logger.error("Error while updating the database") - for x in traceback.format_exc().splitlines(): - logger.error(x) - raise UpdaterError -- cgit v1.2.3-1-g7c22