From 44638176067df5231bf0be30801e36863391cd1f Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 8 Oct 2012 10:38:02 -0500 Subject: Reporting: Merge new reporting data Move reporting data to a new schema Use south for django migrations Add bcfg2-report-collector daemon Conflicts: doc/development/index.txt doc/server/plugins/connectors/properties.txt doc/server/plugins/generators/packages.txt setup.py src/lib/Bcfg2/Client/Tools/SELinux.py src/lib/Bcfg2/Compat.py src/lib/Bcfg2/Encryption.py src/lib/Bcfg2/Options.py src/lib/Bcfg2/Server/Admin/Init.py src/lib/Bcfg2/Server/Admin/Reports.py src/lib/Bcfg2/Server/BuiltinCore.py src/lib/Bcfg2/Server/Core.py src/lib/Bcfg2/Server/FileMonitor/Inotify.py src/lib/Bcfg2/Server/Plugin/base.py src/lib/Bcfg2/Server/Plugin/interfaces.py src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py src/lib/Bcfg2/Server/Plugins/FileProbes.py src/lib/Bcfg2/Server/Plugins/Ohai.py src/lib/Bcfg2/Server/Plugins/Packages/Collection.py src/lib/Bcfg2/Server/Plugins/Packages/Source.py src/lib/Bcfg2/Server/Plugins/Packages/Yum.py src/lib/Bcfg2/Server/Plugins/Packages/__init__.py src/lib/Bcfg2/Server/Plugins/Probes.py src/lib/Bcfg2/Server/Plugins/Properties.py src/lib/Bcfg2/Server/Reports/backends.py src/lib/Bcfg2/Server/Reports/manage.py src/lib/Bcfg2/Server/Reports/nisauth.py src/lib/Bcfg2/settings.py src/sbin/bcfg2-crypt src/sbin/bcfg2-yum-helper testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py --- .../Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 2 + src/lib/Bcfg2/Server/Plugins/Reporting.py | 105 +++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Plugins/Reporting.py (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 31c3d79b0..01f590b06 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -21,6 +21,8 @@ except ImportError: LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py new file mode 100644 index 000000000..883b95ba4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -0,0 +1,105 @@ +import time +import platform +import traceback +from lxml import etree + +from Bcfg2.Reporting.Transport import load_transport_from_config, \ + TransportError, TransportImportError + +try: + import cPickle as pickle +except: + import pickle + +from Bcfg2.Options import REPORTING_COMMON_OPTIONS +from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \ + PluginExecutionError + +def _rpc_call(method): + def _real_rpc_call(self, *args, **kwargs): + """Wrapper for calls to the reporting collector""" + + try: + return self.transport.rpc(method, *args, **kwargs) + except TransportError: + # this is needed for Admin.Pull + raise PluginExecutionError + return _real_rpc_call + +class Reporting(Statistics, PullSource): + + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + + CLIENT_METADATA_FILEDS = ('profile', 'bundles', 'aliases', 'addresses', + 'groups', 'categories', 'uuid', 'version') + + def __init__(self, core, datastore): + Statistics.__init__(self, core, datastore) + PullSource.__init__(self) + self.core = core + self.experimental = True + + self.whoami = platform.node() + self.transport = None + + core.setup.update(REPORTING_COMMON_OPTIONS) + core.setup.reparse() + self.logger.error("File limit: %s" % core.setup['reporting_file_limit']) + + try: + self.transport = load_transport_from_config(core.setup) + except TransportError: + self.logger.error("%s: Failed to load transport: %s" % + (self.name, traceback.format_exc().splitlines()[-1])) + raise PluginInitError + + + def process_statistics(self, client, xdata): + stats = xdata.find("Statistics") + stats.set('time', time.asctime(time.localtime())) + + cdata = { 'server': self.whoami } + for field in self.CLIENT_METADATA_FILEDS: + try: + value = getattr(client, field) + except AttributeError: + continue + if value: + if isinstance(value, set): + value = [v for v in value] + cdata[field] = value + + try: + interaction_data = pickle.dumps({ 'hostname': client.hostname, + 'metadata': cdata, 'stats': + etree.tostring(stats, xml_declaration=False).decode('UTF-8') }) + except: + self.logger.error("%s: Failed to build interaction object: %s" % + (self.__class__.__name__, + traceback.format_exc().splitlines()[-1])) + + # try 3 times to store the data + for i in [1, 2, 3]: + try: + self.transport.store(client.hostname, interaction_data) + self.logger.debug("%s: Queued statistics data for %s" % + (self.__class__.__name__, client.hostname)) + return + except TransportError: + continue + except: + self.logger.error("%s: Attempt %s: Failed to add statistic %s" % + (self.__class__.__name__, i, + traceback.format_exc().splitlines()[-1])) + self.logger.error("%s: Retry limit reached for %s" % + (self.__class__.__name__, client.hostname)) + + def shutdown(self): + super(Reporting, self).shutdown() + if self.transport: + self.transport.shutdown() + + Ping = _rpc_call('Ping') + GetExtra = _rpc_call('GetExtra') + GetCurrentEntry = _rpc_call('GetCurrentEntry') + -- cgit v1.2.3-1-g7c22