diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r-- | src/lib/Bcfg2/Server/Admin.py | 50 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 34 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Info.py | 91 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/TemplateHelper.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/AWSTags.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bundler.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 13 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Ohai.py | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Properties.py | 41 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Reporting.py | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSHbase.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 12 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/models.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/updatefix.py | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/models.py | 29 |
18 files changed, 243 insertions, 90 deletions
diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c294e6be5..c32b1989b 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -25,15 +25,19 @@ import Bcfg2.Server.Plugins.Metadata try: from django.core.exceptions import ImproperlyConfigured from django.core import management + import django import django.conf import Bcfg2.Server.models HAS_DJANGO = True - try: - import south # pylint: disable=W0611 + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: HAS_REPORTS = True - except ImportError: - HAS_REPORTS = False + else: + try: + import south # pylint: disable=W0611 + HAS_REPORTS = True + except ImportError: + HAS_REPORTS = False except ImportError: HAS_DJANGO = False HAS_REPORTS = False @@ -439,6 +443,25 @@ class Compare(AdminCmd): print("") +class ExpireCache(_ProxyAdminCmd): + """ Expire the metadata cache """ + + options = _ProxyAdminCmd.options + [ + Bcfg2.Options.PositionalArgument( + "hostname", nargs="*", default=[], + help="Expire cache for the given host(s)")] + + def run(self, setup): + clients = None + if setup.hostname is not None and len(setup.hostname) > 0: + clients = setup.hostname + + try: + self.proxy.expire_metadata_cache(clients) + except Bcfg2.Client.Proxy.ProxyError: + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) + + class Init(AdminCmd): """Interactively initialize a new repository.""" @@ -877,6 +900,7 @@ if HAS_DJANGO: Django management system """ command = None args = [] + kwargs = {} def run(self, _): '''Call a django command''' @@ -885,7 +909,7 @@ if HAS_DJANGO: else: command = self.__class__.__name__.lower() args = [command] + self.args - management.call_command(*args) + management.call_command(*args, **self.kwargs) class DBShell(_DjangoProxyCmd): """ Call the Django 'dbshell' command on the database """ @@ -901,7 +925,6 @@ if HAS_DJANGO: """ Sync the Django ORM with the configured database """ def run(self, setup): - Bcfg2.Server.models.load_models() try: Bcfg2.DBSettings.sync_databases( interactive=False, @@ -915,6 +938,17 @@ if HAS_DJANGO: self.logger.error("Database update failed: %s" % err) raise SystemExit(1) + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + class Makemigrations(_DjangoProxyCmd): + """ Call the 'makemigrations' command on the database """ + args = ['Reporting'] + + else: + class Schemamigration(_DjangoProxyCmd): + """ Call the South 'schemamigration' command on the database """ + args = ['Bcfg2.Reporting'] + kwargs = {'auto': True} + if HAS_REPORTS: import datetime @@ -1194,6 +1228,10 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + # this has been introduced in django 1.7, so pylint fails with + # older django releases + django.setup() # pylint: disable=E1101 def run(self): """ Run bcfg2-admin """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 7e35d19a8..ad78800fb 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -27,6 +27,7 @@ from Bcfg2.Server.Statistics import track_statistics try: from django.core.exceptions import ImproperlyConfigured + import django import django.conf HAS_DJANGO = True except ImportError: @@ -83,10 +84,14 @@ def close_db_connection(func): """ The decorated function """ rv = func(self, *args, **kwargs) if self._database_available: # pylint: disable=W0212 - from django import db self.logger.debug("%s: Closing database connection" % threading.current_thread().getName()) - db.close_connection() + + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + for connection in django.db.connections.all(): + connection.close() + else: + django.db.close_connection() return rv return inner @@ -114,7 +119,8 @@ class DefaultACL(Plugin, ClientACLs): def check_acl_ip(self, address, rmi): return (("." not in rmi and not rmi.endswith("_debug") and - rmi != 'get_statistics') or + rmi != 'get_statistics' and + rmi != 'expire_metadata_cache') or address[0] == "127.0.0.1") # in core we frequently want to catch all exceptions, regardless of @@ -430,6 +436,7 @@ class Core(object): self.logger.error("Unexpected instantiation failure for plugin %s" % plugin, exc_info=1) + @close_db_connection def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ if not self._running: @@ -444,10 +451,6 @@ class Core(object): for plugin in list(self.plugins.values()): plugin.shutdown() self.logger.info("%s: All plugins shut down" % self.name) - if self._database_available: - from django import db - self.logger.info("%s: Closing database connection" % self.name) - db.close_connection() @property def metadata_cache_mode(self): @@ -682,7 +685,7 @@ class Core(object): self.logger.debug("Building configuration for %s" % client) start = time.time() config = lxml.etree.Element("Configuration", version='2.0', - revision=self.revision) + revision=str(self.revision)) try: meta = self.build_metadata(client) except MetadataConsistencyError: @@ -1366,6 +1369,21 @@ class Core(object): return "This method is deprecated and will be removed in a future " + \ "release\n%s" % self.fam.set_debug(debug) + @exposed + def expire_metadata_cache(self, _, hostnames=None): + """ Expire the metadata cache for one or all clients + + :param hostnames: A list of hostnames to expire the metadata + cache for or None. If None the cache of + all clients will be expired. + :type hostnames: None or list of strings + """ + if hostnames is not None: + for hostname in hostnames: + self.metadata_cache.expire(hostname) + else: + self.metadata_cache.expire() + class NetworkCore(Core): """ A server core that actually listens on the network, can be diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index c4b34a469..8f6e136fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -214,6 +214,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): def shutdown(self): if self.started and self.notifier: self.notifier.stop() + Pseudo.shutdown(self) shutdown.__doc__ = Pseudo.shutdown.__doc__ def list_watches(self): diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index 6af561089..418d984a1 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -12,6 +12,7 @@ import fnmatch import argparse import operator import lxml.etree +import traceback from code import InteractiveConsole import Bcfg2.Logger import Bcfg2.Options @@ -369,6 +370,7 @@ class Automatch(InfoCmd): class ExpireCache(InfoCmd): """ Expire the metadata cache """ + only_interactive = True options = [ Bcfg2.Options.PositionalArgument( @@ -376,12 +378,20 @@ class ExpireCache(InfoCmd): help="Expire cache for the given host(s)")] def run(self, setup): - if setup.clients: - for client in self.get_client_list(setup.clients): - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) + if setup.hostname: + for client in self.get_client_list(setup.hostname): + self.core.metadata_cache.expire(client) else: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.core.metadata_cache.expire() + + +class EventDebug(InfoCmd): + """ Enable debugging output for FAM events """ + only_interactive = True + aliases = ['event_debug'] + + def run(self, _): + self.core.fam.set_debug(True) class Bundles(InfoCmd): @@ -672,18 +682,35 @@ class Query(InfoCmd): print("\n".join(res)) +class Quit(InfoCmd): + """ Exit program """ + only_interactive = True + aliases = ['exit', 'EOF'] + + def run(self, _): + raise SystemExit(0) + + class Shell(InfoCmd): """ Open an interactive shell to run multiple bcfg2-info commands """ interactive = False def run(self, setup): try: - self.core.cmdloop('Welcome to bcfg2-info\n' - 'Type "help" for more information') + self.core.cli.cmdloop('Welcome to bcfg2-info\n' + 'Type "help" for more information') except KeyboardInterrupt: print("\nCtrl-C pressed, exiting...") +class Update(InfoCmd): + """ Process pending filesystem events """ + only_interactive = True + + def run(self, _): + self.core.fam.handle_events_in_interval(0.1) + + class ProfileTemplates(InfoCmd): """ Benchmark template rendering times """ @@ -796,36 +823,18 @@ if HAS_PROFILE: display_trace(prof) -class InfoCore(cmd.Cmd, Bcfg2.Server.Core.Core): +class InfoCore(Bcfg2.Server.Core.Core): """Main class for bcfg2-info.""" - def __init__(self): - cmd.Cmd.__init__(self) + def __init__(self, cli): Bcfg2.Server.Core.Core.__init__(self) - self.prompt = 'bcfg2-info> ' + self.cli = cli def get_locals(self): """ Expose the local variables of the core to subcommands that need to reference them (i.e., the interactive interpreter) """ return locals() - def do_quit(self, _): - """ quit|exit - Exit program """ - raise SystemExit(0) - - do_EOF = do_quit - do_exit = do_quit - - def do_eventdebug(self, _): - """ eventdebug - Enable debugging output for FAM events """ - self.fam.set_debug(True) - - do_event_debug = do_eventdebug - - def do_update(self, _): - """ update - Process pending filesystem events """ - self.fam.handle_events_in_interval(0.1) - def run(self): self.load_plugins() self.block_for_fam_events(handle_events=True) @@ -840,12 +849,15 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.Core): Bcfg2.Server.Core.Core.shutdown(self) -class CLI(Bcfg2.Options.CommandRegistry): +class CLI(cmd.Cmd, Bcfg2.Options.CommandRegistry): """ The bcfg2-info CLI """ options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] def __init__(self): + cmd.Cmd.__init__(self) Bcfg2.Options.CommandRegistry.__init__(self) + self.prompt = 'bcfg2-info> ' + self.register_commands(globals().values(), parent=InfoCmd) parser = Bcfg2.Options.get_parser( description="Inspect a running Bcfg2 server", @@ -860,7 +872,7 @@ class CLI(Bcfg2.Options.CommandRegistry): else: if Bcfg2.Options.setup.profile: print("Profiling functionality not available.") - self.core = InfoCore() + self.core = InfoCore(self) for command in self.commands.values(): command.core = self.core @@ -877,3 +889,22 @@ class CLI(Bcfg2.Options.CommandRegistry): def shutdown(self): Bcfg2.Options.CommandRegistry.shutdown(self) self.core.shutdown() + + def get_names(self): + """ Overwrite cmd.Cmd.get_names to use the instance to get the + methods and not the class, because the CommandRegistry + dynamically adds methods for the registed subcommands. """ + return dir(self) + + def onecmd(self, line): + """ Overwrite cmd.Cmd.onecmd to catch all exceptions (except + SystemExit) print an error message and continue cmdloop. """ + try: + cmd.Cmd.onecmd(self, line) + except SystemExit: + raise + except: # pylint: disable=W0702 + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, + exc_traceback) + self.stdout.write(''.join(lines)) diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index ce6fdca74..98faa269d 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -26,7 +26,7 @@ class TemplateHelper(ServerPlugin): ServerPlugin.__init__(self, *args, **kwargs) # we instantiate a dummy helper to discover which keywords and # defaults are reserved - dummy = HelperModule("foo.py") + dummy = HelperModule("foo.py", None) self.reserved_keywords = dir(dummy) self.reserved_defaults = dummy.reserved_defaults diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 0d6eefaaa..556805bde 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -172,6 +172,11 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, def start_client_run(self, metadata): self.expire_cache(key=metadata.hostname) + if self.core.metadata_cache_mode == 'aggressive': + self.logger.warning("AWSTags is incompatible with aggressive " + "client metadata caching, try 'cautious' " + "or 'initial'") + self.core.metadata_cache.expire(metadata.hostname) def get_additional_data(self, metadata): return self.get_tags(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 6c35ada59..f5bcbe797 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -128,7 +128,7 @@ class Bundler(Plugin, # dependent bundle -- add it to the list of # bundles for this client if child.get("name") not in bundles_added: - bundles.append(child.get("name")) + bundles.add(child.get("name")) bundles_added.add(child.get("name")) if child.get('inherit_modification', 'false') == 'true': if metadata.version_info >= \ diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index b60f60e65..184b362f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -13,6 +13,17 @@ class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) + def __init__(self, filename, core): + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, + should_monitor=True) + self.core = core + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def _match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] @@ -39,7 +50,7 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), - should_monitor=True) + core=core) self._local = local() def get_additional_groups(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 461be9ba8..b314e60a0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -94,7 +94,12 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, return [self.probe] def ReceiveData(self, meta, datalist): - self.cache[meta.hostname] = datalist[0].text + if meta.hostname not in self.cache or \ + self.cache[meta.hostname] != datalist[0].text: + self.cache[meta.hostname] = datalist[0].text + + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire(meta.hostname) def get_additional_data(self, meta): if meta.hostname in self.cache: diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 76aab69b5..573c9af71 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -74,6 +74,7 @@ class ProbeStore(Debuggable): def __init__(self, core, datadir): # pylint: disable=W0613 Debuggable.__init__(self) + self.core = core self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") @@ -134,7 +135,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) self._groupcache[hostname] = list(set(r.group for r in groupdata)) - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock def set_groups(self, hostname, groups): @@ -155,7 +156,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): ProbesGroupsModel.objects.filter( hostname=hostname).exclude(group__in=groups).delete() if olddata != groups: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) def _load_data(self, hostname): Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) @@ -168,7 +169,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): time.mktime(pdata.timestamp.timetuple()) ts_set = True self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock def set_data(self, hostname, data): @@ -198,7 +199,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): qset.delete() expire_metadata = True if expire_metadata: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) class XMLProbeStore(ProbeStore): @@ -234,7 +235,7 @@ class XMLProbeStore(ProbeStore): self._groupcache[client.get('name')].append( pdata.get('name')) - Bcfg2.Server.Cache.expire("Metadata") + self.core.metadata_cache.expire() def _load_groups(self, hostname): self._load_data(hostname) @@ -274,7 +275,7 @@ class XMLProbeStore(ProbeStore): olddata = self._groupcache.get(hostname, []) self._groupcache[hostname] = groups if olddata != groups: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) def set_data(self, hostname, data): Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) @@ -285,7 +286,7 @@ class XMLProbeStore(ProbeStore): self._datacache[hostname][probe] = pdata expire_metadata |= olddata != data if expire_metadata: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) class ClientProbeDataSet(dict): diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index c4dd75e60..e6549b714 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -35,13 +35,17 @@ LOGGER = logging.getLogger(__name__) class PropertyFile(object): """ Base Properties file handler """ - def __init__(self, name): + def __init__(self, name, core): """ :param name: The filename of this properties file. + :type name: string + :param core: The Bcfg2.Server.Core initializing the Properties plugin + :type core: Bcfg2.Server.Core .. automethod:: _write """ self.name = name + self.core = core def write(self): """ Write the data in this data structure back to the property @@ -69,6 +73,12 @@ class PropertyFile(object): file. """ raise NotImplementedError + def _expire_metadata_cache(self): + """ Expires the metadata cache, if it is required by the caching + mode. """ + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def validate_data(self): """ Verify that the data in this file is valid. """ raise NotImplementedError @@ -81,9 +91,9 @@ class PropertyFile(object): class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """Handle JSON Properties files.""" - def __init__(self, name): + def __init__(self, name, core): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) self.json = None def Index(self): @@ -93,10 +103,13 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Could not load JSON data from %s: %s" % (self.name, err)) + self._expire_metadata_cache() + Index.__doc__ = Bcfg2.Server.Plugin.FileBacked.Index.__doc__ def _write(self): json.dump(self.json, open(self.name, 'wb')) return True + _write.__doc__ = PropertyFile._write.__doc__ def validate_data(self): try: @@ -105,6 +118,7 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Data for %s cannot be dumped to JSON: " "%s" % (self.name, err)) + validate_data.__doc__ = PropertyFile.validate_data.__doc__ def __str__(self): return str(self.json) @@ -116,11 +130,10 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle YAML Properties files. """ - def __init__(self, name): + def __init__(self, name, core): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) self.yaml = None - __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ def Index(self): try: @@ -129,6 +142,7 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Could not load YAML data from %s: %s" % (self.name, err)) + self._expire_metadata_cache() Index.__doc__ = Bcfg2.Server.Plugin.FileBacked.Index.__doc__ def _write(self): @@ -155,10 +169,15 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): """ Handle XML Properties files. """ - def __init__(self, name, should_monitor=False): + def __init__(self, name, core, should_monitor=False): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + self._expire_metadata_cache() + Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _write(self): open(self.name, "wb").write( @@ -258,11 +277,11 @@ class Properties(Bcfg2.Server.Plugin.Plugin, :class:`PropertyFile` """ if fname.endswith(".xml"): - return XMLPropertyFile(fname) + return XMLPropertyFile(fname, self.core) elif HAS_JSON and fname.endswith(".json"): - return JSONPropertyFile(fname) + return JSONPropertyFile(fname, self.core) elif HAS_YAML and (fname.endswith(".yaml") or fname.endswith(".yml")): - return YAMLPropertyFile(fname) + return YAMLPropertyFile(fname, self.core) else: raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 59fbe6f03..e2d8a058f 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -117,7 +117,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.core.metadata_cache.expire() def end_statistics(self, metadata): self.end_client_run(self, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 5c73546b4..e372006c7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -9,12 +9,15 @@ from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ PluginInitError, PluginExecutionError -# required for reporting try: - import south # pylint: disable=W0611 - HAS_SOUTH = True + import django + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + HAS_REPORTING = True + else: + import south # pylint: disable=W0611 + HAS_REPORTING = True except ImportError: - HAS_SOUTH = False + HAS_REPORTING = False def _rpc_call(method): @@ -48,8 +51,8 @@ class Reporting(Statistics, Threaded, PullSource): self.whoami = platform.node() self.transport = None - if not HAS_SOUTH: - msg = "Django south is required for Reporting" + if not HAS_REPORTING: + msg = "Django 1.7+ or Django south is required for Reporting" self.logger.error(msg) raise PluginInitError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 7736bd050..7f20e72eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -286,6 +286,10 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.debug_log("New public key %s; invalidating " "ssh_known_hosts cache" % event.filename) self.skn = False + + if self.core.metadata_cache_mode in ['cautious', + 'aggressive']: + self.core.metadata_cache.expire() return if event.filename == 'info.xml': diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index b0b8d0061..ff67571fa 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -15,9 +15,10 @@ MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') class HelperModule(Debuggable): """ Representation of a TemplateHelper module """ - def __init__(self, name): + def __init__(self, name, core): Debuggable.__init__(self) self.name = name + self.core = core #: The name of the module as used by get_additional_data(). #: the name of the file with .py stripped off. @@ -44,6 +45,10 @@ class HelperModule(Debuggable): if event and event.code2str() not in ['exists', 'changed', 'created']: return + # expire the metadata cache, because the module might have changed + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + try: module = imp.load_source( safe_module_name('TemplateHelper', self._module_name), @@ -101,7 +106,6 @@ class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): __author__ = 'chris.a.st.pierre@gmail.com' ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE - __child__ = HelperModule def __init__(self, core): Plugin.__init__(self, core) @@ -109,6 +113,10 @@ class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): DirectoryBacked.__init__(self, self.data) TemplateDataProvider.__init__(self) + # The HelperModule needs access to the core, so we have to construct + # it manually and add the custom argument. + self.__child__ = lambda fname: HelperModule(fname, core) + def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index ac4c8eac4..67aa425d9 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -266,7 +266,7 @@ class Reason(models.Model): current_to = models.CharField(max_length=1024, blank=True) version = models.CharField(max_length=1024, blank=True) current_version = models.CharField(max_length=1024, blank=True) - current_exists = models.BooleanField() # False means its missing. Default True + current_exists = models.BooleanField(default=True) # False means its missing. current_diff = models.TextField(max_length=1024*1024, blank=True) is_binary = models.BooleanField(default=False) is_sensitive = models.BooleanField(default=False) diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py index 91c370994..09b218464 100644 --- a/src/lib/Bcfg2/Server/Reports/updatefix.py +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -4,7 +4,7 @@ import django.core.management import sys import logging import traceback -from Bcfg2.Server.models import InternalDatabaseVersion +from Bcfg2.Server.models import internal_database_version logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') @@ -138,7 +138,7 @@ def rollupdate(current_version): exc_info=1) # since array start at 0 but version start at 1 # we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i + 1) + ret = internal_database_version().create(version=i + 1) return ret else: return None @@ -149,10 +149,10 @@ def update_database(): try: logger.debug("Running upgrade of models to the new one") django.core.management.call_command("syncdb", interactive=False, verbosity=0) - know_version = InternalDatabaseVersion.objects.order_by('-version') + know_version = internal_database_version().order_by('-version') if not know_version: logger.debug("No version, creating initial version") - know_version = InternalDatabaseVersion.objects.create(version=lastversion) + know_version = internal_database_version().create(version=lastversion) else: know_version = know_version[0] logger.debug("Presently at %s" % know_version) diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 8d6642a25..9c0153c74 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -8,6 +8,7 @@ import Bcfg2.Server.Plugins LOGGER = logging.getLogger(__name__) MODELS = [] +INTERNAL_DATABASE_VERSION = None class _OptionContainer(object): @@ -56,15 +57,23 @@ def load_models(plugins=None): setattr(sys.modules[__name__], sym, obj) MODELS.append(sym) - class InternalDatabaseVersion(models.Model): - """ Object that tell us to which version the database is """ - version = models.IntegerField() - updated = models.DateTimeField(auto_now_add=True) +def internal_database_version(): + global INTERNAL_DATABASE_VERSION - def __str__(self): - return "version %d updated %s" % (self.version, - self.updated.isoformat()) + if INTERNAL_DATABASE_VERSION is None: + from django.db import models + class InternalDatabaseVersion(models.Model): + """ Object that tell us to which version the database is """ + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) - class Meta: # pylint: disable=C0111,W0232 - app_label = "reports" - get_latest_by = "version" + def __str__(self): + return "version %d updated %s" % (self.version, + self.updated.isoformat()) + + class Meta: # pylint: disable=C0111,W0232 + app_label = "reports" + get_latest_by = "version" + INTERNAL_DATABASE_VERSION = InternalDatabaseVersion + + return INTERNAL_DATABASE_VERSION.objects |