diff options
Diffstat (limited to 'src/lib')
29 files changed, 325 insertions, 212 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index f230aacb7..baf8a14f2 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,14 +1,12 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ -import os -import sys import time -import select import fnmatch import logging import Bcfg2.Client.Tools -from Bcfg2.Compat import input, any, all # pylint: disable=W0622 +from Bcfg2.Client import prompt +from Bcfg2.Compat import any, all # pylint: disable=W0622 def cmpent(ent1, ent2): @@ -154,7 +152,7 @@ class Frame(object): for entry in multi: self.logger.debug(entry) - def promptFilter(self, prompt, entries): + def promptFilter(self, msg, entries): """Filter a supplied list based on user input.""" ret = [] entries.sort(cmpent) @@ -165,20 +163,9 @@ class Frame(object): if 'qtext' in entry.attrib: iprompt = entry.get('qtext') else: - iprompt = prompt % (entry.tag, entry.get('name')) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - try: - ans = input(iprompt.encode(sys.stdout.encoding, 'replace')) - if ans in ['y', 'Y']: - ret.append(entry) - except EOFError: - # python 2.4.3 on CentOS doesn't like ^C for some reason - break - except: - print("Error while reading input") - continue + iprompt = msg % (entry.tag, entry.get('name')) + if prompt(iprompt): + ret.append(entry) return ret def __getattr__(self, name): @@ -281,7 +268,7 @@ class Frame(object): def Decide(self): # pylint: disable=R0912 """Set self.whitelist based on user interaction.""" - prompt = "Install %s: %s? (y/N): " + iprompt = "Install %s: %s? (y/N): " rprompt = "Remove %s: %s? (y/N): " if self.setup['remove']: if self.setup['remove'] == 'all': @@ -353,7 +340,7 @@ class Frame(object): (bmodified or a.get('when') == 'always'))] # now we process all "always actions" if self.setup['interactive']: - self.promptFilter(prompt, actions) + self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) if bundle.tag != 'Bundle': @@ -379,7 +366,7 @@ class Frame(object): if b.get("name"))) if self.setup['interactive']: - self.whitelist = self.promptFilter(prompt, self.whitelist) + self.whitelist = self.promptFilter(iprompt, self.whitelist) self.removal = self.promptFilter(rprompt, self.removal) for entry in candidates: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py index 896ca5f49..64a0b1e15 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py @@ -12,5 +12,4 @@ class POSIXHardlink(POSIXLinkTool): return os.path.samefile(entry.get('name'), entry.get('to')) def _link(self, entry): - ## TODO: set permissions return os.link(entry.get('to'), entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 08d943251..451495be2 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -360,7 +360,7 @@ class SELinuxEntryHandler(object): """ find extra entries of this entry type """ specified = [self._key(e) for e in self.tool.getSupportedEntries() - if e.get("type") == self.etype] + if e.tag == "SE%s" % self.etype.title()] try: records = self.custom_records except ValueError: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index a0dfe6dd9..4539a6a36 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -603,17 +603,18 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if stat['verify'] != {}: stat['verify_fail'] = True package_fail = True - self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.debug(" Verify Problems:") + self.logger.info("It is suggested that you either manage " + "these files, revert the changes, or ignore " + "false failures:") + self.logger.info(" Verify Problems: %s" % + stat['pkg'].get('name')) for fname, probs in list(stat['verify'].items()): if len(probs) > 1: - self.logger.debug(" %s" % fname) + self.logger.info(" %s" % fname) for prob in probs: - self.logger.debug(" %s" % prob) + self.logger.info(" %s" % prob[1]) else: - self.logger.debug(" %s: %s" % (fname, probs[0])) + self.logger.info(" %s: %s" % (fname, probs[0])) if len(all_pkg_objs) > 0: # Is this an install only package? We just look at the first one diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd86a2a4b..41759655d 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -3,10 +3,10 @@ import os import sys import stat -import select +import Bcfg2.Client import Bcfg2.Client.XML from Bcfg2.Utils import Executor, ClassName -from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 +from Bcfg2.Compat import walk_packages # pylint: disable=W0622 __all__ = [m[1] for m in walk_packages(path=__path__)] @@ -113,25 +113,34 @@ class Tool(object): #: A list of all entries handled by this tool self.handled = [] - for struct in config: + self._analyze_config() + self._check_execs() + + def _analyze_config(self): + """ Analyze the config at tool initialization-time for + important and handled entries """ + for struct in self.config: for entry in struct: if (entry.tag == 'Path' and entry.get('important', 'false').lower() == 'true'): self.__important__.append(entry.get('name')) - if self.handlesEntry(entry): - self.handled.append(entry) + self.handled = self.getSupportedEntries() + + def _check_execs(self): + """ Check all executables used by this tool to ensure that + they exist and are executable """ for filename in self.__execs__: try: mode = stat.S_IMODE(os.stat(filename)[stat.ST_MODE]) - if mode & stat.S_IEXEC != stat.S_IEXEC: - raise ToolInstantiationError("%s: %s not executable" % - (self.name, filename)) except OSError: raise ToolInstantiationError(sys.exc_info()[1]) except: raise ToolInstantiationError("%s: Failed to stat %s" % - (self.name, filename), - exc_info=1) + (self.name, filename)) + if not mode & stat.S_IEXEC: + raise ToolInstantiationError("%s: %s not executable" % + (self.name, filename)) + def BundleUpdated(self, bundle, states): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -185,11 +194,13 @@ class Tool(object): if self.canVerify(entry): try: func = getattr(self, "Verify%s" % entry.tag) - states[entry] = func(entry, mods) except AttributeError: self.logger.error("%s: Cannot verify %s entries" % (self.name, entry.tag)) - except: + continue + try: + states[entry] = func(entry, mods) + except: # pylint: disable=W0702 self.logger.error("%s: Unexpected failure verifying %s" % (self.name, self.primarykey(entry)), @@ -213,14 +224,16 @@ class Tool(object): :returns: None """ for entry in entries: try: - func = getattr(self, "Install%s" % (entry.tag)) - states[entry] = func(entry) - if states[entry]: - self.modified.append(entry) + func = getattr(self, "Install%s" % entry.tag) except AttributeError: self.logger.error("%s: Cannot install %s entries" % (self.name, entry.tag)) - except: + continue + try: + states[entry] = func(entry) + if states[entry]: + self.modified.append(entry) + except: # pylint: disable=W0702 self.logger.error("%s: Unexpected failure installing %s" % (self.name, self.primarykey(entry)), exc_info=1) @@ -409,6 +422,19 @@ class PkgTool(Tool): """ raise NotImplementedError + def _get_package_command(self, packages): + """ Get the command to install the given list of packages. + + :param packages: The Package entries to install + :type packages: list of lxml.etree._Element + :returns: string - the command to run + """ + pkgargs = " ".join(self.pkgtool[1][0] % + tuple(pkg.get(field) + for field in self.pkgtool[1][1]) + for pkg in packages) + return self.pkgtool[0] % pkgargs + def Install(self, packages, states): """ Run a one-pass install where all required packages are installed with a single command, followed by single package @@ -422,11 +448,6 @@ class PkgTool(Tool): self.logger.info("Trying single pass package install for pkgtype %s" % self.pkgtype) - data = [tuple([pkg.get(field) for field in self.pkgtool[1][1]]) - for pkg in packages] - pkgargs = " ".join([self.pkgtool[1][0] % datum for datum in data]) - - self.logger.debug("Installing packages: %s" % pkgargs) if self.cmd.run(self.pkgtool[0] % pkgargs): self.logger.info("Single Pass Succeded") # set all package states to true and flush workqueues @@ -436,7 +457,7 @@ class PkgTool(Tool): and entry.get('type') == self.pkgtype and entry.get('name') in pkgnames): self.logger.debug('Setting state to true for pkg %s' % - (entry.get('name'))) + entry.get('name')) states[entry] = True self.RefreshPackages() else: @@ -452,18 +473,13 @@ class PkgTool(Tool): else: self.logger.info("Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) - if self.cmd.run( - self.pkgtool[0] % - (self.pkgtool[1][0] % - tuple([pkg.get(field) - for field in self.pkgtool[1][1]]))): + if self.cmd.run(self._get_package_command([pkg])): states[pkg] = True else: self.logger.error("Failed to install package %s" % - (pkg.get('name'))) + pkg.get('name')) self.RefreshPackages() - for entry in [ent for ent in packages if states[ent]]: - self.modified.append(entry) + self.modified.extend(entry for entry in packages if states[entry]) def RefreshPackages(self): """ Refresh the internal representation of the package @@ -557,11 +573,13 @@ class SvcTool(Tool): if self.setup['servicemode'] == 'disabled': return - for entry in [ent for ent in bundle if self.handlesEntry(ent)]: - restart = entry.get("restart", "true") - if (restart.lower() == "false" or - (restart.lower() == "interactive" and - not self.setup['interactive'])): + for entry in bundle: + if not self.handlesEntry(entry): + continue + + restart = entry.get("restart", "true").lower() + if (restart == "false" or + (restart == "interactive" and not self.setup['interactive'])): continue success = False @@ -570,14 +588,8 @@ class SvcTool(Tool): success = self.stop_service(entry) elif entry.get('name') not in self.restarted: if self.setup['interactive']: - prompt = ('Restart service %s?: (y/N): ' % - entry.get('name')) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input(prompt) - if ans not in ['y', 'Y']: + if not Bcfg2.Client.prompt('Restart service %s? (y/N) ' + % entry.get('name')): continue success = self.restart_service(entry) if success: @@ -593,8 +605,8 @@ class SvcTool(Tool): install_entries = [] for entry in entries: if entry.get('install', 'true').lower() == 'false': - self.logger.info("Service %s installation is false. Skipping " - "installation." % (entry.get('name'))) + self.logger.info("Installation is false for %s:%s, skipping" % + (entry.tag, entry.get('name'))) else: install_entries.append(entry) return Tool.Install(self, install_entries, states) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index c03021f14..dd5ae1e83 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -1,3 +1,31 @@ """This contains all Bcfg2 Client modules""" __all__ = ["Frame", "Tools", "XML", "Client"] + +import os +import sys +import select +from Bcfg2.Compat import input # pylint: disable=W0622 + + +def prompt(msg): + """ Helper to give a yes/no prompt to the user. Flushes input + buffers, handles exceptions, etc. Returns True if the user + answers in the affirmative, False otherwise. + + :param msg: The message to show to the user. The message is not + altered in any way for display; i.e., it should + contain "[y/N]" if desired, etc. + :type msg: string + :returns: bool - True if yes, False if no """ + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + try: + ans = input(msg.encode(sys.stdout.encoding, 'replace')) + return ans in ['y', 'Y'] + except EOFError: + # python 2.4.3 on CentOS doesn't like ^C for some reason + return False + except: + print("Error while reading input: %s" % sys.exc_info()[1]) + return False diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index b0f0ef5cf..beb534791 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -260,3 +260,10 @@ def oct_mode(mode): :type mode: int :returns: string """ return oct(mode).replace('o', '') + + +try: + long = long +except NameError: + # longs are just ints in py3k + long = int diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index fb7af7465..bca4a9c1e 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -224,7 +224,11 @@ class DjangoORM(StorageBase): inter.extra_count = counter_fields[TYPE_EXTRA] inter.save() for entry_type in updates.keys(): - getattr(inter, entry_type).add(*updates[entry_type]) + # batch this for sqlite + i = 0 + while(i < len(updates[entry_type])): + getattr(inter, entry_type).add(*updates[entry_type][i:i+100]) + i += 100 # performance metrics for times in stats.findall('OpStamps'): diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index c7850f4af..ab2dc8418 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -392,7 +392,13 @@ class BaseEntry(models.Model): @classmethod def prune_orphans(cls): '''Remove unused entries''' - cls.objects.filter(interaction__isnull=True).delete() + # yeat another sqlite hack + cls_orphans = [x['id'] \ + for x in cls.objects.filter(interaction__isnull=True).values("id")] + i = 0 + while i < len(cls_orphans): + cls.objects.filter(id__in=cls_orphans[i:i+100]).delete() + i += 100 class SuccessEntry(BaseEntry): diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 1ec6b8c24..533dcc79e 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -88,7 +88,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.0rc1</span> + <span>Bcfg2 Version 1.3.0rc2</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index 45ba20b86..d9c415c20 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -30,6 +30,9 @@ {% endif %} {% endfor %} </table> -{% else %}<p>No client records are available.</p> +{% else %} + <div class='client_list_box'> + <p>No client records are available.</p> + </div> {% endif %} {% endblock %} diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 8ab3f8e59..0341a18af 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -213,8 +213,11 @@ def entry_status(request, entry_type, pk, timestamp=None, **kwargs): # There is no good way to do this... items = [] - for it in cls.objects.filter(interaction__in=current_clients, name=item.name).distinct("id").select_related(): - items.append((it, it.interaction_set.filter(pk__in=current_clients).order_by('client__name').select_related('client'))) + seen = [] + for it in cls.objects.filter(interaction__in=current_clients, name=item.name).select_related(): + if it.pk not in seen: + items.append((it, it.interaction_set.filter(pk__in=current_clients).order_by('client__name').select_related('client'))) + seen.append(it.pk) return render_to_response('config_items/entry_status.html', {'entry': item, diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index f93ca0a7b..1b0ecadf5 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -332,9 +332,23 @@ class BaseCore(object): self.fam.handle_event_set(self.lock) except: continue - # VCS plugin periodic updates - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): - self.revision = plugin.get_revision() + self._update_vcs_revision() + + @track_statistics() + def _update_vcs_revision(self): + """ Update the revision of the current configuration on-disk + from the VCS plugin """ + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + try: + newrev = plugin.get_revision() + if newrev != self.revision: + self.logger.debug("Updated to revision %s" % newrev) + self.revision = newrev + break + except: + self.logger.warning("Error getting revision from %s: %s" % + (plugin.name, sys.exc_info()[1])) + self.revision = '-1' def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is @@ -360,8 +374,8 @@ class BaseCore(object): try: plug = getattr(mod, plugin.split('.')[-1]) except AttributeError: - self.logger.error("Failed to load plugin %s (AttributeError)" % - plugin) + self.logger.error("Failed to load plugin %s: %s" % + (plugin, sys.exc_info()[1])) return # Blacklist conflicting plugins cplugs = [conflict for conflict in plug.conflicts @@ -536,18 +550,16 @@ class BaseCore(object): continue try: self.Bind(entry, metadata) - except PluginExecutionError: - exc = sys.exc_info()[1] - if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) - self.logger.error("Failed to bind entry %s:%s: %s" % - (entry.tag, entry.get('name'), exc)) - except Exception: + except: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % exc) - self.logger.error("Unexpected failure in BindStructure: %s %s" - % (entry.tag, entry.get('name')), exc_info=1) + if isinstance(exc, PluginExecutionError): + msg = "Failed to bind entry" + else: + msg = "Unexpected failure binding entry" + self.logger.error("%s %s:%s: %s" % + (msg, entry.tag, entry.get('name'), exc)) def Bind(self, entry, metadata): """ Bind a single entry using the appropriate generator. diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index c931ed2a7..b46337eb0 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -30,6 +30,9 @@ CFG_SECTION = "encryption" #: The config option used to store the algorithm CFG_ALGORITHM = "algorithm" +#: The config option used to store the decryption strictness +CFG_DECRYPT = "decrypt" + #: Default cipher algorithm. To get a full list of valid algorithms, #: you can run:: #: @@ -40,6 +43,7 @@ ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 CFG_ALGORITHM, default="aes_256_cbc").lower().replace("-", "_") + Rand.rand_seed(os.urandom(1024)) @@ -165,7 +169,7 @@ def get_passphrases(): if setup.cfp.has_section(CFG_SECTION): return dict([(o, setup.cfp.get(CFG_SECTION, o)) for o in setup.cfp.options(CFG_SECTION) - if o != CFG_ALGORITHM]) + if o not in [CFG_ALGORITHM, CFG_DECRYPT]]) else: return dict() diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index ecea1ad1b..8bfb76461 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -130,7 +130,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype) def check_plaintext(self, filename, data, rtype): - """ check generic plaintex files for required headers """ + """ check generic plaintext files for required headers """ self.check_lines(filename, data.splitlines(), rtype) def check_lines(self, filename, lines, rtype): diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 437e69d82..ed0d9930f 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,8 +1,11 @@ """ Check Genshi templates for syntax errors """ import sys -import genshi.template import Bcfg2.Server.Lint +from genshi.template import TemplateLoader, NewTextTemplate, MarkupTemplate, \ + TemplateSyntaxError +from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator class Genshi(Bcfg2.Server.Lint.ServerPlugin): @@ -10,29 +13,40 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): def Run(self): """ run plugin """ - loader = genshi.template.TemplateLoader() if 'Cfg' in self.core.plugins: - self.check_files(self.core.plugins['Cfg'].entries, - loader=loader) + self.check_cfg() + if 'Bundler' in self.core.plugins: + self.check_bundler() @classmethod def Errors(cls): return {"genshi-syntax-error": "error"} - def check_files(self, entries, loader=None): - """ Check genshi templates in a list of entries for syntax - errors """ - if loader is None: - loader = genshi.template.TemplateLoader() - - for eset in entries.values(): - for fname, sdata in list(eset.entries.items()): - if (self.HandlesFile(fname) and - (fname.endswith(".genshi") or fname.endswith(".newtxt"))): + def check_cfg(self): + """ Check genshi templates in Cfg for syntax errors """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, CfgGenshiGenerator) and + not entry.template): try: - loader.load(sdata.name, - cls=genshi.template.NewTextTemplate) - except genshi.template.TemplateSyntaxError: + entry.loader.load(entry.name, + cls=NewTextTemplate) + except TemplateSyntaxError: err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) + + def check_bundler(self): + """ Check templates in Bundler for syntax errors """ + loader = TemplateLoader() + + for entry in self.core.plugins['Bundler'].entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, BundleTemplateFile)): + try: + loader.load(entry.name, cls=MarkupTemplate) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("genshi-syntax-error", + "Genshi syntax error: %s" % err) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index b702ac899..313e53ee9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -23,10 +23,15 @@ class CfgExternalCommandVerifier(CfgVerifier): def verify_entry(self, entry, metadata, data): try: proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - err = proc.communicate(input=data)[1] + out, err = proc.communicate(input=data) rv = proc.wait() if rv != 0: - raise CfgVerificationError(err) + # pylint: disable=E1103 + raise CfgVerificationError(err.strip() or out.strip() or + "Non-zero return value %s" % rv) + # pylint: enable=E1103 + except CfgVerificationError: + raise except: err = sys.exc_info()[1] raise CfgVerificationError("Error running external command " diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index c11939bd5..5f10879be 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -166,6 +166,7 @@ class CfgGenshiGenerator(CfgGenerator): raise def handle_event(self, event): + CfgGenerator.handle_event(self, event) try: self.template = self.loader.load(self.name, cls=NewTextTemplate, encoding=self.encoding) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 81adea42f..2301de725 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -553,11 +553,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, try: self._validate_data(entry, metadata, data) except CfgVerificationError: - msg = "Data for %s for %s failed to verify: %s" % \ - (entry.get('name'), metadata.hostname, - sys.exc_info()[1]) - self.logger.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("Failed to verify %s for %s: %s" % + (entry.get('name'), + metadata.hostname, + sys.exc_info()[1])) if entry.get('encoding') == 'base64': data = b64encode(data) diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index f4d86a64f..04c14aa96 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -5,7 +5,7 @@ import Bcfg2.Server.Plugins.Rules class Defaults(Bcfg2.Server.Plugins.Rules.Rules, - Bcfg2.Server.Plugin.StructureValidator): + Bcfg2.Server.Plugin.GoalValidator): """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' @@ -22,27 +22,18 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - def validate_structures(self, metadata, structures): + def validate_goals(self, metadata, config): """ Apply defaults """ - for struct in structures: + for struct in config.getchildren(): for entry in struct.getchildren(): - if entry.tag.startswith("Bound"): - is_bound = True - entry.tag = entry.tag[5:] - else: - is_bound = False try: - try: - self.BindEntry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - # either no matching defaults (which is okay), - # or multiple matching defaults (which is not - # okay, but is logged). either way, we don't - # care about the error. - pass - finally: - if is_bound: - entry.tag = "Bound" + entry.tag + self.BindEntry(entry, metadata) + except Bcfg2.Server.Plugin.PluginExecutionError: + # either no matching defaults (which is okay), + # or multiple matching defaults (which is not + # okay, but is logged). either way, we don't + # care about the error. + pass @property def _regex_enabled(self): diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 8cc63a46f..c8362db41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -2,7 +2,7 @@ git. """ import sys -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import Version, PluginExecutionError from subprocess import Popen, PIPE try: @@ -12,16 +12,16 @@ except ImportError: HAS_GITPYTHON = False -class Git(Bcfg2.Server.Plugin.Version): +class Git(Version): """ The Git plugin provides a revision interface for Bcfg2 repos using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".git" if HAS_GITPYTHON: - __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] + __rmi__ = Version.__rmi__ + ['Update'] def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + Version.__init__(self, core, datastore) if HAS_GITPYTHON: self.repo = git.Repo(self.vcs_root) else: @@ -31,6 +31,11 @@ class Git(Bcfg2.Server.Plugin.Version): self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) + def _log_git_cmd(self, output): + """ Send output from a GitPython command to the debug log """ + for line in output.strip().splitlines(): + self.debug_log("Git: %s" % line) + def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: @@ -46,11 +51,9 @@ class Git(Bcfg2.Server.Plugin.Version): raise Exception(err) return rv except: - err = sys.exc_info()[1] - msg = "Git: Error getting revision from %s: %s" % (self.vcs_root, - err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Git: Error getting revision from %s: " + "%s" % (self.vcs_root, + sys.exc_info()[1])) def Update(self, ref=None): """ Git.Update() => True|False @@ -60,20 +63,25 @@ class Git(Bcfg2.Server.Plugin.Version): self.debug_log("Git: Performing garbage collection on repo at %s" % self.vcs_root) try: - self.repo.git.gc('--auto') + self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) + self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + try: + self._log_git_cmd(self.repo.git.fetch('--all')) + except git.GitCommandError: + self.logger.warning("Git: Failed to fetch refs: %s" % + sys.exc_info()[1]) + if ref: self.debug_log("Git: Checking out %s" % ref) try: - self.repo.git.checkout('-f', ref) + self._log_git_cmd(self.repo.git.checkout('-f', ref)) except git.GitCommandError: - err = sys.exc_info()[1] - msg = "Git: Failed to checkout %s: %s" % (ref, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Git: Failed to checkout %s: %s" % + (ref, sys.exc_info()[1])) # determine if we should try to pull to get the latest commit # on this head @@ -87,12 +95,10 @@ class Git(Bcfg2.Server.Plugin.Version): self.debug_log("Git: %s is a tracking branch, pulling from %s" % (self.repo.head.ref.name, tracking)) try: - self.repo.git.pull("--rebase") - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - msg = "Git: Failed to pull from upstream: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self._log_git_cmd(self.repo.git.pull("--rebase")) + except git.GitCommandError: + raise PluginExecutionError("Git: Failed to pull from " + "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % (self.vcs_root, self.get_revision())) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index e568c5c65..b053e65d3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1452,7 +1452,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default"): + if grp.get("default", "false").lower() == "true": defaults.append(self.RenderXML(grp)) if len(defaults) > 1: self.LintError("multiple-default-groups", diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 3ad64b242..59eefe143 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -533,7 +533,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): # should be resolved current = pkgs.pop() self.debug_log("Packages: handling package requirement %s" % - current) + (current,)) packages.add(current) deps = self.get_deps(current) newdeps = set(deps).difference(examined) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4057ed230..775caaa08 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -684,6 +684,21 @@ class YumCollection(Collection): return self.call_helper("get_groups", inputdata=gdicts) + def _element_to_pkg(self, el, name): + """ Convert a Package or Instance element to a package tuple """ + rv = (name, el.get("arch"), el.get("epoch"), + el.get("version"), el.get("release")) + if rv[3] in ['any', 'auto']: + rv = (rv[0], rv[1], rv[2], None, None) + # if a package requires no specific version, we just use + # the name, not the tuple. this limits the amount of JSON + # encoding/decoding that has to be done to pass the + # package list to bcfg2-yum-helper. + if rv[1:] == (None, None, None, None): + return name + else: + return rv + def packages_from_entry(self, entry): """ When using the Python yum libraries, convert a Package entry to a list of package tuples. See :ref:`yum-pkg-objects` @@ -693,32 +708,42 @@ class YumCollection(Collection): :type entry: lxml.etree._Element :returns: list of tuples """ + if not self.use_yum: + return Collection.packages_from_entry(self, entry) + rv = set() name = entry.get("name") - def _tag_to_pkg(tag): - """ Convert a Package or Instance tag to a package tuple """ - rv = (name, tag.get("arch"), tag.get("epoch"), - tag.get("version"), tag.get("release")) - if rv[3] in ['any', 'auto']: - rv = (rv[0], rv[1], rv[2], None, None) - # if a package requires no specific version, we just use - # the name, not the tuple. this limits the amount of JSON - # encoding/decoding that has to be done to pass the - # package list to bcfg2-yum-helper. - if rv[1:] == (None, None, None, None): - return name - else: - return rv - for inst in entry.getchildren(): if inst.tag != "Instance": continue - rv.add(_tag_to_pkg(inst)) + rv.add(self._element_to_pkg(inst, name)) if not rv: - rv.add(_tag_to_pkg(entry)) + rv.add(self._element_to_pkg(entry, name)) return list(rv) + def _get_entry_attrs(self, pkgtup): + """ Given a package tuple, return a dict of attributes + suitable for applying to either a Package or an Instance + tag """ + attrs = dict(version=self.setup.cfp.get("packages", "version", + default="auto")) + if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): + return attrs + + try: + if pkgtup[1]: + attrs['arch'] = pkgtup[1] + if pkgtup[2]: + attrs['epoch'] = pkgtup[2] + if pkgtup[3]: + attrs['version'] = pkgtup[3] + if pkgtup[4]: + attrs['release'] = pkgtup[4] + except IndexError: + self.logger.warning("Malformed package tuple: %s" % pkgtup) + return attrs + def packages_to_entry(self, pkglist, entry): """ When using the Python yum libraries, convert a list of package tuples to a Package entry. See :ref:`yum-pkg-objects` @@ -738,28 +763,8 @@ class YumCollection(Collection): :type entry: lxml.etree._Element :returns: None """ - def _get_entry_attrs(pkgtup): - """ Given a package tuple, return a dict of attributes - suitable for applying to either a Package or an Instance - tag """ - attrs = dict(version=self.setup.cfp.get("packages", - "version", - default="auto")) - if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): - return attrs - - try: - if pkgtup[1]: - attrs['arch'] = pkgtup[1] - if pkgtup[2]: - attrs['epoch'] = pkgtup[2] - if pkgtup[3]: - attrs['version'] = pkgtup[3] - if pkgtup[4]: - attrs['release'] = pkgtup[4] - except IndexError: - self.logger.warning("Malformed package tuple: %s" % pkgtup) - return attrs + if not self.use_yum: + return Collection.packages_to_entry(self, pkglist, entry) packages = dict() for pkg in pkglist: @@ -776,9 +781,9 @@ class YumCollection(Collection): **pkgattrs) for inst in instances: lxml.etree.SubElement(pkg_el, "Instance", - _get_entry_attrs(inst)) + self._get_entry_attrs(inst)) else: - attrs = _get_entry_attrs(instances[0]) + attrs = self._get_entry_attrs(instances[0]) attrs.update(pkgattrs) lxml.etree.SubElement(entry, 'BoundPackage', **attrs) @@ -803,7 +808,11 @@ class YumCollection(Collection): initial_names.append(pkg) new = [] for pkg in complete: - if pkg[0] not in initial_names: + if isinstance(pkg, tuple): + name = pkg[0] + else: + name = pkg + if name not in initial_names: new.append(pkg) return new diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index cc1a2ceac..ab2f80552 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -180,7 +180,7 @@ class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): self.logger.error("SSLCA: Failed to unlink temporary files: %s" % sys.exc_info()[1]) if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(self.parent.get_ca(ca)['chaincert']).read() + cert += open(ca['chaincert']).read() open(os.path.join(self.path, filename), 'w').write(cert) return cert @@ -363,7 +363,8 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' - es_cls = lambda self, *args: SSLCAEntrySet(*args, parent=self) + # python 2.5 doesn't support mixing *magic and keyword arguments + es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile def get_ca(self, name): diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index e3f8ed749..ad3eb65bc 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -12,12 +12,25 @@ LOGGER = logging.getLogger(__name__) MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') +def safe_module_name(module): + """ Munge the name of a TemplateHelper module to avoid collisions + with other Python modules. E.g., if someone has a helper named + 'ldap.py', it should not be added to ``sys.modules`` as ``ldap``, + but rather as something more obscure. """ + return '__TemplateHelper_%s' % module + + class HelperModule(object): """ Representation of a TemplateHelper module """ def __init__(self, name): self.name = name + + #: The name of the module as used by get_additional_data(). + #: the name of the file with .py stripped off. self._module_name = MODULE_RE.search(self.name).group('module') + + #: The attributes exported by this module self._attrs = [] def HandleEvent(self, event=None): @@ -31,7 +44,8 @@ class HelperModule(object): return try: - module = imp.load_source(self._module_name, self.name) + module = imp.load_source(safe_module_name(self._module_name), + self.name) except: # pylint: disable=W0702 err = sys.exc_info()[1] LOGGER.error("TemplateHelper: Failed to import %s: %s" % @@ -97,7 +111,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): module_name = MODULE_RE.search(helper).group(1) try: - module = imp.load_source(module_name, helper) + module = imp.load_source(safe_module_name(module_name), helper) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("templatehelper-import-error", diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py index b377806ab..cb131c29d 100644 --- a/src/lib/Bcfg2/Server/Reports/updatefix.py +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -80,6 +80,9 @@ def _populate_interaction_entry_counts(): cursor.close() +def update_noop(): + return True + # be sure to test your upgrade query before reflecting the change in the models # the list of function and sql command to do should go here _fixes = [_merge_database_table_entries, @@ -103,6 +106,8 @@ _fixes = [_merge_database_table_entries, _interactions_constraint_or_idx, 'alter table reports_reason add is_binary bool NOT NULL default False;', 'alter table reports_reason add is_sensitive bool NOT NULL default False;', + update_noop, #_remove_table_column('reports_interaction', 'client_version'), + "alter table reports_reason add unpruned varchar(1280) not null default 'N/A';", ] # this will calculate the last possible version of the database diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index da6e2036e..d819ee534 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -136,6 +136,8 @@ if HAS_SOUTH: 'south', 'Bcfg2.Reporting', ) +if 'BCFG2_LEGACY_MODELS' in os.environ: + INSTALLED_APPS += ('Bcfg2.Server.Reports.reports',) # Imported from Bcfg2.Server.Reports MEDIA_ROOT = '' diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 86803afcb..8223d7543 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.0rc1" +__version__ = "1.3.0rc2" class Bcfg2VersionInfo(tuple): |