From da53b2b3fe5f5e1ba8511da9ed7adbf23800a9eb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 18 Jan 2013 14:36:07 -0500 Subject: fixed syntax errors for py < 2.5 --- src/lib/Bcfg2/Server/Plugins/SSLCA.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 0d51adf18..b5cc11aa4 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -362,7 +362,8 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' - es_cls = lambda self, *args: SSLCAEntrySet(*args, parent=self) + # python 2.5 doesn't support mixing *magic and keyword arguments + es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile def get_ca(self, name): -- cgit v1.2.3-1-g7c22 From 01df50ad3db5f1fec41c9e1ef2d7f78184a4abfb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 23 Jan 2013 14:41:00 -0500 Subject: Packages: fixed bug with display of package entries that have been converted to tuples --- src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index f9bb9e1a2..b25cb0fc4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -264,7 +264,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): support multiple package types in package groups (e.g., "recommended," "optional," etc.) :type ptype: string - :returns: list of strings - package names, but see + :returns: list of strings - package names, but see :ref:`pkg-objects` """ if not self.__package_groups__: @@ -467,7 +467,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): included in the client configuration. See :ref:`pkg-objects` for more details. - :param pkglist: A list of packages as returned by + :param pkglist: A list of packages as returned by :func:`complete` :type pkglist: list of strings, but see :ref:`pkg-objects` :param entry: The base XML entry to add all of the Package @@ -562,7 +562,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): # should be resolved current = pkgs.pop() self.debug_log("Packages: handling package requirement %s" % - current) + (current,)) packages.add(current) deps = self.get_deps(current) newdeps = set(deps).difference(examined) -- cgit v1.2.3-1-g7c22 From 43b8714fadb3d7b60198e2b73792f28cddae713b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 23 Jan 2013 14:46:18 -0500 Subject: Packages: only convert package entry to tuple if yum libraries are used --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 37171e1b1..7b86a19a9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -692,6 +692,9 @@ class YumCollection(Collection): :type entry: lxml.etree._Element :returns: list of tuples """ + if not self.use_yum: + return Collection.packages_from_entry(self, entry) + rv = set() name = entry.get("name") @@ -737,6 +740,9 @@ class YumCollection(Collection): :type entry: lxml.etree._Element :returns: None """ + if not self.use_yum: + return Collection.packages_to_entry(self, pkglist, entry) + def _get_entry_attrs(pkgtup): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance -- cgit v1.2.3-1-g7c22 From b2c25cf9275cc39fde42c96d71bab6b200997e92 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 23 Jan 2013 14:47:31 -0500 Subject: Packages: removed unnecessary nested functions --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 83 ++++++++++++++-------------- 1 file changed, 41 insertions(+), 42 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7b86a19a9..31e583768 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -683,6 +683,21 @@ class YumCollection(Collection): return self.call_helper("get_groups", inputdata=gdicts) + def _element_to_pkg(self, el, name): + """ Convert a Package or Instance element to a package tuple """ + rv = (name, el.get("arch"), el.get("epoch"), + el.get("version"), el.get("release")) + if rv[3] in ['any', 'auto']: + rv = (rv[0], rv[1], rv[2], None, None) + # if a package requires no specific version, we just use + # the name, not the tuple. this limits the amount of JSON + # encoding/decoding that has to be done to pass the + # package list to bcfg2-yum-helper. + if rv[1:] == (None, None, None, None): + return name + else: + return rv + def packages_from_entry(self, entry): """ When using the Python yum libraries, convert a Package entry to a list of package tuples. See :ref:`yum-pkg-objects` @@ -698,29 +713,36 @@ class YumCollection(Collection): rv = set() name = entry.get("name") - def _tag_to_pkg(tag): - """ Convert a Package or Instance tag to a package tuple """ - rv = (name, tag.get("arch"), tag.get("epoch"), - tag.get("version"), tag.get("release")) - if rv[3] in ['any', 'auto']: - rv = (rv[0], rv[1], rv[2], None, None) - # if a package requires no specific version, we just use - # the name, not the tuple. this limits the amount of JSON - # encoding/decoding that has to be done to pass the - # package list to bcfg2-yum-helper. - if rv[1:] == (None, None, None, None): - return name - else: - return rv - for inst in entry.getchildren(): if inst.tag != "Instance": continue - rv.add(_tag_to_pkg(inst)) + rv.add(self._element_to_pkg(inst, name)) if not rv: - rv.add(_tag_to_pkg(entry)) + rv.add(self._element_to_pkg(entry, name)) return list(rv) + def _get_entry_attrs(self, pkgtup): + """ Given a package tuple, return a dict of attributes + suitable for applying to either a Package or an Instance + tag """ + attrs = dict(version=self.setup.cfp.get("packages", "version", + default="auto")) + if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): + return attrs + + try: + if pkgtup[1]: + attrs['arch'] = pkgtup[1] + if pkgtup[2]: + attrs['epoch'] = pkgtup[2] + if pkgtup[3]: + attrs['version'] = pkgtup[3] + if pkgtup[4]: + attrs['release'] = pkgtup[4] + except IndexError: + self.logger.warning("Malformed package tuple: %s" % pkgtup) + return attrs + def packages_to_entry(self, pkglist, entry): """ When using the Python yum libraries, convert a list of package tuples to a Package entry. See :ref:`yum-pkg-objects` @@ -743,29 +765,6 @@ class YumCollection(Collection): if not self.use_yum: return Collection.packages_to_entry(self, pkglist, entry) - def _get_entry_attrs(pkgtup): - """ Given a package tuple, return a dict of attributes - suitable for applying to either a Package or an Instance - tag """ - attrs = dict(version=self.setup.cfp.get("packages", - "version", - default="auto")) - if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): - return attrs - - try: - if pkgtup[1]: - attrs['arch'] = pkgtup[1] - if pkgtup[2]: - attrs['epoch'] = pkgtup[2] - if pkgtup[3]: - attrs['version'] = pkgtup[3] - if pkgtup[4]: - attrs['release'] = pkgtup[4] - except IndexError: - self.logger.warning("Malformed package tuple: %s" % pkgtup) - return attrs - packages = dict() for pkg in pkglist: try: @@ -781,9 +780,9 @@ class YumCollection(Collection): **pkgattrs) for inst in instances: lxml.etree.SubElement(pkg_el, "Instance", - _get_entry_attrs(inst)) + self._get_entry_attrs(inst)) else: - attrs = _get_entry_attrs(instances[0]) + attrs = self._get_entry_attrs(instances[0]) attrs.update(pkgattrs) lxml.etree.SubElement(entry, 'BoundPackage', **attrs) -- cgit v1.2.3-1-g7c22 From 282d6d7056bf8481d6f194e436dfb96ede6d559d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 24 Jan 2013 07:59:11 -0500 Subject: Git: log output of GitPython commands --- src/lib/Bcfg2/Server/Plugins/Git.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 8cc63a46f..e46c331b3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -31,6 +31,11 @@ class Git(Bcfg2.Server.Plugin.Version): self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) + def _log_git_cmd(self, output): + """ Send output from a GitPython command to the debug log """ + for line in output.strip().splitlines(): + self.debug_log("Git: %s" % line) + def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: @@ -60,7 +65,7 @@ class Git(Bcfg2.Server.Plugin.Version): self.debug_log("Git: Performing garbage collection on repo at %s" % self.vcs_root) try: - self.repo.git.gc('--auto') + self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) @@ -68,7 +73,7 @@ class Git(Bcfg2.Server.Plugin.Version): if ref: self.debug_log("Git: Checking out %s" % ref) try: - self.repo.git.checkout('-f', ref) + self._log_git_cmd(self.repo.git.checkout('-f', ref)) except git.GitCommandError: err = sys.exc_info()[1] msg = "Git: Failed to checkout %s: %s" % (ref, err) @@ -87,7 +92,7 @@ class Git(Bcfg2.Server.Plugin.Version): self.debug_log("Git: %s is a tracking branch, pulling from %s" % (self.repo.head.ref.name, tracking)) try: - self.repo.git.pull("--rebase") + self._log_git_cmd(self.repo.git.pull("--rebase")) except: # pylint: disable=W0702 err = sys.exc_info()[1] msg = "Git: Failed to pull from upstream: %s" % err -- cgit v1.2.3-1-g7c22 From 53f1c977e2d9ce23d5292fa71e95c27ba6aa59ce Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 24 Jan 2013 07:59:33 -0500 Subject: Git: fetch refs before checking out a ref on Git.Update --- src/lib/Bcfg2/Server/Plugins/Git.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index e46c331b3..759146059 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -70,6 +70,13 @@ class Git(Bcfg2.Server.Plugin.Version): self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) + self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + try: + self._log_git_cmd(self.repo.git.fetch('--all')) + except git.GitCommandError: + self.logger.warning("Git: Failed to fetch refs: %s" % + sys.exc_info()[1]) + if ref: self.debug_log("Git: Checking out %s" % ref) try: -- cgit v1.2.3-1-g7c22 From b9f1c40d4f581a1af37b7f15a5438e19d6df618b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 24 Jan 2013 08:02:52 -0500 Subject: Git: fixed overly verbose error reporting --- src/lib/Bcfg2/Server/Plugins/Git.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 759146059..c8362db41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -2,7 +2,7 @@ git. """ import sys -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import Version, PluginExecutionError from subprocess import Popen, PIPE try: @@ -12,16 +12,16 @@ except ImportError: HAS_GITPYTHON = False -class Git(Bcfg2.Server.Plugin.Version): +class Git(Version): """ The Git plugin provides a revision interface for Bcfg2 repos using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".git" if HAS_GITPYTHON: - __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] + __rmi__ = Version.__rmi__ + ['Update'] def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + Version.__init__(self, core, datastore) if HAS_GITPYTHON: self.repo = git.Repo(self.vcs_root) else: @@ -51,11 +51,9 @@ class Git(Bcfg2.Server.Plugin.Version): raise Exception(err) return rv except: - err = sys.exc_info()[1] - msg = "Git: Error getting revision from %s: %s" % (self.vcs_root, - err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Git: Error getting revision from %s: " + "%s" % (self.vcs_root, + sys.exc_info()[1])) def Update(self, ref=None): """ Git.Update() => True|False @@ -82,10 +80,8 @@ class Git(Bcfg2.Server.Plugin.Version): try: self._log_git_cmd(self.repo.git.checkout('-f', ref)) except git.GitCommandError: - err = sys.exc_info()[1] - msg = "Git: Failed to checkout %s: %s" % (ref, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Git: Failed to checkout %s: %s" % + (ref, sys.exc_info()[1])) # determine if we should try to pull to get the latest commit # on this head @@ -100,11 +96,9 @@ class Git(Bcfg2.Server.Plugin.Version): (self.repo.head.ref.name, tracking)) try: self._log_git_cmd(self.repo.git.pull("--rebase")) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - msg = "Git: Failed to pull from upstream: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + except git.GitCommandError: + raise PluginExecutionError("Git: Failed to pull from " + "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % (self.vcs_root, self.get_revision())) -- cgit v1.2.3-1-g7c22 From f301da8af95a184c6302df2d6e4418787c73b6bd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 25 Jan 2013 09:14:45 -0500 Subject: Decisions: fix thinko --- src/lib/Bcfg2/Server/Plugins/Decisions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 5521ea045..eae18fdfe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -62,6 +62,6 @@ class Decisions(Bcfg2.Server.Plugin.EntrySet, def GetDecisions(self, metadata, mode): ret = [] for cdt in self.get_matching(metadata): - if os.path.basename(cdt).startswith(mode): + if os.path.basename(cdt.name).startswith(mode): ret.extend(cdt.get_decisions()) return ret -- cgit v1.2.3-1-g7c22 From 257dff5cbe26a111b719201a0031da39a34b0fe0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 29 Jan 2013 08:08:51 -0500 Subject: TemplateHelper: import helper modules with munged names to avoid collisions --- src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 9c8314f50..ea7454e11 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -12,13 +12,26 @@ LOGGER = logging.getLogger(__name__) MODULE_RE = re.compile(r'(?P(?P[^\/]+)\.py)$') +def safe_module_name(module): + """ Munge the name of a TemplateHelper module to avoid collisions + with other Python modules. E.g., if someone has a helper named + 'ldap.py', it should not be added to ``sys.modules`` as ``ldap``, + but rather as something more obscure. """ + return '__TemplateHelper_%s' % module + + class HelperModule(object): """ Representation of a TemplateHelper module """ def __init__(self, name, fam=None): self.name = name self.fam = fam + + #: The name of the module as used by get_additional_data(). + #: the name of the file with .py stripped off. self._module_name = MODULE_RE.search(self.name).group('module') + + #: The attributes exported by this module self._attrs = [] def HandleEvent(self, event=None): @@ -32,7 +45,8 @@ class HelperModule(object): return try: - module = imp.load_source(self._module_name, self.name) + module = imp.load_source(safe_module_name(self._module_name), + self.name) except: # pylint: disable=W0702 err = sys.exc_info()[1] LOGGER.error("TemplateHelper: Failed to import %s: %s" % @@ -98,7 +112,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): module_name = MODULE_RE.search(helper).group(1) try: - module = imp.load_source(module_name, helper) + module = imp.load_source(safe_module_name(module_name), helper) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("templatehelper-import-error", -- cgit v1.2.3-1-g7c22 From 277701bbf6d44d0ad6dee2e2d7b4cd66676454e5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 29 Jan 2013 11:36:14 -0500 Subject: SSLCA: fixed appending chain certs --- src/lib/Bcfg2/Server/Plugins/SSLCA.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index b5cc11aa4..7d00201da 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -179,7 +179,7 @@ class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): self.logger.error("SSLCA: Failed to unlink temporary files: %s" % sys.exc_info()[1]) if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(self.parent.get_ca(ca)['chaincert']).read() + cert += open(ca['chaincert']).read() open(os.path.join(self.path, filename), 'w').write(cert) return cert -- cgit v1.2.3-1-g7c22 From 03b9d1774ad1c89eb51e7fc24f6b47267f9bb474 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 6 Feb 2013 13:59:47 -0500 Subject: Packages: fixed several bugs that could cause duplicate Package entries --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 6 +++++- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 26 +++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 31e583768..46231c636 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -807,7 +807,11 @@ class YumCollection(Collection): initial_names.append(pkg) new = [] for pkg in complete: - if pkg[0] not in initial_names: + if isinstance(pkg, tuple): + name = pkg[0] + else: + name = pkg + if name not in initial_names: new.append(pkg) return new diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f30e060bd..f112c65cd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -314,19 +314,16 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if collection is None: collection = self.get_collection(metadata) - # initial is the set of packages that are explicitly specified - # in the configuration - initial = set() - # base is the set of initial packages with groups expanded + # base is the set of initial packages -- explicitly + # given in the specification, from expanded package groups, + # and essential to the distribution base = set() - # essential pkgs are those marked as such by the distribution - essential = collection.get_essential() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - initial.update(collection.packages_from_entry(pkg)) + base.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -338,21 +335,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pkg, xml_declaration=False).decode('UTF-8')) + # remove package groups + for el in to_remove: + el.getparent().remove(el) + gpkgs = collection.get_groups(groups) for pkgs in gpkgs.values(): base.update(pkgs) - base.update(initial | essential) - for el in to_remove: - el.getparent().remove(el) + # essential pkgs are those marked as such by the distribution + base.update(collection.get_essential()) packages, unknown = collection.complete(base) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = collection.get_new_packages(initial, packages) - self.debug_log("Packages: %d initial, %d complete, %d new" % - (len(initial), len(packages), len(newpkgs))) + newpkgs = collection.get_new_packages(base, packages) + self.debug_log("Packages: %d base, %d complete, %d new" % + (len(base), len(packages), len(newpkgs))) newpkgs.sort() collection.packages_to_entry(newpkgs, independent) -- cgit v1.2.3-1-g7c22 From e1edce98761782fe51c451be4f5e114c75f46bd5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 8 Feb 2013 08:12:47 -0500 Subject: Cfg: better error handling from verifiers, :test --- .../Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py | 12 ++++++++++-- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 9 ++++----- 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index b702ac899..8e2d98db9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -23,10 +23,18 @@ class CfgExternalCommandVerifier(CfgVerifier): def verify_entry(self, entry, metadata, data): try: proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - err = proc.communicate(input=data)[1] + out, err = proc.communicate(input=data) rv = proc.wait() if rv != 0: - raise CfgVerificationError(err) + if not err: + if out: + # got nothing on stderr, try stdout + err = out + else: + err = "Non-zero return value %s" % rv + raise CfgVerificationError(err.strip()) + except CfgVerificationError: + raise except: err = sys.exc_info()[1] raise CfgVerificationError("Error running external command " diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index fcfaa393b..ec3ba222c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -589,11 +589,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, try: self._validate_data(entry, metadata, data) except CfgVerificationError: - msg = "Data for %s for %s failed to verify: %s" % \ - (entry.get('name'), metadata.hostname, - sys.exc_info()[1]) - self.logger.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("Failed to verify %s for %s: %s" % + (entry.get('name'), + metadata.hostname, + sys.exc_info()[1])) if entry.get('encoding') == 'base64': data = b64encode(data) -- cgit v1.2.3-1-g7c22 From 7241066deea16a9ebba718d7ecd157891e23cf33 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 13 Feb 2013 11:28:26 -0500 Subject: Defaults: change to GoalValidator to apply defaults after structures are bound (#1136) --- src/lib/Bcfg2/Server/Plugins/Defaults.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index f4d86a64f..04c14aa96 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -5,7 +5,7 @@ import Bcfg2.Server.Plugins.Rules class Defaults(Bcfg2.Server.Plugins.Rules.Rules, - Bcfg2.Server.Plugin.StructureValidator): + Bcfg2.Server.Plugin.GoalValidator): """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' @@ -22,27 +22,18 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - def validate_structures(self, metadata, structures): + def validate_goals(self, metadata, config): """ Apply defaults """ - for struct in structures: + for struct in config.getchildren(): for entry in struct.getchildren(): - if entry.tag.startswith("Bound"): - is_bound = True - entry.tag = entry.tag[5:] - else: - is_bound = False try: - try: - self.BindEntry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - # either no matching defaults (which is okay), - # or multiple matching defaults (which is not - # okay, but is logged). either way, we don't - # care about the error. - pass - finally: - if is_bound: - entry.tag = "Bound" + entry.tag + self.BindEntry(entry, metadata) + except Bcfg2.Server.Plugin.PluginExecutionError: + # either no matching defaults (which is okay), + # or multiple matching defaults (which is not + # okay, but is logged). either way, we don't + # care about the error. + pass @property def _regex_enabled(self): -- cgit v1.2.3-1-g7c22 From b4d6a72ae4db7a5b83415b6fcf632ed04475fb09 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 13 Feb 2013 16:30:47 -0500 Subject: fixed unit tests --- .../Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 8e2d98db9..313e53ee9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -26,13 +26,10 @@ class CfgExternalCommandVerifier(CfgVerifier): out, err = proc.communicate(input=data) rv = proc.wait() if rv != 0: - if not err: - if out: - # got nothing on stderr, try stdout - err = out - else: - err = "Non-zero return value %s" % rv - raise CfgVerificationError(err.strip()) + # pylint: disable=E1103 + raise CfgVerificationError(err.strip() or out.strip() or + "Non-zero return value %s" % rv) + # pylint: enable=E1103 except CfgVerificationError: raise except: -- cgit v1.2.3-1-g7c22 From e3f871022b9cc4bb4916c23920c6621be8c33e7d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Feb 2013 11:17:32 -0500 Subject: fixed checking Genshi templates for comments (#1141) --- src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 73550cd9d..4fa2fb894 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -190,6 +190,7 @@ class CfgGenshiGenerator(CfgGenerator): raise def handle_event(self, event): + CfgGenerator.handle_event(self, event) try: self.template = self.loader.load(self.name, cls=NewTextTemplate, encoding=self.encoding) -- cgit v1.2.3-1-g7c22 From c7fbaf3552b61cb272188a9cfc7590c0f14934a8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Feb 2013 11:17:40 -0500 Subject: better Genshi syntax lint checker --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index b200346bc..7030c1574 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -109,8 +109,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """ Add the correct child entry type to Bundler depending on whether the XML file in question is a plain XML file or a templated bundle """ - bundle = lxml.etree.parse(name, - parser=Bcfg2.Server.XMLParser) + bundle = lxml.etree.parse(name, parser=Bcfg2.Server.XMLParser) nsmap = bundle.getroot().nsmap if (name.endswith('.genshi') or ('py' in nsmap and -- cgit v1.2.3-1-g7c22 From ab208819c734f6cf46e88b24653fd02b1805773a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Feb 2013 13:07:30 -0500 Subject: bcfg2-lint: Fixed erroneous detection of multiple default groups (#1142) --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 5519d42b5..ef0c152fd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1469,7 +1469,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default"): + if grp.get("default", "false").lower() == "true": defaults.append(self.RenderXML(grp)) if len(defaults) > 1: self.LintError("multiple-default-groups", -- cgit v1.2.3-1-g7c22