From 03ae03918aea6ed1a8f1ada1f02b7af0ad84c3ea Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 May 2013 08:10:29 -0400 Subject: Metadata: don't require all profile groups to be public when using metadata db --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bdf3b87fe..8aeda9f81 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +957,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1128,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client -- cgit v1.2.3-1-g7c22 From 11fe02d3358aa7ff4634f924a04fc1f5274ed6d4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 May 2013 09:34:56 -0400 Subject: fixed pylint test --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8aeda9f81..71e81c1fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair, require_public=True): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) -- cgit v1.2.3-1-g7c22 From 8d2ea0ad333b023aed1be7e2059c4200aa80ee76 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 2 May 2013 13:05:15 +0200 Subject: SSHbase: add support for ipv6 addresses in known_hosts file --- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index fc07a90e9..5aa7c4d9e 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -288,7 +288,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: -- cgit v1.2.3-1-g7c22 From 69f5a49af88de48a9c64f7c57abec8fc5dc41f45 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 7 May 2013 10:46:27 -0400 Subject: fixed long lines --- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 5aa7c4d9e..d8b3104b7 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -288,7 +288,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)]) + ipaddr = set([info[4][0] + for info in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: -- cgit v1.2.3-1-g7c22 From 4f745cc2731f7035f02566ba8bc1a0e9ae1b1a71 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 10 May 2013 11:03:35 -0500 Subject: Probes: Fix failing nosetests Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Probes.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 309b96475..f8baddb4b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data.encode('utf-8')) + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -222,15 +222,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: -- cgit v1.2.3-1-g7c22 From e55b0dc3b96cdac318dcf07309d18e763ccb7229 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 10 May 2013 12:28:58 -0500 Subject: APT: Allow specification of deb-src lines (Resolves #1148) Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 5 +++++ src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index bc2928fa6..a82a183d8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -40,6 +40,11 @@ class AptCollection(Collection): else: lines.append("deb %s %s %s" % (source.url, source.version, " ".join(source.components))) + if source.debsrc: + lines.append("deb-src %s %s %s" % + (source.url, + source.version, + " ".join(source.components))) lines.append("") return "\n".join(lines) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 7ba374dd3..22073493c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -158,6 +158,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = xsource.get('debsrc', 'false') == 'true' + #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is #: applicable; most backends do not generate any sort of -- cgit v1.2.3-1-g7c22 From 0887f4c929ba1354dd1fa78eccd1a7c5151a0ab7 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 10 May 2013 15:40:49 -0400 Subject: Packages: handle URLErrors (e.g., timeouts) when downloading GPG keys better --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index efbca28cd..d5773de97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,7 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -459,7 +459,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) - except HTTPError: + except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) -- cgit v1.2.3-1-g7c22 From 34aff9f18f7a8ee59e8e07ceaf89d79bd6e96509 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 14 May 2013 11:43:14 -0400 Subject: doc: added devel docs for bcfg2-lint plugins --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 13 +++-- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 7 ++- src/lib/Bcfg2/Server/Plugins/Metadata.py | 76 +++++++++++++++----------- src/lib/Bcfg2/Server/Plugins/Pkgmgr.py | 16 ++++-- src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 19 ++++++- 5 files changed, 87 insertions(+), 44 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 5c5e3da0c..eef176cca 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -144,10 +144,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ + """ Perform various :ref:`Bundler + ` checks. """ def Run(self): - """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and @@ -161,7 +161,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): "inconsistent-bundle-name": "warning"} def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ + """ Find bundles listed in Metadata but not implemented in + Bundler. """ if self.files is None: # when given a list of files on stdin, this check is # useless, so skip it @@ -180,7 +181,11 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): bundle) def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ + """ Verify bundle name attribute matches filename. + + :param bundle: The bundle to verify + :type bundle: Bcfg2.Server.Plugins.Bundler.BundleFile + """ try: xdata = lxml.etree.XML(bundle.data) except AttributeError: diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 8d1e50526..09685d972 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -129,7 +129,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for GroupPatterns """ + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + ` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 71e81c1fe..ceb1d9080 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1479,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for Metadata """ + """ ``bcfg2-lint`` plugin for :ref:`Metadata + `. This checks for several things: + + * ```` tags nested inside other ```` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ def Run(self): self.nested_clients() @@ -1502,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "default-is-not-profile": "error"} def deprecated_options(self): - """ check for the location='floating' option, which has been - deprecated in favor of floating='true' """ + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1521,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (loc, floating, self.RenderXML(el))) def nested_clients(self): - """ check for a Client tag inside a Client tag, which doesn't - make any sense """ + """ Check for a ```` tag inside a ```` tag, + which is either redundant or will never match. """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", @@ -1530,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (el.get("name"), self.RenderXML(el))) def bogus_profiles(self): - """ check for clients that have profiles that are either not - flagged as public groups in groups.xml, or don't exist """ + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1549,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (profile, client.get("name"), profile, self.RenderXML(client))) - def duplicate_groups(self): - """ check for groups that are defined twice. We count a group - tag as a definition if it a) has profile or public set; or b) - has any children. """ - self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), - "group", - include=lambda g: (g.get("profile") or - g.get("public") or - g.getchildren())) - def duplicate_default_groups(self): - """ check for multiple default groups """ + """ Check for multiple default groups. """ defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): @@ -1574,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "\n".join(defaults)) def duplicate_clients(self): - """ check for clients that are defined twice. """ + """ Check for clients that are defined more than once. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1582,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.metadata.clients_xml.xdata.xpath("//Client"), "client") - def duplicate_entries(self, allentries, etype, include=None): - """ generic duplicate entry finder """ - if include is None: - include = lambda e: True + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-``) and for + display to the end user. + :type etype: string + """ entries = dict() for el in allentries: - if include(el): - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] for ename, els in entries.items(): if len(els) > 1: self.LintError("duplicate-%s" % etype, @@ -1600,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (etype.title(), ename, "\n".join(els))) def default_is_profile(self): - """ ensure that the default group is a profile group """ + """ Ensure that the default group is a profile group. """ if (self.metadata.default and not self.metadata.groups[self.metadata.default].is_profile): xdata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 7dac907e1..a1dcb575f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ Find duplicate :ref:`Pkgmgr + ` entries with the same + priority. """ + def Run(self): pset = set() for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', @@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) - + @classmethod def Errors(cls): - return {"duplicate-packages":"error"} + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 7dd15f7b5..fcd73bae2 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -97,7 +97,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + ` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.reserved_keywords = dir(HelperModule("foo.py")) @@ -108,7 +119,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): self.check_helper(helper.name) def check_helper(self, helper): - """ check a helper module for export errors """ + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ module_name = MODULE_RE.search(helper).group(1) try: -- cgit v1.2.3-1-g7c22