diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins')
32 files changed, 613 insertions, 382 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py index d662da60a..a18204d60 100644 --- a/src/lib/Bcfg2/Server/Plugins/Base.py +++ b/src/lib/Bcfg2/Server/Plugins/Base.py @@ -20,13 +20,8 @@ class Base(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Base repository") - raise Bcfg2.Server.Plugin.PluginInitError + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, + self.core.fam) def BuildStructures(self, metadata): """Build structures for client described by metadata.""" diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 7030c1574..eef176cca 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -13,7 +13,7 @@ import Bcfg2.Server.Lint try: import genshi.template.base - import Bcfg2.Server.Plugins.TGenshi + from Bcfg2.Server.Plugins.TGenshi import removecomment, TemplateFile HAS_GENSHI = True except ImportError: HAS_GENSHI = False @@ -34,14 +34,12 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): if HAS_GENSHI: - class BundleTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile, + class BundleTemplateFile(TemplateFile, Bcfg2.Server.Plugin.StructFile): """ Representation of a Genshi-templated bundle XML file """ def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name, - specific, - encoding) + TemplateFile.__init__(self, name, specific, encoding) Bcfg2.Server.Plugin.StructFile.__init__(self, name) self.logger = logging.getLogger(name) @@ -52,9 +50,9 @@ if HAS_GENSHI: msg = "No parsed template information for %s" % self.name self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate(metadata=metadata, - repo=SETUP['repo']).filter( - Bcfg2.Server.Plugins.TGenshi.removecomment) + stream = self.template.generate( + metadata=metadata, + repo=SETUP['repo']).filter(removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), parser=Bcfg2.Server.XMLParser) @@ -85,23 +83,15 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$') + patterns = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) self.encoding = core.setup['encoding'] self.__child__ = self.template_dispatch - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - err = sys.exc_info()[1] - msg = "Failed to load Bundle repository %s: %s" % (self.data, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, + self.core.fam) global SETUP SETUP = core.setup @@ -154,10 +144,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ def Run(self): - """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and @@ -171,7 +161,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): "inconsistent-bundle-name": "warning"} def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ + """ Find bundles listed in Metadata but not implemented in + Bundler. """ if self.files is None: # when given a list of files on stdin, this check is # useless, so skip it @@ -190,7 +181,11 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): bundle) def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ + """ Verify bundle name attribute matches filename. + + :param bundle: The bundle to verify + :type bundle: Bcfg2.Server.Plugins.Bundler.BundleFile + """ try: xdata = lxml.etree.XML(bundle.data) except AttributeError: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index c2e5afbad..83a5c1165 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -90,7 +90,7 @@ class CfgGenshiGenerator(CfgGenerator): #: exception in a Genshi template so we can provide a decent error #: message that actually tells the end user where an error #: occurred. - pyerror_re = re.compile('<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') + pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index aaeb65cd6..c7b62f352 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -48,9 +48,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if (HAS_CRYPTO and SETUP.cfp.has_section("sshkeys") and SETUP.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases(SETUP)[SETUP.cfp.get( - "sshkeys", - "passphrase")] + return Bcfg2.Encryption.get_passphrases(SETUP)[ + SETUP.cfp.get("sshkeys", "passphrase")] return None def handle_event(self, event): @@ -70,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): the given client metadata, and may be obtained by doing ``self.XMLMatch(metadata)`` :type spec: lxml.etree._Element - :returns: None + :returns: string - The filename of the private key """ if spec is None: spec = self.XMLMatch(metadata) @@ -141,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if spec is None: spec = self.XMLMatch(metadata) category = spec.get("category", self.category) - print("category=%s" % category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index ec3ba222c..154cd5e63 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -35,6 +35,24 @@ SETUP = None #: facility for passing it otherwise. CFG = None +_HANDLERS = [] + + +def handlers(): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if not _HANDLERS: + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + hdlr = getattr(module, mname) + if issubclass(hdlr, CfgBaseFileMatcher): + _HANDLERS.append(hdlr) + _HANDLERS.sort(key=operator.attrgetter("__priority__")) + return _HANDLERS + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable): @@ -87,7 +105,7 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable.__init__(self) self.encoding = encoding __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ -""" + """ .. ----- .. autoattribute:: CfgBaseFileMatcher.__basenames__ .. autoattribute:: CfgBaseFileMatcher.__extensions__ @@ -111,12 +129,12 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, components = ['^(?P<basename>%s)' % '|'.join(re.escape(b) for b in basenames)] if cls.__specific__: - components.append('(|\\.H_(?P<hostname>\S+?)|' + - '\.G(?P<prio>\d+)_(?P<group>\S+?))') + components.append(r'(|\.H_(?P<hostname>\S+?)|' + + r'\.G(?P<prio>\d+)_(?P<group>\S+?))') if cls.__extensions__: - components.append('\\.(?P<extension>%s)' % - '|'.join(cls.__extensions__)) - components.append('$') + components.append(r'\.(?P<extension>%s)' % + r'|'.join(cls.__extensions__)) + components.append(r'$') return re.compile("".join(components)) @classmethod @@ -459,7 +477,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry_type, encoding) Bcfg2.Server.Plugin.Debuggable.__init__(self) self.specific = None - self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): @@ -468,24 +485,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if self._handlers is None: - self._handlers = [] - for submodule in walk_packages(path=__path__, - prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if CfgBaseFileMatcher in hdlr.__mro__: - self._handlers.append(hdlr) - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -502,7 +501,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in handlers(): if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -520,7 +519,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, return elif hdlr.ignore(event, basename=self.path): return - elif action == 'changed': + # we only get here if event.filename in self.entries, so handle + # created event like changed + elif action == 'changed' or action == 'created': self.entries[event.filename].handle_event(event) return elif action == 'deleted': @@ -580,10 +581,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, def bind_entry(self, entry, metadata): self.bind_info_to_entry(entry, metadata) - data = self._generate_data(entry, metadata) - - for fltr in self.get_handlers(metadata, CfgFilter): - data = fltr.modify_data(entry, metadata, data) + data, generator = self._generate_data(entry, metadata) + + if generator is not None: + # apply no filters if the data was created by a CfgCreator + for fltr in self.get_handlers(metadata, CfgFilter): + if fltr.specific <= generator.specific: + # only apply filters that are as specific or more + # specific than the generator used for this entry. + # Note that specificity comparison is backwards in + # this sense, since it's designed to sort from + # most specific to least specific. + data = fltr.modify_data(entry, metadata, data) if SETUP['validate']: try: @@ -599,6 +608,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, else: try: if not isinstance(data, unicode): + if not isinstance(data, str): + data = data.decode('utf-8') data = u_str(data, self.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), @@ -690,7 +701,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :type entry: lxml.etree._Element :param metadata: The client metadata to generate data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: string - the data for the entry + :returns: tuple of (string, generator) - the data for the + entry and the generator used to generate it (or + None, if data was created) """ try: generator = self.best_matching(metadata, @@ -699,7 +712,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, except PluginExecutionError: # if no creators or generators exist, _create_data() # raises an appropriate exception - return self._create_data(entry, metadata) + return (self._create_data(entry, metadata), None) if entry.get('mode').lower() == 'inherit': # use on-disk permissions @@ -709,7 +722,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set('mode', oct_mode(stat.S_IMODE(os.stat(fname).st_mode))) try: - return generator.get_data(entry, metadata) + return (generator.get_data(entry, metadata), generator) except: msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), sys.exc_info()[1]) @@ -758,8 +771,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, pass if not rv or not rv[0].hostname: - rv.append(Bcfg2.Server.Plugin.Specificity( - hostname=metadata.hostname)) + rv.append( + Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) return rv def build_filename(self, specific): @@ -884,12 +897,15 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): self.check_delta(basename, entry) self.check_pubkey(basename, entry) + self.check_missing_files() @classmethod def Errors(cls): return {"cat-file-used": "warning", "diff-file-used": "warning", - "no-pubkey-xml": "warning"} + "no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} def check_delta(self, basename, entry): """ check that no .cat or .diff files are in use """ @@ -923,3 +939,41 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-pubkey-xml", "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = [] + for hdlr in handlers(): + ignore.extend(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + all_files.update(os.path.join(root, fname) + for fname in files + if not any(fname.endswith("." + i) + for i in ignore)) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index ba1559a1a..22cacaa76 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -20,9 +20,9 @@ class Cvs(Bcfg2.Server.Plugin.Version): """Read cvs revision information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() return data[3].strip('\n') except IndexError: msg = "Failed to read CVS log" diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 0033e00f3..b4abafb0e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -20,9 +20,9 @@ class Darcs(Bcfg2.Server.Plugin.Version): """Read Darcs changeset information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() revision = data[0].strip('\n') except: msg = "Failed to read darcs repository" diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index eae18fdfe..66f299bc9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -2,7 +2,6 @@ blacklist certain entries. """ import os -import sys import lxml.etree import Bcfg2.Server.Plugin @@ -40,18 +39,10 @@ class Decisions(Bcfg2.Server.Plugin.EntrySet, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Decision.__init__(self) - Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', - self.data, - DecisionFile, + self.data, DecisionFile, core.setup['encoding']) - try: - core.fam.AddMonitor(self.data, self) - except OSError: - err = sys.exc_info()[1] - msg = 'Adding filemonitor for %s failed: %s' % (self.data, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) + core.fam.AddMonitor(self.data, self) def HandleEvent(self, event): """ Handle events on Decision files by passing them off to diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 5ec0d7280..8e074118f 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -24,7 +24,11 @@ import sys import pwd import grp import Bcfg2.Client.XML -from Bcfg2.Compat import b64encode, oct_mode +try: + from Bcfg2.Compat import b64encode, oct_mode +except ImportError: + from base64 import b64encode + oct_mode = oct path = "%s" @@ -67,7 +71,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, 'config.xml'), fam=core.fam, - should_monitor=True) + should_monitor=True, + create=self.name) self.entries = dict() self.probes = dict() @@ -225,11 +230,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, root = lxml.etree.Element("FileInfo") root.append(info) try: - open(infoxml, - "w").write( - lxml.etree.tostring(root, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) + root.getroottree().write(infoxml, xml_declaration=False, + pretty_print=True) except IOError: err = sys.exc_info()[1] self.logger.error("Could not write %s: %s" % (infoxml, err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index f6735df12..6165ac651 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -20,11 +20,11 @@ class Fossil(Bcfg2.Server.Plugin.Version): """Read fossil revision information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data if \ - line.split(': ')[0].strip() == 'checkout'][-1] + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() + revline = [line.split(': ')[1].strip() for line in data + if line.split(': ')[0].strip() == 'checkout'][-1] return revline.split(' ')[0] except IndexError: msg = "Failed to read fossil info" diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index c8362db41..44971aba7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -44,7 +44,7 @@ class Git(Version): else: cmd = ["git", "--git-dir", self.vcs_path, "--work-tree", self.vcs_root, "rev-parse", "HEAD"] - self.debug_log("Git: Running cmd") + self.debug_log("Git: Running %s" % cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) rv, err = proc.communicate() if proc.wait(): diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py new file mode 100644 index 000000000..810b273af --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -0,0 +1,47 @@ +""" GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. """ + +import os +import lxml.etree +import Bcfg2.Server.Plugin +try: + from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +except ImportError: + # BundleTemplateFile missing means that genshi is missing. we + # import genshi to get the _real_ error + import genshi # pylint: disable=W0611 + + +class GroupLogicConfig(BundleTemplateFile): + """ Representation of the GroupLogic groups.xml file """ + create = lxml.etree.Element("GroupLogic", + nsmap=dict(py="http://genshi.edgewall.org/")) + + def __init__(self, name, fam): + BundleTemplateFile.__init__(self, name, + Bcfg2.Server.Plugin.Specificity(), None) + self.fam = fam + self.should_monitor = True + self.fam.AddMonitor(self.name, self) + + def _match(self, item, metadata): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return BundleTemplateFile._match(self, item, metadata) + + +class GroupLogic(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ GroupLogic is a connector plugin that lets you use an XML + Genshi template to dynamically set additional groups for + clients. """ + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), + core.fam) + + def get_additional_groups(self, metadata): + return [el.get("name") + for el in self.config.get_xml_value(metadata).findall("Group")] diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 5716a134f..09685d972 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -3,7 +3,6 @@ import os import re import sys -import logging import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from Bcfg2.Utils import PackedDigitRange @@ -16,16 +15,16 @@ class PatternMap(object): self.pattern = pattern self.rangestr = rangestr self.groups = groups - if pattern != None: + if pattern is not None: self.re = re.compile(pattern) self.process = self.process_re - elif rangestr != None: + elif rangestr is not None: if '\\' in rangestr: raise Exception("Backslashes are not allowed in NameRanges") range_finder = r'\[\[[\d\-,]+\]\]' self.process = self.process_range - self.re = re.compile('^' + re.sub(range_finder, '(\d+)', - rangestr)) + self.re = re.compile(r'^' + re.sub(range_finder, r'(\d+)', + rangestr)) dmatcher = re.compile(re.sub(range_finder, r'\[\[([\d\-,]+)\]\]', rangestr)) @@ -67,6 +66,7 @@ class PatternMap(object): class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): """ representation of GroupPatterns config.xml """ __identifier__ = None + create = 'GroupPatterns' def __init__(self, filename, core=None): try: @@ -77,7 +77,6 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): should_monitor=True) self.core = core self.patterns = [] - self.logger = logging.getLogger(self.__class__.__name__) def Index(self): Bcfg2.Server.Plugin.XMLFileBacked.Index(self) @@ -130,7 +129,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for GroupPatterns """ + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8fb3a0998..4ed3dede5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -40,6 +40,8 @@ if HAS_DJANGO: """ dict-like object to make it easier to access client bcfg2 versions from the database """ + create = False + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version @@ -75,6 +77,7 @@ if HAS_DJANGO: yield client.hostname def keys(self): + """ Get keys for the mapping """ return [c.hostname for c in MetadataClientModel.objects.all()] def __contains__(self, key): @@ -94,9 +97,11 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): # then we immediately set should_monitor to the proper value, # so that XInclude'd files get properly watched fpath = os.path.join(metadata.data, basefile) + toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, fam=metadata.core.fam, - should_monitor=False) + should_monitor=False, + create=toptag) self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile @@ -326,6 +331,11 @@ class ClientMetadata(object): return grp return '' + def __repr__(self): + return "%s(%s, profile=%s, groups=%s)" % (self.__class__.__name__, + self.hostname, + self.profile, self.groups) + class MetadataQuery(object): """ This class provides query methods for the metadata of all @@ -439,7 +449,7 @@ class MetadataQuery(object): return [self.by_name(name) for name in self.all_clients()] -class MetadataGroup(tuple): +class MetadataGroup(tuple): # pylint: disable=E0012,R0924 """ representation of a metadata group. basically just a named tuple """ # pylint: disable=R0913,W0613 @@ -549,6 +559,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, open(os.path.join(repo, cls.name, fname), "w").write(kwargs[aname]) + @property + def use_database(self): + """ Expose self._use_db publicly for use in + :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` """ + return self._use_db + def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ @@ -595,7 +611,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _add_xdata(self, config, tag, name, attribs=None, alias=False): """ Generic method to add XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node != None: + if node is not None: raise Bcfg2.Server.Plugin.MetadataConsistencyError("%s \"%s\" " "already exists" % (tag, name)) @@ -655,7 +671,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _update_xdata(self, config, tag, name, attribs, alias=False): """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node == None: + if node is None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % @@ -672,7 +688,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """Update a groups attributes.""" if self._use_db: msg = "Metadata does not support updating groups with " + \ - "use_database enabled" + "use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: @@ -700,7 +716,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _remove_xdata(self, config, tag, name): """ Generic method to remove XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata) - if node == None: + if node is None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % @@ -936,16 +952,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if group not in self.groups: self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - for gname, ginfo in list(self.groups.items()): - for group in ginfo.groups: - if group not in self.groups: - self.debug_log("Group %s set as nonexistent group %s" % - (gname, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" - self.logger.info("Asserting client %s profile to %s" % - (client, profile)) + self.logger.info("Asserting client %s profile to %s" % (client, + profile)) if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") @@ -954,7 +966,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -996,19 +1008,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.clients_xml.write() def set_version(self, client, version): - """Set group parameter for provided client.""" - if client in self.clients: - if client not in self.versions or version != self.versions[client]: - self.logger.info("Setting client %s version to %s" % - (client, version)) - if not self._use_db: - self.update_client(client, dict(version=version)) - self.clients_xml.write() - self.versions[client] = version - else: - msg = "Cannot set version on non-existent client %s" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + """Set version for provided client.""" + if client not in self.clients: + # this creates the client as a side effect + self.get_initial_metadata(client) + + if client not in self.versions or version != self.versions[client]: + self.logger.info("Setting client %s version to %s" % (client, + version)) + if not self._use_db: + self.update_client(client, dict(version=version)) + self.clients_xml.write() + self.versions[client] = version def resolve_client(self, addresspair, cleanup_cache=False): """Lookup address locally or in DNS to get a hostname.""" @@ -1085,7 +1096,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: return self.core.metadata_cache[client] @@ -1096,6 +1106,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() profile = None + def _add_group(grpname): + """ Add a group to the set of groups for this client. + Handles setting categories and category suppression. + Returns the new profile for the client (which might be + unchanged). """ + groups.add(grpname) + if grpname in self.groups: + group = self.groups[grpname] + category = group.category + if category: + if category in categories: + self.logger.warning("%s: Group %s suppressed by " + "category %s; %s already a member " + "of %s" % + (self.name, grpname, category, + client, categories[category])) + return + categories[category] = grpname + if not profile and group.is_profile: + return grpname + else: + return profile + if client not in self.clients: pgroup = None if client in self.clientgroups: @@ -1104,42 +1137,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) - groups.add(pgroup) - category = self.groups[pgroup].category - if category: - categories[category] = pgroup - if (pgroup in self.groups and self.groups[pgroup].is_profile): - profile = pgroup + self.set_profile(client, pgroup, (None, None), + require_public=False) + profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) - if client in self.clientgroups: - for cgroup in self.clientgroups[client]: - if cgroup in groups: - continue - if cgroup not in self.groups: - self.groups[cgroup] = MetadataGroup(cgroup) - category = self.groups[cgroup].category - if category and category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, cgroup, category, - client, categories[category])) - continue - if category: - categories[category] = cgroup - groups.add(cgroup) - # favor client groups for setting profile - if not profile and self.groups[cgroup].is_profile: - profile = cgroup + for cgroup in self.clientgroups.get(client, []): + if cgroup in groups: + continue + if cgroup not in self.groups: + self.groups[cgroup] = MetadataGroup(cgroup) + profile = _add_group(cgroup) groups, categories = self._merge_groups(client, groups, categories=categories) + if len(groups) == 0 and self.default: + # no initial groups; add the default profile + profile = _add_group(self.default) + groups, categories = self._merge_groups(client, groups, + categories=categories) + bundles = set() for group in groups: try: @@ -1466,7 +1487,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for Metadata """ + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ def Run(self): self.nested_clients() @@ -1475,6 +1505,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.duplicate_groups() self.duplicate_default_groups() self.duplicate_clients() + self.default_is_profile() @classmethod def Errors(cls): @@ -1484,11 +1515,15 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "non-profile-set-as-profile": "error", "duplicate-group": "error", "duplicate-client": "error", - "multiple-default-groups": "error"} + "multiple-default-groups": "error", + "default-is-not-profile": "error"} def deprecated_options(self): - """ check for the location='floating' option, which has been - deprecated in favor of floating='true' """ + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return clientdata = self.metadata.clients_xml.xdata for el in clientdata.xpath("//Client"): loc = el.get("location") @@ -1503,8 +1538,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (loc, floating, self.RenderXML(el))) def nested_clients(self): - """ check for a Client tag inside a Client tag, which doesn't - make any sense """ + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", @@ -1512,8 +1547,11 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (el.get("name"), self.RenderXML(el))) def bogus_profiles(self): - """ check for clients that have profiles that are either not - flagged as public groups in groups.xml, or don't exist """ + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return for client in self.metadata.clients_xml.xdata.findall('.//Client'): profile = client.get("profile") if profile not in self.metadata.groups: @@ -1528,20 +1566,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (profile, client.get("name"), profile, self.RenderXML(client))) - def duplicate_groups(self): - """ check for groups that are defined twice. We count a group - tag as a definition if it a) has profile or public set; or b) - has any children. """ - self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), - "group", - include=lambda g: (g.get("profile") or - g.get("public") or - g.getchildren())) - def duplicate_default_groups(self): - """ check for multiple default groups """ + """ Check for multiple default groups. """ defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): @@ -1553,24 +1579,55 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "\n".join(defaults)) def duplicate_clients(self): - """ check for clients that are defined twice. """ + """ Check for clients that are defined more than once. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return self.duplicate_entries( self.metadata.clients_xml.xdata.xpath("//Client"), "client") - def duplicate_entries(self, allentries, etype, include=None): - """ generic duplicate entry finder """ - if include is None: - include = lambda e: True + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ entries = dict() for el in allentries: - if include(el): - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] for ename, els in entries.items(): if len(els) > 1: self.LintError("duplicate-%s" % etype, "%s %s is defined multiple times:\n%s" % (etype.title(), ename, "\n".join(els))) + + def default_is_profile(self): + """ Ensure that the default group is a profile group. """ + if (self.metadata.default and + not self.metadata.groups[self.metadata.default].is_profile): + xdata = \ + self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % + self.metadata.default)[0] + self.LintError("default-is-not-profile", + "Default group is not a profile group:\n%s" % + self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index c39bd4c42..466665382 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -5,26 +5,9 @@ import re import sys import glob import socket -import logging import Bcfg2.Server import Bcfg2.Server.Plugin -LOGGER = logging.getLogger(__name__) - - -class NagiosGenConfig(Bcfg2.Server.Plugin.StructFile): - """ NagiosGen config file handler """ - - def __init__(self, filename, fam): - # create config.xml if missing - if not os.path.exists(filename): - LOGGER.warning("NagiosGen: %s missing. " - "Creating empty one for you." % filename) - open(filename, "w").write("<NagiosGen></NagiosGen>") - - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, - should_monitor=True) - class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): @@ -36,8 +19,11 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Generator.__init__(self) - self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'), - core.fam) + self.config = \ + Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, + 'config.xml'), + core.fam, should_monitor=True, + create=self.name) self.Entries = {'Path': {'/etc/nagiosgen.status': self.createhostconfig, '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ebc03197e..1ec3cbd60 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -2,8 +2,10 @@ operating system using ohai (http://wiki.opscode.com/display/chef/Ohai) """ -import lxml.etree import os +import sys +import glob +import lxml.etree import Bcfg2.Server.Plugin try: @@ -31,22 +33,39 @@ class OhaiCache(object): self.dirname = dirname self.cache = dict() + def hostpath(self, host): + """ Get the path to the file that contains Ohai data for the + given host """ + return os.path.join(self.dirname, "%s.json" % host) + def __setitem__(self, item, value): - if value == None: + if value is None: # simply return if the client returned nothing return self.cache[item] = json.loads(value) - open("%s/%s.json" % (self.dirname, item), 'w').write(value) + open(self.hostpath(item), 'w').write(value) def __getitem__(self, item): if item not in self.cache: try: - data = open("%s/%s.json" % (self.dirname, item)).read() + data = open(self.hostpath(item)).read() except: raise KeyError(item) self.cache[item] = json.loads(data) return self.cache[item] + def __delitem__(self, item): + if item in self.cache: + del self.cache[item] + try: + os.unlink(self.hostpath(item)) + except: + raise IndexError("Could not unlink %s: %s" % (self.hostpath(item), + sys.exc_info()[1])) + + def __len__(self): + return len(glob.glob(self.hostpath('*'))) + def __iter__(self): data = list(self.cache.keys()) data.extend([x[:-5] for x in os.listdir(self.dirname)]) @@ -69,10 +88,6 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', interpreter='/bin/sh') self.probe.text = PROBECODE - try: - os.stat(self.data) - except OSError: - os.makedirs(self.data) self.cache = OhaiCache(self.data) def GetProbes(self, _): diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 490ee6f20..71128d64c 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -9,13 +9,15 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """POSIXCompat is a goal validator plugin for POSIX entries.""" + create = False + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.GoalValidator.__init__(self) def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" - if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0): # do not care about a client that is _any_ 1.3.0 release # (including prereleases and RCs) return diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 27f493677..a82a183d8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -30,8 +30,8 @@ class AptCollection(Collection): """ Get an APT configuration file (i.e., ``sources.list``). :returns: string """ - lines = ["# This config was generated automatically by the Bcfg2 " \ - "Packages plugin", ''] + lines = ["# This config was generated automatically by the Bcfg2 " + "Packages plugin", ''] for source in self: if source.rawurl: @@ -40,6 +40,11 @@ class AptCollection(Collection): else: lines.append("deb %s %s %s" % (source.url, source.version, " ".join(source.components))) + if source.debsrc: + lines.append("deb-src %s %s %s" % + (source.url, + source.version, + " ".join(source.components))) lines.append("") return "\n".join(lines) @@ -93,6 +98,8 @@ class AptSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() @@ -104,8 +111,8 @@ class AptSource(Source): vindex = 0 for dep in words[1].split(','): if '|' in dep: - cdeps = [re.sub('\s+', '', - re.sub('\(.*\)', '', cdep)) + cdeps = [re.sub(r'\s+', '', + re.sub(r'\(.*\)', '', cdep)) for cdep in dep.split('|')] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, @@ -114,7 +121,7 @@ class AptSource(Source): bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: - raw_dep = re.sub('\(.*\)', '', dep) + raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 2735e389a..332f0bbab 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -7,6 +7,7 @@ import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Packages.Source import SourceInitError +# pylint: disable=E0012,R0924 class PackagesSources(Bcfg2.Server.Plugin.StructFile, Bcfg2.Server.Plugin.Debuggable): """ PackagesSources handles parsing of the @@ -16,6 +17,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, each ``Source`` tag. """ __identifier__ = None + create = "Sources" def __init__(self, filename, cachepath, fam, packages, setup): """ @@ -39,14 +41,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, If ``sources.xml`` cannot be read """ Bcfg2.Server.Plugin.Debuggable.__init__(self) - try: - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, - should_monitor=True) - except OSError: - err = sys.exc_info()[1] - msg = "Packages: Failed to read configuration file: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + should_monitor=True) #: The full path to the directory where #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data @@ -129,7 +125,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass object from XML representation of a source in ``sources.xml``. - ``source_from-xml`` determines the appropriate subclass of + ``source_from_xml`` determines the appropriate subclass of ``Source`` to instantiate according to the ``type`` attribute of the ``Source`` tag. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 985405e65..22073493c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -52,8 +52,8 @@ import re import sys import Bcfg2.Server.Plugin from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ - urlopen, cPickle, md5 + HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ + cPickle, md5 def fetch_url(url): @@ -65,7 +65,7 @@ def fetch_url(url): :raises: URLError - Failure fetching URL :returns: string - the content of the page at the given URL """ if '@' in url: - mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) + mobj = re.match(r'(\w+://)([^:]+):([^@]+)@(.*)$', url) if not mobj: raise ValueError("Invalid URL") user = mobj.group(2) @@ -158,6 +158,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = xsource.get('debsrc', 'false') == 'true' + #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is #: applicable; most backends do not generate any sort of @@ -315,7 +319,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: OSError - If the saved data cannot be read :raises: cPickle.UnpicklingError - If the saved data is corrupt """ - data = open(self.cachefile) + data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, self.essentialpkgs) = cPickle.load(data) @@ -615,7 +619,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'w').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6b8ed1f7d..4608bcca5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -66,7 +66,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ # pylint: enable=W0622 from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ - fetch_url + fetch_url LOGGER = logging.getLogger(__name__) @@ -281,7 +281,7 @@ class YumCollection(Collection): #: Define a unique cache file for this collection to use #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, - "cache-%s" % self.cachekey) + "cache-%s" % self.cachekey) if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) @@ -422,7 +422,7 @@ class YumCollection(Collection): config.add_section(reponame) added = True except ConfigParser.DuplicateSectionError: - match = re.search("-(\d+)", reponame) + match = re.search(r'-(\d+)', reponame) if match: rid = int(match.group(1)) + 1 else: @@ -675,7 +675,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -838,12 +841,13 @@ class YumCollection(Collection): return Collection.complete(self, packagelist) if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -874,11 +878,16 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - verbose = self.debug_flag or self.setup['verbose'] - if verbose: + if self.setup['verbose']: + cmd.append("-v") + if self.debug_flag: + if not self.setup['verbose']: + # ensure that running in debug gets -vv, even if + # verbose is not enabled + cmd.append("-v") cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) + self.debug_log("Packages: running %s" % " ".join(cmd)) try: helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) except OSError: @@ -893,19 +902,27 @@ class YumCollection(Collection): else: (stdout, stderr) = helper.communicate() rv = helper.wait() + errlines = stderr.splitlines() if rv: + if not errlines: + errlines.append("No error output") self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) - else: + "(returned %d): %s" % (rv, errlines[0])) + for line in errlines[1:]: + self.logger.error("Packages: %s" % line) + elif errlines: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr, flag=verbose) + errlines[0]) + for line in errlines[1:]: + self.debug_log("Packages: %s" % line) + try: return json.loads(stdout) except ValueError: err = sys.exc_info()[1] self.logger.error("Packages: error reading bcfg2-yum-helper " "output: %s" % err) - return None + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -931,13 +948,21 @@ class YumCollection(Collection): if force_update: # we call this twice: one to clean up data from the old # config, and once to clean up data from the new config - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass os.unlink(self.cfgfile) self.write_config() if force_update: - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): @@ -1120,9 +1145,9 @@ class YumSource(Source): self.packages['global'] = copy.deepcopy(sdata.pop()) except IndexError: self.logger.error("Packages: No packages in repo") + self.packages['global'] = set() while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) + self.packages['global'].update(sdata.pop()) for key in self.packages: if key == 'global': @@ -1169,7 +1194,7 @@ class YumSource(Source): if entry.get('name').startswith('/'): self.needed_paths.add(entry.get('name')) pro = pdata.find(RP + 'provides') - if pro != None: + if pro is not None: for entry in pro.getchildren(): prov = entry.get('name') if prov not in self.provides[arch]: @@ -1185,9 +1210,9 @@ class YumSource(Source): try: groupid = group.xpath('id')[0].text self.yumgroups[groupid] = {'mandatory': list(), - 'default': list(), - 'optional': list(), - 'conditional': list()} + 'default': list(), + 'optional': list(), + 'conditional': list()} except IndexError: continue try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index c3eadc6bb..f82b8a392 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,7 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" +APT_CONFIG_DEFAULT = \ + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, @@ -184,6 +185,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) + def get_config(self, metadata): + """ Get yum/apt config, as a string, for the specified client. + + :param metadata: The client to create the config for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ + return self.get_collection(metadata).get_config() + def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: @@ -239,14 +248,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == \ - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == \ - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if (entry.get("name") == + self.core.setup.cfp.get("packages", + "yum_config", + default=YUM_CONFIG_DEFAULT) or + entry.get("name") == + self.core.setup.cfp.get("packages", + "apt_config", + default=APT_CONFIG_DEFAULT)): return True return False @@ -450,7 +459,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) - except HTTPError: + except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) @@ -518,8 +527,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection = cclass(metadata, relevant, self.cachepath, self.data, self.core.fam, debug=self.debug_flag) ckey = collection.cachekey - self.clients[metadata.hostname] = ckey - self.collections[ckey] = collection + if cclass != Collection: + self.clients[metadata.hostname] = ckey + self.collections[ckey] = collection return collection def get_additional_data(self, metadata): @@ -536,7 +546,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data()) + return dict(sources=collection.get_additional_data(), + get_config=self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 7dac907e1..a1dcb575f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + def Run(self): pset = set() for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', @@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) - + @classmethod def Errors(cls): - return {"duplicate-packages":"error"} + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index f106b75a4..0974184b4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -12,6 +12,7 @@ import Bcfg2.Server.Plugin try: from django.db import models + from django.core.exceptions import MultipleObjectsReturned HAS_DJANGO = True class ProbesDataModel(models.Model, @@ -58,7 +59,7 @@ class ClientProbeDataSet(dict): dict.__init__(self, *args, **kwargs) -class ProbeData(str): +class ProbeData(str): # pylint: disable=E0012,R0924 """ a ProbeData object emulates a str object, but also has .xdata, .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ @@ -111,15 +112,15 @@ class ProbeData(str): class ProbeSet(Bcfg2.Server.Plugin.EntrySet): """ Handle universal and group- and host-specific probe files """ - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") + ignore = re.compile(r'^(\.#.*|.*~|\..*\.(tmp|sw[px])|probed\.xml)$') probename = \ - re.compile("(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$") - bangline = re.compile('^#!\s*(?P<interpreter>.*)$') + re.compile(r'(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$') + bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$') basename_is_regex = True def __init__(self, path, fam, encoding, plugin_name): self.plugin_name = plugin_name - Bcfg2.Server.Plugin.EntrySet.__init__(self, '[0-9A-Za-z_\-]+', path, + Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, Bcfg2.Server.Plugin.SpecificData, encoding) fam.AddMonitor(path, self) @@ -153,7 +154,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - probe.text = entry.data + if (metadata.version_info and + metadata.version_info > (1, 3, 1, '', 0)): + try: + probe.text = entry.data.decode('utf-8') + except AttributeError: + probe.text = entry.data + else: + try: + probe.text = entry.data + except: # pylint: disable=W0702 + self.logger.error("Client unable to handle unicode " + "probes. Skipping %s" % + probe.get('name')) + continue match = self.bangline.match(entry.data.split('\n')[0]) if match: probe.set('interpreter', match.group('interpreter')) @@ -209,15 +223,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: - datafile = open(os.path.join(self.data, 'probed.xml'), 'w') - datafile.write(lxml.etree.tostring( - top, xml_declaration=False, - pretty_print='true').decode('UTF-8')) + top.getroottree().write(os.path.join(self.data, 'probed.xml'), + xml_declaration=False, + pretty_print='true') except IOError: err = sys.exc_info()[1] self.logger.error("Failed to write probed.xml: %s" % err) @@ -232,21 +246,25 @@ class Probes(Bcfg2.Server.Plugin.Probing, if pdata.data != data: pdata.data = data pdata.save() + ProbesDataModel.objects.filter( hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: - ProbesGroupsModel.objects.get(hostname=client.hostname, - group=group) - except ProbesGroupsModel.DoesNotExist: - grp = ProbesGroupsModel(hostname=client.hostname, - group=group) - grp.save() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) + except MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=client.hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() + group__in=self.cgroups[client.hostname]).delete() def load_data(self): """ Load probe data from the appropriate backend (probed.xml @@ -320,7 +338,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, def ReceiveDataItem(self, client, data, cgroups, cprobedata): """Receive probe results pertaining to client.""" - if data.text == None: + if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) cprobedata[data.get('name')] = ProbeData('') diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 3ebad40e3..e97f66675 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -266,8 +266,13 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): return repr(self.xdata) -class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - """ A collection of properties files. """ +class Properties(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.DirectoryBacked): + """ The properties plugin maps property files into client metadata + instances. """ + + #: Extensions that are understood by Properties. extensions = ["xml"] if HAS_JSON: extensions.append("json") @@ -284,14 +289,18 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): #: Ignore XML schema (``.xsd``) files ignore = re.compile(r'.*\.xsd$') - def __init__(self, data, fam): - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, data, fam) + def __init__(self, core, datastore): + global SETUP # pylint: disable=W0603 + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) + SETUP = core.setup #: Instead of creating children of this object with a static #: object, we use :func:`property_dispatcher` to create a #: child of the appropriate subclass of :class:`PropertyFile` self.__child__ = self.property_dispatcher - __init__.__doc__ = Bcfg2.Server.Plugin.DirectoryBacked.__init__.__doc__ + __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def property_dispatcher(self, fname, fam): """ Dispatch an event on a Properties file to the @@ -314,30 +323,9 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) - -class Properties(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """ The properties plugin maps property files into client metadata - instances. """ - - def __init__(self, core, datastore): - global SETUP # pylint: disable=W0603 - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - SETUP = core.setup - try: - self.store = PropDirectoryBacked(self.data, core.fam) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s" % - err) - raise Bcfg2.Server.Plugin.PluginInitError - - __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def get_additional_data(self, metadata): rv = dict() - for fname, pfile in self.store.entries.items(): + for fname, pfile in self.entries.items(): rv[fname] = pfile.get_additional_data(metadata) return rv get_additional_data.__doc__ = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index a6dc2c1ef..3354763d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -92,10 +92,11 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): # try 3 times to store the data for i in [1, 2, 3]: try: - self.transport.store(client.hostname, cdata, - lxml.etree.tostring( + self.transport.store( + client.hostname, cdata, + lxml.etree.tostring( stats, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False)) self.debug_log("%s: Queued statistics data for %s" % (self.__class__.__name__, client.hostname)) return diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index c7db67301..d8b3104b7 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -201,10 +201,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, if specific.hostname and specific.hostname in names: hostnames = names[specific.hostname] elif specific.group: - hostnames = list(chain( + hostnames = list( + chain( *[names[cmeta.hostname] - for cmeta in \ - mquery.by_groups([specific.group])])) + for cmeta in + mquery.by_groups([specific.group])])) elif specific.all: # a generic key for all hosts? really? hostnames = list(chain(*list(names.values()))) @@ -287,7 +288,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([info[4][0] + for info in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7d00201da..f111ffc60 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec): def get_spec(self, metadata): rv = SSLCAXMLSpec.get_spec(self, metadata) rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "SubjectAltName"] + if e.tag == "subjectAltName"] return rv diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index 0aea439f9..41e6bf8b5 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -6,7 +6,9 @@ import Bcfg2.Server.Plugin class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ - name = 'ServiceCompat' + + create = False + __author__ = 'bcfg-dev@mcs.anl.gov' mode_map = {('true', 'true'): 'default', ('interactive', 'true'): 'interactive_only', @@ -14,7 +16,7 @@ class ServiceCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, config): """ Apply defaults """ - if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0): # do not care about a client that is _any_ 1.3.0 release # (including prereleases and RCs) return diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 51f44c52d..240fd7f89 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -59,9 +59,48 @@ class Svn(Bcfg2.Server.Plugin.Version): self.client.callback_conflict_resolver = \ self.get_conflict_resolver(choice) + try: + if self.core.setup.cfp.get( + "svn", + "always_trust").lower() == "true": + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.debug("Svn: Using subversion cache for SSL " + "certificate trust") + + try: + if (self.core.setup.cfp.get("svn", "user") and + self.core.setup.cfp.get("svn", "password")): + self.client.callback_get_login = \ + self.get_login + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.info("Svn: Using subversion cache for " + "password-based authetication") + self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) + # pylint: disable=W0613 + def get_login(self, realm, username, may_save): + """ PySvn callback to get credentials for HTTP basic authentication """ + self.logger.debug("Svn: Logging in with username: %s" % + self.core.setup.cfp.get("svn", "user")) + return True, \ + self.core.setup.cfp.get("svn", "user"), \ + self.core.setup.cfp.get("svn", "password"), \ + False + # pylint: enable=W0613 + + def ssl_server_trust_prompt(self, trust_dict): + """ PySvn callback to always trust SSL certificates from SVN server """ + self.logger.debug("Svn: Trusting SSL certificate from %s, " + "issued by %s for realm %s" % + (trust_dict['hostname'], + trust_dict['issuer_dname'], + trust_dict['realm'])) + return True, trust_dict['failures'], False + def get_conflict_resolver(self, choice): """ Get a PySvn conflict resolution callback """ def callback(conflict_description): diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index ea7454e11..db7370f01 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -82,7 +82,7 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to provide helper classes and functions to templates """ __author__ = 'chris.a.st.pierre@gmail.com' - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE __child__ = HelperModule @@ -97,18 +97,33 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.reserved_keywords = dir(HelperModule("foo.py")) def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper): + if self.HandlesFile(helper.name): self.check_helper(helper.name) def check_helper(self, helper): - """ check a helper module for export errors """ + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ module_name = MODULE_RE.search(helper).group(1) try: diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index b33eeba28..ad51cf368 100644 --- a/src/lib/Bcfg2/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,32 +1,5 @@ """Imports for Bcfg2.Server.Plugins.""" -__all__ = [ - 'Account', - 'Base', - 'Bundler', - 'Bzr', - 'Cfg', - 'Cvs', - 'Darcs', - 'Decisions', - 'Fossil', - 'Git', - 'GroupPatterns', - 'Hg', - 'Hostbase', - 'Metadata', - 'NagiosGen', - 'Ohai', - 'Packages', - 'Properties', - 'Probes', - 'Pkgmgr', - 'Rules', - 'SSHbase', - 'Snapshots', - 'Statistics', - 'Svn', - 'TCheetah', - 'Trigger', - 'TGenshi', - ] +from Bcfg2.Compat import walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] |