diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins')
66 files changed, 2527 insertions, 4177 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/ACL.py b/src/lib/Bcfg2/Server/Plugins/ACL.py new file mode 100644 index 000000000..1c1e54312 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/ACL.py @@ -0,0 +1,145 @@ +""" Support for client ACLs based on IP address and client metadata """ + +import os +import struct +import socket +import Bcfg2.Server.Plugin + + +def rmi_names_equal(first, second): + """ Compare two XML-RPC method names and see if they match. + Resolves some limited wildcards; see + :ref:`server-plugins-misc-acl-wildcards` for details. + + :param first: One of the ACLs to compare + :type first: string + :param second: The other ACL to compare + :type second: string + :returns: bool """ + if first == second: + # single wildcard is special, and matches everything + return True + if first is None or second is None: + return False + if '*' not in first + second: + # no wildcards, and not exactly equal + return False + first_parts = first.split('.') + second_parts = second.split('.') + if len(first_parts) != len(second_parts): + return False + for i in range(len(first_parts)): + if (first_parts[i] != second_parts[i] and first_parts[i] != '*' and + second_parts[i] != '*'): + return False + return True + + +def ip2int(ip): + """ convert a dotted-quad IP address into an integer + representation of the same """ + return struct.unpack('>L', socket.inet_pton(socket.AF_INET, ip))[0] + + +def ip_matches(ip, entry): + """ Return True if the given IP matches the IP or IP and netmask + in the given ACL entry; False otherwise """ + if entry.get("netmask"): + try: + mask = int("1" * int(entry.get("netmask")) + + "0" * (32 - int(entry.get("netmask"))), 2) + except ValueError: + mask = ip2int(entry.get("netmask")) + return ip2int(ip) & mask == ip2int(entry.get("address")) & mask + elif entry.get("address") is None: + # no address, no netmask -- match all + return True + elif ip == entry.get("address"): + # just a plain ip address + return True + return False + + +class IPACLFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of ACL ip.xml, for IP-based ACLs """ + actions = dict(Allow=True, + Deny=False, + Defer=None) + + def check_acl(self, address, rmi): + """ Check a client address against the ACL list """ + if not len(self.entries): + # default defer if no ACLs are defined. + self.debug_log("ACL: %s requests %s: No IP ACLs, defer" % + (address, rmi)) + return self.actions["Defer"] + for entry in self.entries: + if (ip_matches(address, entry) and + rmi_names_equal(entry.get("method"), rmi)): + self.debug_log("ACL: %s requests %s: Found matching IP ACL, " + "%s" % (address, rmi, entry.tag.lower())) + return self.actions[entry.tag] + if address == "127.0.0.1": + self.debug_log("ACL: %s requests %s: No matching IP ACLs, " + "localhost allowed" % (address, rmi)) + return self.actions['Allow'] # default allow for localhost + + self.debug_log("ACL: %s requests %s: No matching IP ACLs, defer" % + (address, rmi)) + return self.actions["Defer"] # default defer for other machines + + +class MetadataACLFile(Bcfg2.Server.Plugin.StructFile): + """ representation of ACL metadata.xml, for metadata-based ACLs """ + def check_acl(self, metadata, rmi): + """ check client metadata against the ACL list """ + if not len(self.entries): + # default allow if no ACLs are defined. + self.debug_log("ACL: %s requests %s: No metadata ACLs, allow" % + (metadata.hostname, rmi)) + return True + for el in self.Match(metadata): + if rmi_names_equal(el.get("method"), rmi): + self.debug_log("ACL: %s requests %s: Found matching metadata " + "ACL, %s" % (metadata.hostname, rmi, + el.tag.lower())) + return el.tag == "Allow" + if metadata.hostname in ['localhost', 'localhost.localdomain']: + # default allow for localhost + self.debug_log("ACL: %s requests %s: No matching metadata ACLs, " + "localhost allowed" % (metadata.hostname, rmi)) + return True + self.debug_log("ACL: %s requests %s: No matching metadata ACLs, deny" % + (metadata.hostname, rmi)) + return False # default deny for other machines + + +class ACL(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.ClientACLs): + """ allow connections to bcfg-server based on IP address """ + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.ClientACLs.__init__(self) + self.ip_acls = IPACLFile(os.path.join(self.data, 'ip.xml'), + should_monitor=True) + self.metadata_acls = MetadataACLFile(os.path.join(self.data, + 'metadata.xml'), + should_monitor=True) + + def check_acl_ip(self, address, rmi): + self.debug_log("ACL: %s requests %s: Checking IP ACLs" % + (address[0], rmi)) + return self.ip_acls.check_acl(address[0], rmi) + + def check_acl_metadata(self, metadata, rmi): + self.debug_log("ACL: %s requests %s: Checking metadata ACLs" % + (metadata.hostname, rmi)) + return self.metadata_acls.check_acl(metadata, rmi) + + def set_debug(self, debug): + rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) + self.ip_acls.set_debug(debug) + self.metadata_acls.set_debug(debug) + return rv + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 147f37fbf..3f92542e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -3,10 +3,9 @@ import os import re import sys -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from boto import connect_ec2 -from Bcfg2.Cache import Cache +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import ConfigParser @@ -66,11 +65,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): create = 'AWSTags' def __init__(self, filename, core=None): - try: - fam = core.fam - except AttributeError: - fam = None - Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, should_monitor=True) self.core = core self.tags = [] @@ -107,15 +102,13 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): class AWSTags(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.Connector): """ Query tags from AWS via boto, optionally setting group membership """ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) try: @@ -178,6 +171,8 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, return self._tagcache[metadata.hostname] def expire_cache(self, key=None): + """ Expire the cache for one host, or for all hosts. This is + exposed as an XML-RPC RMI. """ self._tagcache.expire(key=key) def start_client_run(self, metadata): @@ -189,29 +184,3 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, def get_additional_groups(self, metadata): return self.config.get_groups(metadata.hostname, self.get_tags(metadata)) - - -class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags - <server-plugins-connectors-awstags>` patterns for validity. """ - - def Run(self): - cfg = self.core.plugins['AWSTags'].config - for entry in cfg.xdata.xpath('//Tag'): - self.check(entry, "name") - if entry.get("value"): - self.check(entry, "value") - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, attr): - """ Check a single attribute (``name`` or ``value``) of a - single entry for validity. """ - try: - re.compile(entry.get(attr)) - except re.error: - self.LintError("pattern-fails-to-initialize", - "'%s' regex could not be compiled: %s\n %s" % - (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Plugins/Account.py b/src/lib/Bcfg2/Server/Plugins/Account.py deleted file mode 100644 index fd49d3655..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Account.py +++ /dev/null @@ -1,102 +0,0 @@ -"""This handles authentication setup.""" - -import Bcfg2.Server.Plugin - - -class Account(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """This module generates account config files, - based on an internal data repo: - static.(passwd|group|limits.conf) -> static entries - dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch) - useraccess -> users to be granted login access on some hosts - superusers -> users to be granted root privs on all hosts - rootlike -> users to be granted root privs on some hosts - - """ - name = 'Account' - __author__ = 'bcfg-dev@mcs.anl.gov' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb, - '/etc/group': self.from_yp_cb, - '/etc/security/limits.conf': self.gen_limits_cb, - '/root/.ssh/authorized_keys': self.gen_root_keys_cb, - '/etc/sudoers': self.gen_sudoers}} - try: - self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, - self.core.fam) - except: - self.logger.error("Failed to load repos: %s, %s" % \ - (self.data, "%s/ssh" % (self.data))) - raise Bcfg2.Server.Plugin.PluginInitError - - def from_yp_cb(self, entry, metadata): - """Build password file from cached yp data.""" - fname = entry.attrib['name'].split('/')[-1] - entry.text = self.repository.entries["static.%s" % (fname)].data - entry.text += self.repository.entries["dyn.%s" % (fname)].data - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0644'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - - def gen_limits_cb(self, entry, metadata): - """Build limits entries based on current ACLs.""" - entry.text = self.repository.entries["static.limits.conf"].data - superusers = self.repository.entries["superusers"].data.split() - useraccess = [line.split(':') for line in \ - self.repository.entries["useraccess"].data.split()] - users = [user for (user, host) in \ - useraccess if host == metadata.hostname.split('.')[0]] - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users]) - if "*" not in users: - entry.text += "* hard maxlogins 0\n" - - def gen_root_keys_cb(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - rdata = self.repository.entries - entry.text = "".join([rdata["%s.key" % user].data for user \ - in superusers if \ - ("%s.key" % user) in rdata]) - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] - - def gen_sudoers(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - entry.text = self.repository.entries['static.sudoers'].data - entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \ - for uname in superusers]) - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0440'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py deleted file mode 100644 index a18204d60..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Base.py +++ /dev/null @@ -1,33 +0,0 @@ -"""This module sets up a base list of configuration entries.""" - -import copy -import lxml.etree -import Bcfg2.Server.Plugin -from itertools import chain - - -class Base(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """This Structure is good for the pile of independent configs - needed for most actual systems. - """ - name = 'Base' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = Bcfg2.Server.Plugin.StructFile - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, - self.core.fam) - - def BuildStructures(self, metadata): - """Build structures for client described by metadata.""" - ret = lxml.etree.Element("Independent", version='2.0') - fragments = list(chain(*[base.Match(metadata) - for base in list(self.entries.values())])) - for frag in fragments: - ret.append(copy.copy(frag)) - return [ret] diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 58f8f4430..41ee57b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -1,201 +1,132 @@ """This provides bundle clauses with translation functionality.""" -import copy -import logging -import lxml.etree import os -import os.path import re import sys -import Bcfg2.Server -import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint - -try: - import genshi.template.base - from Bcfg2.Server.Plugins.TGenshi import removecomment, TemplateFile - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False - - -SETUP = None +import copy +import fnmatch +import lxml.etree +from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ + StructureValidator, XMLDirectoryBacked, Generator +from genshi.template import TemplateError -class BundleFile(Bcfg2.Server.Plugin.StructFile): +class BundleFile(StructFile): """ Representation of a bundle XML file """ - def get_xml_value(self, metadata): - """ get the XML data that applies to the given client """ - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata): - bundle.append(copy.copy(item)) - return bundle - - -if HAS_GENSHI: - class BundleTemplateFile(TemplateFile, - Bcfg2.Server.Plugin.StructFile): - """ Representation of a Genshi-templated bundle XML file """ - - def __init__(self, name, specific, encoding, fam=None): - TemplateFile.__init__(self, name, specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam) - self.logger = logging.getLogger(name) - - def get_xml_value(self, metadata): - """ get the rendered XML data that applies to the given - client """ - if not hasattr(self, 'template'): - msg = "No parsed template information for %s" % self.name - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate( - metadata=metadata, - repo=SETUP['repo']).filter(removecomment) - data = lxml.etree.XML( - stream.render('xml', strip_whitespace=False).encode(), - parser=Bcfg2.Server.XMLParser) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - - def Match(self, metadata, xdata): # pylint: disable=W0221 - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - self.logger.debug("File %s got %d match(es)" % (self.name, - len(rv))) - return rv - - class SGenshiTemplateFile(BundleTemplateFile): - """ provided for backwards compat with the deprecated SGenshi - plugin """ - pass - - -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): + bundle_name_re = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') + + def __init__(self, filename, should_monitor=False): + StructFile.__init__(self, filename, should_monitor=should_monitor) + if self.name.endswith(".genshi"): + self.logger.warning("Bundler: %s: Bundle filenames ending with " + ".genshi are deprecated; add the Genshi XML " + "namespace to a .xml bundle instead" % + self.name) + + def Index(self): + StructFile.Index(self) + if self.xdata.get("name"): + self.logger.warning("Bundler: %s: Explicitly specifying bundle " + "names is deprecated" % self.name) + + @property + def bundle_name(self): + """ The name of the bundle, as determined from the filename """ + return self.bundle_name_re.match( + os.path.basename(self.name)).group("name") + + +class Bundler(Plugin, + Structure, + StructureValidator, + XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.setup['encoding'] - self.__child__ = self.template_dispatch - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, - self.core.fam) - global SETUP - SETUP = core.setup - - def template_dispatch(self, name, _): - """ Add the correct child entry type to Bundler depending on - whether the XML file in question is a plain XML file or a - templated bundle """ - bundle = lxml.etree.parse(name, parser=Bcfg2.Server.XMLParser) - nsmap = bundle.getroot().nsmap - if (name.endswith('.genshi') or - ('py' in nsmap and - nsmap['py'] == 'http://genshi.edgewall.org/')): - if HAS_GENSHI: - spec = Bcfg2.Server.Plugin.Specificity() - return BundleTemplateFile(name, spec, self.encoding, - fam=self.core.fam) - else: - raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " - "available: %s" - % name) - else: - return BundleFile(name, fam=self.fam) + __child__ = BundleFile + patterns = re.compile(r'^.*\.(?:xml|genshi)$') + + def __init__(self, core): + Plugin.__init__(self, core) + Structure.__init__(self) + StructureValidator.__init__(self) + XMLDirectoryBacked.__init__(self, self.data) + #: Bundles by bundle name, rather than filename + self.bundles = dict() + + def HandleEvent(self, event): + XMLDirectoryBacked.HandleEvent(self, event) + self.bundles = dict([(b.bundle_name, b) + for b in self.entries.values()]) + + def validate_structures(self, metadata, structures): + """ Translate <Path glob='...'/> entries into <Path name='...'/> + entries """ + for struct in structures: + for pathglob in struct.xpath("//Path[@glob]"): + for plugin in self.core.plugins_by_type(Generator): + for match in fnmatch.filter(plugin.Entries['Path'].keys(), + pathglob.get("glob")): + lxml.etree.SubElement(pathglob.getparent(), + "Path", name=match) + pathglob.getparent().remove(pathglob) def BuildStructures(self, metadata): - """Build all structures for client (metadata).""" bundleset = [] - - bundle_entries = {} - for key, item in self.entries.items(): - bundle_entries.setdefault( - self.patterns.match(os.path.basename(key)).group('name'), - []).append(item) - - for bundlename in metadata.bundles: + bundles = copy.copy(metadata.bundles) + bundles_added = set(bundles) + while bundles: + bundlename = bundles.pop() try: - entries = bundle_entries[bundlename] + bundle = self.bundles[bundlename] except KeyError: self.logger.error("Bundler: Bundle %s does not exist" % bundlename) continue + try: - bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: + data = bundle.XMLMatch(metadata) + except TemplateError: err = sys.exc_info()[1] self.logger.error("Bundler: Failed to render templated bundle " "%s: %s" % (bundlename, err)) + continue except: self.logger.error("Bundler: Unexpected bundler error for %s" % bundlename, exc_info=1) - return bundleset - + continue -class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ - - def Run(self): - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - (not HAS_GENSHI or - not isinstance(bundle, BundleTemplateFile))): - self.bundle_names(bundle) - - @classmethod - def Errors(cls): - return {"bundle-not-found": "error", - "inconsistent-bundle-name": "warning"} - - def missing_bundles(self): - """ Find bundles listed in Metadata but not implemented in - Bundler. """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - def bundle_names(self, bundle): - """ Verify bundle name attribute matches filename. - - :param bundle: The bundle to verify - :type bundle: Bcfg2.Server.Plugins.Bundler.BundleFile - """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - fname = os.path.splitext(os.path.basename(bundle.name))[0] - bname = xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, " - "bundle name is %s" % (fname, bname)) + if data.get("independent", "false").lower() == "true": + data.tag = "Independent" + del data.attrib['independent'] + + data.set("name", bundlename) + + for child in data.findall("Bundle"): + if child.getchildren(): + # XInclude'd bundle -- "flatten" it so there + # aren't extra Bundle tags, since other bits in + # Bcfg2 only handle the direct children of the + # top-level Bundle tag + if data.get("name"): + self.logger.warning("Bundler: In file XIncluded from " + "%s: Explicitly specifying " + "bundle names is deprecated" % + self.name) + for el in child.getchildren(): + data.append(el) + data.remove(child) + elif child.get("name"): + # dependent bundle -- add it to the list of + # bundles for this client + if child.get("name") not in bundles_added: + bundles.append(child.get("name")) + bundles_added.add(child.get("name")) + data.remove(child) + else: + # neither name or children -- wat + self.logger.warning("Bundler: Useless empty Bundle tag " + "in %s" % self.name) + data.remove(child) + bundleset.append(data) + return bundleset diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index e0cbdf72a..01b51ace4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -11,19 +11,19 @@ class Bzr(Bcfg2.Server.Plugin.Version): using bazaar. """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.logger.debug("Initialized Bazaar plugin with directory %s at " - "revision = %s" % (self.vcs_root, + "revision = %s" % (Bcfg2.Options.setup.vcs_root, self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: - working_tree = WorkingTree.open(self.vcs_root) + working_tree = WorkingTree.open(Bcfg2.Options.setup.vcs_root) revision = str(working_tree.branch.revno()) if (working_tree.has_changes(working_tree.basis_tree()) or - working_tree.unknowns()): + working_tree.unknowns()): revision += "+" except errors.NotBranchError: msg = "Failed to read Bazaar branch" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index 41d5588e4..7792d7e5c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -3,8 +3,9 @@ based on an XML specification of which SSH keypairs should granted access. """ import lxml.etree +import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP, CFG +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, get_cfg from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -20,26 +21,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): #: Handle authorized keys XML files __basenames__ = ['authorizedkeys.xml', 'authorized_keys.xml'] - #: This handler is experimental, in part because it depends upon - #: the (experimental) CfgPrivateKeyCreator handler - experimental = True - def __init__(self, fname): - CfgGenerator.__init__(self, fname, None, None) + CfgGenerator.__init__(self, fname, None) StructFile.__init__(self, fname) self.cache = dict() - self.core = CFG.core + self.core = get_cfg().core __init__.__doc__ = CfgGenerator.__init__.__doc__ - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "category")): - return SETUP.cfp.get("sshkeys", "category") - return None - def handle_event(self, event): CfgGenerator.handle_event(self, event) StructFile.HandleEvent(self, event) @@ -51,12 +39,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): rv = [] for allow in spec.findall("Allow"): options = [] - if allow.find("Params") is not None: - self.logger.warning("Use of <Params> in authorized_keys.xml " - "is deprecated; use <Option> instead") - options.extend("=".join(p) - for p in allow.find("Params").attrib.items()) - for opt in allow.findall("Option"): if opt.get("value"): options.append("%s=%s" % (opt.get("name"), @@ -68,7 +50,8 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): if pubkey_name: host = allow.get("host") group = allow.get("group") - category = allow.get("category", self.category) + category = allow.get("category", + Bcfg2.Options.setup.sshkeys_category) if host: key_md = self.core.build_metadata(host) elif group: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py deleted file mode 100644 index 49a5a85b3..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py +++ /dev/null @@ -1,28 +0,0 @@ -""" Handle .cat files, which append lines to and remove lines from -plaintext files """ - -from Bcfg2.Server.Plugins.Cfg import CfgFilter - - -class CfgCatFilter(CfgFilter): - """ CfgCatFilter appends lines to and remove lines from plaintext - :ref:`server-plugins-generators-Cfg` files""" - - #: Handle .cat files - __extensions__ = ['cat'] - - #: .cat files are deprecated - deprecated = True - - def modify_data(self, entry, metadata, data): - datalines = data.strip().split('\n') - for line in self.data.split('\n'): - if not line: - continue - if line.startswith('+'): - datalines.append(line[1:]) - elif line.startswith('-'): - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 724164cf5..84309b5dd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,8 +2,10 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: from Cheetah.Template import Template @@ -12,6 +14,18 @@ except ImportError: HAS_CHEETAH = False +class DefaultCheetahDataProvider(DefaultTemplateDataProvider): + """ Template data provider for Cheetah templates. Cheetah and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + class CfgCheetahGenerator(CfgGenerator): """ The CfgCheetahGenerator allows you to use the `Cheetah <http://www.cheetahtemplate.org/>`_ templating system to generate @@ -27,19 +41,18 @@ class CfgCheetahGenerator(CfgGenerator): #: :class:`Cheetah.Template.Template` compiler settings settings = dict(useStackFrames=False) - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CHEETAH: raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): - template = Template(self.data.decode(self.encoding), + template = Template(self.data.decode(Bcfg2.Options.setup.encoding), compilerSettings=self.settings) - template.metadata = metadata - template.name = entry.get('realname', entry.get('name')) - template.path = entry.get('realname', entry.get('name')) - template.source_path = self.name - template.repo = SETUP['repo'] + for key, val in get_template_data( + entry, metadata, self.name, + default=DefaultCheetahDataProvider()).items(): + setattr(template, key, val) return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py deleted file mode 100644 index da506a195..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ /dev/null @@ -1,35 +0,0 @@ -""" Handle .diff files, which apply diffs to plaintext files """ - -import os -import tempfile -from Bcfg2.Server.Plugin import PluginExecutionError -from subprocess import Popen, PIPE -from Bcfg2.Server.Plugins.Cfg import CfgFilter - - -class CfgDiffFilter(CfgFilter): - """ CfgDiffFilter applies diffs to plaintext - :ref:`server-plugins-generators-Cfg` files """ - - #: Handle .diff files - __extensions__ = ['diff'] - - #: .diff files are deprecated - deprecated = True - - def modify_data(self, entry, metadata, data): - basehandle, basename = tempfile.mkstemp() - open(basename, 'w').write(data) - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basename] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=self.data)[1] - ret = patch.wait() - output = open(basename, 'r').read() - os.unlink(basename) - if ret != 0: - raise PluginExecutionError("Error applying diff %s: %s" % - (self.name, stderr)) - return output - modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index cf7eae75b..e2a2f696a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,12 +1,10 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ -import Bcfg2.Server.Plugins.Cfg from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, EVPError, \ - get_algorithm + from Bcfg2.Server.Encryption import bruteforce_decrypt, EVPError HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -23,8 +21,8 @@ class CfgEncryptedGenerator(CfgGenerator): #: .genshi.crypt and .cheetah.crypt files __priority__ = 50 - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ @@ -35,10 +33,7 @@ class CfgEncryptedGenerator(CfgGenerator): return # todo: let the user specify a passphrase by name try: - self.data = bruteforce_decrypt( - self.data, - setup=Bcfg2.Server.Plugins.Cfg.SETUP, - algorithm=get_algorithm(Bcfg2.Server.Plugins.Cfg.SETUP)) + self.data = bruteforce_decrypt(self.data) except EVPError: raise PluginExecutionError("Failed to decrypt %s" % self.name) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 130652aef..f69ab8e5f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -1,32 +1,24 @@ """ Handle encrypted Genshi templates (.crypt.genshi or .genshi.crypt files) """ +from genshi.template import TemplateLoader from Bcfg2.Compat import StringIO from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import SETUP from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, get_algorithm + from Bcfg2.Server.Encryption import bruteforce_decrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False -try: - from genshi.template import TemplateLoader -except ImportError: - # CfgGenshiGenerator will raise errors if genshi doesn't exist - TemplateLoader = object # pylint: disable=C0103 - class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt the data on the fly as it's read in using - :func:`Bcfg2.Encryption.bruteforce_decrypt` """ + :func:`Bcfg2.Server.Encryption.bruteforce_decrypt` """ def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): - plaintext = \ - StringIO(bruteforce_decrypt(fileobj.read(), - algorithm=get_algorithm(SETUP))) + plaintext = StringIO(bruteforce_decrypt(fileobj.read())) return TemplateLoader._instantiate(self, cls, plaintext, filepath, filename, encoding=encoding) @@ -45,7 +37,7 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): #: when it's read in __loader_cls__ = EncryptedTemplateLoader - def __init__(self, fname, spec, encoding): - CfgGenshiGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenshiGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 313e53ee9..953473a12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -3,8 +3,8 @@ import os import sys import shlex +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError -from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError @@ -15,27 +15,19 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] - def __init__(self, name, specific, encoding): - CfgVerifier.__init__(self, name, specific, encoding) + def __init__(self, name, specific): + CfgVerifier.__init__(self, name, specific) self.cmd = [] + self.exc = Executor(timeout=30) __init__.__doc__ = CfgVerifier.__init__.__doc__ def verify_entry(self, entry, metadata, data): try: - proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - out, err = proc.communicate(input=data) - rv = proc.wait() - if rv != 0: - # pylint: disable=E1103 - raise CfgVerificationError(err.strip() or out.strip() or - "Non-zero return value %s" % rv) - # pylint: enable=E1103 - except CfgVerificationError: - raise - except: - err = sys.exc_info()[1] - raise CfgVerificationError("Error running external command " - "verifier: %s" % err) + result = self.exc.run(self.cmd, inputdata=data) + if not result.success: + raise CfgVerificationError(result.error) + except OSError: + raise CfgVerificationError(sys.exc_info()[1]) verify_entry.__doc__ = CfgVerifier.verify_entry.__doc__ def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 83a5c1165..ef4e6a656 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -5,63 +5,54 @@ import re import sys import traceback -from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP - -try: - import genshi.core - from genshi.template import TemplateLoader, NewTextTemplate - from genshi.template.eval import UndefinedError, Suite - #: True if Genshi libraries are available - HAS_GENSHI = True - - def _genshi_removes_blank_lines(): - """ Genshi 0.5 uses the Python :mod:`compiler` package to - compile genshi snippets to AST. Genshi 0.6 uses some bespoke - magic, because compiler has been deprecated. - :func:`compiler.parse` produces an AST that removes all excess - whitespace (e.g., blank lines), while - :func:`genshi.template.astutil.parse` does not. In order to - determine which actual line of code an error occurs on, we - need to know which is in use and how it treats blank lines. - I've beat my head against this for hours and the best/only way - I can find is to compile some genshi code with an error and - see which line it's on.""" - code = """d = dict() - +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, removecomment, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from genshi.template import TemplateLoader, NewTextTemplate +from genshi.template.eval import UndefinedError, Suite + + +def _genshi_removes_blank_lines(): + """ Genshi 0.5 uses the Python :mod:`compiler` package to + compile genshi snippets to AST. Genshi 0.6 uses some bespoke + magic, because compiler has been deprecated. + :func:`compiler.parse` produces an AST that removes all excess + whitespace (e.g., blank lines), while + :func:`genshi.template.astutil.parse` does not. In order to + determine which actual line of code an error occurs on, we + need to know which is in use and how it treats blank lines. + I've beat my head against this for hours and the best/only way + I can find is to compile some genshi code with an error and + see which line it's on.""" + code = """d = dict() d['a']""" - try: - Suite(code).execute(dict()) - except KeyError: - line = traceback.extract_tb(sys.exc_info()[2])[-1][1] - if line == 2: - return True - else: - return False - - #: True if Genshi removes all blank lines from a code block before - #: executing it; False indicates that Genshi only removes leading - #: and trailing blank lines. See - #: :func:`_genshi_removes_blank_lines` for an explanation of this. - GENSHI_REMOVES_BLANK_LINES = _genshi_removes_blank_lines() -except ImportError: - TemplateLoader = None # pylint: disable=C0103 - HAS_GENSHI = False - - -def removecomment(stream): - """ A Genshi filter that removes comments from the stream. This - function is a generator. - - :param stream: The Genshi stream to remove comments from - :type stream: genshi.core.Stream - :returns: tuple of ``(kind, data, pos)``, as when iterating - through a Genshi stream - """ - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos + try: + Suite(code).execute(dict()) + except KeyError: + line = traceback.extract_tb(sys.exc_info()[2])[-1][1] + if line == 2: + return True + else: + return False + +#: True if Genshi removes all blank lines from a code block before +#: executing it; False indicates that Genshi only removes leading +#: and trailing blank lines. See +#: :func:`_genshi_removes_blank_lines` for an explanation of this. +GENSHI_REMOVES_BLANK_LINES = _genshi_removes_blank_lines() + + +class DefaultGenshiDataProvider(DefaultTemplateDataProvider): + """ Template data provider for Genshi templates. Cheetah and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = template + return rv class CfgGenshiGenerator(CfgGenerator): @@ -92,10 +83,8 @@ class CfgGenshiGenerator(CfgGenerator): #: occurred. pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) - if not HAS_GENSHI: - raise PluginExecutionError("Genshi is not available") + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) self.template = None self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ @@ -105,19 +94,18 @@ class CfgGenshiGenerator(CfgGenerator): raise PluginExecutionError("Failed to load template %s" % self.name) - fname = entry.get('realname', entry.get('name')) - stream = \ - self.template.generate(name=fname, - metadata=metadata, - path=self.name, - source_path=self.name, - repo=SETUP['repo']).filter(removecomment) + stream = self.template.generate( + **get_template_data( + entry, metadata, self.name, + default=DefaultGenshiDataProvider())).filter(removecomment) try: try: - return stream.render('text', encoding=self.encoding, + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding, strip_whitespace=False) except TypeError: - return stream.render('text', encoding=self.encoding) + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding) except UndefinedError: # a failure in a genshi expression _other_ than %{ python ... %} err = sys.exc_info()[1] @@ -196,8 +184,9 @@ class CfgGenshiGenerator(CfgGenerator): def handle_event(self, event): CfgGenerator.handle_event(self, event) try: - self.template = self.loader.load(self.name, cls=NewTextTemplate, - encoding=self.encoding) + self.template = \ + self.loader.load(self.name, cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) except: raise PluginExecutionError("Failed to load template: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index 3b6fc8fa0..886b3993b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -1,6 +1,6 @@ """ Handle info.xml files """ -from Bcfg2.Server.Plugin import PluginExecutionError, InfoXML +from Bcfg2.Server.Plugin import InfoXML from Bcfg2.Server.Plugins.Cfg import CfgInfo @@ -17,21 +17,9 @@ class CfgInfoXML(CfgInfo): __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): - mdata = dict() - self.infoxml.pnode.Match(metadata, mdata, entry=entry) - if 'Info' not in mdata: - raise PluginExecutionError("Failed to set metadata for file %s" % - entry.get('name')) - self._set_info(entry, mdata['Info'][None]) + self.infoxml.BindEntry(entry, metadata) bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ def handle_event(self, event): self.infoxml.HandleEvent() handle_event.__doc__ = CfgInfo.handle_event.__doc__ - - def _set_info(self, entry, info): - CfgInfo._set_info(self, entry, info) - if '__children__' in info: - for child in info['__children__']: - entry.append(child) - _set_info.__doc__ = CfgInfo._set_info.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py deleted file mode 100644 index 5122d9aa1..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ /dev/null @@ -1,46 +0,0 @@ -""" Handle info and :info files """ - -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Cfg import CfgInfo - - -class CfgLegacyInfo(CfgInfo): - """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for - :ref:`server-plugins-generators-cfg` """ - - #: Handle :file:`info` and :file:`:info` - __basenames__ = ['info', ':info'] - - #: CfgLegacyInfo is deprecated. Use - #: :class:`Bcfg2.Server.Plugins.Cfg.CfgInfoXML.CfgInfoXML` instead. - deprecated = True - - def __init__(self, path): - CfgInfo.__init__(self, path) - self.path = path - - #: The set of info metadata stored in the file - self.metadata = None - __init__.__doc__ = CfgInfo.__init__.__doc__ - - def bind_info_to_entry(self, entry, metadata): - self._set_info(entry, self.metadata) - bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ - - def handle_event(self, event): - if event.code2str() == 'deleted': - return - self.metadata = dict() - for line in open(self.path).readlines(): - match = Bcfg2.Server.Plugin.INFO_REGEX.match(line) - if not match: - self.logger.warning("Failed to parse line in %s: %s" % - (event.filename, line)) - continue - else: - for key, value in list(match.groupdict().items()): - if value: - self.metadata[key] = value - if ('mode' in self.metadata and len(self.metadata['mode']) == 3): - self.metadata['mode'] = "0%s" % self.metadata['mode'] - handle_event.__doc__ = CfgInfo.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index e890fdecb..e9698f526 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -3,18 +3,13 @@ import os import shutil import tempfile -import subprocess -from Bcfg2.Server.Plugin import PluginExecutionError, StructFile -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, SETUP +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import XMLCfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False -class CfgPrivateKeyCreator(CfgCreator, StructFile): +class CfgPrivateKeyCreator(XMLCfgCreator): """The CfgPrivateKeyCreator creates SSH keys on the fly. """ #: Different configurations for different clients/groups can be @@ -24,38 +19,22 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] - def __init__(self, fname): - CfgCreator.__init__(self, fname) - StructFile.__init__(self, fname) + cfg_section = "sshkeys" + options = [ + Bcfg2.Options.Option( + cf=("sshkeys", "category"), dest="sshkeys_category", + help="Metadata category that generated SSH keys are specific to"), + Bcfg2.Options.Option( + cf=("sshkeys", "passphrase"), dest="sshkeys_passphrase", + help="Passphrase used to encrypt generated SSH private keys")] + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) - __init__.__doc__ = CfgCreator.__init__.__doc__ - - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "category")): - return SETUP.cfp.get("sshkeys", "category") - return None - - @property - def passphrase(self): - """ The passphrase used to encrypt private keys """ - if (HAS_CRYPTO and - SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases(SETUP)[ - SETUP.cfp.get("sshkeys", "passphrase")] - return None - - def handle_event(self, event): - CfgCreator.handle_event(self, event) - StructFile.HandleEvent(self, event) - handle_event.__doc__ = CfgCreator.handle_event.__doc__ + self.cmd = Executor() + __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata @@ -102,62 +81,22 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): log_cmd.append("''") self.debug_log("Cfg: Generating new SSH key pair: %s" % " ".join(log_cmd)) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - err = proc.communicate()[1] - if proc.wait(): + result = self.cmd.run(cmd) + if not result.success: raise CfgCreationError("Cfg: Failed to generate SSH key pair " "at %s for %s: %s" % - (filename, metadata.hostname, err)) - elif err: + (filename, metadata.hostname, + result.error)) + elif result.stderr: self.logger.warning("Cfg: Generated SSH key pair at %s for %s " "with errors: %s" % (filename, metadata.hostname, - err)) + result.stderr)) return filename except: shutil.rmtree(tempdir) raise - def get_specificity(self, metadata, spec=None): - """ Get config settings for key generation specificity - (per-host or per-group). - - :param metadata: The client metadata to create data for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param spec: The key specification to follow when creating the - keys. This should be an XML document that only - contains key specification data that applies to - the given client metadata, and may be obtained by - doing ``self.XMLMatch(metadata)`` - :type spec: lxml.etree._Element - :returns: dict - A dict of specificity arguments suitable for - passing to - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` - or - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` - """ - if spec is None: - spec = self.XMLMatch(metadata) - category = spec.get("category", self.category) - if category is None: - per_host_default = "true" - else: - per_host_default = "false" - per_host = spec.get("perhost", per_host_default).lower() == "true" - - specificity = dict(host=metadata.hostname) - if category and not per_host: - group = metadata.group_in_category(category) - if group: - specificity = dict(group=group, - prio=int(spec.get("priority", 50))) - else: - self.logger.info("Cfg: %s has no group in category %s, " - "creating host-specific key" % - (metadata.hostname, category)) - return specificity - # pylint: disable=W0221 def create_data(self, entry, metadata): """ Create data for the given entry on the given client @@ -170,7 +109,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): :returns: string - The private key data """ spec = self.XMLMatch(metadata) - specificity = self.get_specificity(metadata, spec) + specificity = self.get_specificity(metadata) filename = self._gen_keypair(metadata, spec) try: @@ -184,63 +123,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): # encrypt the private key, write to the proper place, and # return it privkey = open(filename).read() - if HAS_CRYPTO and self.passphrase: - self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Encryption.ssl_encrypt( - privkey, - self.passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - specificity['ext'] = '.crypt' - self.write_data(privkey, **specificity) return privkey finally: shutil.rmtree(os.path.dirname(filename)) # pylint: enable=W0221 - - def Index(self): - StructFile.Index(self) - if HAS_CRYPTO: - strict = self.xdata.get( - "decrypt", - SETUP.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict")) == "strict" - for el in self.xdata.xpath("//*[@encrypted]"): - try: - el.text = self._decrypt(el).encode('ascii', - 'xmlcharrefreplace') - except UnicodeDecodeError: - self.logger.info("Cfg: Decrypted %s to gibberish, skipping" - % el.tag) - except Bcfg2.Encryption.EVPError: - msg = "Cfg: Failed to decrypt %s element in %s" % \ - (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: - self.logger.info(msg) - Index.__doc__ = StructFile.Index.__doc__ - - def _decrypt(self, element): - """ Decrypt a single encrypted element """ - if not element.text or not element.text.strip(): - return - passes = Bcfg2.Encryption.get_passphrases(SETUP) - try: - passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Encryption.ssl_decrypt( - element.text, - passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - except Bcfg2.Encryption.EVPError: - # error is raised below - pass - except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Encryption.bruteforce_decrypt( - element.text, - passphrases=passes.values(), - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 4bd8690ed..3f2d1030b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -8,7 +8,7 @@ import tempfile import lxml.etree from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG +from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, get_cfg class CfgPublicKeyCreator(CfgCreator, StructFile): @@ -21,17 +21,20 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): creation of a keypair when a public key is created. """ #: Different configurations for different clients/groups can be - #: handled with Client and Group tags within privkey.xml + #: handled with Client and Group tags within pubkey.xml __specific__ = False #: Handle XML specifications of private keys __basenames__ = ['pubkey.xml'] + #: No text content on any tags, so encryption support disabled + encryption = False + def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) - self.cfg = CFG - self.core = CFG.core + self.cfg = get_cfg() + self.core = self.cfg.core self.cmd = Executor() def create_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py new file mode 100644 index 000000000..92fcc4cd8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py @@ -0,0 +1,255 @@ +""" Cfg creator that creates SSL certs """ + +import os +import sys +import tempfile +import lxml.etree +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Compat import ConfigParser +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator, \ + CfgCreator, CfgVerifier, CfgVerificationError, get_cfg + + +class CfgSSLCACertCreator(XMLCfgCreator, CfgVerifier): + """ This class acts as both a Cfg creator that creates SSL certs, + and as a Cfg verifier that verifies SSL certs. """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within pubkey.xml + __specific__ = False + + #: Handle XML specifications of private keys + __basenames__ = ['sslcert.xml'] + + cfg_section = "sslca" + options = [ + Bcfg2.Options.Option( + cf=("sslca", "category"), dest="sslca_category", + help="Metadata category that generated SSL keys are specific to"), + Bcfg2.Options.Option( + cf=("sslca", "passphrase"), dest="sslca_passphrase", + help="Passphrase used to encrypt generated SSL keys"), + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"), + prefix="")] + + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) + CfgVerifier.__init__(self, fname, None) + self.cmd = Executor() + self.cfg = get_cfg() + + def build_req_config(self, metadata): + """ Generates a temporary openssl configuration file that is + used to generate the required certificate request. """ + fd, fname = tempfile.mkstemp() + cfp = ConfigParser.ConfigParser({}) + cfp.optionxform = str + defaults = dict( + req=dict( + default_md='sha1', + distinguished_name='req_distinguished_name', + req_extensions='v3_req', + x509_extensions='v3_req', + prompt='no'), + req_distinguished_name=dict(), + v3_req=dict(subjectAltName='@alt_names'), + alt_names=dict()) + for section in list(defaults.keys()): + cfp.add_section(section) + for key in defaults[section]: + cfp.set(section, key, defaults[section][key]) + spec = self.XMLMatch(metadata) + cert = spec.find("Cert") + altnamenum = 1 + altnames = spec.findall('subjectAltName') + altnames.extend(list(metadata.aliases)) + altnames.append(metadata.hostname) + for altname in altnames: + cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) + altnamenum += 1 + for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: + if cert.get(item): + cfp.set('req_distinguished_name', item, cert.get(item)) + cfp.set('req_distinguished_name', 'CN', metadata.hostname) + self.debug_log("Cfg: Writing temporary CSR config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise CfgCreationError("Cfg: Failed to write temporary CSR config " + "file: %s" % sys.exc_info()[1]) + return fname + + def build_request(self, keyfile, metadata): + """ Create the certificate request """ + req_config = self.build_req_config(metadata) + try: + fd, req = tempfile.mkstemp() + os.close(fd) + cert = self.XMLMatch(metadata).find("Cert") + days = cert.get("days", "365") + cmd = ["openssl", "req", "-new", "-config", req_config, + "-days", days, "-key", keyfile, "-text", "-out", req] + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate CSR: %s" % + result.error) + return req + finally: + try: + os.unlink(req_config) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR " + "config: %s" % sys.exc_info()[1]) + + def get_ca(self, name): + """ get a dict describing a CA from the config file """ + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv + + def create_data(self, entry, metadata): + """ generate a new cert """ + self.logger.info("Cfg: Generating new SSL cert for %s" % self.name) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + req = self.build_request(self._get_keyfile(cert, metadata), metadata) + try: + days = cert.get('days', '365') + cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, + "-days", days, "-batch"] + passphrase = ca.get('passphrase') + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate cert: %s" % + result.error) + except KeyError: + raise CfgCreationError("Cfg: [sslca_%s] section has no 'config' " + "option" % cert.get('ca', 'default')) + finally: + try: + os.unlink(req) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR: %s " % + sys.exc_info()[1]) + data = result.stdout + if cert.get('append_chain') and 'chaincert' in ca: + data += open(ca['chaincert']).read() + + self.write_data(data, **self.get_specificity(metadata)) + return data + + def verify_entry(self, entry, metadata, data): + fd, fname = tempfile.mkstemp() + self.debug_log("Cfg: Writing SSL cert %s to temporary file %s for " + "verification" % (entry.get("name"), fname)) + os.fdopen(fd, 'w').write(data) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + try: + if ca.get('chaincert'): + self.verify_cert_against_ca(fname, entry, metadata) + self.verify_cert_against_key(fname, + self._get_keyfile(cert, metadata)) + finally: + os.unlink(fname) + + def _get_keyfile(self, cert, metadata): + """ Given a <Cert/> element and client metadata, return the + full path to the file on the filesystem that the key lives in.""" + keypath = cert.get("key") + eset = self.cfg.entries[keypath] + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + # SSL key needs to be created + try: + creator = eset.best_matching(metadata, + eset.get_handlers(metadata, + CfgCreator)) + except PluginExecutionError: + raise CfgCreationError("Cfg: No SSL key or key creator " + "defined for %s" % keypath) + + keyentry = lxml.etree.Element("Path", name=keypath) + creator.create_data(keyentry, metadata) + + tries = 0 + while True: + if tries >= 10: + raise CfgCreationError("Cfg: Timed out waiting for event " + "on SSL key at %s" % keypath) + get_fam().handle_events_in_interval(1) + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + tries += 1 + continue + + def verify_cert_against_ca(self, filename, entry, metadata): + """ + check that a certificate validates against the ca cert, + and that it has not expired. + """ + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get("ca", "default")) + chaincert = ca.get('chaincert') + cmd = ["openssl", "verify"] + is_root = ca.get('root_ca', "false").lower() == 'true' + if is_root: + cmd.append("-CAfile") + else: + # verifying based on an intermediate cert + cmd.extend(["-purpose", "sslserver", "-untrusted"]) + cmd.extend([chaincert, filename]) + self.debug_log("Cfg: Verifying %s against CA" % entry.get("name")) + result = self.cmd.run(cmd) + if result.stdout == cert + ": OK\n": + self.debug_log("Cfg: %s verified successfully against CA" % + entry.get("name")) + else: + raise CfgVerificationError("%s failed verification against CA: %s" + % (entry.get("name"), result.error)) + + def _get_modulus(self, fname, ftype="x509"): + """ get the modulus from the given file """ + cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] + self.debug_log("Cfg: Getting modulus of %s for verification: %s" % + (fname, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgVerificationError("Failed to get modulus of %s: %s" % + (fname, result.error)) + return result.stdout.strip() + + def verify_cert_against_key(self, filename, keyfile): + """ check that a certificate validates against its private + key. """ + cert = self._get_modulus(filename) + key = self._get_modulus(keyfile, ftype="rsa") + if cert == key: + self.debug_log("Cfg: %s verified successfully against key %s" % + (filename, keyfile)) + else: + raise CfgVerificationError("%s failed verification against key %s" + % (filename, keyfile)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py new file mode 100644 index 000000000..a158302be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py @@ -0,0 +1,36 @@ +""" Cfg creator that creates SSL keys """ + +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator + + +class CfgSSLCAKeyCreator(XMLCfgCreator): + """ Cfg creator that creates SSL keys """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within sslkey.xml + __specific__ = False + + __basenames__ = ["sslkey.xml"] + + cfg_section = "sslca" + + def create_data(self, entry, metadata): + self.logger.info("Cfg: Generating new SSL key for %s" % self.name) + spec = self.XMLMatch(metadata) + key = spec.find("Key") + if not key: + key = dict() + ktype = key.get('type', 'rsa') + bits = key.get('bits', '2048') + if ktype == 'rsa': + cmd = ["openssl", "genrsa", bits] + elif ktype == 'dsa': + cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + result = Executor().run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate key %s for %s: %s" % + (self.name, metadata.hostname, + result.error)) + self.write_data(result.stdout, **self.get_specificity(metadata)) + return result.stdout diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index c6e2d0acb..5dc3d98eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -3,60 +3,39 @@ import re import os import sys -import stat import errno import operator import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint -from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ - any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, walk_packages # pylint: enable=W0622 -#: SETUP contains a reference to the -#: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for -#: parsing command-line and config file options. -#: :class:`Bcfg2.Server.Plugins.Cfg.Cfg` stores it in a module global -#: so that the handler objects can access it, because there is no other -#: facility for passing a setup object from a -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` to its -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` objects and thence to -#: the EntrySet children. -SETUP = None - -#: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` -#: plugin object created by the Bcfg2 core. This is provided so that -#: the handler objects can access it as necessary, since the existing -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no -#: facility for passing it otherwise. -CFG = None - -_HANDLERS = [] - - -def handlers(): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if not _HANDLERS: - for submodule in walk_packages(path=__path__, prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if issubclass(hdlr, CfgBaseFileMatcher): - _HANDLERS.append(hdlr) - _HANDLERS.sort(key=operator.attrgetter("__priority__")) - return _HANDLERS - - -class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, - Bcfg2.Server.Plugin.Debuggable): +try: + import Bcfg2.Server.Encryption + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +_handlers = [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + +_CFG = None + + +def get_cfg(): + """ Get the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` plugin object + created by the Bcfg2 core. This is provided so that the handler + objects can access it as necessary, since the existing + :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and + :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no + facility for passing it otherwise.""" + return _CFG + + +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ .. currentmodule:: Bcfg2.Server.Plugins.Cfg CfgBaseFileMatcher is the parent class for all Cfg handler @@ -100,13 +79,10 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, #: Flag to indicate an experimental handler. experimental = False - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): if not self.__specific__ and not specific: specific = Bcfg2.Server.Plugin.Specificity(all=True) - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.encoding = encoding + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ """ .. ----- @@ -197,7 +173,7 @@ class CfgGenerator(CfgBaseFileMatcher): client. See :class:`Bcfg2.Server.Plugin.helpers.EntrySet` for more details on how the best handler is chosen.""" - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -205,7 +181,7 @@ class CfgGenerator(CfgBaseFileMatcher): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def get_data(self, entry, metadata): # pylint: disable=W0613 @@ -225,9 +201,9 @@ class CfgFilter(CfgBaseFileMatcher): """ CfgFilters modify the initial content of a file after it has been generated by a :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def modify_data(self, entry, metadata, data): @@ -249,10 +225,7 @@ class CfgFilter(CfgBaseFileMatcher): class CfgInfo(CfgBaseFileMatcher): """ CfgInfo handlers provide metadata (owner, group, paranoid, - etc.) for a file entry. - - .. private-include: _set_info - """ + etc.) for a file entry. """ #: Whether or not the files handled by this handler are permitted #: to have specificity indicators in their filenames -- e.g., @@ -268,7 +241,7 @@ class CfgInfo(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def bind_info_to_entry(self, entry, metadata): """ Assign the appropriate attributes to the entry, modifying @@ -282,20 +255,6 @@ class CfgInfo(CfgBaseFileMatcher): """ raise NotImplementedError - def _set_info(self, entry, info): - """ Helper function to assign a dict of info attributes to an - entry object. ``entry`` is modified in-place. - - :param entry: The abstract entry to bind the info to - :type entry: lxml.etree._Element - :param info: A dict of attribute: value pairs - :type info: dict - :returns: None - """ - for key, value in list(info.items()): - if not key.startswith("__"): - entry.attrib[key] = value - class CfgVerifier(CfgBaseFileMatcher): """ CfgVerifier handlers validate entry data once it has been @@ -305,9 +264,9 @@ class CfgVerifier(CfgBaseFileMatcher): etc.), or both. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def verify_entry(self, entry, metadata, data): @@ -338,18 +297,15 @@ class CfgCreator(CfgBaseFileMatcher): #: file, and are thus not specific __specific__ = False - #: The CfgCreator interface is experimental at this time - experimental = True - def __init__(self, fname): """ :param name: The full path to the file :type name: string .. ----- - .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ + .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def create_data(self, entry, metadata): """ Create new data for the given entry and write it to disk @@ -369,7 +325,9 @@ class CfgCreator(CfgBaseFileMatcher): ``host`` is given, it will be host-specific. It will be group-specific if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, the filename will be - non-specific. + non-specific. In general, this will be called as:: + + self.get_filename(**self.get_specificity(metadata)) :param host: The file applies to the given host :type host: bool @@ -400,6 +358,9 @@ class CfgCreator(CfgBaseFileMatcher): written as a host-specific file, or as a group-specific file if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, it will be written as a non-specific file. + In general, this will be called as:: + + self.write_data(data, **self.get_specificity(metadata)) :param data: The data to write :type data: string @@ -419,7 +380,7 @@ class CfgCreator(CfgBaseFileMatcher): :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgCreationError` """ fileloc = self.get_filename(host=host, group=group, prio=prio, ext=ext) - self.debug_log("%s: Writing new file %s" % (self.name, fileloc)) + self.debug_log("Cfg: Writing new file %s" % fileloc) try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -435,6 +396,95 @@ class CfgCreator(CfgBaseFileMatcher): raise CfgCreationError("Could not write %s: %s" % (fileloc, err)) +class XMLCfgCreator(CfgCreator, # pylint: disable=W0223 + Bcfg2.Server.Plugin.StructFile): + """ A CfgCreator that uses XML to describe how data should be + generated. """ + + #: Whether or not the created data from this class can be + #: encrypted + encryptable = True + + #: Encryption and creation settings can be stored in bcfg2.conf, + #: either under the [cfg] section, or under the named section. + cfg_section = None + + def __init__(self, name): + CfgCreator.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + + def handle_event(self, event): + CfgCreator.handle_event(self, event) + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event) + + @property + def passphrase(self): + """ The passphrase used to encrypt created data """ + if self.cfg_section: + localopt = "%s_passphrase" % self.cfg_section + passphrase = getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_passphrase) + else: + passphrase = Bcfg2.Options.setup.cfg_passphrase + if passphrase is None: + return None + try: + return Bcfg2.Options.setup.passphrases[passphrase] + except KeyError: + raise CfgCreationError("%s: No such passphrase: %s" % + (self.__class__.__name__, passphrase)) + + @property + def category(self): + """ The category to which created data is specific """ + if self.cfg_section: + localopt = "%s_category" % self.cfg_section + return getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_category) + else: + return Bcfg2.Options.setup.cfg_category + + def write_data(self, data, host=None, group=None, prio=0, ext=''): + if HAS_CRYPTO and self.encryptable and self.passphrase: + self.debug_log("Cfg: Encrypting created data") + data = Bcfg2.Server.Encryption.ssl_encrypt(data, self.passphrase) + ext = '.crypt' + CfgCreator.write_data(self, data, host=host, group=group, prio=prio, + ext=ext) + + def get_specificity(self, metadata): + """ Get config settings for key generation specificity + (per-host or per-group). + + :param metadata: The client metadata to create data for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: dict - A dict of specificity arguments suitable for + passing to + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` + or + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` + """ + category = self.xdata.get("category", self.category) + if category is None: + per_host_default = "true" + else: + per_host_default = "false" + per_host = self.xdata.get("perhost", + per_host_default).lower() == "true" + + specificity = dict(host=metadata.hostname) + if category and not per_host: + group = metadata.group_in_category(category) + if group: + specificity = dict(group=group, + prio=int(self.xdata.get("priority", 50))) + else: + self.logger.info("Cfg: %s has no group in category %s, " + "creating host-specific data" % + (metadata.hostname, category)) + return specificity + + class CfgVerificationError(Exception): """ Raised by :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an @@ -453,37 +503,27 @@ class CfgDefaultInfo(CfgInfo): """ :class:`Bcfg2.Server.Plugins.Cfg.Cfg` handler that supplies a default set of file metadata """ - def __init__(self, defaults): + def __init__(self): CfgInfo.__init__(self, '') - self.defaults = defaults __init__.__doc__ = CfgInfo.__init__.__doc__.split(".. -----")[0] - def bind_info_to_entry(self, entry, metadata): - self._set_info(entry, self.defaults) + def bind_info_to_entry(self, entry, _): + for key, value in Bcfg2.Server.Plugin.default_path_metadata().items(): + entry.attrib[key] = value bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ -#: A :class:`CfgDefaultInfo` object instantiated with -#: :attr:`Bcfg2.Server.Plugin.helper.DEFAULT_FILE_METADATA` as its -#: default metadata. This is used to set a default file metadata set -#: on an entry before a "real" :class:`CfgInfo` handler applies its -#: metadata to the entry. -DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.DEFAULT_FILE_METADATA) - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Debuggable): +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Handle a collection of host- and group-specific Cfg files with multiple different Cfg handlers in a single directory. """ - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) + def __init__(self, basename, path, entry_type): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): - rv = Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + rv = Bcfg2.Server.Plugin.EntrySet.set_debug(self, debug) for entry in self.entries.values(): entry.set_debug(debug) return rv @@ -504,7 +544,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in handlers(): + for hdlr in Bcfg2.Options.setup.cfg_handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -597,7 +637,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # most specific to least specific. data = fltr.modify_data(entry, metadata, data) - if SETUP['validate']: + if Bcfg2.Options.setup.cfg_validation: try: self._validate_data(entry, metadata, data) except CfgVerificationError: @@ -613,7 +653,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, if not isinstance(data, unicode): if not isinstance(data, str): data = data.decode('utf-8') - data = u_str(data, self.encoding) + data = u_str(data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) @@ -652,7 +692,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, rv = [] for ent in self.entries.values(): if (isinstance(ent, handler_type) and - (not ent.__specific__ or ent.specific.matches(metadata))): + (not ent.__specific__ or ent.specific.matches(metadata))): rv.append(ent) return rv @@ -668,7 +708,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :returns: None """ info_handlers = self.get_handlers(metadata, CfgInfo) - DEFAULT_INFO.bind_info_to_entry(entry, metadata) + CfgDefaultInfo().bind_info_to_entry(entry, metadata) if len(info_handlers) > 1: self.logger.error("More than one info supplier found for %s: %s" % (entry.get("name"), info_handlers)) @@ -717,13 +757,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # raises an appropriate exception return (self._create_data(entry, metadata), None) - if entry.get('mode').lower() == 'inherit': - # use on-disk permissions - self.logger.warning("Cfg: %s: Use of mode='inherit' is deprecated" - % entry.get("name")) - fname = os.path.join(self.path, generator.name) - entry.set('mode', - oct_mode(stat.S_IMODE(os.stat(fname).st_mode))) try: return (generator.get_data(entry, metadata), generator) except: @@ -801,10 +834,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, self.logger.error(msg) raise PluginExecutionError(msg) try: - etext = new_entry['text'].encode(self.encoding) + etext = new_entry['text'].encode(Bcfg2.Options.setup.encoding) except: msg = "Cfg: Cannot encode content of %s as %s" % \ - (name, self.encoding) + (name, Bcfg2.Options.setup.encoding) self.logger.error(msg) raise PluginExecutionError(msg) open(name, 'w').write(etext) @@ -812,13 +845,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, badattr = [attr for attr in ['owner', 'group', 'mode'] if attr in new_entry] if badattr: - # check for info files and inform user of their removal - for ifile in ['info', ':info']: - info = os.path.join(self.path, ifile) - if os.path.exists(info): - self.logger.info("Removing %s and replacing with info.xml" - % info) - os.remove(info) metadata_updates = {} metadata_updates.update(self.metadata) for attr in badattr: @@ -836,28 +862,45 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, flag=log) +class CfgHandlerAction(Bcfg2.Options.ComponentAction): + """ Option parser action to load Cfg handlers """ + bases = ['Bcfg2.Server.Plugins.Cfg'] + + class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client - machines. - """ + machines. """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData - def __init__(self, core, datastore): - global SETUP, CFG # pylint: disable=W0603 - Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.BooleanOption( + '--cfg-validation', cf=('cfg', 'validation'), default=True, + help='Run validation on Cfg files'), + Bcfg2.Options.Option( + cf=('cfg', 'category'), dest="cfg_category", + help='The default name of the metadata category that created data ' + 'is specific to'), + Bcfg2.Options.Option( + cf=('cfg', 'passphrase'), dest="cfg_passphrase", + help='The default passphrase name used to encrypt created data'), + Bcfg2.Options.Option( + cf=("cfg", "handlers"), dest="cfg_handlers", + help="Cfg handlers to load", + type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, + default=_handlers)] + + def __init__(self, core): + global _CFG # pylint: disable=W0603 + Bcfg2.Server.Plugin.GroupSpool.__init__(self, core) Bcfg2.Server.Plugin.PullTarget.__init__(self) - - CFG = self - - SETUP = core.setup - if 'validate' not in SETUP: - SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) - SETUP.reparse() + Bcfg2.Options.setup.cfg_handlers.sort( + key=operator.attrgetter("__priority__")) + _CFG = self __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ def has_generator(self, entry, metadata): @@ -891,127 +934,3 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, log) AcceptPullData.__doc__ = \ Bcfg2.Server.Plugin.PullTarget.AcceptPullData.__doc__ - - -class CfgLint(Bcfg2.Server.Lint.ServerPlugin): - """ warn about usage of .cat and .diff files """ - - def Run(self): - for basename, entry in list(self.core.plugins['Cfg'].entries.items()): - self.check_delta(basename, entry) - self.check_pubkey(basename, entry) - self.check_missing_files() - self.check_conflicting_handlers() - - @classmethod - def Errors(cls): - return {"cat-file-used": "warning", - "diff-file-used": "warning", - "no-pubkey-xml": "warning", - "unknown-cfg-files": "error", - "extra-cfg-files": "error", - "multiple-global-handlers": "error"} - - def check_delta(self, basename, entry): - """ check that no .cat or .diff files are in use """ - for fname, handler in entry.entries.items(): - path = handler.name - if self.HandlesFile(path) and isinstance(handler, CfgFilter): - extension = fname.split(".")[-1] - if extension in ["cat", "diff"]: - self.LintError("%s-file-used" % extension, - "%s file used on %s: %s" % (extension, - basename, - fname)) - - def check_pubkey(self, basename, entry): - """ check that privkey.xml files have corresponding pubkey.xml - files """ - if "privkey.xml" not in entry.entries: - return - privkey = entry.entries["privkey.xml"] - if not self.HandlesFile(privkey.name): - return - - pubkey = basename + ".pub" - if pubkey not in self.core.plugins['Cfg'].entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - else: - pubset = self.core.plugins['Cfg'].entries[pubkey] - if "pubkey.xml" not in pubset.entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - - def _list_path_components(self, path): - """ Get a list of all components of a path. E.g., - ``self._list_path_components("/foo/bar/foobaz")`` would return - ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed - to be in order.""" - rv = [] - remaining, component = os.path.split(path) - while component != '': - rv.append(component) - remaining, component = os.path.split(remaining) - return rv - - def check_conflicting_handlers(self): - """ Check that a single entryset doesn't have multiple - non-specific (i.e., 'all') handlers. """ - cfg = self.core.plugins['Cfg'] - for eset in cfg.entries.values(): - alls = [e for e in eset.entries.values() - if (e.specific.all and - issubclass(e.__class__, CfgGenerator))] - if len(alls) > 1: - self.LintError("multiple-global-handlers", - "%s has multiple global handlers: %s" % - (eset.path, ", ".join(os.path.basename(e.name) - for e in alls))) - - def check_missing_files(self): - """ check that all files on the filesystem are known to Cfg """ - cfg = self.core.plugins['Cfg'] - - # first, collect ignore patterns from handlers - ignore = set() - for hdlr in handlers(): - ignore.update(hdlr.__ignore__) - - # next, get a list of all non-ignored files on the filesystem - all_files = set() - for root, _, files in os.walk(cfg.data): - for fname in files: - fpath = os.path.join(root, fname) - # check against the handler ignore patterns and the - # global FAM ignore list - if (not any(fname.endswith("." + i) for i in ignore) and - not any(fnmatch(fpath, p) - for p in self.config['ignore']) and - not any(fnmatch(c, p) - for p in self.config['ignore'] - for c in self._list_path_components(fpath))): - all_files.add(fpath) - - # next, get a list of all files known to Cfg - cfg_files = set() - for root, eset in cfg.entries.items(): - cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) - for fname in eset.entries.keys()) - - # finally, compare the two - unknown_files = all_files - cfg_files - extra_files = cfg_files - all_files - if unknown_files: - self.LintError( - "unknown-cfg-files", - "Files on the filesystem could not be understood by Cfg: %s" % - "; ".join(unknown_files)) - if extra_files: - self.LintError( - "extra-cfg-files", - "Cfg has entries for files that do not exist on the " - "filesystem: %s\nThis is probably a bug." % - "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 22cacaa76..35bff0835 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -1,7 +1,7 @@ """ The Cvs plugin provides a revision interface for Bcfg2 repos using cvs. """ -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,22 +11,19 @@ class Cvs(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "CVSROOT" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized cvs plugin with CVS directory %s" % self.vcs_path) def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" + result = self.cmd.run(["env LC_ALL=C", "cvs", "log"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: - data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - return data[3].strip('\n') - except IndexError: - msg = "Failed to read CVS log" + return result.stdout.splitlines()[0].strip() + except (IndexError, AttributeError): + msg = "Failed to read revision from CVS: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "cvs log" from directory %s' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e6ef50fa1..88cb6d17c 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -6,8 +6,8 @@ import Bcfg2.Server.Plugin class DBStats(Bcfg2.Server.Plugin.Plugin): """ DBstats provides a database-backed statistics handler """ - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) self.logger.error("DBStats has been replaced with Reporting") self.logger.error("DBStats: Be sure to migrate your data " "before running the report collector") diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index b4abafb0e..01e42b2d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -1,7 +1,7 @@ """ Darcs is a version plugin for dealing with Bcfg2 repos stored in the Darcs VCS. """ -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,23 +11,19 @@ class Darcs(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "_darcs" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized Darcs plugin with darcs directory %s" % self.vcs_path) def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - revision = data[0].strip('\n') - except: - msg = "Failed to read darcs repository" + result = self.cmd.run(["env LC_ALL=C", "darcs", "changes"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) + if result.success: + return result.stdout.splitlines()[0].strip() + else: + msg = "Failed to read revision from darcs: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "darcs changes" from directory %s' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return revision diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 66f299bc9..3d3ef8f8c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -2,57 +2,33 @@ blacklist certain entries. """ import os -import lxml.etree import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor -class DecisionFile(Bcfg2.Server.Plugin.SpecificData): +class DecisionFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a Decisions XML file """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - self.contents = None - - def handle_event(self, event): - Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) - self.contents = lxml.etree.XML(self.data) - - def get_decisions(self): + def get_decisions(self, metadata): """ Get a list of whitelist or blacklist tuples """ + if self.xdata is None: + # no white/blacklist has been read yet, probably because + # it doesn't exist + return [] return [(x.get('type'), x.get('name')) - for x in self.contents.xpath('.//Decision')] + for x in self.XMLMatch(metadata).xpath('.//Decision')] -class Decisions(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Plugin, +class Decisions(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Decision): - """ Decisions plugin - - Arguments: - - `core`: Bcfg2.Core instance - - `datastore`: File repository location - """ - basename_is_regex = True + """ Decisions plugin """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Decision.__init__(self) - Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', - self.data, DecisionFile, - core.setup['encoding']) - core.fam.AddMonitor(self.data, self) - - def HandleEvent(self, event): - """ Handle events on Decision files by passing them off to - EntrySet.handle_event """ - if event.filename != self.path: - return self.handle_event(event) + self.whitelist = DecisionFile(os.path.join(self.data, "whitelist.xml")) + self.blacklist = DecisionFile(os.path.join(self.data, "blacklist.xml")) def GetDecisions(self, metadata, mode): - ret = [] - for cdt in self.get_matching(metadata): - if os.path.basename(cdt.name).startswith(mode): - ret.extend(cdt.get_decisions()) - return ret + return getattr(self, mode).get_decision(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 04c14aa96..79e2ca0e2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -9,6 +9,8 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + # Rules is a Generator that happens to implement all of the # functionality we want, so we overload it, but Defaults should # _not_ handle any entries; it does its stuff in the structure diff --git a/src/lib/Bcfg2/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index d3a1ee871..1872e68af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -1,43 +1,20 @@ """This plugin provides automatic dependency handling.""" import lxml.etree - import Bcfg2.Server.Plugin - - -class DNode(Bcfg2.Server.Plugin.INode): - """DNode provides supports for single predicate types for dependencies.""" - def _load_children(self, data, idict): - for item in data.getchildren(): - if item.tag in self.containers: - self.children.append(self.__class__(item, idict, self)) - else: - data = [(child.tag, child.get('name')) - for child in item.getchildren()] - try: - self.contents[item.tag][item.get('name')] = data - except KeyError: - self.contents[item.tag] = {item.get('name'): data} - - -class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): - __node__ = DNode +from Bcfg2.Server.Plugin import PluginExecutionError class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.StructureValidator): - name = 'Deps' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = DepXMLSrc - # Override the default sort_order (of 500) so that this plugin # gets handled after others running at the default. In particular, # we want to run after Packages, so we can see the final set of # packages that will be installed on the client. sort_order = 750 - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.PrioDir.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) self.cache = {} @@ -55,63 +32,59 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, tag = entry.tag if tag.startswith('Bound'): tag = tag[5:] - if (tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): + if ((tag, entry.get('name')) not in entries + and not isinstance(entry, lxml.etree._Comment)): entries.append((tag, entry.get('name'))) entries.sort() entries = tuple(entries) - gdata = list(metadata.groups) - gdata.sort() - gdata = tuple(gdata) + groups = list(metadata.groups) + groups.sort() + groups = tuple(groups) # Check to see if we have cached the prereqs already - if (entries, gdata) in self.cache: - prereqs = self.cache[(entries, gdata)] + if (entries, groups) in self.cache: + prereqs = self.cache[(entries, groups)] else: prereqs = self.calculate_prereqs(metadata, entries) - self.cache[(entries, gdata)] = prereqs + self.cache[(entries, groups)] = prereqs - newstruct = lxml.etree.Element("Independent") + newstruct = lxml.etree.Element("Independent", + name=self.__class__.__name__) for tag, name in prereqs: - try: - lxml.etree.SubElement(newstruct, tag, name=name) - except: - self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) + lxml.etree.SubElement(newstruct, tag, name=name) structures.append(newstruct) - def calculate_prereqs(self, metadata, entries): """Calculate the prerequisites defined in Deps for the passed set of entries. """ prereqs = [] - [src.Cache(metadata) for src in self.entries.values()] - toexamine = list(entries[:]) while toexamine: entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] + # tuples of (PriorityStructFile, element) for each + # matching element and the structfile that contains it + matching = [] + for deps in self.entries.values(): + el = deps.find("/%s[name='%s']" % (entry.tag, + entry.get("name"))) + if el: + matching.append((deps, el)) if len(matching) > 1: - prio = [int(src.priority) for src in matching] + prio = [int(m[0].priority) for m in matching] if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError + raise PluginExecutionError( + "Deps: Found conflicting dependencies with same " + "priority for %s:%s for %s: %s" % + (entry.tag, entry.get("name"), + metadata.hostname, [m[0].name for m in matching])) index = prio.index(max(prio)) matching = [matching[index]] - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - # XML comments seem to show up in the cache as a - # tuple with item 0 being callable. The logic - # below filters them out. Would be better to - # exclude them when we load the cache in the first - # place. - if prq not in prereqs and prq not in entries and not callable(prq[0]): - toexamine.append(prq) - prereqs.append(prq) - else: + if not matching: continue + for prq in matching[0][1].getchildren(): + if prq not in prereqs and prq not in entries: + toexamine.append(prq) + prereqs.append(prq) return prereqs diff --git a/src/lib/Bcfg2/Server/Plugins/Editor.py b/src/lib/Bcfg2/Server/Plugins/Editor.py deleted file mode 100644 index f82e0f1dd..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Editor.py +++ /dev/null @@ -1,80 +0,0 @@ -import Bcfg2.Server.Plugin -import re -import lxml.etree - - -def linesub(pattern, repl, filestring): - """Substitutes instances of pattern with repl in filestring.""" - if filestring == None: - filestring = '' - output = list() - fileread = filestring.split('\n') - for line in fileread: - output.append(re.sub(pattern, repl, filestring)) - return '\n'.join(output) - - -class EditDirectives(Bcfg2.Server.Plugin.SpecificData): - """This object handles the editing directives.""" - def ProcessDirectives(self, input): - """Processes a list of edit directives on input.""" - temp = input - for directive in self.data.split('\n'): - directive = directive.split(',') - temp = linesub(directive[0], directive[1], temp) - return temp - - -class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path, entry_type, encoding): - self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1]) - Bcfg2.Server.Plugin.EntrySet.__init__(self, - basename, - path, - entry_type, - encoding) - self.inputs = dict() - - def bind_entry(self, entry, metadata): - client = metadata.hostname - filename = entry.get('name') - permdata = {'owner': 'root', - 'group': 'root', - 'mode': '0644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) - if not entry.text: - entry.set('empty', 'true') - try: - f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w') - f.write(entry.text) - f.close() - except: - pass - - def get_client_data(self, client): - return self.inputs[client] - - -class Editor(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.Probing): - name = 'Editor' - __author__ = 'bcfg2-dev@mcs.anl.gov' - filename_pattern = 'edits' - es_child_cls = EditDirectives - es_cls = EditEntrySet - - def GetProbes(self, _): - '''Return a set of probes for execution on client''' - probelist = list() - for name in list(self.entries.keys()): - probe = lxml.etree.Element('probe') - probe.set('name', name) - probe.set('source', "Editor") - probe.text = "cat %s" % name - probelist.append(probe) - return probelist - - def ReceiveData(self, client, datalist): - for data in datalist: - self.entries[data.get('name')].inputs[client.hostname] = data.text diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 8e074118f..38f9403f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -8,9 +8,9 @@ import os import sys import errno import lxml.etree -import Bcfg2.Options import Bcfg2.Server import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor from Bcfg2.Compat import b64decode #: The probe we send to clients to get the file data. Returns an XML @@ -64,13 +64,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, the client """ __author__ = 'chris.a.st.pierre@gmail.com' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Probing.__init__(self) self.config = \ Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, 'config.xml'), - fam=core.fam, should_monitor=True, create=self.name) self.entries = dict() @@ -88,7 +87,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, # for which update is false; we can't possibly do # anything with the data we get from such a probe if (entry.get('update', 'false').lower() == "false" and - not cfg.has_generator(entry, metadata)): + not cfg.has_generator(entry, metadata)): continue self.entries[metadata.hostname][path] = entry probe = lxml.etree.Element('probe', name=path, @@ -148,7 +147,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, self.write_file(fileloc, contents) self.verify_file(filename, contents, metadata) infoxml = os.path.join(cfg.data, filename.lstrip("/"), "info.xml") - self.write_infoxml(infoxml, entry, data) + self.write_infoxml(infoxml, data) elif entrydata == contents: self.debug_log("Existing %s contents match probed contents" % filename) @@ -198,7 +197,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if tries >= 10: self.logger.error("%s still not registered" % filename) return - self.core.fam.handle_events_in_interval(1) + Bcfg2.Server.FileMonitor.get_fam().handle_events_in_interval(1) try: cfg.entries[filename].bind_entry(entry, metadata) except Bcfg2.Server.Plugin.PluginExecutionError: @@ -214,18 +213,18 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, updated = True tries += 1 - def write_infoxml(self, infoxml, entry, data): + def write_infoxml(self, infoxml, data): """ write an info.xml for the file """ if os.path.exists(infoxml): return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) + default_mdata = Bcfg2.Server.Plugin.default_path_metadata() info = lxml.etree.Element( "Info", - owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), - mode=data.get("mode", Bcfg2.Options.MDATA_MODE.value), - encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) + owner=data.get("owner", default_mdata['owner']), + group=data.get("group", default_mdata['group']), + mode=data.get("mode", default_mdata['mode'])) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 6165ac651..c4d9af4a4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -1,7 +1,7 @@ """ The Fossil plugin provides a revision interface for Bcfg2 repos using fossil.""" -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,24 +11,24 @@ class Fossil(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "_FOSSIL_" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized Fossil plugin with fossil directory %s" % self.vcs_path) def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" + result = self.cmd.run(["env LC_ALL=C", "fossil", "info"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: - data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data - if line.split(': ')[0].strip() == 'checkout'][-1] - return revline.split(' ')[0] - except IndexError: - msg = "Failed to read fossil info" + revision = None + for line in result.stdout.splitlines(): + ldata = line.split(': ') + if ldata[0].strip() == 'checkout': + revision = line[1].strip().split(' ')[0] + return revision + except (IndexError, AttributeError): + msg = "Failed to read revision from Fossil: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "fossil info" from directory "%s"' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 44971aba7..9012fceb0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -2,13 +2,14 @@ git. """ import sys +import Bcfg2.Options from Bcfg2.Server.Plugin import Version, PluginExecutionError -from subprocess import Popen, PIPE try: import git HAS_GITPYTHON = True except ImportError: + from Bcfg2.Utils import Executor HAS_GITPYTHON = False @@ -20,14 +21,16 @@ class Git(Version): if HAS_GITPYTHON: __rmi__ = Version.__rmi__ + ['Update'] - def __init__(self, core, datastore): - Version.__init__(self, core, datastore) + def __init__(self, core): + Version.__init__(self, core) if HAS_GITPYTHON: - self.repo = git.Repo(self.vcs_root) + self.repo = git.Repo(Bcfg2.Options.setup.vcs_root) + self.cmd = None else: self.logger.debug("Git: GitPython not found, using CLI interface " "to Git") self.repo = None + self.cmd = Executor() self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) @@ -43,16 +46,16 @@ class Git(Version): return self.repo.head.commit.hexsha else: cmd = ["git", "--git-dir", self.vcs_path, - "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + "--work-tree", Bcfg2.Options.setup.vcs_root, + "rev-parse", "HEAD"] self.debug_log("Git: Running %s" % cmd) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - raise Exception(err) - return rv + result = self.cmd.run(cmd) + if not result.success: + raise Exception(result.stderr) + return result.stdout except: raise PluginExecutionError("Git: Error getting revision from %s: " - "%s" % (self.vcs_root, + "%s" % (Bcfg2.Options.setup.vcs_root, sys.exc_info()[1])) def Update(self, ref=None): @@ -61,14 +64,15 @@ class Git(Version): """ self.logger.info("Git: Git.Update(ref='%s')" % ref) self.debug_log("Git: Performing garbage collection on repo at %s" % - self.vcs_root) + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) - self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + self.debug_log("Git: Fetching all refs for repo at %s" % + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.fetch('--all')) except git.GitCommandError: @@ -101,5 +105,5 @@ class Git(Version): "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) + (Bcfg2.Options.setup.vcs_root, self.get_revision())) return True diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index d74c16e8b..b60f60e65 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -6,30 +6,24 @@ import lxml.etree from threading import local import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Metadata import MetadataGroup -try: - from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile -except ImportError: - # BundleTemplateFile missing means that genshi is missing. we - # import genshi to get the _real_ error - import genshi # pylint: disable=W0611 -class GroupLogicConfig(BundleTemplateFile): +class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): """ Representation of the GroupLogic groups.xml file """ create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) - def __init__(self, name, fam): - BundleTemplateFile.__init__(self, name, - Bcfg2.Server.Plugin.Specificity(), None) - self.fam = fam - self.should_monitor = True - self.fam.AddMonitor(self.name, self) + def _match(self, item, metadata, *args): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata, + *args) - def _match(self, item, metadata): + def _xml_match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] - return BundleTemplateFile._match(self, item, metadata) + return Bcfg2.Server.Plugin.StructFile._xml_match(self, item, metadata, + *args) class GroupLogic(Bcfg2.Server.Plugin.Plugin, @@ -41,11 +35,11 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, # use groups set by them sort_order = 1000 - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), - core.fam) + should_monitor=True) self._local = local() def get_additional_groups(self, metadata): @@ -66,7 +60,7 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, return [] self._local.building.add(metadata.hostname) rv = [] - for el in self.config.get_xml_value(metadata).findall("Group"): + for el in self.config.XMLMatch(metadata).findall("Group"): if el.get("category"): rv.append(MetadataGroup(el.get("name"), category=el.get("category"))) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 09685d972..7fa95fd05 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -69,11 +69,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): create = 'GroupPatterns' def __init__(self, filename, core=None): - try: - fam = core.fam - except AttributeError: - fam = None - Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, should_monitor=True) self.core = core self.patterns = [] @@ -81,7 +77,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): def Index(self): Bcfg2.Server.Plugin.XMLFileBacked.Index(self) if (self.core and - self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache_mode in ['cautious', 'aggressive']): self.core.metadata_cache.expire() self.patterns = [] for entry in self.xdata.xpath('//GroupPattern'): @@ -118,48 +114,11 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """ set group membership based on client hostnames """ - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = PatternFile(os.path.join(self.data, 'config.xml'), core=core) def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) - - -class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns - <server-plugins-grouping-grouppatterns>` patterns for validity. - This is simply done by trying to create a - :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for - each pattern, and catching exceptions and presenting them as - ``bcfg2-lint`` errors.""" - - def Run(self): - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, groups, ptype="NamePattern"): - """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index 3c9b8a459..8427a56c3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -32,15 +32,12 @@ from guppy.heapy import Remote class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" - name = 'Guppy' __author__ = 'bcfg-dev@mcs.anl.gov' - - experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] __child_rmi__ = __rmi__[:] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) self.Enable() diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 3fd3918bd..7554b4d52 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -12,15 +12,15 @@ class Hg(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".hg" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.logger.debug("Initialized hg plugin with hg directory %s" % self.vcs_path) def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = self.vcs_root + "/" + repo_path = Bcfg2.Options.setup.vcs_root + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() return repo.changelog.rev(tip) diff --git a/src/lib/Bcfg2/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py deleted file mode 100644 index 55757e0b4..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Hostbase.py +++ /dev/null @@ -1,599 +0,0 @@ -""" -This file provides the Hostbase plugin. -It manages dns/dhcp/nis host information -""" - -from lxml.etree import Element, SubElement -import os -import re -from time import strftime -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError -from django.template import Context, loader -from django.db import connection -# Compatibility imports -from Bcfg2.Compat import StringIO - -try: - set -except NameError: - # deprecated since python 2.6 - from sets import Set as set - - -class Hostbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.Generator): - """The Hostbase plugin handles host/network info.""" - name = 'Hostbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - filepath = '/my/adm/hostbase/files/bind' - deprecated = True - - def __init__(self, core, datastore): - - self.ready = False - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - files = ['zone.tmpl', - 'reversesoa.tmpl', - 'named.tmpl', - 'reverseappend.tmpl', - 'dhcpd.tmpl', - 'hosts.tmpl', - 'hostsappend.tmpl'] - self.filedata = {} - self.dnsservers = [] - self.dhcpservers = [] - self.templates = {'zone': loader.get_template('zone.tmpl'), - 'reversesoa': loader.get_template('reversesoa.tmpl'), - 'named': loader.get_template('named.tmpl'), - 'namedviews': loader.get_template('namedviews.tmpl'), - 'reverseapp': loader.get_template('reverseappend.tmpl'), - 'dhcp': loader.get_template('dhcpd.tmpl'), - 'hosts': loader.get_template('hosts.tmpl'), - 'hostsapp': loader.get_template('hostsappend.tmpl'), - } - self.Entries['ConfigFile'] = {} - self.__rmi__ = ['rebuildState'] - try: - self.rebuildState(None) - except: - raise PluginInitError - - def FetchFile(self, entry, metadata): - """Return prebuilt file data.""" - fname = entry.get('name').split('/')[-1] - if not fname in self.filedata: - raise PluginExecutionError - perms = {'owner': 'root', - 'group': 'root', - 'mode': '644'} - [entry.attrib.__setitem__(key, value) - for (key, value) in list(perms.items())] - entry.text = self.filedata[fname] - - def BuildStructures(self, metadata): - """Build hostbase bundle.""" - if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: - return [] - output = Element("Bundle", name='hostbase') - if metadata.hostname in self.dnsservers: - for configfile in self.Entries['ConfigFile']: - if re.search('/etc/bind/', configfile): - SubElement(output, "ConfigFile", name=configfile) - if metadata.hostname in self.dhcpservers: - SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") - return [output] - - def rebuildState(self, _): - """Pre-cache all state information for hostbase config files - callable as an XMLRPC function. - - """ - self.buildZones() - self.buildDHCP() - self.buildHosts() - self.buildHostsLPD() - self.buildPrinters() - self.buildNetgroups() - return True - - def buildZones(self): - """Pre-build and stash zone files.""" - cursor = connection.cursor() - - cursor.execute("SELECT id, serial FROM hostbase_zone") - zones = cursor.fetchall() - - for zone in zones: - # update the serial number for all zone files - todaydate = (strftime('%Y%m%d')) - try: - if todaydate == str(zone[1])[:8]: - serial = zone[1] + 1 - else: - serial = int(todaydate) * 100 - except (KeyError): - serial = int(todaydate) * 100 - cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") - zones = cursor.fetchall() - - iplist = [] - hosts = {} - - for zone in zones: - zonefile = StringIO() - externalzonefile = StringIO() - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - nameservers = cursor.fetchall() - cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z - INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - addresses = cursor.fetchall() - cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z - INNER JOIN hostbase_mx m ON z.mx_id = m.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - mxs = cursor.fetchall() - context = Context({ - 'zone': zone, - 'nameservers': nameservers, - 'addresses': addresses, - 'mxs': mxs - }) - zonefile.write(self.templates['zone'].render(context)) - externalzonefile.write(self.templates['zone'].render(context)) - - querystring = """SELECT h.hostname, p.ip_addr, - n.name, c.cname, m.priority, m.mx, n.dns_view - FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE n.name LIKE '%%%%%s' - AND h.status = 'active' - ORDER BY h.hostname, n.name, p.ip_addr - """ % zone[1] - cursor.execute(querystring) - zonehosts = cursor.fetchall() - prevhost = (None, None, None, None) - cnames = StringIO() - cnamesexternal = StringIO() - for host in zonehosts: - if not host[2].split(".", 1)[1] == zone[1]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - continue - if not prevhost[1] == host[1] or not prevhost[2] == host[2]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - zonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - elif not prevhost[5] == host[5]: - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - - if host[3]: - try: - if host[3].split(".", 1)[1] == zone[1]: - cnames.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - else: - cnames.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - - except: - pass - prevhost = host - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - zonefile.write("\n\n%s" % zone[9]) - externalzonefile.write("\n\n%s" % zone[9]) - self.filedata[zone[1]] = zonefile.getvalue() - self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile - self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") - reversezones = cursor.fetchall() - - reversenames = [] - for reversezone in reversezones: - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % reversezone[0]) - reverse_nameservers = cursor.fetchall() - - context = Context({ - 'inaddr': reversezone[1].rstrip('.rev'), - 'zone': reversezone, - 'nameservers': reverse_nameservers, - }) - - self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1]] += reversezone[9] - self.filedata[reversezone[1] + '.external'] += reversezone[9] - - subnet = reversezone[1].split(".") - subnet.reverse() - reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) - - for filename in reversenames: - cursor.execute(""" - SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON n.ip_id = p.id - WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr - """ % filename[1]) - reversehosts = cursor.fetchall() - zonefile = StringIO() - externalzonefile = StringIO() - if len(filename[0].split(".")) == 2: - originlist = [] - [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]]))) - for ip in reversehosts - if (".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] - for origin in originlist: - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if host[1].rstrip('0123456789').rstrip('.') == origin[1]] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].rstrip('0123456789').rstrip('.') == origin[1] - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - else: - originlist = [filename[0]] - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].split("."), host[0]) not in hosts] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if ((host[1].split("."), host[0]) not in hosts_external - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': filename[0], - 'fileorigin': None, - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': filename[0], - 'fileorigin': None, - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() - self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile - self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile - - ## here's where the named.conf file gets written - context = Context({ - 'zones': zones, - 'reverses': reversenames, - }) - self.filedata['named.conf'] = self.templates['named'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile - self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile - - def buildDHCP(self): - """Pre-build dhcpd.conf and stash in the filedata table.""" - - # fetches all the hosts with DHCP == True - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname, mac_addr, ip_addr - FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip ip ON i.id = ip.interface_id - WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' - AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' - ORDER BY h.hostname, i.mac_addr - """) - - dhcphosts = cursor.fetchall() - count = 0 - hosts = [] - hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] - if len(dhcphosts) > 1: - for x in range(1, len(dhcphosts)): - # if an interface has 2 or more ip addresses - # adds the ip to the current interface - if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: - hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) - # if a host has 2 or more interfaces - # writes the current one and grabs the next - elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: - hosts.append(hostdata) - count += 1 - hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] - # new host found, writes current data to the template - else: - hosts.append(hostdata) - count = 0 - hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] - #makes sure the last of the data gets written out - if hostdata not in hosts: - hosts.append(hostdata) - - context = Context({ - 'hosts': hosts, - 'numips': len(hosts), - }) - - self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile - - def buildHosts(self): - """Pre-build and stash /etc/hosts file.""" - - append_data = [] - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host ORDER BY hostname - """) - hostbase = cursor.fetchall() - domains = [host[0].split(".", 1)[1] for host in hostbase] - domains_set = set(domains) - domain_data = [(domain, domains.count(domain)) for domain in domains_set] - domain_data.sort() - - cursor.execute(""" - SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr - """) - ips = cursor.fetchall() - three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ - for ip in ips] - three_octets_set = set(three_octets) - three_octets_data = [(octet, three_octets.count(octet)) \ - for octet in three_octets_set] - three_octets_data.sort() - - for three_octet in three_octets_data: - querystring = """SELECT h.hostname, h.primary_user, - p.ip_addr, n.name, c.cname - FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] - cursor.execute(querystring) - tosort = list(cursor.fetchall()) - tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) - append_data.append((three_octet, tuple(tosort))) - - two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] - two_octets_set = set(two_octets) - two_octets_data = [(octet, two_octets.count(octet)) - for octet in two_octets_set] - two_octets_data.sort() - - context = Context({ - 'domain_data': domain_data, - 'three_octets_data': three_octets_data, - 'two_octets_data': two_octets_data, - 'three_octets': three_octets, - 'num_ips': len(three_octets), - }) - - self.filedata['hosts'] = self.templates['hosts'].render(context) - - for subnet in append_data: - ips = [] - simple = True - namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] - cnamelist = [] - if subnet[1][0][4]: - cnamelist.append(subnet[1][0][4].split('.', 1)[0]) - simple = False - appenddata = subnet[1][0] - for ip in subnet[1][1:]: - if appenddata[2] == ip[2]: - namelist.append(ip[3].split('.', 1)[0]) - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - appenddata = ip - else: - if appenddata[0] == ip[0]: - simple = False - ips.append((appenddata[2], appenddata[0], set(namelist), - cnamelist, simple, appenddata[1])) - appenddata = ip - simple = True - namelist = [ip[3].split('.', 1)[0]] - cnamelist = [] - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - ips.append((appenddata[2], appenddata[0], set(namelist), - cnamelist, simple, appenddata[1])) - context = Context({ - 'subnet': subnet[0], - 'ips': ips, - }) - self.filedata['hosts'] += self.templates['hostsapp'].render(context) - self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile - - def buildPrinters(self): - """The /mcs/etc/printers.data file""" - header = """# This file is automatically generated. DO NOT EDIT IT! -# -Name Room User Type Notes -============== ========== ============================== ======================== ==================== -""" - - cursor = connection.cursor() - # fetches all the printers from the database - cursor.execute(""" - SELECT printq, location, primary_user, comments - FROM hostbase_host - WHERE whatami='printer' AND printq <> '' AND status = 'active' - ORDER BY printq - """) - printers = cursor.fetchall() - - printersfile = header - for printer in printers: - # splits up the printq line and gets the - # correct description out of the comments section - temp = printer[3].split('\n') - for printq in re.split(',[ ]*', printer[0]): - if len(temp) > 1: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], temp[1], temp[0])) - else: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], '', printer[3])) - self.filedata['printers.data'] = printersfile - self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile - - def buildHostsLPD(self): - """Creates the /mcs/etc/hosts.lpd file""" - - # this header needs to be changed to be more generic - header = """+@machines -+@all-machines -achilles.ctd.anl.gov -raven.ops.anl.gov -seagull.hr.anl.gov -parrot.ops.anl.gov -condor.ops.anl.gov -delphi.esh.anl.gov -anlcv1.ctd.anl.gov -anlvms.ctd.anl.gov -olivia.ctd.anl.gov\n\n""" - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' - ORDER BY hostname""") - redmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' - """) - redmachines.extend(list(cursor.fetchall())) - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' - ORDER BY hostname""") - winmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' - """) - winmachines.__add__(list(cursor.fetchall())) - hostslpdfile = header - for machine in redmachines: - hostslpdfile += machine[0] + "\n" - hostslpdfile += "\n" - for machine in winmachines: - hostslpdfile += machine[0] + "\n" - self.filedata['hosts.lpd'] = hostslpdfile - self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile - - def buildNetgroups(self): - """Makes the *-machine files""" - header = """################################################################### -# This file lists hosts in the '%s' machine netgroup, it is -# automatically generated. DO NOT EDIT THIS FILE! -# -# Number of hosts in '%s' machine netgroup: %i -#\n\n""" - - cursor = connection.cursor() - # fetches all the hosts that with valid netgroup entries - cursor.execute(""" - SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' - ORDER BY h.netgroup, h.hostname - """) - nameslist = cursor.fetchall() - # gets the first host and initializes the hash - hostdata = nameslist[0] - netgroups = {hostdata[2]: [hostdata[0]]} - for row in nameslist: - # if new netgroup, create it - if row[2] not in netgroups: - netgroups.update({row[2]: []}) - # if it belongs in the netgroup and has multiple interfaces, put them in - if hostdata[0] == row[0] and row[3]: - netgroups[row[2]].append(row[1]) - hostdata = row - # if its a new host, write the old one to the hash - elif hostdata[0] != row[0]: - netgroups[row[2]].append(row[0]) - hostdata = row - - for netgroup in netgroups: - fileoutput = StringIO() - fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) - for each in netgroups[netgroup]: - fileoutput.write(each + "\n") - self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() - fileoutput.close() - self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile - - cursor.execute(""" - UPDATE hostbase_host SET dirty=0 - """) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index f724402d0..553ddbc47 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -3,7 +3,6 @@ import logging import sys import time import traceback -import Bcfg2.Options import Bcfg2.Server.Plugin logger = logging.getLogger('Bcfg2.Plugins.Ldap') @@ -44,10 +43,10 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): The approach implemented here is having the user call a registering decorator that updates a global variable in this module. """ - def __init__(self, filename, fam): + def __init__(self, filename): self.filename = filename Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename) - fam.AddMonitor(self.filename, self) + self.fam.AddMonitor(self.filename, self) def Index(self): """ @@ -69,12 +68,12 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): experimental = True debug_flag = False - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = ConfigFile(self.data + "/config.py", core.fam) + self.config = ConfigFile(self.data + "/config.py") - def debug_log(self, message, flag = None): + def debug_log(self, message, flag=None): if (flag is None) and self.debug_flag or flag: self.logger.error(message) @@ -83,37 +82,39 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): try: data = {} self.debug_log("LdapPlugin debug: found queries " + - str(LDAP_QUERIES)) + str(LDAP_QUERIES)) for QueryClass in LDAP_QUERIES: query = QueryClass() if query.is_applicable(metadata): self.debug_log("LdapPlugin debug: processing query '" + - query.name + "'") + query.name + "'") data[query.name] = query.get_result(metadata) else: self.debug_log("LdapPlugin debug: query '" + query.name + - "' not applicable to host '" + metadata.hostname + "'") + "' not applicable to host '" + + metadata.hostname + "'") return data except Exception: if hasattr(query, "name"): logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + - str(query.name) + - "', query results will be empty" + - " and may cause bind failures") + "Exception during processing of query named '" + + str(query.name) + + "', query results will be empty" + + " and may cause bind failures") for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): logger.error("LdapPlugin error: " + - line.replace("\n", "")) + line.replace("\n", "")) return {} + class LdapConnection(object): """ Connection to an LDAP server. """ - def __init__(self, host = "localhost", port = 389, - binddn = None, bindpw = None): + def __init__(self, host="localhost", port=389, + binddn=None, bindpw=None): self.host = host self.port = port self.binddn = binddn @@ -134,8 +135,8 @@ class LdapConnection(object): for attempt in range(RETRY_COUNT + 1): if attempt >= 1: logger.error("LdapPlugin error: " + - "LDAP server down (retry " + str(attempt) + "/" + - str(RETRY_COUNT) + ")") + "LDAP server down (retry " + str(attempt) + "/" + + str(RETRY_COUNT) + ")") try: if not self.conn: self.init_conn() @@ -155,6 +156,7 @@ class LdapConnection(object): def url(self): return "ldap://" + self.host + ":" + str(self.port) + class LdapQuery(object): """ Query referencing an LdapConnection and providing several @@ -211,9 +213,10 @@ class LdapQuery(object): return self.result else: logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) + "No valid connection defined for query " + str(self)) return None + class LdapSubQuery(LdapQuery): """ SubQueries are meant for internal use only and are not added @@ -244,5 +247,5 @@ class LdapSubQuery(LdapQuery): return self.process_result(metadata, **kwargs) else: logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index d6febcff6..bf51ff678 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -12,41 +12,52 @@ import socket import logging import lxml.etree import Bcfg2.Server -import Bcfg2.Server.Lint +import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked +from Bcfg2.Server.Cache import Cache # pylint: disable=W0622 from Bcfg2.Compat import MutableMapping, all, any, wraps # pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo -try: - from django.db import models - HAS_DJANGO = True -except ImportError: - HAS_DJANGO = False +# pylint: disable=C0103 +ClientVersions = None +MetadataClientModel = None +# pylint: enable=C0103 +HAS_DJANGO = False -LOGGER = logging.getLogger(__name__) +def load_django_models(): + """ Load models for Django after option parsing has completed """ + # pylint: disable=W0602 + global MetadataClientModel, ClientVersions, HAS_DJANGO + # pylint: enable=W0602 -if HAS_DJANGO: - class MetadataClientModel(models.Model, + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return + + class MetadataClientModel(models.Model, # pylint: disable=W0621 Bcfg2.Server.Plugin.PluginDatabaseModel): """ django model for storing clients in the database """ hostname = models.CharField(max_length=255, primary_key=True) version = models.CharField(max_length=31, null=True) - class ClientVersions(MutableMapping, + class ClientVersions(MutableMapping, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.DatabaseBacked): """ dict-like object to make it easier to access client bcfg2 versions from the database """ - create = False def __getitem__(self, key): try: - return MetadataClientModel.objects.get(hostname=key).version + return MetadataClientModel.objects.get( + hostname=key).version except MetadataClientModel.DoesNotExist: raise KeyError(key) @@ -80,7 +91,7 @@ if HAS_DJANGO: def keys(self): """ Get keys for the mapping """ - return [c.hostname for c in MetadataClientModel.objects.all()] + return list(iter(self)) def __contains__(self, key): try: @@ -93,25 +104,19 @@ if HAS_DJANGO: class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - # we tell XMLFileBacked _not_ to add a monitor for this file, - # because the main Metadata plugin has already added one. - # then we immediately set should_monitor to the proper value, - # so that XInclude'd files get properly watched + def __init__(self, metadata, basefile): fpath = os.path.join(metadata.data, basefile) toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, - fam=metadata.core.fam, should_monitor=False, create=toptag) - self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None self.basedata = None self.basedir = metadata.data self.logger = metadata.logger - self.pseudo_monitor = isinstance(metadata.core.fam, + self.pseudo_monitor = isinstance(Bcfg2.Server.FileMonitor.get_fam(), Bcfg2.Server.FileMonitor.Pseudo) def _get_xdata(self): @@ -250,8 +255,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def add_monitor(self, fpath): self.extras.append(fpath) - if self.fam and self.should_monitor: - self.fam.AddMonitor(fpath, self.metadata) + self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event=None): """Handle fam events""" @@ -354,6 +358,8 @@ class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): + self.logger = logging.getLogger(self.__class__.__name__) + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` #: object for the given hostname. #: @@ -406,8 +412,9 @@ class MetadataQuery(object): @wraps(func) def inner(arg): if isinstance(arg, str): - LOGGER.warning("%s: %s takes a list as argument, not a string" - % (self.__class__.__name__, func.__name__)) + self.logger.warning("%s: %s takes a list as argument, not a " + "string" % (self.__class__.__name__, + func.__name__)) return func(arg) # pylint: enable=C0111 @@ -490,25 +497,33 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' sort_order = 500 - def __init__(self, core, datastore, watch_clients=True): + options = Bcfg2.Server.Plugin.DatabaseBacked.options + [ + Bcfg2.Options.Common.password, + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin"), + Bcfg2.Options.Option( + cf=('communication', 'authentication'), default='cert+password', + choices=['cert', 'bootstrap', 'cert+password'], + help='Default client authentication method')] + options_parsed_hook = staticmethod(load_django_models) + + def __init__(self, core): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) - self.watch_clients = watch_clients + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) self.states = dict() self.extra = dict() self.handlers = dict() self.groups_xml = self._handle_file("groups.xml") if (self._use_db and - os.path.exists(os.path.join(self.data, "clients.xml"))): + os.path.exists(os.path.join(self.data, "clients.xml"))): self.logger.warning("Metadata: database enabled but clients.xml " "found, parsing in compatibility mode") self.clients_xml = self._handle_file("clients.xml") @@ -540,15 +555,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: - self.versions = ClientVersions(core, datastore) + self.versions = ClientVersions(core) # pylint: disable=E1102 else: self.versions = dict() self.uuid = {} self.session_cache = {} + self.cache = Cache("Metadata") self.default = None self.pdirty = False - self.password = core.setup['password'] + self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, self.list_clients, self.get_client_names_by_groups, @@ -576,16 +592,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ - if self.watch_clients: - try: - self.core.fam.AddMonitor(os.path.join(self.data, fname), self) - except: - err = sys.exc_info()[1] - msg = "Unable to add file monitor for %s: %s" % (fname, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.states[fname] = False - xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) + try: + Bcfg2.Server.FileMonitor.get_fam().AddMonitor( + os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + self.states[fname] = False + xmlcfg = XMLMetadataConfig(self, fname) aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) self.handlers[xmlcfg.HandleEvent] = getattr(self, "_handle_%s_event" % aname) @@ -600,7 +616,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, elif alias: for child in node: if (child.tag == "Alias" and - child.attrib["name"] == name): + child.attrib["name"] == name): return node return None @@ -666,7 +682,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: + # pylint: disable=E1102 client = MetadataClientModel(hostname=client_name) + # pylint: enable=E1102 client.save() self.update_client_list() return client @@ -814,7 +832,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client.get('secure', 'false').lower() == 'true': self.secure.append(clname) if (client.get('location', 'fixed') == 'floating' or - client.get('floating', 'false').lower() == 'true'): + client.get('floating', 'false').lower() == 'true'): self.floating.append(clname) if 'password' in client.attrib: self.passwords[clname] = client.get('password') @@ -940,16 +958,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self._aggregate_conditions(conditions)) self.states['groups.xml'] = True - def expire_cache(self, key=None): - self.core.metadata_cache.expire(key) - def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.expire_cache() + self.cache.expire() # clear out the list of category suppressions that # have been warned about, since this may change when @@ -1103,7 +1118,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for p in self.group_membership[grpname]): newgroups.add(grpname) if (grpname in self.groups and - self.groups[grpname].category): + self.groups[grpname].category): categories[self.groups[grpname].category] = grpname groups.update(newgroups) for grpname, predicates in self.negated_groups.items(): @@ -1112,7 +1127,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if any(p(client, groups, categories) for p in predicates): removegroups.add(grpname) if (grpname in self.groups and - self.groups[grpname].category): + self.groups[grpname].category): del categories[self.groups[grpname].category] groups.difference_update(removegroups) return (groups, categories) @@ -1171,8 +1186,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: - return self.core.metadata_cache[client] + if client in self.cache: + return self.cache[client] if client in self.aliases: client = self.aliases[client] @@ -1269,7 +1284,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': - self.core.metadata_cache[client] = rv + self.cache[client] = rv return rv def get_all_group_names(self): @@ -1362,6 +1377,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): + self.logger.debug("Client %s address validates" % client) return True else: self.logger.error("Got request for %s from incorrect address %s" % @@ -1381,7 +1397,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) auth_type = self.auth.get(client, - self.core.setup['authentication']) + Bcfg2.Options.setup.authentication) elif user == 'root': id_method = 'address' try: @@ -1405,13 +1421,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # next we validate the address if (id_method != 'uuid' and - not self.validate_client_address(client, address)): + not self.validate_client_address(client, address)): return False if id_method == 'cert' and auth_type != 'cert+password': # remember the cert-derived client name for this connection if client in self.floating: self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s certificate validates" % client) # we are done if cert+password not required return True @@ -1438,6 +1455,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # populate the session cache if user != 'root': self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s authenticated successfully" % client) return True # pylint: enable=R0911,R0912 @@ -1468,7 +1486,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup['authentication']) == 'bootstrap': + Bcfg2.Options.setup.authentication) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): @@ -1576,7 +1594,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for group in egroups: for parent in group.findall('Group'): if (parent.get('name') not in gseen and - include_group(parent.get('name'))): + include_group(parent.get('name'))): rv.append(gfmt % (parent.get('name'), parent.get('name'))) gseen.append(parent.get("name")) @@ -1584,171 +1602,3 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, rv.append('"group-%s" -> "group-%s";' % (group.get('name'), parent.get('name'))) return rv - - -class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin for :ref:`Metadata - <server-plugins-grouping-metadata>`. This checks for several things: - - * ``<Client>`` tags nested inside other ``<Client>`` tags; - * Deprecated options (like ``location="floating"``); - * Profiles that don't exist, or that aren't profile groups; - * Groups or clients that are defined multiple times; - * Multiple default groups or a default group that isn't a profile - group. - """ - - def Run(self): - self.nested_clients() - self.deprecated_options() - self.bogus_profiles() - self.duplicate_groups() - self.duplicate_default_groups() - self.duplicate_clients() - self.default_is_profile() - - @classmethod - def Errors(cls): - return {"nested-client-tags": "warning", - "deprecated-clients-options": "warning", - "nonexistent-profile-group": "error", - "non-profile-set-as-profile": "error", - "duplicate-group": "error", - "duplicate-client": "error", - "multiple-default-groups": "error", - "default-is-not-profile": "error"} - - def deprecated_options(self): - """ Check for the ``location='floating'`` option, which has - been deprecated in favor of ``floating='true'``. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - clientdata = self.metadata.clients_xml.xdata - for el in clientdata.xpath("//Client"): - loc = el.get("location") - if loc: - if loc == "floating": - floating = True - else: - floating = False - self.LintError("deprecated-clients-options", - "The location='%s' option is deprecated. " - "Please use floating='%s' instead:\n%s" % - (loc, floating, self.RenderXML(el))) - - def nested_clients(self): - """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, - which is either redundant or will never match. """ - groupdata = self.metadata.groups_xml.xdata - for el in groupdata.xpath("//Client//Client"): - self.LintError("nested-client-tags", - "Client %s nested within Client tag: %s" % - (el.get("name"), self.RenderXML(el))) - - def bogus_profiles(self): - """ Check for clients that have profiles that are either not - flagged as profile groups in ``groups.xml``, or don't exist. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - for client in self.metadata.clients_xml.xdata.findall('.//Client'): - profile = client.get("profile") - if profile not in self.metadata.groups: - self.LintError("nonexistent-profile-group", - "%s has nonexistent profile group %s:\n%s" % - (client.get("name"), profile, - self.RenderXML(client))) - elif not self.metadata.groups[profile].is_profile: - self.LintError("non-profile-set-as-profile", - "%s is set as profile for %s, but %s is not a " - "profile group:\n%s" % - (profile, client.get("name"), profile, - self.RenderXML(client))) - - def duplicate_default_groups(self): - """ Check for multiple default groups. """ - defaults = [] - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default", "false").lower() == "true": - defaults.append(self.RenderXML(grp)) - if len(defaults) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined:\n%s" % - "\n".join(defaults)) - - def duplicate_clients(self): - """ Check for clients that are defined more than once. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - self.duplicate_entries( - self.metadata.clients_xml.xdata.xpath("//Client"), - "client") - - def duplicate_groups(self): - """ Check for groups that are defined more than once. There are two - ways this can happen: - - 1. The group is listed twice with contradictory options. - 2. The group is listed with no options *first*, and then with - options later. - - In this context, 'first' refers to the order in which groups - are parsed; see the loop condition below and - _handle_groups_xml_event above for details. """ - groups = dict() - duplicates = dict() - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - grpname = grp.get("name") - if grpname in duplicates: - duplicates[grpname].append(grp) - elif set(grp.attrib.keys()).difference(['negate', 'name']): - # group has options - if grpname in groups: - duplicates[grpname] = [grp, groups[grpname]] - else: - groups[grpname] = grp - else: # group has no options - groups[grpname] = grp - for grpname, grps in duplicates.items(): - self.LintError("duplicate-group", - "Group %s is defined multiple times:\n%s" % - (grpname, - "\n".join(self.RenderXML(g) for g in grps))) - - def duplicate_entries(self, allentries, etype): - """ Generic duplicate entry finder. - - :param allentries: A list of all entries to check for - duplicates. - :type allentries: list of lxml.etree._Element - :param etype: The entry type. This will be used to determine - the error name (``duplicate-<etype>``) and for - display to the end user. - :type etype: string - """ - entries = dict() - for el in allentries: - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] - for ename, els in entries.items(): - if len(els) > 1: - self.LintError("duplicate-%s" % etype, - "%s %s is defined multiple times:\n%s" % - (etype.title(), ename, "\n".join(els))) - - def default_is_profile(self): - """ Ensure that the default group is a profile group. """ - if (self.metadata.default and - not self.metadata.groups[self.metadata.default].is_profile): - xdata = \ - self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % - self.metadata.default)[0] - self.LintError("default-is-not-profile", - "Default group is not a profile group:\n%s" % - self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 8f1d03586..045e46350 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -5,28 +5,25 @@ import re import sys import glob import socket -import Bcfg2.Server -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import Plugin, Generator, StructFile, \ + PluginExecutionError -class NagiosGen(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): +class NagiosGen(Plugin, Generator): """ NagiosGen is a Bcfg2 plugin that dynamically generates Nagios configuration file based on Bcfg2 data. """ __author__ = 'bcfg-dev@mcs.anl.gov' line_fmt = '\t%-32s %s' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) + def __init__(self, core): + Plugin.__init__(self, core) + Generator.__init__(self) self.config = \ - Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, - 'config.xml'), - core.fam, should_monitor=True, - create=self.name) + StructFile(os.path.join(self.data, 'config.xml'), + should_monitor=True, create=self.name) self.Entries = { 'Path': {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', @@ -42,11 +39,11 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, def createhostconfig(self, entry, metadata): """Build host specific configuration file.""" try: - host_address = socket.gethostbyname(metadata.hostname) - except socket.gaierror: - self.logger.error("Failed to find IP address for %s" % - metadata.hostname) - raise Bcfg2.Server.Plugin.PluginExecutionError + host_address = socket.getaddrinfo(metadata.hostname, None)[0][4][0] + except socket.error: + self.logger.error() + raise PluginExecutionError("Failed to find IP address for %s" % + metadata.hostname) host_groups = [grp for grp in metadata.groups if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] host_config = ['define host {', @@ -56,7 +53,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, if host_groups: host_config.append(self.line_fmt % ("hostgroups", - ",".join(host_groups))) + ",".join(sorted(host_groups)))) # read the config xtra = dict() diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 0853ea993..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -80,11 +80,9 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, """The Ohai plugin is used to detect information about the client operating system. """ - name = 'Ohai' - experimental = True - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Probing.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 71128d64c..cf53c5866 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -11,8 +11,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, create = False - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.GoalValidator.__init__(self) def validate_goals(self, metadata, goals): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 27a725f23..c1d53a78e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -13,8 +13,7 @@ class AptCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -22,7 +21,7 @@ class AptCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -53,10 +52,6 @@ class AptCollection(Collection): class AptSource(Source): """ Handle APT sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``AptSource`` are "apt", "debian", "ubuntu", and "nexenta" - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 39c51f351..8b20df58a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -73,17 +73,17 @@ The Collection Module --------------------- """ -import sys import copy -import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Statistics import track_statistics -LOGGER = logging.getLogger(__name__) - -class Collection(list, Bcfg2.Server.Plugin.Debuggable): +class Collection(list, Debuggable): """ ``Collection`` objects represent the set of :class:`Bcfg2.Server.Plugins.Packages.Source` objects that apply to a given client, and can be used to query all software @@ -93,8 +93,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): #: Whether or not this Packages backend supports package groups __package_groups__ = False - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): """ :param metadata: The client metadata for this collection :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata @@ -111,29 +110,24 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): directory, where more permanent data can be stored :type basepath: string - :param fam: A file monitor object to use if this Collection - needs to monitor for file activity - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param debug: Enable debugging output :type debug: bool .. ----- .. autoattribute:: __package_groups__ """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) list.__init__(self, sources) - self.debug_flag = debug + self.debug_flag = self.debug_flag or debug self.metadata = metadata self.basepath = basepath self.cachepath = cachepath self.virt_pkgs = dict() - self.fam = fam + self.fam = get_fam() try: - self.setup = sources[0].setup self.ptype = sources[0].ptype except IndexError: - self.setup = None self.ptype = "unknown" @property @@ -204,19 +198,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return sorted(list(set(groups))) @property - def basegroups(self): - """ Get a list of group names used by this Collection type in - resolution of - :ref:`server-plugins-generators-packages-magic-groups`. - - The base implementation simply aggregates the results of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.basegroups`.""" - groups = set() - for source in self: - groups.update(source.basegroups) - return list(groups) - - @property def cachefiles(self): """ A list of the full path to all cachefiles used by this collection. @@ -229,7 +210,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ Given a list of package group names, return a dict of ``<group name>: <list of packages>``. This method is provided @@ -250,7 +231,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): rv[group] = self.get_group(group, ptype) return rv - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_group(self, group, ptype=None): """ Get the list of packages of the given type in a package group. @@ -386,20 +367,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): for source in self: source.filter_unknown(unknown) - def magic_groups_match(self): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups for any of the sources contained in this - Collection. - - The base implementation returns True if any source - :func:`Bcfg2.Server.Plugins.Packages.Source.Source.magic_groups_match` - returns True. - - :returns: bool - """ - return any(s.magic_groups_match(self.metadata) for s in self) - def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This can be used to handle, e.g., @@ -476,9 +443,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ for pkg in pkglist: lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, - version=self.setup.cfp.get("packages", - "version", - default="auto"), + version=Bcfg2.Options.setup.packages_version, type=self.ptype, origin='Packages') def get_new_packages(self, initial, complete): @@ -499,7 +464,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. @@ -630,22 +595,8 @@ def get_collection_class(source_type): :type source_type: string :returns: type - the Collection subclass that should be used to instantiate an object to contain sources of the given type. """ - modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() - try: - module = sys.modules[modname] - except KeyError: - try: - module = __import__(modname).Server.Plugins.Packages - except ImportError: - msg = "Packages: Unknown source type %s" % source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - try: - cclass = getattr(module, source_type.title() + "Collection") - except AttributeError: - msg = "Packages: No collection class found for %s sources" % \ - source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return cclass + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % source_type.title()): + return getattr(mod, "%sCollection" % source_type.title()) + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Packages: No collection class found for %s sources" % source_type) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 99aed5ce5..5f4d2ea41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -12,8 +12,7 @@ class PacCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -21,7 +20,7 @@ class PacCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -29,10 +28,6 @@ class PacCollection(Collection): class PacSource(Source): """ Handle Pacman sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``PacSource`` are "arch" and "parabola" - basegroups = ['arch', 'parabola'] - #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index c47e18201..1af046ec0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,12 +4,12 @@ import os import sys import Bcfg2.Server.Plugin +from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError # pylint: disable=E0012,R0924 -class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): +class PackagesSources(Bcfg2.Server.Plugin.StructFile): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate @@ -19,7 +19,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, __identifier__ = None create = "Sources" - def __init__(self, filename, cachepath, fam, packages, setup): + def __init__(self, filename, cachepath, packages): """ :param filename: The full path to ``sources.xml`` :type filename: string @@ -27,21 +27,15 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data will be cached :type cachepath: string - :param fam: The file access monitor to use to create watches - on ``sources.xml`` and any XIncluded files. - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param packages: The Packages plugin object ``sources.xml`` is being parsed on behalf of (i.e., the calling object) :type packages: Bcfg2.Server.Plugins.Packages.Packages - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - If ``sources.xml`` cannot be read """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=True) #: The full path to the directory where @@ -57,8 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, err = sys.exc_info()[1] self.logger.error("Could not create Packages cache at %s: %s" % (self.cachepath, err)) - #: The Bcfg2 options dict - self.setup = setup #: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that #: instantiated this ``PackagesSources`` object @@ -72,10 +64,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.parsed = set() def set_debug(self, debug): - Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + Bcfg2.Server.Plugin.StructFile.set_debug(self, debug) for source in self.entries: source.set_debug(debug) - set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -106,7 +97,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, load its data. """ return sorted(list(self.parsed)) == sorted(self.extras) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] @@ -120,7 +111,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, ``Index`` is responsible for calling :func:`source_from_xml` for each ``Source`` tag in each file. """ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def source_from_xml(self, xsource): """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass @@ -141,19 +132,17 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, xsource.get("url")))) return None - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown source type %s (%s)" % (stype, - err)) + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % stype.title()): + cls = getattr(mod, "%sSource" % stype.title()) + break + else: + self.logger.error("Packages: Unknown source type %s" % stype) return None try: - source = cls(self.cachepath, xsource, self.setup) + source = cls(self.cachepath, xsource) except SourceInitError: err = sys.exc_info()[1] self.logger.error("Packages: %s" % err) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 22073493c..4b6130f72 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -27,7 +27,6 @@ in your ``Source`` subclass: * :func:`Source.urls` * :func:`Source.read_files` -* :attr:`Source.basegroups` Additionally, you may want to consider overriding the following methods and attributes: @@ -50,10 +49,11 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys -import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ cPickle, md5 +from Bcfg2.Server.Statistics import track_statistics def fetch_url(url): @@ -92,7 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -106,28 +106,21 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 those features. """ - #: The list of - #: :ref:`server-plugins-generators-packages-magic-groups` that - #: make sources of this type available to clients. - basegroups = [] - #: The Package type handled by this Source class. The ``type`` #: attribute of Package entries will be set to the value ``ptype`` #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None - def __init__(self, basepath, xsource, setup): # pylint: disable=R0912 + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache data for this source should be stored :type basepath: string :param xsource: The XML tag that describes this source :type source: lxml.etree._Element - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -136,9 +129,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The XML tag that describes this source self.xsource = xsource - #: A Bcfg2 options dict - self.setup = setup - #: A set of package names that are deemed "essential" by this #: source self.essentialpkgs = set() @@ -308,8 +298,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :return: list of strings - group names """ return sorted(list(set([g for g in metadata.groups - if (g in self.basegroups or - g in self.groups or + if (g in self.groups or g in self.arches)]))) def load_state(self): @@ -332,7 +321,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.essentialpkgs), cache, 2) cache.close() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def setup_data(self, force_update=False): """ Perform all data fetching and setup tasks. For most backends, this involves downloading all metadata from the @@ -636,16 +625,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 def applies(self, metadata): """ Return true if this source applies to the given client, - i.e., the client is in all necessary groups and - :ref:`server-plugins-generators-packages-magic-groups`. + i.e., the client is in all necessary groups. :param metadata: The client metadata to check to see if this source applies :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: bool """ - # check base groups - if not self.magic_groups_match(metadata): + # check arch groups + if not self.arch_groups_match(metadata): return False # check Group/Client tags from sources.xml @@ -716,29 +704,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return [] - def magic_groups_match(self, metadata): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups this source. Also returns True if magic - groups are off in the configuration and the client's - architecture matches (i.e., architecture groups are *always* - checked). + def arch_groups_match(self, metadata): + """ Returns True if the client is in an arch group that + matches the arch of this source. :returns: bool """ - found_arch = False for arch in self.arches: if arch in metadata.groups: - found_arch = True - break - if not found_arch: - return False - - if not self.setup.cfp.getboolean("packages", "magic_groups", - default=False): - return True - else: - for group in self.basegroups: - if group in metadata.groups: - return True - return False + return True + return False diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 67ff05ca1..3cfda9e9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -60,8 +60,10 @@ import socket import logging import lxml.etree import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor +from distutils.spawn import find_executable # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -69,6 +71,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -107,13 +110,36 @@ PULPSERVER = None PULPCONFIG = None -def _setup_pulp(setup): +options = [ # pylint: disable=C0103 + Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.PathOption( + cf=("packages:yum", "helper"), dest="yum_helper", + help="Path to the bcfg2-yum-helper executable"), + Bcfg2.Options.BooleanOption( + cf=("packages:yum", "use_yum_libraries"), + help="Use Python yum libraries"), + Bcfg2.Options.PathOption( + cf=("packages:yum", "gpg_keypath"), default="/etc/pki/rpm-gpg", + help="GPG key path on the client"), + Bcfg2.Options.Option( + cf=("packages:yum", "*"), dest="yum_options", + help="Other yum options to include in generated yum configs")] +if HAS_PULP: + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "username"), dest="pulp_username", + help="Username for Pulp authentication")) + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "password"), dest="pulp_password", + help="Password for Pulp authentication")) + + +def _setup_pulp(): """ Connect to a Pulp server and pass authentication credentials. This only needs to be called once, but multiple calls won't hurt anything. - :param setup: A Bcfg2 options dict - :type setup: dict :returns: :class:`pulp.client.api.server.PulpServer` """ global PULPSERVER, PULPCONFIG @@ -124,19 +150,6 @@ def _setup_pulp(setup): raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: - try: - username = setup.cfp.get("packages:pulp", "username") - password = setup.cfp.get("packages:pulp", "password") - except ConfigParser.NoSectionError: - msg = "Packages: No [pulp] section found in bcfg2.conf" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - except ConfigParser.NoOptionError: - msg = "Packages: Required option not found in bcfg2.conf: %s" % \ - sys.exc_info()[1] - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -144,7 +157,9 @@ def _setup_pulp(setup): int(serveropts['port']), serveropts['scheme'], serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) + PULPSERVER.set_basic_auth_credentials( + Bcfg2.Options.setup.pulp_username, + Bcfg2.Options.setup.pulp_password) server.set_active_server(PULPSERVER) return PULPSERVER @@ -175,7 +190,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): #: The path to certificates on consumer machines certpath = "/etc/pki/consumer/cert.pem" - def __init__(self, path, fam): + def __init__(self, path): """ :param path: The path to the directory where Pulp consumer certificates will be stored @@ -193,7 +208,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): important='true', sensitive='true', paranoid=self.metadata['paranoid']) - self.fam = fam + self.fam = Bcfg2.Server.FileMonitor.get_fam() self.fam.AddMonitor(path, self) def HandleEvent(self, event): @@ -274,9 +289,8 @@ class YumCollection(Collection): #: :class:`PulpCertificateSet` object used to handle Pulp certs pulp_cert_set = None - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) self.keypath = os.path.join(self.cachepath, "keys") @@ -297,13 +311,15 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: self.setup_data() + self.cmd = Executor() else: self.cachefile = None + self.cmd = None if HAS_PULP and self.has_pulp_sources: - _setup_pulp(self.setup) + _setup_pulp() if self.pulp_cert_set is None: certdir = os.path.join( self.basepath, @@ -319,28 +335,7 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.__class__.pulp_cert_set = PulpCertificateSet(certdir, - self.fam) - - @property - def disableMetaData(self): # pylint: disable=C0103 - """ Report whether or not metadata processing is enabled. - This duplicates code in Packages/__init__.py, and can probably - be removed in Bcfg2 1.4 when we have a module-level setup - object. """ - if self.setup is None: - return True - try: - return not self.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" + self.__class__.pulp_cert_set = PulpCertificateSet(certdir) @property def __package_groups__(self): @@ -348,24 +343,18 @@ class YumCollection(Collection): @property def helper(self): - """ The full path to :file:`bcfg2-yum-helper`. First, we - check in the config file to see if it has been explicitly - specified; next we see if it's in $PATH (which we do by making - a call to it; I wish there was a way to do this without - forking, but apparently not); finally we check in /usr/sbin, - the default location. """ - # pylint: disable=W0212 - if not self.__class__._helper: - try: - self.__class__._helper = self.setup.cfp.get("packages:yum", - "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + """The full path to :file:`bcfg2-yum-helper`. First, we check in the + config file to see if it has been explicitly specified; next + we see if it's in $PATH; finally we default to /usr/sbin, the + default location. """ + if not self._helper: + # pylint: disable=W0212 + self.__class__._helper = Bcfg2.Options.setup.yum_helper + if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH - try: - self.debug_log("Checking for bcfg2-yum-helper in $PATH") - self.cmd.run(['bcfg2-yum-helper']) - self.__class__._helper = 'bcfg2-yum-helper' - except OSError: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + self.__class__._helper = find_executable('bcfg2-yum-helper') + if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" return self.__class__._helper # pylint: enable=W0212 @@ -374,9 +363,7 @@ class YumCollection(Collection): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries @property def has_pulp_sources(self): @@ -393,7 +380,7 @@ class YumCollection(Collection): cachefiles.add(self.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_config(self): """ Write the server-side config file to :attr:`cfgfile` based on the data from :func:`get_config`""" @@ -412,15 +399,15 @@ class YumCollection(Collection): debuglevel="0", sslverify="0", reposdir="/dev/null") - if self.setup['debug']: + if Bcfg2.Options.setup.debug: mainopts['debuglevel'] = "5" - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: mainopts['debuglevel'] = "2" try: - for opt in self.setup.cfp.options("packages:yum"): + for opt, val in Bcfg2.Options.setup.yum_options.items(): if opt not in self.option_blacklist: - mainopts[opt] = self.setup.cfp.get("packages:yum", opt) + mainopts[opt] = val except ConfigParser.NoSectionError: pass @@ -496,7 +483,7 @@ class YumCollection(Collection): return "# This config was generated automatically by the Bcfg2 " \ "Packages plugin\n\n" + buf.getvalue() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This adds several kinds of @@ -542,8 +529,7 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.setup.cfp.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = Bcfg2.Options.setup.gpg_keypath remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() @@ -588,7 +574,7 @@ class YumCollection(Collection): # each pulp source can only have one arch, so we don't # have to check the arch in url_map if (source.pulp_id and - source.pulp_id not in consumer['repoids']): + source.pulp_id not in consumer['repoids']): try: consumerapi.bind(self.metadata.hostname, source.pulp_id) @@ -603,7 +589,7 @@ class YumCollection(Collection): name=self.pulp_cert_set.certpath) self.pulp_cert_set.bind_entry(crt, self.metadata) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _get_pulp_consumer(self, consumerapi=None): """ Get a Pulp consumer object for the client. @@ -632,7 +618,7 @@ class YumCollection(Collection): "%s" % err) return consumer - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None): """ Add GPG keys instances to a ``Package`` entry. This is called from :func:`build_extra_structures` to add GPG keys to @@ -675,7 +661,7 @@ class YumCollection(Collection): self.logger.error("Packages: Could not read GPG key %s: %s" % (localkey, err)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ If using the yum libraries, given a list of package group names, return a dict of ``<group name>: <list of packages>``. @@ -762,8 +748,7 @@ class YumCollection(Collection): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance tag """ - attrs = dict(version=self.setup.cfp.get("packages", "version", - default="auto")) + attrs = dict(version=Bcfg2.Options.setup.packages_version) if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): return attrs @@ -852,7 +837,7 @@ class YumCollection(Collection): new.append(pkg) return new - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): """ Build a complete list of all packages and their dependencies. @@ -904,7 +889,7 @@ class YumCollection(Collection): else: return set(), set() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get @@ -922,22 +907,20 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - if self.setup['verbose']: + if Bcfg2.Options.setup.verbose: cmd.append("-v") if self.debug_flag: - if not self.setup['verbose']: - # ensure that running in debug gets -vv, even if - # verbose is not enabled - cmd.append("-v") - cmd.append("-v") + cmd.append("-d") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout, inputdata=json.dumps(inputdata)) else: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout) if not result.success: self.logger.error("Packages: error running bcfg2-yum-helper: %s" % result.error) @@ -1000,20 +983,16 @@ class YumCollection(Collection): class YumSource(Source): """ Handle yum sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``YumSource`` are "yum", "redhat", "centos", and "fedora" - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - #: YumSource sets the ``type`` on Package entries to "yum" ptype = 'yum' - def __init__(self, basepath, xsource, setup): - Source.__init__(self, basepath, xsource, setup) + def __init__(self, basepath, xsource): + Source.__init__(self, basepath, xsource) self.pulp_id = None if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") - _setup_pulp(self.setup) + _setup_pulp() repoapi = RepositoryAPI() try: self.repo = repoapi.repository(self.pulp_id) @@ -1056,9 +1035,7 @@ class YumSource(Source): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries def save_state(self): """ If using the builtin yum parser, save state to @@ -1140,7 +1117,7 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally downloaded metadata files. This diverges from the stock @@ -1188,7 +1165,7 @@ class YumSource(Source): self.packages[key].difference(self.packages['global']) self.save_state() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_filelist(self, data, arch): """ parse filelists.xml.gz data """ if arch not in self.filemap: @@ -1202,7 +1179,7 @@ class YumSource(Source): self.filemap[arch][fentry.text] = \ set([pkg.get('name')]) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_primary(self, data, arch): """ parse primary.xml.gz data """ if arch not in self.packages: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py new file mode 100644 index 000000000..48304d26e --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -0,0 +1,399 @@ +""" Libraries for bcfg2-yum-helper plugin, used if yum library support +is enabled. The yum libs have horrific memory leaks, so apparently +the right way to get around that in long-running processes it to have +a short-lived helper. No, seriously -- check out the yum-updatesd +code. It's pure madness. """ + +import os +import sys +import yum +import logging +import Bcfg2.Options +import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout +try: + import json +except ImportError: + import simplejson as json + + +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + + +def pkgtup_to_string(package): + """ given a package tuple, return a human-readable string + describing the package """ + if package[3] in ['auto', 'any']: + return package[0] + + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + + +class YumHelper(object): + """ Yum helper base object """ + + def __init__(self, cfgfile, verbose=1): + self.cfgfile = cfgfile + self.yumbase = yum.YumBase() + # pylint: disable=E1121,W0212 + try: + self.yumbase.preconf.debuglevel = verbose + self.yumbase.preconf.fn = cfgfile + self.yumbase._getConfig() + except AttributeError: + self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121,W0212 + self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 + self._groups = None + + def get_groups(self): + """ getter for the groups property """ + if self._groups is not None: + return self._groups + else: + return ["noarch"] + + def set_groups(self, groups): + """ setter for the groups property """ + self._groups = set(groups).union(["noarch"]) + + groups = property(get_groups, set_groups) + + def get_package_object(self, pkgtup, silent=False): + """ given a package tuple, get a yum package object """ + try: + matches = yum.packageSack.packagesNewestByName( + self.yumbase.pkgSack.searchPkgTuple(pkgtup)) + except yum.Errors.PackageSackError: + if not silent: + self.logger.warning("Package '%s' not found" % + self.get_package_name(pkgtup)) + matches = [] + except yum.Errors.RepoError: + err = sys.exc_info()[1] + self.logger.error("Temporary failure loading metadata for %s: %s" % + (self.get_package_name(pkgtup), err)) + matches = [] + + pkgs = self._filter_arch(matches) + if pkgs: + return pkgs[0] + else: + return None + + def get_group(self, group, ptype="default"): + """ Resolve a package group name into a list of packages """ + if group.startswith("@"): + group = group[1:] + + try: + if self.yumbase.comps.has_group(group): + group = self.yumbase.comps.return_group(group) + else: + self.logger.error("%s is not a valid group" % group) + return [] + except yum.Errors.GroupsError: + err = sys.exc_info()[1] + self.logger.warning(err) + return [] + + if ptype == "default": + return [p + for p, d in list(group.default_packages.items()) + if d] + elif ptype == "mandatory": + return [p + for p, m in list(group.mandatory_packages.items()) + if m] + elif ptype == "optional" or ptype == "all": + return group.packages + else: + self.logger.warning("Unknown group package type '%s'" % ptype) + return [] + + def _filter_arch(self, packages): + """ filter packages in the given list that do not have an + architecture in the list of groups for this client """ + matching = [] + for pkg in packages: + if pkg.arch in self.groups: + matching.append(pkg) + else: + self.logger.debug("%s has non-matching architecture (%s)" % + (pkg, pkg.arch)) + if matching: + return matching + else: + # no packages match architecture; we'll assume that the + # user knows what s/he is doing and this is a multiarch + # box. + return packages + + def get_package_name(self, package): + """ get the name of a package or virtual package from the + internal representation used by this Collection class """ + if isinstance(package, tuple): + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) + else: + return str(package) + + def complete(self, packagelist): + """ resolve dependencies and generate a complete package list + from the given list of initial packages """ + packages = set() + unknown = set() + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + pkgobj = self.get_package_object(pkgtup) + if not pkgobj: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): + self.logger.debug("%s added to transaction multiple times" + % pkgobj) + else: + self.logger.debug("Adding %s to transaction" % pkgobj) + self.yumbase.tsInfo.addInstall(pkgobj) + self.yumbase.resolveDeps() + + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) + + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock + def clean_cache(self): + """ clean the yum cache """ + for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", + "ExpireCache"]: + # for reasons that are entirely obvious, all of the yum + # API clean* methods return a tuple of 0 (zero, always + # zero) and a list containing a single message about how + # many files were deleted. so useful. thanks, yum. + msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] + if not msg.startswith("0 "): + self.logger.info(msg) + + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + + +class HelperSubcommand(Bcfg2.Options.Subcommand): + """ Base class for all yum helper subcommands """ + + # the value to JSON encode and print out if the command fails + fallback = None + + # whether or not this command accepts input on stdin + accept_input = True + + def __init__(self): + Bcfg2.Options.Subcommand.__init__(self) + self.verbosity = 0 + if Bcfg2.Options.setup.debug: + self.verbosity = 5 + elif Bcfg2.Options.setup.verbose: + self.verbosity = 1 + + def run(self, setup): + try: + data = json.loads(sys.stdin.read()) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + print(json.dumps(self.fallback)) + return 2 + + try: + print(json.dumps(self._run(setup, data))) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error running %s: %s" % + self.__class__.__name__.lower(), + sys.exc_info()[1], exc_info=1) + print(json.dumps(self.fallback)) + return 2 + return 0 + + def _run(self, setup, data): + """ Actually run the command """ + raise NotImplementedError + + +class DepSolverSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the depsolver (i.e., + only resolve dependencies, don't modify the cache) """ + + def __init__(self): + HelperSubcommand.__init__(self) + self.depsolver = DepSolver(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class CacheManagerSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the cachemanager + (i.e., modify the cache) """ + fallback = False + accept_input = False + + def __init__(self): + HelperSubcommand.__init__(self) + self.cachemgr = CacheManager(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class Clean(CacheManagerSubcommand): + """ Clean the cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.clean_cache() + return True + + +class MakeCache(CacheManagerSubcommand): + """ Update the on-disk cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.populate_cache() + return True + + +class Complete(DepSolverSubcommand): + """ Given an initial set of packages, get a complete set of + packages with all dependencies resolved """ + fallback = dict(packages=[], unknown=[]) + + def _run(self, _, data): + self.depsolver.groups = data['groups'] + self.fallback['unknown'] = data['packages'] + (packages, unknown) = self.depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + return dict(packages=list(packages), unknown=list(unknown)) + + +class GetGroups(DepSolverSubcommand): + """ Resolve the given package groups """ + def _run(self, _, data): + rv = dict() + for gdata in data: + if "type" in gdata: + packages = self.depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = self.depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + return rv + + +Get_Groups = GetGroups # pylint: disable=C0103 + + +class CLI(Bcfg2.Options.CommandRegistry): + """ The bcfg2-yum-helper CLI """ + options = [ + Bcfg2.Options.PathOption( + "-c", "--yum-config", help="Yum config file"), + Bcfg2.Options.PositionalArgument( + "command", help="Yum helper command", + choices=['clean', 'complete', 'get_groups'])] + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=HelperSubcommand) + parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", + components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run bcfg2-yum-helper """ + if not os.path.exists(Bcfg2.Options.setup.yum_config): + self.logger.error("Config file %s not found" % + Bcfg2.Options.setup.yum_config) + return 1 + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4b58c0fdb..49f64bdf3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -7,20 +7,32 @@ import sys import glob import shutil import lxml.etree -import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ - MutableMapping +from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Statistics import track_statistics -#: The default path for generated yum configs -YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" -#: The default path for generated apt configs -APT_CONFIG_DEFAULT = \ - "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +def packages_boolean(value): + """ For historical reasons, the Packages booleans 'resolver' and + 'metadata' both accept "enabled" in addition to the normal boolean + values. """ + if value == 'disabled': + return False + elif value == 'enabled': + return True + else: + return value + + +class PackagesBackendAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages backends """ + bases = ['Bcfg2.Server.Plugins.Packages'] + module = True class OnDemandDict(MutableMapping): @@ -70,7 +82,6 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -85,6 +96,38 @@ class Packages(Bcfg2.Server.Plugin.Plugin, .. private-include: _build_packages""" + options = [ + Bcfg2.Options.Option( + cf=("packages", "backends"), dest="packages_backends", + help="Packages backends to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + Bcfg2.Options.PathOption( + cf=("packages", "cache"), dest="packages_cache", + help="Path to the Packages cache", + default='<repository>/Packages/cache'), + Bcfg2.Options.Option( + cf=("packages", "resolver"), dest="packages_resolver", + help="Disable the Packages resolver", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "metadata"), dest="packages_metadata", + help="Disable all Packages metadata processing", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "version"), dest="packages_version", + help="Set default Package entry version", default="auto", + choices=["auto", "any"]), + Bcfg2.Options.PathOption( + cf=("packages", "yum_config"), + help="The default path for generated yum configs", + default="/etc/yum.repos.d/bcfg2.repo"), + Bcfg2.Options.PathOption( + cf=("packages", "apt_config"), + help="The default path for generated apt configs", + default= + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list")] + #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] @@ -93,12 +136,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ - [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -107,9 +146,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. - self.cachepath = \ - self.core.setup.cfp.get("packages", "cache", - default=os.path.join(self.data, 'cache')) + self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') @@ -117,14 +154,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # create key directory if needed os.makedirs(self.keypath) - # warn about deprecated magic groups - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - self.logger.warning("Packages: Magic groups are deprecated and " - "will be removed in a future release") - self.logger.warning("You can disable magic groups by setting " - "magic_groups=0 in [packages] in bcfg2.conf") - # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` @@ -132,8 +161,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self, - self.core.setup) + self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -153,7 +181,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. - self.collections = dict() + self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` @@ -161,21 +189,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. - self.clients = dict() - - #: groupcache caches group lookups. It maps Collections (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of package groups, and thence to the packages - #: indicated by those groups. - self.groupcache = dict() - - #: pkgcache caches complete package sets. It maps Collections - #: (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of initial packages, and thence to the final - #: (complete) package selections resolved from the initial - #: packages - self.pkgcache = dict() + self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ @@ -187,48 +202,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return rv set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ - @property - def disableResolver(self): # pylint: disable=C0103 - """ Report the state of the resolver. This can be disabled in - the configuration. Note that disabling metadata (see - :attr:`disableMetaData`) implies disabling the resolver. - - This property cannot be set. """ - if self.disableMetaData: - # disabling metadata without disabling the resolver Breaks - # Things - return True - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.core.setup.cfp.get( - "packages", - "resolver", - default="enabled").lower() == "disabled" - - @property - def disableMetaData(self): # pylint: disable=C0103 - """ Report whether or not metadata processing is enabled. - - This property cannot be set. """ - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.core.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" - def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. @@ -276,9 +249,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if entry.tag == 'Package': collection = self.get_collection(metadata) - entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) @@ -304,27 +275,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - collection = self.get_collection(metadata) - if collection.magic_groups_match(): - return True - else: - return True + return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if entry.get("name") in [Bcfg2.Options.setup.apt_config, + Bcfg2.Options.setup.yum_config]: return True return False - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: @@ -353,13 +312,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :returns: None """ collection = self.get_collection(metadata) - indep = lxml.etree.Element('Independent') + indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list @@ -382,8 +341,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ - if self.disableResolver: - # Config requests no resolver + if (not Bcfg2.Options.setup.packages_metadata or + not Bcfg2.Options.setup.packages_resolver): + # Config requests no resolver. Note that disabling + # metadata implies disabling the resolver. for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("group"): @@ -422,11 +383,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups.sort() # check for this set of groups in the group cache + gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", + collection.cachekey) gkey = hash(tuple(groups)) - if gkey not in self.groupcache[collection.cachekey]: - self.groupcache[collection.cachekey][gkey] = \ - collection.get_groups(groups) - for pkgs in self.groupcache[collection.cachekey][gkey].values(): + if gkey not in gcache: + gcache[gkey] = collection.get_groups(groups) + for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution @@ -434,10 +396,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # check for this set of packages in the package cache pkey = hash(tuple(base)) - if pkey not in self.pkgcache[collection.cachekey]: - self.pkgcache[collection.cachekey][pkey] = \ - collection.complete(base) - packages, unknown = self.pkgcache[collection.cachekey][pkey] + pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", + collection.cachekey) + if pkey not in pcache: + pcache[pkey] = collection.complete(base) + packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -447,7 +410,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() collection.packages_to_entry(newpkgs, independent) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False @@ -455,7 +418,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config(force_update=True) return True - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Reload(self): """ Packages.Refresh() => True|False @@ -463,7 +426,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True - def expire_cache(self, _=None): + def child_reload(self, _=None): + """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): @@ -490,18 +454,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: collection.setup_data(force_update) # clear Collection and package caches - self.clients = dict() - self.collections = dict() - self.groupcache = dict() - self.pkgcache = dict() + Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): @@ -533,7 +494,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or - not os.path.exists(localfile)): + not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: @@ -556,7 +517,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -573,12 +534,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - collection = Collection(metadata, [], self.cachepath, self.data, - self.core.fam) - ckey = collection.cachekey - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) - return collection + return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -610,13 +566,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, - self.core.fam, debug=self.debug_flag) + debug=self.debug_flag) ckey = collection.cachekey if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): @@ -665,8 +619,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - if metadata.hostname in self.clients: - del self.clients[metadata.hostname] + self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index a1dcb575f..c7d8986ed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,24 +1,18 @@ '''This module implements a package management scheme for all images''' -import os import re -import glob +import sys import logging import lxml.etree import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint +from Bcfg2.Server.Plugin import PluginExecutionError -try: - set -except NameError: - # deprecated since python 2.6 - from sets import Set as set logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') class FuzzyDict(dict): - fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)') + fuzzy = re.compile(r'(?P<name>.*):(?P<alist>\S+(,\S+)*)') def __getitem__(self, key): if isinstance(key, str): @@ -47,95 +41,217 @@ class FuzzyDict(dict): raise -class PNode(Bcfg2.Server.Plugin.INode): +class PNode(object): """PNode has a list of packages available at a particular group intersection. """ - splitters = {'rpm': re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ - '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'), - 'encap': re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')} + splitters = dict( + rpm=re.compile( + r'^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + + r'(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'), + encap=re.compile( + r'^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')) + raw = dict( + Client="lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' in m.groups and predicate(m, e)") + nraw = dict( + Client="lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' not in m.groups and predicate(m, e)") + containers = ['Group', 'Client'] ignore = ['Package'] - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = FuzzyDict() - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data) - def __init__(self, data, pdict, parent=None): # copy local attributes to all child nodes if no local attribute exists if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) + attrs = set(data.attrib.keys()).difference( + child.attrib.keys() + ['name']) for attr in attrs: try: child.set(attr, data.get(attr)) except: - # don't fail on things like comments and other immutable elements + # don't fail on things like comments and other + # immutable elements pass - Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) + self.data = data + self.contents = {} + if parent is None: + self.predicate = lambda m, e: True + else: + predicate = parent.predicate + if data.get('negate', 'false').lower() == 'true': + psrc = self.nraw + else: + psrc = self.raw + if data.tag in list(psrc.keys()): + self.predicate = eval(psrc[data.tag] % + {'name': data.get('name')}, + {'predicate': predicate}) + else: + raise PluginExecutionError("Unknown tag: %s" % data.tag) + self.children = [] + self._load_children(data, pdict) + if 'Package' not in self.contents: self.contents['Package'] = FuzzyDict() for pkg in data.findall('./Package'): - if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: + if ('name' in pkg.attrib and + pkg.get('name') not in pdict['Package']): pdict['Package'].add(pkg.get('name')) - if pkg.get('name') != None: + if pkg.get('name') is not None: self.contents['Package'][pkg.get('name')] = {} if pkg.getchildren(): self.contents['Package'][pkg.get('name')]['__children__'] \ - = pkg.getchildren() + = pkg.getchildren() if 'simplefile' in pkg.attrib: - pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) + pkg.set('url', + "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) self.contents['Package'][pkg.get('name')].update(pkg.attrib) else: if 'file' in pkg.attrib: if 'multiarch' in pkg.attrib: archs = pkg.get('multiarch').split() srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), - pkg.get('file') % {'src':srcs[idx], - 'arch':archs[idx]}) - for idx in range(len(archs))]) + url = ' '.join( + ["%s/%s" % (pkg.get('uri'), + pkg.get('file') % {'src': srcs[idx], + 'arch': archs[idx]}) + for idx in range(len(archs))]) pkg.set('url', url) else: pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) - if pkg.get('type') in self.splitters and pkg.get('file') != None: - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) + if (pkg.get('type') in self.splitters and + pkg.get('file') is not None): + mdata = \ + self.splitters[pkg.get('type')].match(pkg.get('file')) if not mdata: - logger.error("Failed to match pkg %s" % pkg.get('file')) + logger.error("Failed to match pkg %s" % + pkg.get('file')) continue pkgname = mdata.group('name') self.contents['Package'][pkgname] = mdata.groupdict() self.contents['Package'][pkgname].update(pkg.attrib) if pkg.attrib.get('file'): - self.contents['Package'][pkgname]['url'] = pkg.get('url') - self.contents['Package'][pkgname]['type'] = pkg.get('type') + self.contents['Package'][pkgname]['url'] = \ + pkg.get('url') + self.contents['Package'][pkgname]['type'] = \ + pkg.get('type') if pkg.get('verify'): - self.contents['Package'][pkgname]['verify'] = pkg.get('verify') + self.contents['Package'][pkgname]['verify'] = \ + pkg.get('verify') if pkg.get('multiarch'): - self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch') + self.contents['Package'][pkgname]['multiarch'] = \ + pkg.get('multiarch') if pkgname not in pdict['Package']: pdict['Package'].add(pkgname) if pkg.getchildren(): - self.contents['Package'][pkgname]['__children__'] = pkg.getchildren() + self.contents['Package'][pkgname]['__children__'] = \ + pkg.getchildren() else: - self.contents['Package'][pkg.get('name')].update(pkg.attrib) + self.contents['Package'][pkg.get('name')].update( + pkg.attrib) + def _load_children(self, data, idict): + """ load children """ + for item in data.getchildren(): + if item.tag in self.ignore: + continue + elif item.tag in self.containers: + self.children.append(self.__class__(item, idict, self)) + else: + try: + self.contents[item.tag][item.get('name')] = \ + dict(item.attrib) + except KeyError: + self.contents[item.tag] = \ + {item.get('name'): dict(item.attrib)} + if item.text: + self.contents[item.tag][item.get('name')]['__text__'] = \ + item.text + if item.getchildren(): + self.contents[item.tag][item.get('name')]['__children__'] \ + = item.getchildren() + try: + idict[item.tag].append(item.get('name')) + except KeyError: + idict[item.tag] = [item.get('name')] -class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that - returns matching package entries. - """ + def Match(self, metadata, data, entry=lxml.etree.Element("None")): + """Return a dictionary of package mappings.""" + if self.predicate(metadata, entry): + for key in self.contents: + try: + data[key].update(self.contents[key]) + except: # pylint: disable=W0702 + data[key] = FuzzyDict() + data[key].update(self.contents[key]) + for child in self.children: + child.Match(metadata, data) + + +class PkgSrc(Bcfg2.Server.Plugin.XMLFileBacked): + """ XMLSrc files contain a + :class:`Bcfg2.Server.Plugin.helpers.INode` hierarchy that returns + matching entries. XMLSrc objects are deprecated and + :class:`Bcfg2.Server.Plugin.helpers.StructFile` should be + preferred where possible.""" __node__ = PNode __cacheobj__ = FuzzyDict + __priority_required__ = True + + def __init__(self, filename, should_monitor=False): + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, + should_monitor) + self.items = {} + self.cache = None + self.pnode = None + self.priority = -1 + + def HandleEvent(self, _=None): + """Read file upon update.""" + try: + data = open(self.name).read() + except IOError: + msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.items = {} + try: + xdata = lxml.etree.XML(data, parser=Bcfg2.Server.XMLParser) + except lxml.etree.XMLSyntaxError: + msg = "Failed to parse file %s: %s" % (self.name, + sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.pnode = self.__node__(xdata, self.items) + self.cache = None + try: + self.priority = int(xdata.get('priority')) + except (ValueError, TypeError): + if self.__priority_required__: + msg = "Got bogus priority %s for file %s" % \ + (xdata.get('priority'), self.name) + logger.error(msg) + raise PluginExecutionError(msg) + + del xdata, data + + def Cache(self, metadata): + """Build a package dict for a given host.""" + if self.cache is None or self.cache[0] != metadata: + cache = (metadata, self.__cacheobj__()) + if self.pnode is None: + logger.error("Cache method called early for %s; " + "forcing data load" % self.name) + self.HandleEvent() + return + self.pnode.Match(metadata, cache[1]) + self.cache = cache + + def __str__(self): + return str(self.items) class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): @@ -165,53 +281,14 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): mdata = FuzzyDict.fuzzy.match(pname) if mdata: arches = mdata.group('alist').split(',') - [entry.remove(inst) for inst in \ - entry.findall('Instance') \ - if inst.get('arch') not in arches] + for inst in entry.findall('Instance'): + if inst.get('arch') not in arches: + entry.remove(inst) def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) + return ( + entry.tag == 'Package' and + entry.get('name').split(':')[0] in self.Entries['Package'].keys()) def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) - - -class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ Find duplicate :ref:`Pkgmgr - <server-plugins-generators-pkgmgr>` entries with the same - priority. """ - - def Run(self): - pset = set() - for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', - '*.xml')): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError( - "duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) - - @classmethod - def Errors(cls): - return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 48be1ac26..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -8,15 +8,33 @@ import copy import operator import lxml.etree import Bcfg2.Server +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode # pylint: disable=W0622 - -try: - from django.db import models - from django.core.exceptions import MultipleObjectsReturned - HAS_DJANGO = True +from Bcfg2.Compat import unicode, any # pylint: disable=W0622 +import Bcfg2.Server.FileMonitor +from Bcfg2.Logger import Debuggable +from Bcfg2.Server.Statistics import track_statistics + +HAS_DJANGO = False +# pylint: disable=C0103 +ProbesDataModel = None +ProbesGroupsModel = None +# pylint: enable=C0103 + + +def load_django_models(): + """ Load models for Django after option parsing has completed """ + # pylint: disable=W0602 + global ProbesDataModel, ProbesGroupsModel, HAS_DJANGO + # pylint: enable=W0602 + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return - class ProbesDataModel(models.Model, + class ProbesDataModel(models.Model, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.PluginDatabaseModel): """ The database model for storing probe data """ hostname = models.CharField(max_length=255) @@ -24,13 +42,12 @@ try: timestamp = models.DateTimeField(auto_now=True) data = models.TextField(null=True) - class ProbesGroupsModel(models.Model, + class ProbesGroupsModel(models.Model, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.PluginDatabaseModel): """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) -except ImportError: - HAS_DJANGO = False + try: import json @@ -51,6 +68,226 @@ except ImportError: HAS_YAML = False +class ProbeStore(Debuggable): + """ Caching abstraction layer between persistent probe data + storage and the Probes plugin.""" + + def __init__(self, core, datadir): # pylint: disable=W0613 + Debuggable.__init__(self) + self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") + + def get_groups(self, hostname): + """ Get the list of groups for the given host """ + if hostname not in self._groupcache: + self._load_groups(hostname) + return self._groupcache.get(hostname, []) + + def set_groups(self, hostname, groups): + """ Set the list of groups for the given host """ + raise NotImplementedError + + def get_data(self, hostname): + """ Get a dict of probe data for the given host """ + if hostname not in self._datacache: + self._load_data(hostname) + return self._datacache.get(hostname, dict()) + + def set_data(self, hostname, data): + """ Set probe data for the given host """ + raise NotImplementedError + + def _load_groups(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def _load_data(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def commit(self): + """ Commit the current data in the cache to the persistent + backend store. This is not used with the + :class:`Bcfg2.Server.Plugins.Probes.DBProbeStore`, because it + commits on every change. """ + pass + + +class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): + """ Caching abstraction layer between the database and the Probes + plugin. """ + create = False + + def __init__(self, core, datadir): + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) + ProbeStore.__init__(self, core, datadir) + + @property + def _use_db(self): + return True + + def _load_groups(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) + self._groupcache[hostname] = list(set(r.group for r in groupdata)) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + for group in groups: + try: + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + except ProbesGroupsModel.MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + ProbesGroupsModel.objects.filter( + hostname=hostname).exclude(group__in=groups).delete() + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def _load_data(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + ts_set = False + for pdata in ProbesDataModel.objects.filter(hostname=hostname): + if not ts_set: + self._datacache[hostname].timestamp = \ + time.mktime(pdata.timestamp.timetuple()) + ts_set = True + self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + self._datacache[hostname][probe] = pdata + try: + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + except ProbesDataModel.MultipleObjectsReturned: + ProbesDataModel.objects.filter(hostname=hostname, + probe=probe).delete() + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + expire_metadata |= created + if record.data != pdata: + record.data = pdata + record.save() + expire_metadata = True + qset = ProbesDataModel.objects.filter( + hostname=hostname).exclude(probe__in=data.keys()) + if len(qset): + qset.delete() + expire_metadata = True + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + +class XMLProbeStore(ProbeStore): + """ Caching abstraction layer between ``probed.xml`` and the + Probes plugin.""" + def __init__(self, core, datadir): + ProbeStore.__init__(self, core, datadir) + self._fname = os.path.join(datadir, 'probed.xml') + self._load_data() + + def _load_data(self, _=None): + """ Load probe data from probed.xml """ + Bcfg2.Server.Cache.expire("Probes", "probegroups") + Bcfg2.Server.Cache.expire("Probes", "probedata") + if not os.path.exists(self._fname): + self.commit() + try: + data = lxml.etree.parse(self._fname, + parser=Bcfg2.Server.XMLParser).getroot() + except (IOError, lxml.etree.XMLSyntaxError): + err = sys.exc_info()[1] + self.logger.error("Failed to read file probed.xml: %s" % err) + return + for client in data.getchildren(): + self._datacache[client.get('name')] = \ + ClientProbeDataSet(timestamp=client.get("timestamp")) + self._groupcache[client.get('name')] = [] + for pdata in client: + if pdata.tag == 'Probe': + self._datacache[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get("value")) + elif pdata.tag == 'Group': + self._groupcache[client.get('name')].append( + pdata.get('name')) + + Bcfg2.Server.Cache.expire("Metadata") + + def _load_groups(self, hostname): + self._load_data(hostname) + + def commit(self): + """ Write received probe data to probed.xml """ + top = lxml.etree.Element("Probed") + for client, probed in sorted(self._datacache.items()): + # make a copy of probe data for this client in case it + # submits probe data while we're trying to write + # probed.xml + probedata = copy.copy(probed) + ctag = \ + lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probedata.timestamp))) + for probe in sorted(probedata): + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe].decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe]) + for group in sorted(self._groupcache[client]): + lxml.etree.SubElement(ctag, "Group", name=group) + try: + top.getroottree().write(self._fname, + xml_declaration=False, + pretty_print='true') + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to write %s: %s" % (self._fname, err)) + + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + olddata = self._datacache[hostname].get(probe, ProbeData('')) + self._datacache[hostname][probe] = pdata + expire_metadata |= olddata != data + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + class ClientProbeDataSet(dict): """ dict of probe => [probe data] that records a timestamp for each host """ @@ -124,17 +361,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$') basename_is_regex = True - def __init__(self, path, fam, encoding, plugin_name): + def __init__(self, path, plugin_name): self.plugin_name = plugin_name Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, - Bcfg2.Server.Plugin.SpecificData, - encoding) - fam.AddMonitor(path, self) + Bcfg2.Server.Plugin.SpecificData) + Bcfg2.Server.FileMonitor.get_fam().AddMonitor(path, self) def HandleEvent(self, event): """ handle events on everything but probed.xml """ if (event.filename != self.path and - not event.filename.endswith("probed.xml")): + not event.filename.endswith("probed.xml")): return self.handle_event(event) def get_probe_data(self, metadata): @@ -161,7 +397,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) if (metadata.version_info and - metadata.version_info > (1, 3, 1, '', 0)): + metadata.version_info > (1, 3, 1, '', 0)): try: probe.text = entry.data.decode('utf-8') except AttributeError: @@ -187,235 +423,90 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): + groupline_re = re.compile(r'^group:\s*(?P<groupname>\S+)\s*') + + options = [ + Bcfg2.Options.BooleanOption( + cf=('probes', 'use_database'), dest="probes_db", + help="Use database capabilities of the Probes plugin"), + Bcfg2.Options.Option( + cf=('probes', 'allowed_groups'), dest="probes_allowed_groups", + help="Whitespace-separated list of group name regexps to which " + "probes can assign a client", + default=[re.compile('.*')], + type=Bcfg2.Options.Types.anchored_regex_list)] + options_parsed_hook = staticmethod(load_django_models) + + def __init__(self, core): Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) try: - self.probes = ProbeSet(self.data, core.fam, core.setup['encoding'], - self.name) + self.probes = ProbeSet(self.data, self.name) except: err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) - self.allowed_cgroups = core.setup['probe_allowed_groups'] - self.probedata = dict() - self.cgroups = dict() - self.load_data() - __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - - @Bcfg2.Server.Plugin.track_statistics() - def write_data(self, client): - """ Write probe data out for use with bcfg2-info """ if self._use_db: - return self._write_data_db(client) + self.probestore = DBProbeStore(core, self.data) else: - return self._write_data_xml(client) - - def _write_data_xml(self, _): - """ Write received probe data to probed.xml """ - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - # make a copy of probe data for this client in case it - # submits probe data while we're trying to write - # probed.xml - probedata = copy.copy(probed) - ctag = \ - lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probedata.timestamp))) - for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(ctag, "Group", name=group) - try: - top.getroottree().write(os.path.join(self.data, 'probed.xml'), - xml_declaration=False, - pretty_print='true') - except IOError: - err = sys.exc_info()[1] - self.logger.error("Failed to write probed.xml: %s" % err) + self.probestore = XMLProbeStore(core, self.data) - @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock - def _write_data_db(self, client): - """ Write received probe data to the database """ - for probe, data in self.probedata[client.hostname].items(): - try: - pdata = ProbesDataModel.objects.get_or_create( - hostname=client.hostname, - probe=probe)[0] - except MultipleObjectsReturned: - ProbesDataModel.objects.filter(hostname=client.hostname, - probe=probe).delete() - ProbesDataModel.objects.get_or_create( - hostname=client.hostname, - probe=probe) - if pdata.data != data: - pdata.data = data - pdata.save() + @track_statistics() + def GetProbes(self, metadata): + return self.probes.get_probe_data(metadata) - ProbesDataModel.objects.filter( - hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() - - for group in self.cgroups[client.hostname]: - try: - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - except MultipleObjectsReturned: - ProbesGroupsModel.objects.filter(hostname=client.hostname, - group=group).delete() - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - ProbesGroupsModel.objects.filter( - hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() - - def expire_cache(self, key=None): - self.load_data(client=key) - - def load_data(self, client=None): - """ Load probe data from the appropriate backend (probed.xml - or the database) """ - if self._use_db: - return self._load_data_db(client=client) - else: - # the XML backend doesn't support loading data for single - # clients, so it reloads all data - return self._load_data_xml() - - def _load_data_xml(self): - """ Load probe data from probed.xml """ - try: - data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), - parser=Bcfg2.Server.XMLParser).getroot() - except (IOError, lxml.etree.XMLSyntaxError): - err = sys.exc_info()[1] - self.logger.error("Failed to read file probed.xml: %s" % err) - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if pdata.tag == 'Probe': - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get("value")) - elif pdata.tag == 'Group': - self.cgroups[client.get('name')].append(pdata.get('name')) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def _load_data_db(self, client=None): - """ Load probe data from the database """ - if client is None: - self.probedata = {} - self.cgroups = {} - probedata = ProbesDataModel.objects.all() - groupdata = ProbesGroupsModel.objects.all() - else: - self.probedata.pop(client, None) - self.cgroups.pop(client, None) - probedata = ProbesDataModel.objects.filter(hostname=client) - groupdata = ProbesGroupsModel.objects.filter(hostname=client) - - for pdata in probedata: - if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = ClientProbeDataSet( - timestamp=time.mktime(pdata.timestamp.timetuple())) - self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in groupdata: - if pgroup.hostname not in self.cgroups: - self.cgroups[pgroup.hostname] = [] - self.cgroups[pgroup.hostname].append(pgroup.group) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) - - @Bcfg2.Server.Plugin.track_statistics() - def GetProbes(self, meta): - return self.probes.get_probe_data(meta) - GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ - - @Bcfg2.Server.Plugin.track_statistics() def ReceiveData(self, client, datalist): - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - if client.hostname in self.cgroups: - olddata = copy.copy(self.cgroups[client.hostname]) - else: - olddata = [] - - cgroups = [] - cprobedata = ClientProbeDataSet() + cgroups = set() + cdata = dict() for data in datalist: - self.ReceiveDataItem(client, data, cgroups, cprobedata) - self.cgroups[client.hostname] = cgroups - self.probedata[client.hostname] = cprobedata - - if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and - olddata != self.cgroups[client.hostname]): - self.core.metadata_cache.expire(client.hostname) - self.write_data(client) - ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ - - def ReceiveDataItem(self, client, data, cgroups, cprobedata): - """Receive probe results pertaining to client.""" + groups, cdata[data.get("name")] = \ + self.ReceiveDataItem(client, data) + cgroups.update(groups) + self.probestore.set_groups(client.hostname, list(cgroups)) + self.probestore.set_data(client.hostname, cdata) + self.probestore.commit() + + def ReceiveDataItem(self, client, data): + """ Receive probe results pertaining to client. Returns a + tuple of (<probe groups>, <probe data>). """ if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) - cprobedata[data.get('name')] = ProbeData('') - return + return [], '' dlines = data.text.split('\n') self.logger.debug("Processing probe from %s: %s:%s" % (client.hostname, data.get('name'), [line.strip() for line in dlines])) + groups = [] for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in cgroups: - if self._group_allowed(newgroup): - cgroups.append(newgroup) - else: - self.logger.info( - "Disallowed group assignment %s from %s" % - (newgroup, client.hostname)) + match = self.groupline_re.match(line) + if match: + newgroup = match.group("groupname") + if self._group_allowed(newgroup): + groups.append(newgroup) + else: + self.logger.warning( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - cprobedata[data.get('name')] = dobj + return (groups, ProbeData("\n".join(dlines))) + + def get_additional_groups(self, metadata): + return self.probestore.get_groups(metadata.hostname) + + def get_additional_data(self, metadata): + return self.probestore.get_data(metadata.hostname) def _group_allowed(self, group): """ Determine if the named group can be set as a probe group by checking the regexes listed in the [probes] groups_allowed setting """ - return any(r.match(group) for r in self.allowed_cgroups) - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - get_additional_groups.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) - get_additional_data.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ + return any(r.match(group) + for r in Bcfg2.Options.setup.probes_allowed_groups) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 8c6cf799a..04314218c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -7,13 +7,9 @@ import sys import copy import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False try: import json @@ -35,8 +31,6 @@ except ImportError: LOGGER = logging.getLogger(__name__) -SETUP = None - class PropertyFile(object): """ Base Properties file handler """ @@ -53,12 +47,9 @@ class PropertyFile(object): """ Write the data in this data structure back to the property file. This public method performs checking to ensure that writing is possible and then calls :func:`_write`. """ - if not SETUP.cfp.getboolean("properties", "writes_enabled", - default=True): - msg = "Properties files write-back is disabled in the " + \ - "configuration" - LOGGER.error(msg) - raise PluginExecutionError(msg) + if not Bcfg2.Options.setup.writes_enabled: + raise PluginExecutionError("Properties files write-back is " + "disabled in the configuration") try: self.validate_data() except PluginExecutionError: @@ -90,8 +81,8 @@ class PropertyFile(object): class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle JSON Properties files. """ - def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + def __init__(self, name): + Bcfg2.Server.Plugin.FileBacked.__init__(self, name) PropertyFile.__init__(self, name) self.json = None __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ @@ -129,8 +120,8 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle YAML Properties files. """ - def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + def __init__(self, name): + Bcfg2.Server.Plugin.FileBacked.__init__(self, name) PropertyFile.__init__(self, name) self.yaml = None __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ @@ -168,8 +159,8 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): """ Handle XML Properties files. """ - def __init__(self, name, fam=None, should_monitor=False): - Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam, + def __init__(self, name, should_monitor=False): + Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) PropertyFile.__init__(self, name) __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ @@ -205,53 +196,8 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): return True validate_data.__doc__ = PropertyFile.validate_data.__doc__ - def Index(self): - Bcfg2.Server.Plugin.StructFile.Index(self) - if HAS_CRYPTO: - strict = self.xdata.get( - "decrypt", - SETUP.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict")) == "strict" - for el in self.xdata.xpath("//*[@encrypted]"): - try: - el.text = self._decrypt(el).encode('ascii', - 'xmlcharrefreplace') - except UnicodeDecodeError: - LOGGER.info("Properties: Decrypted %s to gibberish, " - "skipping" % el.tag) - except Bcfg2.Encryption.EVPError: - msg = "Properties: Failed to decrypt %s element in %s" % \ - (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: - LOGGER.info(msg) - Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ - - def _decrypt(self, element): - """ Decrypt a single encrypted properties file element """ - if not element.text or not element.text.strip(): - return - passes = Bcfg2.Encryption.get_passphrases(SETUP) - try: - passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Encryption.ssl_decrypt( - element.text, passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - except Bcfg2.Encryption.EVPError: - # error is raised below - pass - except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Encryption.bruteforce_decrypt( - element.text, passphrases=passes.values(), - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") - def get_additional_data(self, metadata): - if SETUP.cfp.getboolean("properties", "automatch", default=False): + if Bcfg2.Options.setup.automatch: default_automatch = "true" else: default_automatch = "false" @@ -273,6 +219,13 @@ class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ The properties plugin maps property files into client metadata instances. """ + options = [ + Bcfg2.Options.BooleanOption( + cf=("properties", "writes_enabled"), default=True, + help="Enable or disable Properties write-back"), + Bcfg2.Options.BooleanOption( + cf=("properties", "automatch"), + help="Enable Properties automatch")] #: Extensions that are understood by Properties. extensions = ["xml"] @@ -291,12 +244,10 @@ class Properties(Bcfg2.Server.Plugin.Plugin, #: Ignore XML schema (``.xsd``) files ignore = re.compile(r'.*\.xsd$') - def __init__(self, core, datastore): - global SETUP # pylint: disable=W0603 - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) - SETUP = core.setup + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) #: Instead of creating children of this object with a static #: object, we use :func:`property_dispatcher` to create a @@ -304,23 +255,21 @@ class Properties(Bcfg2.Server.Plugin.Plugin, self.__child__ = self.property_dispatcher __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def property_dispatcher(self, fname, fam): + def property_dispatcher(self, fname): """ Dispatch an event on a Properties file to the appropriate object. :param fname: The name of the file that received the event :type fname: string - :param fam: The file monitor the event was received by - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :returns: An object of the appropriate subclass of :class:`PropertyFile` """ if fname.endswith(".xml"): - return XMLPropertyFile(fname, fam) + return XMLPropertyFile(fname) elif HAS_JSON and fname.endswith(".json"): - return JSONPropertyFile(fname, fam) + return JSONPropertyFile(fname) elif HAS_YAML and (fname.endswith(".yaml") or fname.endswith(".yml")): - return YAMLPropertyFile(fname, fam) + return YAMLPropertyFile(fname) else: raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 072f3f7e7..59fbe6f03 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -4,7 +4,7 @@ import os import sys import Bcfg2.Server import Bcfg2.Server.Plugin -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor try: from syck import load as yaml_load, error as yaml_error @@ -28,16 +28,15 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to run Puppet external node classifiers (http://docs.puppetlabs.com/guides/external_nodes.html) """ - experimental = True __child__ = PuppetENCFile - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, - self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) self.cache = dict() + self.cmd = Executor() def _run_encs(self, metadata): """ Run all Puppet ENCs """ @@ -46,20 +45,17 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, epath = os.path.join(self.data, enc) self.debug_log("PuppetENC: Running ENC %s for %s" % (enc, metadata.hostname)) - proc = Popen([epath, metadata.hostname], stdin=PIPE, stdout=PIPE, - stderr=PIPE) - (out, err) = proc.communicate() - rv = proc.wait() - if rv != 0: - msg = "PuppetENC: Error running ENC %s for %s (%s): %s" % \ - (enc, metadata.hostname, rv, err) + result = self.cmd.run([epath, metadata.hostname]) + if not result.success: + msg = "PuppetENC: Error running ENC %s for %s: %s" % \ + (enc, metadata.hostname, result.error) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - if err: - self.debug_log("ENC Error: %s" % err) + if result.stderr: + self.debug_log("ENC Error: %s" % result.stderr) try: - yaml = yaml_load(out) + yaml = yaml_load(result.stdout) self.debug_log("Loaded data from %s for %s: %s" % (enc, metadata.hostname, yaml)) except yaml_error: @@ -69,13 +65,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - groups = dict() - if "classes" in yaml: - # stock Puppet ENC output format - groups = yaml['classes'] - elif "groups" in yaml: - # more Bcfg2-ish output format - groups = yaml['groups'] + groups = yaml.get("classes", yaml.get("groups", dict())) if groups: if isinstance(groups, list): self.debug_log("ENC %s adding groups to %s: %s" % diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3354763d4..8b8ada852 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -5,11 +5,10 @@ import time import platform import traceback import lxml.etree -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError -from Bcfg2.Options import REPORTING_COMMON_OPTIONS +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ - Debuggable, PluginInitError, PluginExecutionError + PluginInitError, PluginExecutionError # required for reporting try: @@ -33,25 +32,23 @@ def _rpc_call(method): # pylint: disable=W0223 -class Reporting(Statistics, Threaded, PullSource, Debuggable): +class Reporting(Statistics, Threaded, PullSource): """ Unified statistics and reporting plugin """ - __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Statistics.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + + options = [Bcfg2.Options.Common.reporting_transport] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') - def __init__(self, core, datastore): - Statistics.__init__(self, core, datastore) + def __init__(self, core): + Statistics.__init__(self, core) PullSource.__init__(self) Threaded.__init__(self) - Debuggable.__init__(self) self.whoami = platform.node() self.transport = None - core.setup.update(REPORTING_COMMON_OPTIONS) - core.setup.reparse() - if not HAS_SOUTH: msg = "Django south is required for Reporting" self.logger.error(msg) @@ -59,17 +56,15 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): def start_threads(self): try: - self.transport = load_transport_from_config(self.core.setup) + self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: - msg = "%s: Failed to load transport: %s" % \ - (self.name, traceback.format_exc().splitlines()[-1]) - self.logger.error(msg) - raise PluginInitError(msg) + raise PluginInitError("%s: Failed to instantiate transport: %s" % + (self.name, sys.exc_info()[1])) if self.debug_flag: self.transport.set_debug(self.debug_flag) def set_debug(self, debug): - rv = Debuggable.set_debug(self, debug) + rv = Statistics.set_debug(self, debug) if self.transport is not None: self.transport.set_debug(debug) return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 21862c5db..a3f682ed6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,6 +1,7 @@ """This generator provides rule-based entry mappings.""" import re +import Bcfg2.Options import Bcfg2.Server.Plugin @@ -8,45 +9,44 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) + options = Bcfg2.Server.Plugin.PrioDir.options + [ + Bcfg2.Options.BooleanOption( + cf=("rules", "regex"), dest="rules_regex", + help="Allow regular expressions in Rules")] + + def __init__(self, core): + Bcfg2.Server.Plugin.PrioDir.__init__(self, core) self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) + for src in self.entries.values(): + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + return True return False - def BindEntry(self, entry, metadata): - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - if key not in entry.attrib: - entry.attrib[key] = val + HandleEntry = Bcfg2.Server.Plugin.PrioDir.BindEntry - HandleEntry = BindEntry - - def _matches(self, entry, metadata, rules): - if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules): + def _matches(self, entry, metadata, candidate): + if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, + candidate): return True elif (entry.tag == "Path" and - ((entry.get('name').endswith("/") and - entry.get('name').rstrip("/") in rules) or - (not entry.get('name').endswith("/") and - entry.get('name') + '/' in rules))): + entry.get('name').rstrip("/") == + candidate.get("name").rstrip("/")): # special case for Path tags: # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967 return True elif self._regex_enabled: # attempt regular expression matching - for rule in rules: - if rule not in self._regex_cache: - self._regex_cache[rule] = re.compile("%s$" % rule) - if self._regex_cache[rule].match(entry.get('name')): - return True + rule = candidate.get("name") + if rule not in self._regex_cache: + self._regex_cache[rule] = re.compile("%s$" % rule) + if self._regex_cache[rule].match(entry.get('name')): + return True return False @property def _regex_enabled(self): """ Return True if rules regexes are enabled, False otherwise """ - return self.core.setup.cfp.getboolean("rules", "regex", default=False) + return Bcfg2.Options.setup.rules_regex diff --git a/src/lib/Bcfg2/Server/Plugins/SEModules.py b/src/lib/Bcfg2/Server/Plugins/SEModules.py index fa47f9496..248b662f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/SEModules.py +++ b/src/lib/Bcfg2/Server/Plugins/SEModules.py @@ -43,9 +43,6 @@ class SEModules(Bcfg2.Server.Plugin.GroupSpool): #: SEModules manages ``SEModule`` entries entry_type = 'SEModule' - #: The SEModules plugin is experimental - experimental = True - def _get_module_filename(self, entry): """ GroupSpool stores entries as /foo.pp, but we want people to be able to specify module entries as name='foo' or diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 2deea5f07..89c7107aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -5,27 +5,25 @@ import os import sys import socket import shutil -import logging import tempfile -from itertools import chain -from subprocess import Popen, PIPE +import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin +from itertools import chain +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 - -LOGGER = logging.getLogger(__name__) +try: + from Bcfg2.Server.Encryption import ssl_encrypt, bruteforce_decrypt, \ + EVPError + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, - name, - specific, - encoding) - self.encoding = encoding - def __lt__(self, other): return self.name < other.name @@ -42,49 +40,62 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): entry.text = b64encode(self.data) else: try: - entry.text = u_str(self.data, self.encoding) + entry.text = u_str(self.data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("Please verify you are using the proper encoding") + self.logger.error(msg) + self.logger.error("Please verify you are using the proper " + "encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("You need to specify base64 encoding for %s" % - entry.get('name')) + self.logger.error(msg) + self.logger.error("You need to specify base64 encoding for %s" + % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') + def handle_event(self, event): + Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) + if event.filename.endswith(".crypt"): + if self.data is None: + return + # todo: let the user specify a passphrase by name + try: + self.data = bruteforce_decrypt(self.data) + except EVPError: + raise PluginExecutionError("Failed to decrypt %s" % self.name) + class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - encoding = "base64" - else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding + if basename.startswith("ssh_host_key"): + self.metadata['encoding'] = "base64" if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: self.metadata['mode'] = '0600' + def specificity_from_filename(self, fname, specific=None): + if fname.endswith(".crypt"): + fname = fname[0:-6] + return Bcfg2.Server.Plugin.EntrySet.specificity_from_filename( + self, fname, specific=specific) + class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle the ssh_known_hosts file """ def __init__(self, path): Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) + KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file', @@ -92,7 +103,6 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -124,9 +134,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, "ssh_host_rsa_key.pub", "ssh_host_key.pub"] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + options = [ + Bcfg2.Options.Option( + cf=("sshbase", "passphrase"), dest="sshbase_passphrase", + help="Passphrase used to encrypt generated private SSH host keys")] + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -137,7 +151,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, # do so once self.badnames = dict() - core.fam.AddMonitor(self.data, self) + self.fam = Bcfg2.Server.FileMonitor.get_fam() + self.fam.AddMonitor(self.data, self) self.static = dict() self.entries = dict() @@ -150,9 +165,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.entries["/etc/ssh/" + keypattern] = \ HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + self.cmd = Executor() - def expire_cache(self, key=None): - self.__skn = False + @property + def passphrase(self): + """ The passphrase used to encrypt private keys """ + if HAS_CRYPTO and Bcfg2.Options.setup.sshbase_passphrase: + return Bcfg2.Options.setup.passphrases[ + Bcfg2.Options.setup.sshbase_passphrase] + return None def get_skn(self): """Build memory cache of the ssh known hosts file.""" @@ -252,7 +273,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, return for entry in list(self.entries.values()): - if entry.specific.match(event.filename): + if event.filename.endswith(".crypt"): + fname = event.filename[0:-6] + else: + fname = event.filename + if entry.specific.match(fname): entry.handle_event(event) if any(event.filename.startswith(kp) for kp in self.keypatterns @@ -262,7 +287,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.skn = False return - if event.filename in ['info', 'info.xml', ':info']: + if event.filename == 'info.xml': for entry in list(self.entries.values()): entry.handle_event(event) return @@ -284,12 +309,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, (event.filename, action)) def get_ipcache_entry(self, client): - """Build a cache of dns results.""" + """ Build a cache of dns results. """ if client in self.ipcache: if self.ipcache[client]: return self.ipcache[client] else: - raise socket.gaierror + raise PluginExecutionError("No cached IP address for %s" % + client) else: # need to add entry try: @@ -298,14 +324,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: - ipaddr = Popen(["getent", "hosts", client], - stdout=PIPE).stdout.read().strip().split() - if ipaddr: - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) + result = self.cmd.run(["getent", "hosts", client]) + if result.success: + ipaddr = result.stdout.strip().split() + if ipaddr: + self.ipcache[client] = (ipaddr, client) + return (ipaddr, client) self.ipcache[client] = False - self.logger.error("Failed to find IP address for %s" % client) - raise socket.gaierror + msg = "Failed to find IP address for %s: %s" % (client, + result.error) + self.logger(msg) + raise PluginExecutionError(msg) def get_namecache_entry(self, cip): """Build a cache of name lookups from client IP addresses.""" @@ -375,13 +404,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, msg = "%s still not registered" % filename self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - self.core.fam.handle_events_in_interval(1) + self.fam.handle_events_in_interval(1) tries += 1 try: self.entries[entry.get('name')].bind_entry(entry, metadata) is_bound = True except Bcfg2.Server.Plugin.PluginExecutionError: - pass + print("Failed to bind %s: %s") % ( + lxml.etree.tostring(entry), + sys.exc_info()[1]) def GenerateHostKeyPair(self, client, filename): """Generate new host key pair for client.""" @@ -404,19 +435,34 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", "-t", keytype, "-C", "root@%s" % client] self.debug_log("SSHbase: Running: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - err = proc.communicate()[1] - if proc.wait(): + result = self.cmd.run(cmd) + if not result.success: raise PluginExecutionError("SSHbase: Error running ssh-keygen: %s" - % err) + % result.error) + + if self.passphrase: + self.debug_log("SSHbase: Encrypting private key for %s" % fileloc) + try: + data = ssl_encrypt(open(temploc).read(), self.passphrase) + except IOError: + raise PluginExecutionError("Unable to read temporary SSH key: " + "%s" % sys.exc_info()[1]) + except EVPError: + raise PluginExecutionError("Unable to encrypt SSH key: %s" % + sys.exc_info()[1]) + try: + open("%s.crypt" % fileloc, "wb").write(data) + except IOError: + raise PluginExecutionError("Unable to write encrypted SSH " + "key: %s" % sys.exc_info()[1]) try: - shutil.copy(temploc, fileloc) + if not self.passphrase: + shutil.copy(temploc, fileloc) shutil.copy("%s.pub" % temploc, publoc) except IOError: - err = sys.exc_info()[1] - raise PluginExecutionError("Temporary SSH keys not found: %s" % - err) + raise PluginExecutionError("Unable to copy temporary SSH key: %s" % + sys.exc_info()[1]) try: os.unlink(temploc) diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py deleted file mode 100644 index f111ffc60..000000000 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,371 +0,0 @@ -""" The SSLCA generator handles the creation and management of ssl -certificates and their keys. """ - -import os -import sys -import logging -import tempfile -import lxml.etree -from subprocess import Popen, PIPE, STDOUT -import Bcfg2.Options -import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser -from Bcfg2.Server.Plugin import PluginExecutionError - -LOGGER = logging.getLogger(__name__) - - -class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): - """ Base class to handle key.xml and cert.xml """ - attrs = dict() - tag = None - - def get_spec(self, metadata): - """ Get a specification for the type of object described by - this SSLCA XML file for the given client metadata object """ - entries = [e for e in self.Match(metadata) if e.tag == self.tag] - if len(entries) == 0: - raise PluginExecutionError("No matching %s entry found for %s " - "in %s" % (self.tag, - metadata.hostname, - self.name)) - elif len(entries) > 1: - LOGGER.warning("More than one matching %s entry found for %s in " - "%s; using first match" % (self.tag, - metadata.hostname, - self.name)) - rv = dict() - for attr, default in self.attrs.items(): - val = entries[0].get(attr.lower(), default) - if default in ['true', 'false']: - rv[attr] = val == 'true' - else: - rv[attr] = val - return rv - - -class SSLCAKeySpec(SSLCAXMLSpec): - """ Handle key.xml files """ - attrs = dict(bits='2048', type='rsa') - tag = 'Key' - - -class SSLCACertSpec(SSLCAXMLSpec): - """ Handle cert.xml files """ - attrs = dict(ca='default', - format='pem', - key=None, - days='365', - C=None, - L=None, - ST=None, - OU=None, - O=None, - emailAddress=None, - append_chain='false') - tag = 'Cert' - - def get_spec(self, metadata): - rv = SSLCAXMLSpec.get_spec(self, metadata) - rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "subjectAltName"] - return rv - - -class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): - """ Handle key and cert files """ - def bind_entry(self, entry, _): - """ Bind the data in the file to the given abstract entry """ - entry.text = self.data - entry.set("type", "file") - return entry - - -class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): - """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, encoding, parent=None): - Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type, encoding) - self.parent = parent - self.key = None - self.cert = None - - def handle_event(self, event): - action = event.code2str() - fpath = os.path.join(self.path, event.filename) - - if event.filename == 'key.xml': - if action in ['exists', 'created', 'changed']: - self.key = SSLCAKeySpec(fpath) - self.key.HandleEvent(event) - elif event.filename == 'cert.xml': - if action in ['exists', 'created', 'changed']: - self.cert = SSLCACertSpec(fpath) - self.cert.HandleEvent(event) - else: - Bcfg2.Server.Plugin.EntrySet.handle_event(self, event) - - def build_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # TODO: verify key fits the specs - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new key %s" % filename) - key_spec = self.key.get_spec(metadata) - ktype = key_spec['type'] - bits = key_spec['bits'] - if ktype == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif ktype == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - key, err = proc.communicate() - if proc.wait(): - raise PluginExecutionError("SSLCA: Failed to generate key %s for " - "%s: %s" % (entry.get("name"), - metadata.hostname, err)) - open(os.path.join(self.path, filename), 'w').write(key) - return key - - def build_cert(self, entry, metadata, keyfile): - """ generate a new cert """ - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new cert %s" % filename) - cert_spec = self.cert.get_spec(metadata) - ca = self.parent.get_ca(cert_spec['ca']) - req_config = None - req = None - try: - req_config = self.build_req_config(metadata) - req = self.build_request(keyfile, req_config, metadata) - days = cert_spec['days'] - cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, - "-days", days, "-batch"] - passphrase = ca.get('passphrase') - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a - - self.debug_log("SSLCA: Generating new certificate: %s" % - " ".join(_scrub_pass(a) for a in cmd)) - proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (cert, err) = proc.communicate() - if proc.wait(): - # pylint: disable=E1103 - raise PluginExecutionError("SSLCA: Failed to generate cert: %s" - % err.splitlines()[-1]) - # pylint: enable=E1103 - finally: - try: - if req_config and os.path.exists(req_config): - os.unlink(req_config) - if req and os.path.exists(req): - os.unlink(req) - except OSError: - self.logger.error("SSLCA: Failed to unlink temporary files: %s" - % sys.exc_info()[1]) - if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(ca['chaincert']).read() - - open(os.path.join(self.path, filename), 'w').write(cert) - return cert - - def build_req_config(self, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - fd, fname = tempfile.mkstemp() - cfp = ConfigParser.ConfigParser({}) - cfp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cfp.add_section(section) - for key in defaults[section]: - cfp.set(section, key, defaults[section][key]) - cert_spec = self.cert.get_spec(metadata) - altnamenum = 1 - altnames = cert_spec['subjectaltname'] - altnames.extend(list(metadata.aliases)) - altnames.append(metadata.hostname) - for altname in altnames: - cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) - altnamenum += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if cert_spec[item]: - cfp.set('req_distinguished_name', item, cert_spec[item]) - cfp.set('req_distinguished_name', 'CN', metadata.hostname) - self.debug_log("SSLCA: Writing temporary request config to %s" % fname) - try: - cfp.write(os.fdopen(fd, 'w')) - except IOError: - raise PluginExecutionError("SSLCA: Failed to write temporary CSR " - "config file: %s" % sys.exc_info()[1]) - return fname - - def build_request(self, keyfile, req_config, metadata): - """ - creates the certificate request - """ - fd, req = tempfile.mkstemp() - os.close(fd) - days = self.cert.get_spec(metadata)['days'] - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", keyfile, "-text", "-out", req] - self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - err = proc.communicate()[1] - if proc.wait(): - raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % - err) - return req - - def verify_cert(self, filename, keyfile, entry, metadata): - """ Perform certification verification against the CA and - against the key """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - do_verify = ca.get('chaincert') - if do_verify: - return (self.verify_cert_against_ca(filename, entry, metadata) and - self.verify_cert_against_key(filename, keyfile)) - return True - - def verify_cert_against_ca(self, filename, entry, metadata): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - chaincert = ca.get('chaincert') - cert = os.path.join(self.path, filename) - cmd = ["openssl", "verify"] - is_root = ca.get('root_ca', "false").lower() == 'true' - if is_root: - cmd.append("-CAfile") - else: - # verifying based on an intermediate cert - cmd.extend(["-purpose", "sslserver", "-untrusted"]) - cmd.extend([chaincert, cert]) - self.debug_log("SSLCA: Verifying %s against CA: %s" % - (entry.get("name"), " ".join(cmd))) - res = Popen(cmd, stdout=PIPE, stderr=STDOUT).stdout.read() - if res == cert + ": OK\n": - self.debug_log("SSLCA: %s verified successfully against CA" % - entry.get("name")) - return True - self.logger.warning("SSLCA: %s failed verification against CA: %s" % - (entry.get("name"), res)) - return False - - def verify_cert_against_key(self, filename, keyfile): - """ - check that a certificate validates against its private key. - """ - def _modulus(fname, ftype="x509"): - """ get the modulus from the given file """ - cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] - self.debug_log("SSLCA: Getting modulus of %s for verification: %s" - % (fname, " ".join(cmd))) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - self.logger.warning("SSLCA: Failed to get modulus of %s: %s" % - (fname, err)) - return rv.strip() # pylint: disable=E1103 - - certfile = os.path.join(self.path, filename) - cert = _modulus(certfile) - key = _modulus(keyfile, ftype="rsa") - if cert == key: - self.debug_log("SSLCA: %s verified successfully against key %s" % - (filename, keyfile)) - return True - self.logger.warning("SSLCA: %s failed verification against key %s" % - (filename, keyfile)) - return False - - def bind_entry(self, entry, metadata): - if self.key: - self.bind_info_to_entry(entry, metadata) - try: - return self.best_matching(metadata).bind_entry(entry, metadata) - except PluginExecutionError: - entry.text = self.build_key(entry, metadata) - entry.set("type", "file") - return entry - elif self.cert: - key = self.cert.get_spec(metadata)['key'] - cleanup_keyfile = False - try: - keyfile = self.parent.entries[key].best_matching(metadata).name - except PluginExecutionError: - cleanup_keyfile = True - # create a temp file with the key in it - fd, keyfile = tempfile.mkstemp() - os.chmod(keyfile, 384) # 0600 - el = lxml.etree.Element('Path', name=key) - self.parent.core.Bind(el, metadata) - os.fdopen(fd, 'w').write(el.text) - - try: - self.bind_info_to_entry(entry, metadata) - try: - best = self.best_matching(metadata) - if self.verify_cert(best.name, keyfile, entry, metadata): - return best.bind_entry(entry, metadata) - except PluginExecutionError: - pass - # if we get here, it's because either a) there was no best - # matching entry; or b) the existing cert did not verify - entry.text = self.build_cert(entry, metadata, keyfile) - entry.set("type", "file") - return entry - finally: - if cleanup_keyfile: - try: - os.unlink(keyfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("SSLCA: Failed to unlink temporary " - "key %s: %s" % (keyfile, err)) - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ The SSLCA generator handles the creation and management of ssl - certificates and their keys. """ - __author__ = 'g.hagger@gmail.com' - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) - es_child_cls = SSLCADataFile - - def get_ca(self, name): - """ get a dict describing a CA from the config file """ - return dict(self.core.setup.cfp.items("sslca_%s" % name)) diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py deleted file mode 100644 index cc5946bb2..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py +++ /dev/null @@ -1,129 +0,0 @@ -import logging -import difflib -import Bcfg2.Server.Plugin -import Bcfg2.Server.Snapshots -import Bcfg2.Logger -from Bcfg2.Server.Snapshots.model import Snapshot -import sys -import time -import threading - -# Compatibility import -from Bcfg2.Compat import Queue, u_str, b64decode - -logger = logging.getLogger('Snapshots') - -ftypes = ['ConfigFile', 'SymLink', 'Directory'] -datafields = { - 'Package': ['version'], - 'Path': ['type'], - 'Service': ['status'], - 'ConfigFile': ['owner', 'group', 'mode'], - 'Directory': ['owner', 'group', 'mode'], - 'SymLink': ['to'], - } - - -def build_snap_ent(entry): - basefields = [] - if entry.tag in ['Package', 'Service']: - basefields += ['type'] - desired = dict([(key, u_str(entry.get(key))) for key in basefields]) - state = dict([(key, u_str(entry.get(key))) for key in basefields]) - desired.update([(key, u_str(entry.get(key))) for key in \ - datafields[entry.tag]]) - if entry.tag == 'ConfigFile' or \ - ((entry.tag == 'Path') and (entry.get('type') == 'file')): - if entry.text == None: - desired['contents'] = None - else: - if entry.get('encoding', 'ascii') == 'ascii': - desired['contents'] = u_str(entry.text) - else: - desired['contents'] = u_str(b64decode(entry.text)) - - if 'current_bfile' in entry.attrib: - state['contents'] = u_str(b64decode(entry.get('current_bfile'))) - elif 'current_bdiff' in entry.attrib: - diff = b64decode(entry.get('current_bdiff')) - state['contents'] = u_str( \ - '\n'.join(difflib.restore(diff.split('\n'), 1))) - - state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \ - for key in datafields[entry.tag]]) - if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false': - state = None - return [desired, state] - - -class Snapshots(Bcfg2.Server.Plugin.Statistics): - name = 'Snapshots' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore) - self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) - self.work_queue = Queue() - self.loader = threading.Thread(target=self.load_snapshot) - - def start_threads(self): - self.loader.start() - - def load_snapshot(self): - while self.running: - try: - (metadata, data) = self.work_queue.get(block=True, timeout=5) - except: - continue - self.statistics_from_old_stats(metadata, data) - - def process_statistics(self, metadata, data): - return self.work_queue.put((metadata, data)) - - def statistics_from_old_stats(self, metadata, xdata): - # entries are name -> (modified, correct, start, desired, end) - # not sure we can get all of this from old format stats - t1 = time.time() - entries = dict([('Package', dict()), - ('Service', dict()), ('Path', dict())]) - extra = dict([('Package', dict()), ('Service', dict()), - ('Path', dict())]) - bad = [] - state = xdata.find('.//Statistics') - correct = state.get('state') == 'clean' - revision = u_str(state.get('revision', '-1')) - for entry in state.find('.//Bad'): - data = [False, False, u_str(entry.get('name'))] \ - + build_snap_ent(entry) - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - entries[etag][entry.get('name')] = data - for entry in state.find('.//Modified'): - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - if entry.get('name') in entries[etag]: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - else: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - for entry in state.find('.//Extra'): - if entry.tag in datafields: - data = build_snap_ent(entry)[1] - ename = u_str(entry.get('name')) - data['name'] = ename - extra[entry.tag][ename] = data - else: - print("extra", entry.tag, entry.get('name')) - t2 = time.time() - snap = Snapshot.from_data(self.session, correct, revision, - metadata, entries, extra) - self.session.add(snap) - self.session.commit() - t3 = time.time() - logger.info("Snapshot storage took %fs" % (t3 - t2)) - return True diff --git a/src/lib/Bcfg2/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py deleted file mode 100644 index 7fae445d0..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Statistics.py +++ /dev/null @@ -1,160 +0,0 @@ -'''This file manages the statistics collected by the BCFG2 Server''' - -import copy -import difflib -import logging -import lxml.etree -import os -import sys -from time import asctime, localtime, time, strptime, mktime -import threading -from Bcfg2.Compat import b64decode -import Bcfg2.Server.Plugin - - -class StatisticsStore(object): - """Manages the memory and file copy of statistics collected about client runs.""" - __min_write_delay__ = 0 - - def __init__(self, filename): - self.filename = filename - self.element = lxml.etree.Element('Dummy') - self.dirty = 0 - self.lastwrite = 0 - self.logger = logging.getLogger('Bcfg2.Server.Statistics') - self.ReadFromFile() - - def WriteBack(self, force=0): - """Write statistics changes back to persistent store.""" - if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \ - or force: - try: - fout = open(self.filename + '.new', 'w') - except IOError: - ioerr = sys.exc_info()[1] - self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) - else: - fout.write(lxml.etree.tostring(self.element, - xml_declaration=False).decode('UTF-8')) - fout.close() - os.rename(self.filename + '.new', self.filename) - self.dirty = 0 - self.lastwrite = time() - - def ReadFromFile(self): - """Reads current state regarding statistics.""" - try: - fin = open(self.filename, 'r') - data = fin.read() - fin.close() - self.element = lxml.etree.XML(data) - self.dirty = 0 - except (IOError, lxml.etree.XMLSyntaxError): - self.logger.error("Creating new statistics file %s"%(self.filename)) - self.element = lxml.etree.Element('ConfigStatistics') - self.WriteBack() - self.dirty = 0 - - def updateStats(self, xml, client): - """Updates the statistics of a current node with new data.""" - - # Current policy: - # - Keep anything less than 24 hours old - # - Keep latest clean run for clean nodes - # - Keep latest clean and dirty run for dirty nodes - newstat = xml.find('Statistics') - - if newstat.get('state') == 'clean': - node_dirty = 0 - else: - node_dirty = 1 - - # Find correct node entry in stats data - # The following list comprehension should be guarenteed to return at - # most one result - nodes = [elem for elem in self.element.findall('Node') \ - if elem.get('name') == client] - nummatch = len(nodes) - if nummatch == 0: - # Create an entry for this node - node = lxml.etree.SubElement(self.element, 'Node', name=client) - elif nummatch == 1 and not node_dirty: - # Delete old instance - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if self.isOlderThan24h(elem.get('time'))] - elif nummatch == 1 and node_dirty: - # Delete old dirty statistics entry - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if (elem.get('state') == 'dirty' \ - and self.isOlderThan24h(elem.get('time')))] - else: - # Shouldn't be reached - self.logger.error("Duplicate node entry for %s"%(client)) - - # Set current time for stats - newstat.set('time', asctime(localtime())) - - # Add statistic - node.append(copy.copy(newstat)) - - # Set dirty - self.dirty = 1 - self.WriteBack(force=1) - - def isOlderThan24h(self, testTime): - """Helper function to determine if <time> string is older than 24 hours.""" - now = time() - utime = mktime(strptime(testTime)) - secondsPerDay = 60*60*24 - - return (now-utime) > secondsPerDay - - -class Statistics(Bcfg2.Server.Plugin.ThreadedStatistics, - Bcfg2.Server.Plugin.PullSource): - name = 'Statistics' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) - Bcfg2.Server.Plugin.PullSource.__init__(self) - fpath = "%s/etc/statistics.xml" % datastore - self.data_file = StatisticsStore(fpath) - - def handle_statistic(self, metadata, data): - self.data_file.updateStats(data, metadata.hostname) - - def FindCurrent(self, client): - rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0] - maxtime = max([strptime(stat.get('time')) for stat \ - in rt.findall('Statistics')]) - return [stat for stat in rt.findall('Statistics') \ - if strptime(stat.get('time')) == maxtime][0] - - def GetExtra(self, client): - return [(entry.tag, entry.get('name')) for entry \ - in self.FindCurrent(client).xpath('.//Extra/*')] - - def GetCurrentEntry(self, client, e_type, e_name): - curr = self.FindCurrent(client) - entry = curr.xpath('.//Bad/%s[@name="%s"]' % (e_type, e_name)) - if not entry: - raise Bcfg2.Server.Plugin.PluginExecutionError - cfentry = entry[-1] - - owner = cfentry.get('current_owner', cfentry.get('owner')) - group = cfentry.get('current_group', cfentry.get('group')) - mode = cfentry.get('current_mode', cfentry.get('mode')) - if cfentry.get('sensitive') in ['true', 'True']: - raise Bcfg2.Server.Plugin.PluginExecutionError - elif 'current_bfile' in cfentry.attrib: - contents = b64decode(cfentry.get('current_bfile')) - elif 'current_bdiff' in cfentry.attrib: - diff = b64decode(cfentry.get('current_bdiff')) - contents = '\n'.join(difflib.restore(diff.split('\n'), 1)) - else: - contents = None - - return (owner, group, mode, contents) diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 240fd7f89..b2a16e52e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -4,19 +4,33 @@ additional XML-RPC methods for committing data to the repository and updating the repository. """ import sys +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser try: import pysvn HAS_SVN = True except ImportError: - import pipes - from subprocess import Popen, PIPE + from Bcfg2.Utils import Executor HAS_SVN = False class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" + options = Bcfg2.Server.Plugin.Version.options + [ + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + choices=dir(pysvn.wc_conflict_choice), + default=pysvn.wc_conflict_choice.postpone, + help="SVN conflict resolution method"), + Bcfg2.Options.Option( + cf=("svn", "user"), dest="svn_user", help="SVN username"), + Bcfg2.Options.Option( + cf=("svn", "password"), dest="svn_password", help="SVN password"), + Bcfg2.Options.BooleanOption( + cf=("svn", "always_trust"), dest="svn_trust_ssl", + help="Always trust SSL certs from SVN server")] + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: @@ -24,73 +38,42 @@ class Svn(Bcfg2.Server.Plugin.Version): else: __vcs_metadata_path__ = ".svn" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.revision = None self.svn_root = None + self.client = None + self.cmd = None if not HAS_SVN: self.logger.debug("Svn: PySvn not found, using CLI interface to " "SVN") - self.client = None + self.cmd = Executor() else: self.client = pysvn.Client() - # pylint: disable=E1101 - choice = pysvn.wc_conflict_choice.postpone - try: - resolution = self.core.setup.cfp.get( - "svn", - "conflict_resolution").replace('-', '_') - if resolution in ["edit", "launch", "working"]: - self.logger.warning("Svn: Conflict resolver %s requires " - "manual intervention, using %s" % - choice) - else: - choice = getattr(pysvn.wc_conflict_choice, resolution) - except AttributeError: - self.logger.warning("Svn: Conflict resolver %s does not " - "exist, using %s" % choice) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: No conflict resolution method " - "selected, using %s" % choice) - # pylint: enable=E1101 self.debug_log("Svn: Conflicts will be resolved with %s" % - choice) - self.client.callback_conflict_resolver = \ - self.get_conflict_resolver(choice) + Bcfg2.Options.setup.svn_conflict_resolution) + self.client.callback_conflict_resolver = self.conflict_resolver - try: - if self.core.setup.cfp.get( - "svn", - "always_trust").lower() == "true": - self.client.callback_ssl_server_trust_prompt = \ - self.ssl_server_trust_prompt - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.debug("Svn: Using subversion cache for SSL " - "certificate trust") + if Bcfg2.Options.setup.svn_trust_ssl: + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt - try: - if (self.core.setup.cfp.get("svn", "user") and - self.core.setup.cfp.get("svn", "password")): - self.client.callback_get_login = \ - self.get_login - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: Using subversion cache for " - "password-based authetication") + if (Bcfg2.Options.setup.svn_user and + Bcfg2.Options.setup.svn_password): + self.client.callback_get_login = self.get_login self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) - # pylint: disable=W0613 - def get_login(self, realm, username, may_save): + def get_login(self, realm, username, may_save): # pylint: disable=W0613 """ PySvn callback to get credentials for HTTP basic authentication """ self.logger.debug("Svn: Logging in with username: %s" % - self.core.setup.cfp.get("svn", "user")) - return True, \ - self.core.setup.cfp.get("svn", "user"), \ - self.core.setup.cfp.get("svn", "password"), \ - False - # pylint: enable=W0613 + Bcfg2.Options.setup.svn_user) + return (True, + Bcfg2.Options.setup.svn_user, + Bcfg2.Options.setup.svn_password, + False) def ssl_server_trust_prompt(self, trust_dict): """ PySvn callback to always trust SSL certificates from SVN server """ @@ -101,37 +84,35 @@ class Svn(Bcfg2.Server.Plugin.Version): trust_dict['realm'])) return True, trust_dict['failures'], False - def get_conflict_resolver(self, choice): - """ Get a PySvn conflict resolution callback """ - def callback(conflict_description): - """ PySvn callback function to resolve conflicts """ - self.logger.info("Svn: Resolving conflict for %s with %s" % - (conflict_description['path'], choice)) - return choice, None, False - - return callback + def conflict_resolver(self, conflict_description): + """ PySvn callback function to resolve conflicts """ + self.logger.info("Svn: Resolving conflict for %s with %s" % + (conflict_description['path'], + Bcfg2.Options.setup.svn_conflict_resolution)) + return Bcfg2.Options.setup.svn_conflict_resolution, None, False def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" msg = None if HAS_SVN: try: - info = self.client.info(self.vcs_root) + info = self.client.info(Bcfg2.Options.setup.vcs_root) self.revision = info.revision self.svn_root = info.url return str(self.revision.number) except pysvn.ClientError: # pylint: disable=E1101 msg = "Svn: Failed to get revision: %s" % sys.exc_info()[1] else: - try: - data = Popen("env LC_ALL=C svn info %s" % - pipes.quote(self.vcs_root), shell=True, - stdout=PIPE).communicate()[0].split('\n') - return [line.split(': ')[1] for line in data - if line[:9] == 'Revision:'][-1] - except IndexError: - msg = "Failed to read svn info" - self.logger.error('Ran command "svn info %s"' % self.vcs_root) + result = self.cmd.run(["env LC_ALL=C", "svn", "info", + Bcfg2.Options.setup.vcs_root], + shell=True) + if result.success: + self.revision = [line.split(': ')[1] + for line in result.stdout.splitlines() + if line.startswith('Revision:')][-1] + return self.revision + else: + msg = "Failed to read svn info: %s" % result.error self.revision = None raise Bcfg2.Server.Plugin.PluginExecutionError(msg) @@ -139,7 +120,8 @@ class Svn(Bcfg2.Server.Plugin.Version): '''Svn.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] except pysvn.ClientError: # pylint: disable=E1101 err = sys.exc_info()[1] # try to be smart about the error we got back @@ -161,7 +143,7 @@ class Svn(Bcfg2.Server.Plugin.Version): self.logger.debug("repository is current") else: self.logger.info("Updated %s from revision %s to %s" % - (self.vcs_root, old_revision, + (Bcfg2.Options.setup.vcs_root, old_revision, self.revision.number)) return True @@ -174,10 +156,11 @@ class Svn(Bcfg2.Server.Plugin.Version): return False try: - self.revision = self.client.checkin([self.vcs_root], + self.revision = self.client.checkin([Bcfg2.Options.setup.vcs_root], 'Svn: autocommit', recurse=True) - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] self.logger.info("Svn: Commited changes. At %s" % self.revision.number) return True diff --git a/src/lib/Bcfg2/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py deleted file mode 100644 index f2c59ce29..000000000 --- a/src/lib/Bcfg2/Server/Plugins/TCheetah.py +++ /dev/null @@ -1,79 +0,0 @@ -'''This module implements a templating generator based on Cheetah''' - -import logging -import sys -import traceback -import Bcfg2.Server.Plugin - -from Bcfg2.Compat import unicode, b64encode - -logger = logging.getLogger('Bcfg2.Plugins.TCheetah') - -try: - import Cheetah.Template - import Cheetah.Parser -except: - logger.error("TCheetah: Failed to import Cheetah. Is it installed?") - raise - - -class TemplateFile: - """Template file creates Cheetah template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - self.template = None - self.searchlist = dict() - - def handle_event(self, event): - """Handle all fs events for this template.""" - if event.code2str() == 'deleted': - return - try: - s = {'useStackFrames': False} - self.template = Cheetah.Template.Template(open(self.name).read(), - compilerSettings=s, - searchList=self.searchlist) - except Cheetah.Parser.ParseError: - perror = sys.exc_info()[1] - logger.error("Cheetah parse error for file %s" % (self.name)) - logger.error(perror.report()) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - self.template.metadata = metadata - self.searchlist['metadata'] = metadata - self.template.path = entry.get('realname', entry.get('name')) - self.searchlist['path'] = entry.get('realname', entry.get('name')) - self.template.source_path = self.name - self.searchlist['source_path'] = self.name - - if entry.tag == 'Path': - entry.set('type', 'file') - try: - if type(self.template) == unicode: - entry.text = self.template - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = b64encode(self.template) - else: - entry.text = unicode(str(self.template), self.encoding) - except: - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - logger.error(msg) - logger.error("TCheetah template error for %s" % self.searchlist['path']) - del a, b, c - raise Bcfg2.Server.Plugin.PluginExecutionError - - -class TCheetah(Bcfg2.Server.Plugin.GroupSpool): - """The TCheetah generator implements a templating mechanism for configuration files.""" - name = 'TCheetah' - __author__ = 'bcfg-dev@mcs.anl.gov' - filename_pattern = 'template' - es_child_cls = TemplateFile - deprecated = True diff --git a/src/lib/Bcfg2/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py deleted file mode 100644 index 809587d91..000000000 --- a/src/lib/Bcfg2/Server/Plugins/TGenshi.py +++ /dev/null @@ -1,139 +0,0 @@ -"""This module implements a templating generator based on Genshi.""" - -import logging -import sys -import Bcfg2.Server.Plugin - -from Bcfg2.Compat import unicode, b64encode - -logger = logging.getLogger('Bcfg2.Plugins.TGenshi') - -# try to import genshi stuff -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, \ - TextTemplate, MarkupTemplate, TemplateError -except ImportError: - logger.error("TGenshi: Failed to import Genshi. Is it installed?") - raise -try: - from genshi.template import NewTextTemplate - have_ntt = True -except: - have_ntt = False - -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -class TemplateFile(object): - """Template file creates Genshi template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - if self.specific.all: - matchname = self.name - elif self.specific.group: - matchname = self.name[:self.name.find('.G')] - else: - matchname = self.name[:self.name.find('.H')] - if matchname.endswith('.txt'): - self.template_cls = TextTemplate - elif matchname.endswith('.newtxt'): - if not have_ntt: - logger.error("Genshi NewTextTemplates not supported by this version of Genshi") - else: - self.template_cls = NewTextTemplate - else: - self.template_cls = MarkupTemplate - self.HandleEvent = self.handle_event - - def handle_event(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - try: - self.template = loader.load(self.name, cls=self.template_cls, - encoding=self.encoding) - except LookupError: - lerror = sys.exc_info()[1] - logger.error('Genshi lookup error: %s' % lerror) - except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - except genshi.input.ParseError: - perror = sys.exc_info()[1] - logger.error('Genshi parse error: %s' % perror) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - fname = entry.get('realname', entry.get('name')) - if entry.tag == 'Path': - entry.set('type', 'file') - try: - stream = self.template.generate( \ - name=fname, metadata=metadata, - path=self.name).filter(removecomment) - if have_ntt: - ttypes = [TextTemplate, NewTextTemplate] - else: - ttypes = [TextTemplate] - if True in [isinstance(self.template, t) for t in ttypes]: - try: - textdata = stream.render('text', strip_whitespace=False) - except TypeError: - textdata = stream.render('text') - if type(textdata) == unicode: - entry.text = textdata - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = b64encode(textdata) - else: - entry.text = unicode(textdata, self.encoding) - else: - try: - xmldata = stream.render('xml', strip_whitespace=False) - except TypeError: - xmldata = stream.render('xml') - if type(xmldata) == unicode: - entry.text = xmldata - else: - entry.text = unicode(xmldata, self.encoding) - if entry.text == '': - entry.set('empty', 'true') - except TemplateError: - err = sys.exc_info()[1] - logger.exception('Genshi template error') - raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err) - except AttributeError: - err = sys.exc_info()[1] - logger.exception('Genshi template loading error') - raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err) - - -class TemplateEntrySet(Bcfg2.Server.Plugin.EntrySet): - basename_is_regex = True - - -class TGenshi(Bcfg2.Server.Plugin.GroupSpool): - """ - The TGenshi generator implements a templating - mechanism for configuration files. - - """ - name = 'TGenshi' - __author__ = 'jeff@ocjtech.us' - filename_pattern = 'template\.(txt|newtxt|xml)' - es_cls = TemplateEntrySet - es_child_cls = TemplateFile - deprecated = True diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index db7370f01..047fc062e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -3,11 +3,10 @@ import re import imp import sys -import logging -import Bcfg2.Server.Lint -import Bcfg2.Server.Plugin - -LOGGER = logging.getLogger(__name__) +import lxml.etree +from Bcfg2.Server.Plugin import Plugin, Connector, DirectoryBacked, \ + TemplateDataProvider, DefaultTemplateDataProvider +from Bcfg2.Logger import Debuggable MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') @@ -20,12 +19,12 @@ def safe_module_name(module): return '__TemplateHelper_%s' % module -class HelperModule(object): +class HelperModule(Debuggable): """ Representation of a TemplateHelper module """ - def __init__(self, name, fam=None): + def __init__(self, name): + Debuggable.__init__(self) self.name = name - self.fam = fam #: The name of the module as used by get_additional_data(). #: the name of the file with .py stripped off. @@ -34,6 +33,14 @@ class HelperModule(object): #: The attributes exported by this module self._attrs = [] + #: The attributes added to the template namespace by this module + self.defaults = [] + + default_prov = DefaultTemplateDataProvider() + self.reserved_defaults = default_prov.get_template_data( + lxml.etree.Element("Path", name="/dummy"), + None, None).keys() + ["path"] + def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -49,120 +56,76 @@ class HelperModule(object): self.name) except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("TemplateHelper: Failed to import %s: %s" % - (self.name, err)) + self.logger.error("TemplateHelper: Failed to import %s: %s" % + (self.name, err)) return if not hasattr(module, "__export__"): - LOGGER.error("TemplateHelper: %s has no __export__ list" % - self.name) + self.logger.error("TemplateHelper: %s has no __export__ list" % + self.name) return newattrs = [] - for sym in module.__export__: + for sym in module.__export__ + getattr(module, "__default__", []): + if sym in newattrs: + # already added to attribute list + continue if sym not in self._attrs and hasattr(self, sym): - LOGGER.warning("TemplateHelper: %s: %s is a reserved keyword, " - "skipping export" % (self.name, sym)) + self.logger.warning( + "TemplateHelper: %s: %s is a reserved keyword, " + "skipping export" % (self.name, sym)) continue try: setattr(self, sym, getattr(module, sym)) newattrs.append(sym) except AttributeError: - LOGGER.warning("TemplateHelper: %s exports %s, but has no " - "such attribute" % (self.name, sym)) + self.logger.warning( + "TemplateHelper: %s exports %s, but has no such attribute" + % (self.name, sym)) + # remove old exports for sym in set(self._attrs) - set(newattrs): delattr(self, sym) self._attrs = newattrs + self.defaults = [] + for sym in getattr(module, "__default__", []): + if sym in self.reserved_defaults: + self.logger.warning( + "TemplateHelper: %s: %s is a reserved keyword, not adding " + "as default" % (self.name, sym)) + self.defaults.append(sym) + -class TemplateHelper(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector, - Bcfg2.Server.Plugin.DirectoryBacked): +class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): """ A plugin to provide helper classes and functions to templates """ __author__ = 'chris.a.st.pierre@gmail.com' ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE __child__ = HelperModule - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) + def __init__(self, core): + Plugin.__init__(self, core) + Connector.__init__(self) + DirectoryBacked.__init__(self, self.data) + TemplateDataProvider.__init__(self) def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) - -class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper - <server-plugins-connectors-templatehelper>` modules are valid. - This can check for: - - * A TemplateHelper module that cannot be imported due to syntax or - other compile-time errors; - * A TemplateHelper module that does not have an ``__export__`` - attribute, or whose ``__export__`` is not a list; - * Bogus symbols listed in ``__export__``, including symbols that - don't exist, that are reserved, or that start with underscores. - """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - - def Run(self): - for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper.name): - self.check_helper(helper.name) - - def check_helper(self, helper): - """ Check a single helper module. - - :param helper: The filename of the helper module - :type helper: string - """ - module_name = MODULE_RE.search(helper).group(1) - - try: - module = imp.load_source(safe_module_name(module_name), helper) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("templatehelper-import-error", - "Failed to import %s: %s" % - (helper, err)) - return - - if not hasattr(module, "__export__"): - self.LintError("templatehelper-no-export", - "%s has no __export__ list" % helper) - return - elif not isinstance(module.__export__, list): - self.LintError("templatehelper-nonlist-export", - "__export__ is not a list in %s" % helper) - return - - for sym in module.__export__: - if not hasattr(module, sym): - self.LintError("templatehelper-nonexistent-export", - "%s: exported symbol %s does not exist" % - (helper, sym)) - elif sym in self.reserved_keywords: - self.LintError("templatehelper-reserved-export", - "%s: exported symbol %s is reserved" % - (helper, sym)) - elif sym.startswith("_"): - self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" - % (helper, sym)) - - @classmethod - def Errors(cls): - return {"templatehelper-import-error": "error", - "templatehelper-no-export": "error", - "templatehelper-nonlist-export": "error", - "templatehelper-nonexistent-export": "error", - "templatehelper-reserved-export": "error", - "templatehelper-underscore-export": "warning"} + def get_template_data(self, *_): + rv = dict() + source = dict() + for helper in self.entries.values(): + for key in helper.defaults: + if key not in rv: + rv[key] = getattr(helper, key) + source[key] = helper + else: + self.logger.warning( + "TemplateHelper: Duplicate default variable %s " + "provided by both %s and %s" % + (key, helper.name, source[key].name)) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index f7c82fdb3..12672de7d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -3,18 +3,14 @@ import os import pipes import Bcfg2.Server.Plugin -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor class TriggerFile(Bcfg2.Server.Plugin.FileBacked): """ Representation of a trigger script file """ - def HandleEvent(self, event=None): return - def __str__(self): - return "%s: %s" % (self.__class__.__name__, self.name) - class Trigger(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ClientRunHooks, @@ -22,11 +18,11 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, """Trigger is a plugin that calls external scripts (on the server).""" __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, - self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) + self.cmd = Executor() def async_run(self, args): """ Run the trigger script asynchronously in a forked process @@ -39,14 +35,12 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, if not dpid: self.debug_log("Running %s" % " ".join(pipes.quote(a) for a in args)) - proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - err = proc.communicate()[1] - rv = proc.wait() - if rv != 0: - self.logger.error("Trigger: Error running %s (%s): %s" % - (args[0], rv, err)) - elif err: - self.debug_log("Trigger: Error: %s" % err) + result = self.cmd.run(args) + if not result.success: + self.logger.error("Trigger: Error running %s: %s" % + (args[0], result.error)) + elif result.stderr: + self.debug_log("Trigger: Error: %s" % result.stderr) os._exit(0) # pylint: disable=W0212 def end_client_run(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index ad51cf368..fdb20ed0a 100644 --- a/src/lib/Bcfg2/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,5 +1 @@ """Imports for Bcfg2.Server.Plugins.""" - -from Bcfg2.Compat import walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] |