diff options
-rw-r--r-- | doc/server/plugins/probes/index.txt | 57 | ||||
-rw-r--r-- | man/bcfg2-admin.8 | 6 | ||||
-rw-r--r-- | schemas/fileprobes.xsd | 34 | ||||
-rw-r--r-- | setup.py | 8 | ||||
-rw-r--r-- | src/lib/Server/Lint/InfoXML.py | 10 | ||||
-rw-r--r-- | src/lib/Server/Lint/Validate.py | 4 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Cfg.py | 2 | ||||
-rw-r--r-- | src/lib/Server/Plugins/FileProbes.py | 177 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Svn2.py | 8 |
9 files changed, 291 insertions, 15 deletions
diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index 33ec6f444..b9f698a0f 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -143,3 +143,60 @@ Other examples :hidden: ohai + +.. _server-plugins-probes-fileprobes: + +FileProbes +========== + +The FileProbes plugin allows you to probe a client for a file, +which is then added to the :ref:`server-plugins-generators-cfg` +specification. If the file changes on the client, FileProbes can +either update it in the specification or allow Cfg to replace it. + +FileProbes will not probe a file if there's already a file in Cfg that +will apply to the client. So if, for instance, you have a generic +file in ``Cfg/etc/foo.conf/foo.conf`` that applies to all hosts, +FileProbes will not retrieve ``/etc/foo.conf`` from the client (unless +``update`` is enabled; see Configuration_ below). + +When a new config file is first probed, an ``info.xml`` file is also +written to enforce the permissions from that client. Subsequent +probes from other clients will not modify or overwrite the data in +``info.xml``. (This ensures that any manual changes you make to +``info.xml`` for that file are not circumvented.) + +Configuration +------------- + +FileProbes is configured in ``FileProbes/config.xml``, which might +look something like: + +.. code-block:: xml + + <FileProbes> + <FileProbe name="/etc/foo.conf"/> + <Group name="blah-servers"> + <FileProbe name="/etc/blah.conf" update="true" + </Group> + <Client name="bar.example.com"> + <FileProbe name="/var/lib/bar.gz" base64="true"/> + </Client> + </FileProbes> + +This will result in ``/etc/foo.conf`` being retrieved from all +clients; if it changes on a client, it will be overwritten by the +version that was retrieved initially. + +Clients in the ``blah-servers`` group will be probed for +``/etc/blah.conf``; if it changes on a client, those changes will be +written into the Bcfg2 specification. If the file is deleted from a +client, it will be rewritten from Bcfg2. + +``bar.example.com`` will be probed for ``/var/lib/bar.gz``, which +contains non-ASCII characters and so needs to use base64 encoding when +transferring the file. + +The paths probed by FileProbes must also be included as Path entries +in your bundles in order to be handled properly by Cfg. Permissions +are handled as usual, with ``info.xml`` files in Cfg. diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 829d00f03..4f6528e0e 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -164,11 +164,11 @@ Initialize the database. .RS Load statistics data. .RE -.B purge +.B purge [--client [n]] [--days [n]] [--expired] .RS -Purge records. +Purge historic and expired data. .RE -.B scrub [--client [n]] [--days [n]] [--expired] +.B scrub .RS Scrub the database for duplicate reasons and orphaned entries. .RE diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd new file mode 100644 index 000000000..112047836 --- /dev/null +++ b/schemas/fileprobes.xsd @@ -0,0 +1,34 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + <xsd:annotation> + <xsd:documentation> + FileProbes plugin config schema for bcfg2 + Chris St. Pierre + </xsd:documentation> + </xsd:annotation> + + <xsd:complexType name="GroupType"> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="FileProbe" type="FileProbeType"/> + <xsd:element name="Group" type="GroupType"/> + <xsd:element name="Client" type="GroupType"/> + </xsd:choice> + <xsd:attribute type="xsd:string" name="name" use="required"/> + <xsd:attribute type="xsd:string" name="negate"/> + </xsd:complexType> + + <xsd:complexType name="FileProbeType"> + <xsd:attribute type="xsd:string" name="name" use="required"/> + <xsd:attribute type="xsd:string" name="base64"/> + <xsd:attribute type="xsd:string" name="update"/> + </xsd:complexType> + + <xsd:element name="FileProbes"> + <xsd:complexType> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="FileProbe" type="FileProbeType"/> + <xsd:element name="Group" type="GroupType"/> + <xsd:element name="Client" type="GroupType"/> + </xsd:choice> + </xsd:complexType> + </xsd:element> +</xsd:schema> @@ -4,7 +4,9 @@ from distutils.core import setup from distutils.core import Command from fnmatch import fnmatch from glob import glob +import os import os.path +import sys class BuildDTDDoc (Command): """Build DTD documentation""" @@ -113,6 +115,10 @@ try: except ImportError: pass +py3lib = 'src/lib/Bcfg2Py3Incompat.py' +if sys.hexversion < 0x03000000 and os.path.exists(py3lib): + os.remove(py3lib) + setup(cmdclass=cmdclass, name="Bcfg2", version="1.2.0pre2", @@ -133,7 +139,7 @@ setup(cmdclass=cmdclass, "Bcfg2.Server.Reports.reports.templatetags", "Bcfg2.Server.Snapshots", ], - package_dir = {'Bcfg2':'src/lib'}, + package_dir = {'Bcfg2': 'src/lib'}, package_data = {'Bcfg2.Server.Reports.reports':['fixtures/*.xml', 'templates/*.html', 'templates/*/*.html', 'templates/*/*.inc' ] }, diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 7725ad748..c88e54e95 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -13,12 +13,13 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if self.HandlesFile(infoxml_fname): if (hasattr(entryset, "infoxml") and entryset.infoxml is not None): - self.check_infoxml(entryset.infoxml.pnode.data) + self.check_infoxml(infoxml_fname, + entryset.infoxml.pnode.data) else: self.LintError("no-infoxml", "No info.xml found for %s" % filename) - def check_infoxml(self, xdata): + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -28,8 +29,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), infoxml_fname, - self.RenderXML(info))) + (",".join(missing), fname, self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -39,5 +39,5 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % - (infoxml_fname, self.RenderXML(info))) + (fname, self.RenderXML(info))) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c87c55ee9..834608378 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -24,7 +24,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "%s/Decisions/*.xml":"%s/decisions.xsd", "%s/Packages/config.xml":"%s/packages.xsd", "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"} + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", + "%s/FileProbes/config.xml":"%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 41cf6c9c1..998bacc19 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -187,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if name.endswith(".genshi"): logger.error("Cfg: Unable to pull data for genshi types") - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py new file mode 100644 index 000000000..4df5a7f4a --- /dev/null +++ b/src/lib/Server/Plugins/FileProbes.py @@ -0,0 +1,177 @@ +""" This module allows you to probe a client for a file, which is then +added to the specification. On subsequent runs, the file will be +replaced on the client if it is missing; if it has changed on the +client, it can either be updated in the specification or replaced on +the client """ +__revision__ = '$Revision: 1465 $' + +import os +import errno +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin + +probecode = """#!/usr/bin/env python + +import os +import pwd +import grp +import binascii +import lxml.etree + +path = "%s" + +if not os.path.exists(path): + print "%%s does not exist" %% path + raise SystemExit(1) + +stat = os.stat(path) +data = lxml.etree.Element("ProbedFileData", + name=path, + owner=pwd.getpwuid(stat[4])[0], + group=grp.getgrgid(stat[5])[0], + perms=oct(stat[0] & 07777)) +data.text = binascii.b2a_base64(open(path).read()) +print lxml.etree.tostring(data) +""" + +class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + """ Config file handler for FileProbes """ + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + + +class FileProbes(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Probing): + """ This module allows you to probe a client for a file, which is then + added to the specification. On subsequent runs, the file will be + replaced on the client if it is missing; if it has changed on the + client, it can either be updated in the specification or replaced on + the client """ + + name = 'FileProbes' + experimental = True + __version__ = '$Id$' + __author__ = 'stpierreca@ornl.gov' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Probing.__init__(self) + self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), + core.fam) + self.entries = dict() + self.probes = dict() + + def GetProbes(self, metadata): + """Return a set of probes for execution on client.""" + if metadata.hostname not in self.probes: + cfg = self.core.plugins['Cfg'] + self.entries[metadata.hostname] = dict() + self.probes[metadata.hostname] = [] + for entry in self.config.Match(metadata): + path = entry.get("name") + # do not probe for files that are already in Cfg and + # for which update is false; we can't possibly do + # anything with the data we get from such a probe + try: + if (cfg.entries[path].get_pertinent_entries(metadata) and + entry.get('update', 'false').lower() == "false"): + continue + except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): + pass + self.entries[metadata.hostname][path] = entry + probe = lxml.etree.Element('probe', name=path, + source=self.name, + interpreter="/usr/bin/env python") + probe.text = probecode % path + self.probes[metadata.hostname].append(probe) + self.logger.debug("Adding file probe for %s to %s" % + (path, metadata.hostname)) + return self.probes[metadata.hostname] + + def ReceiveData(self, metadata, datalist): + """Receive data from probe.""" + self.logger.debug("Receiving file probe data from %s" % + metadata.hostname) + + for data in datalist: + if data.text is None: + self.logger.error("Got null response to %s file probe from %s" % + (data.get('name'), metadata.hostname)) + else: + self.logger.debug("%s:fileprobe:%s:%s" % + (metadata.hostname, + data.get("name"), + data.text)) + try: + filedata = lxml.etree.XML(data.text) + self.write_file(filedata, metadata) + except lxml.etree.XMLSyntaxError: + # if we didn't get XML back from the probe, assume + # it's an error message + self.logger.error(data.text) + + def write_file(self, data, metadata): + """Write the probed file data to the bcfg2 specification.""" + filename = data.get("name") + contents = binascii.a2b_base64(data.text) + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) + # we can't use os.path.join() for this because specific + # already has a leading /, which confuses os.path.join() + fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) + if filename not in cfg.entries.keys(): + self.logger.info("Writing new probed file %s" % fileloc) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError, err: + if err.errno == errno.EEXIST: + pass + else: + raise + open(fileloc, 'wb').write(contents) + + infoxml = os.path.join("%s%s" % (cfg.data, filename), + "info.xml") + self.write_infoxml(infoxml, entry, data) + else: + try: + cfgentry = \ + cfg.entries[filename].get_pertinent_entries(metadata)[0] + except Bcfg2.Server.Plugin.PluginExecutionError: + self.logger.info("Writing new probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + return + + if cfgentry.data == contents: + self.logger.debug("Existing %s contents match probed contents" % + filename) + elif (entry.get('update', 'false').lower() == "true"): + self.logger.info("Writing updated probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + else: + self.logger.info("Skipping updated probed file %s" % fileloc) + + def write_infoxml(self, infoxml, entry, data): + """ write an info.xml for the file """ + self.logger.info("Writing info.xml at %s for %s" % + (infoxml, data.get("name"))) + info = \ + lxml.etree.Element("Info", + owner=data.get("owner", + Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", + Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", + Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", + Bcfg2.Options.ENCODING.value)) + + root = lxml.etree.Element("FileInfo") + root.append(info) + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 35f555294..b8a8e6b7e 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: @@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: |