Merge branch 'master' of git.mcs.anl.gov:bcfg2

9 files changed, 291 insertions, 15 deletions

diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt
index 33ec6f444..b9f698a0f 100644
--- a/doc/server/plugins/probes/index.txt
+++ b/doc/server/plugins/probes/index.txt
@@ -143,3 +143,60 @@ Other examples
    :hidden:
 
    ohai
+
+.. _server-plugins-probes-fileprobes:
+
+FileProbes
+==========
+
+The FileProbes plugin allows you to probe a client for a file,
+which is then added to the :ref:`server-plugins-generators-cfg`
+specification.  If the file changes on the client, FileProbes can
+either update it in the specification or allow Cfg to replace it.
+
+FileProbes will not probe a file if there's already a file in Cfg that
+will apply to the client.  So if, for instance, you have a generic
+file in ``Cfg/etc/foo.conf/foo.conf`` that applies to all hosts,
+FileProbes will not retrieve ``/etc/foo.conf`` from the client (unless
+``update`` is enabled; see Configuration_ below).
+
+When a new config file is first probed, an ``info.xml`` file is also
+written to enforce the permissions from that client.  Subsequent
+probes from other clients will not modify or overwrite the data in
+``info.xml``.  (This ensures that any manual changes you make to
+``info.xml`` for that file are not circumvented.)
+
+Configuration
+-------------
+
+FileProbes is configured in ``FileProbes/config.xml``, which might
+look something like:
+
+.. code-block:: xml
+
+    <FileProbes>
+      <FileProbe name="/etc/foo.conf"/>
+      <Group name="blah-servers">
+        <FileProbe name="/etc/blah.conf" update="true"
+      </Group>
+      <Client name="bar.example.com">
+        <FileProbe name="/var/lib/bar.gz" base64="true"/>
+      </Client>
+    </FileProbes>
+
+This will result in ``/etc/foo.conf`` being retrieved from all
+clients; if it changes on a client, it will be overwritten by the
+version that was retrieved initially.
+
+Clients in the ``blah-servers`` group will be probed for
+``/etc/blah.conf``; if it changes on a client, those changes will be
+written into the Bcfg2 specification.  If the file is deleted from a
+client, it will be rewritten from Bcfg2.
+
+``bar.example.com`` will be probed for ``/var/lib/bar.gz``, which
+contains non-ASCII characters and so needs to use base64 encoding when
+transferring the file.
+
+The paths probed by FileProbes must also be included as Path entries
+in your bundles in order to be handled properly by Cfg.  Permissions
+are handled as usual, with ``info.xml`` files in Cfg.
diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8
index 829d00f03..4f6528e0e 100644
--- a/man/bcfg2-admin.8
+++ b/man/bcfg2-admin.8
@@ -164,11 +164,11 @@ Initialize the database.
 .RS
 Load statistics data.
 .RE
-.B purge
+.B purge [--client [n]] [--days [n]] [--expired]
 .RS
-Purge records.
+Purge historic and expired data.
 .RE
-.B scrub [--client [n]] [--days [n]] [--expired]
+.B scrub
 .RS
 Scrub the database for duplicate reasons and orphaned entries.
 .RE
diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd
new file mode 100644
index 000000000..112047836
--- /dev/null
+++ b/schemas/fileprobes.xsd
@@ -0,0 +1,34 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en">  
+  <xsd:annotation>
+    <xsd:documentation>
+      FileProbes plugin config schema for bcfg2
+      Chris St. Pierre
+    </xsd:documentation>
+  </xsd:annotation>
+
+  <xsd:complexType name="GroupType">
+    <xsd:choice minOccurs="1" maxOccurs="unbounded">
+      <xsd:element name="FileProbe" type="FileProbeType"/>
+      <xsd:element name="Group" type="GroupType"/>
+      <xsd:element name="Client" type="GroupType"/>
+    </xsd:choice>
+    <xsd:attribute type="xsd:string" name="name" use="required"/>
+    <xsd:attribute type="xsd:string" name="negate"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="FileProbeType">
+    <xsd:attribute type="xsd:string" name="name" use="required"/>
+    <xsd:attribute type="xsd:string" name="base64"/>
+    <xsd:attribute type="xsd:string" name="update"/>
+  </xsd:complexType>
+
+  <xsd:element name="FileProbes">
+    <xsd:complexType>
+      <xsd:choice minOccurs="1" maxOccurs="unbounded">
+        <xsd:element name="FileProbe" type="FileProbeType"/>
+        <xsd:element name="Group" type="GroupType"/>
+        <xsd:element name="Client" type="GroupType"/>
+      </xsd:choice>
+    </xsd:complexType>
+  </xsd:element>
+</xsd:schema>
diff --git a/setup.py b/setup.py
index 52ec93d10..18590cc34 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,9 @@ from distutils.core import setup
 from distutils.core import Command
 from fnmatch import fnmatch
 from glob import glob
+import os
 import os.path
+import sys
 
 class BuildDTDDoc (Command):
     """Build DTD documentation"""
@@ -113,6 +115,10 @@ try:
 except ImportError:
     pass
 
+py3lib = 'src/lib/Bcfg2Py3Incompat.py'
+if sys.hexversion < 0x03000000 and os.path.exists(py3lib):
+    os.remove(py3lib)
+
 setup(cmdclass=cmdclass,
       name="Bcfg2",
       version="1.2.0pre2",
@@ -133,7 +139,7 @@ setup(cmdclass=cmdclass,
                 "Bcfg2.Server.Reports.reports.templatetags",
                 "Bcfg2.Server.Snapshots",
                 ],
-      package_dir = {'Bcfg2':'src/lib'},
+      package_dir = {'Bcfg2': 'src/lib'},
       package_data = {'Bcfg2.Server.Reports.reports':['fixtures/*.xml',
                 'templates/*.html', 'templates/*/*.html',
                 'templates/*/*.inc' ] },
diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py
index 7725ad748..c88e54e95 100644
--- a/src/lib/Server/Lint/InfoXML.py
+++ b/src/lib/Server/Lint/InfoXML.py
@@ -13,12 +13,13 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
                 if self.HandlesFile(infoxml_fname):
                     if (hasattr(entryset, "infoxml") and
                         entryset.infoxml is not None):
-                        self.check_infoxml(entryset.infoxml.pnode.data)
+                        self.check_infoxml(infoxml_fname,
+                                           entryset.infoxml.pnode.data)
                     else:
                         self.LintError("no-infoxml",
                                        "No info.xml found for %s" % filename)
 
-    def check_infoxml(self, xdata):
+    def check_infoxml(self, fname, xdata):
         for info in xdata.getroottree().findall("//Info"):
             required = []
             if "required_attrs" in self.config:
@@ -28,8 +29,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
             if missing:
                 self.LintError("required-infoxml-attrs-missing",
                                "Required attribute(s) %s not found in %s:%s" %
-                               (",".join(missing), infoxml_fname,
-                                self.RenderXML(info)))
+                               (",".join(missing), fname, self.RenderXML(info)))
 
             if ((Bcfg2.Options.MDATA_PARANOID.value and
                  info.get("paranoid") is not None and
@@ -39,5 +39,5 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
                   info.get("paranoid").lower() != "true"))):
                 self.LintError("paranoid-false",
                                "Paranoid must be true in %s:%s" %
-                               (infoxml_fname, self.RenderXML(info)))
+                               (fname, self.RenderXML(info)))
 
diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py
index c87c55ee9..834608378 100644
--- a/src/lib/Server/Lint/Validate.py
+++ b/src/lib/Server/Lint/Validate.py
@@ -24,7 +24,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
                          "%s/Decisions/*.xml":"%s/decisions.xsd",
                          "%s/Packages/config.xml":"%s/packages.xsd",
                          "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd",
-                         "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"}
+                         "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd",
+                         "%s/FileProbes/config.xml":"%s/fileprobes.xsd",
+                         }
 
         self.filelists = {}
         self.get_filelists()
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 41cf6c9c1..998bacc19 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -187,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
             name = self.build_filename(specific)
             if name.endswith(".genshi"):
                 logger.error("Cfg: Unable to pull data for genshi types")
-                raise PluginExecutionError
+                raise Bcfg2.Server.Plugin.PluginExecutionError
             open(name, 'w').write(new_entry['text'])
             if log:
                 logger.info("Wrote file %s" % name)
diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py
new file mode 100644
index 000000000..4df5a7f4a
--- /dev/null
+++ b/src/lib/Server/Plugins/FileProbes.py
@@ -0,0 +1,177 @@
+""" This module allows you to probe a client for a file, which is then
+added to the specification.  On subsequent runs, the file will be
+replaced on the client if it is missing; if it has changed on the
+client, it can either be updated in the specification or replaced on
+the client """
+__revision__ = '$Revision: 1465 $'
+
+import os
+import errno
+import binascii
+import lxml.etree
+import Bcfg2.Options
+import Bcfg2.Server.Plugin
+
+probecode = """#!/usr/bin/env python
+
+import os
+import pwd
+import grp
+import binascii
+import lxml.etree
+
+path = "%s"
+
+if not os.path.exists(path):
+    print "%%s does not exist" %% path
+    raise SystemExit(1)
+
+stat = os.stat(path)
+data = lxml.etree.Element("ProbedFileData",
+                          name=path,
+                          owner=pwd.getpwuid(stat[4])[0],
+                          group=grp.getgrgid(stat[5])[0],
+                          perms=oct(stat[0] & 07777))
+data.text = binascii.b2a_base64(open(path).read())
+print lxml.etree.tostring(data)
+"""
+
+class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked,
+                       Bcfg2.Server.Plugin.StructFile):
+    """ Config file handler for FileProbes """
+    def __init__(self, filename, fam):
+        Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
+        Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
+
+
+class FileProbes(Bcfg2.Server.Plugin.Plugin,
+                 Bcfg2.Server.Plugin.Probing):
+    """ This module allows you to probe a client for a file, which is then
+    added to the specification.  On subsequent runs, the file will be
+    replaced on the client if it is missing; if it has changed on the
+    client, it can either be updated in the specification or replaced on
+    the client """
+
+    name = 'FileProbes'
+    experimental = True
+    __version__ = '$Id$'
+    __author__ = 'stpierreca@ornl.gov'
+
+    def __init__(self, core, datastore):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Probing.__init__(self)
+        self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'),
+                                       core.fam)
+        self.entries = dict()
+        self.probes = dict()
+
+    def GetProbes(self, metadata):
+        """Return a set of probes for execution on client."""
+        if metadata.hostname not in self.probes:
+            cfg = self.core.plugins['Cfg']
+            self.entries[metadata.hostname] = dict()
+            self.probes[metadata.hostname] = []
+            for entry in self.config.Match(metadata):
+                path = entry.get("name")
+                # do not probe for files that are already in Cfg and
+                # for which update is false; we can't possibly do
+                # anything with the data we get from such a probe
+                try:
+                    if (cfg.entries[path].get_pertinent_entries(metadata) and
+                        entry.get('update', 'false').lower() == "false"):
+                        continue
+                except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError):
+                    pass
+                self.entries[metadata.hostname][path] = entry
+                probe = lxml.etree.Element('probe', name=path,
+                                           source=self.name,
+                                           interpreter="/usr/bin/env python")
+                probe.text = probecode % path
+                self.probes[metadata.hostname].append(probe)
+                self.logger.debug("Adding file probe for %s to %s" %
+                                  (path, metadata.hostname))
+        return self.probes[metadata.hostname]
+
+    def ReceiveData(self, metadata, datalist):
+        """Receive data from probe."""
+        self.logger.debug("Receiving file probe data from %s" %
+                          metadata.hostname)
+
+        for data in datalist:
+            if data.text is None:
+                self.logger.error("Got null response to %s file probe from %s" %
+                                  (data.get('name'), metadata.hostname))
+            else:
+                self.logger.debug("%s:fileprobe:%s:%s" %
+                                  (metadata.hostname,
+                                   data.get("name"),
+                                   data.text))
+                try:
+                    filedata = lxml.etree.XML(data.text)
+                    self.write_file(filedata, metadata)
+                except lxml.etree.XMLSyntaxError:
+                    # if we didn't get XML back from the probe, assume
+                    # it's an error message
+                    self.logger.error(data.text)
+
+    def write_file(self, data, metadata):
+        """Write the probed file data to the bcfg2 specification."""
+        filename = data.get("name")
+        contents = binascii.a2b_base64(data.text)
+        entry = self.entries[metadata.hostname][filename]
+        cfg = self.core.plugins['Cfg']
+        specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname)
+        # we can't use os.path.join() for this because specific
+        # already has a leading /, which confuses os.path.join()
+        fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific))
+        if filename not in cfg.entries.keys():
+            self.logger.info("Writing new probed file %s" % fileloc)    
+            try:
+                os.makedirs(os.path.dirname(fileloc))
+            except OSError, err:
+                if err.errno == errno.EEXIST:
+                    pass
+                else:
+                    raise
+            open(fileloc, 'wb').write(contents)
+
+            infoxml = os.path.join("%s%s" % (cfg.data, filename),
+                                   "info.xml")
+            self.write_infoxml(infoxml, entry, data)
+        else:
+            try:
+                cfgentry = \
+                    cfg.entries[filename].get_pertinent_entries(metadata)[0]
+            except Bcfg2.Server.Plugin.PluginExecutionError:
+                self.logger.info("Writing new probed file %s" % fileloc)
+                open(fileloc, 'wb').write(contents)
+                return
+            
+            if cfgentry.data == contents:
+                self.logger.debug("Existing %s contents match probed contents" %
+                                  filename)
+            elif (entry.get('update', 'false').lower() == "true"):
+                self.logger.info("Writing updated probed file %s" % fileloc)
+                open(fileloc, 'wb').write(contents)
+            else:
+                self.logger.info("Skipping updated probed file %s" % fileloc)
+                        
+    def write_infoxml(self, infoxml, entry, data):
+        """ write an info.xml for the file """
+        self.logger.info("Writing info.xml at %s for %s" %
+                         (infoxml, data.get("name")))
+        info = \
+            lxml.etree.Element("Info",
+                               owner=data.get("owner",
+                                              Bcfg2.Options.MDATA_OWNER.value),
+                               group=data.get("group",
+                                              Bcfg2.Options.MDATA_GROUP.value),
+                               perms=data.get("perms",
+                                              Bcfg2.Options.MDATA_PERMS.value),
+                               encoding=entry.get("encoding",
+                                                  Bcfg2.Options.ENCODING.value))
+        
+        root = lxml.etree.Element("FileInfo")
+        root.append(info)
+        open(infoxml, "w").write(lxml.etree.tostring(root,
+                                                     pretty_print=True))
diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py
index 35f555294..b8a8e6b7e 100644
--- a/src/lib/Server/Plugins/Svn2.py
+++ b/src/lib/Server/Plugins/Svn2.py
@@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin,
         except Exception, err:
             # try to be smart about the error we got back
             details = None
-            if "callback_ssl_server_trust_prompt" in err.message:
+            if "callback_ssl_server_trust_prompt" in str(err):
                 details = "SVN server certificate is not trusted"
-            elif "callback_get_login" in err.message:
+            elif "callback_get_login" in str(err):
                 details = "SVN credentials not cached"
 
             if details is None:
@@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin,
         except Exception, err:
             # try to be smart about the error we got back
             details = None
-            if "callback_ssl_server_trust_prompt" in err.message:
+            if "callback_ssl_server_trust_prompt" in str(err):
                 details = "SVN server certificate is not trusted"
-            elif "callback_get_login" in err.message:
+            elif "callback_get_login" in str(err):
                 details = "SVN credentials not cached"
 
             if details is None: