added bcfg2-lint MergeFiles plugin to suggest config files and probes

  that are very similar and could be merged

added text wrapping to bcfg2-lint error handling

5 files changed, 121 insertions, 11 deletions

diff --git a/examples/bcfg2-lint.conf b/examples/bcfg2-lint.conf
index abf969496..9c0d2c72a 100644
--- a/examples/bcfg2-lint.conf
+++ b/examples/bcfg2-lint.conf
@@ -1,5 +1,5 @@
 [lint]
-plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate
+plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate,MergeFiles
 
 [errors]
 no-infoxml=error
@@ -23,3 +23,6 @@ probe_comments = Maintainer,Purpose,Groups,Other Output
 
 [Validate]
 schema=/usr/share/bcfg2/schema
+
+[MergeFiles]
+threshold=85
diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8
index b1fa9244b..f2d4f9e88 100644
--- a/man/bcfg2-lint.8
+++ b/man/bcfg2-lint.8
@@ -120,6 +120,11 @@ exists for all Cfg files, and that paranoid mode be enabled for all
 files.
 
 .TP
+.BR MergeFiles
+Suggest that similar probes and config files be merged into single
+probes or TGenshi templates.
+
+.TP
 .BR Pkgmgr
 Check for duplicate packages specified in Pkgmgr.
 
diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5
index 49a32bb22..10a812874 100644
--- a/man/bcfg2-lint.conf.5
+++ b/man/bcfg2-lint.conf.5
@@ -152,6 +152,14 @@ A comma-delimited list of attributes to require on
 tags.  Default is "owner,group,perms".
 
 .TP
+.BR MergeFiles
+
+\(bu
+.B threshold
+The threshold at which MergeFiles will suggest merging config files
+and probes.  Default is 75% similar.
+
+.TP
 .BR Validate
 
 \(bu
diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py
new file mode 100644
index 000000000..1e177acff
--- /dev/null
+++ b/src/lib/Server/Lint/MergeFiles.py
@@ -0,0 +1,71 @@
+import os
+from copy import deepcopy
+from difflib import SequenceMatcher
+import Bcfg2.Options
+import Bcfg2.Server.Lint
+
+class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
+    """ find Probes or Cfg files with multiple similar files that
+    might be merged into one """
+
+    @Bcfg2.Server.Lint.returnErrors
+    def Run(self):
+        if 'Cfg' in self.core.plugins:
+            self.check_cfg()
+        if 'Probes' in self.core.plugins:
+            self.check_probes()
+
+    def check_cfg(self):
+        for filename, entryset in self.core.plugins['Cfg'].entries.items():
+            for mset in self.get_similar(entryset.entries):
+                self.LintError("merge-cfg",
+                               "The following files are similar: %s. "
+                               "Consider merging them into a single Genshi "
+                               "template." %
+                               ", ".join([os.path.join(filename, p)
+                                          for p in mset]))
+
+    def check_probes(self):
+        probes = self.core.plugins['Probes'].probes.entries
+        for mset in self.get_similar(probes):
+                self.LintError("merge-cfg",
+                               "The following probes are similar: %s. "
+                               "Consider merging them into a single probe." %
+                               ", ".join([p for p in mset]))
+
+    def get_similar(self, entries):
+        if "threshold" in self.config:
+            # accept threshold either as a percent (e.g., "threshold=75") or
+            # as a ratio (e.g., "threshold=.75")
+            threshold = float(self.config['threshold'])
+            if threshold > 1:
+                threshold /= 100
+        else:
+            threshold = 0.75
+        rv = []
+        elist = entries.items()
+        while elist:
+            result = self._find_similar(elist.pop(0), deepcopy(elist),
+                                        threshold)
+            if len(result) > 1:
+                elist = [(fname, fdata)
+                         for fname, fdata in elist
+                         if fname not in result]
+                rv.append(result)
+        return rv
+
+    def _find_similar(self, ftuple, others, threshold):
+        fname, fdata = ftuple
+        rv = [fname]
+        while others:
+            cname, cdata = others.pop(0)
+            sm = SequenceMatcher(None, fdata.data, cdata.data)
+            # perform progressively more expensive comparisons
+            if (sm.real_quick_ratio() > threshold and
+                sm.quick_ratio() > threshold and
+                sm.ratio() > threshold):
+                rv.extend(self._find_similar((cname, cdata), deepcopy(others),
+                                             threshold))
+        return rv
+
+        
diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py
index 3b89d1f9e..013cbf2ba 100644
--- a/src/lib/Server/Lint/__init__.py
+++ b/src/lib/Server/Lint/__init__.py
@@ -4,6 +4,7 @@ __all__ = ['Bundles',
            'Comments',
            'Duplicates',
            'InfoXML',
+           'MergeFiles',
            'Pkgmgr',
            'RequiredAttrs',
            'Validate']
@@ -11,6 +12,7 @@ __all__ = ['Bundles',
 import logging
 import os.path
 from copy import copy
+import textwrap
 import lxml.etree
 import Bcfg2.Logger
 
@@ -84,7 +86,9 @@ class ErrorHandler (object):
                "properties-schema-not-found":"warning",
                "xml-failed-to-parse":"error",
                "xml-failed-to-read":"error",
-               "xml-failed-to-verify":"error",}
+               "xml-failed-to-verify":"error",
+               "merge-cfg":"warning",
+               "merge-probes":"warning",}
 
     def __init__(self, config=None):
         self.errors = 0
@@ -92,6 +96,9 @@ class ErrorHandler (object):
 
         self.logger = logging.getLogger('bcfg2-lint')
 
+        self._wrapper = textwrap.TextWrapper(initial_indent = "  ",
+                                             subsequent_indent = "  ")
+
         self._handlers = {}
         if config is not None:
             for err, action in config.items():
@@ -116,26 +123,42 @@ class ErrorHandler (object):
             self._handlers[err](msg)
             self.logger.debug("    (%s)" % err)
         else:
-            self.logger.info("Unknown error %s" % err)
+            # assume that it's an error, but complain
+            self.error(msg)
+            self.logger.warning("Unknown error %s" % err)
 
     def error(self, msg):
         """ log an error condition """
         self.errors += 1
-        lines = msg.splitlines()
-        self.logger.error("ERROR: %s" % lines.pop())
-        [self.logger.error("  %s" % l) for l in lines]
+        self._log(msg, self.logger.error, prefix="ERROR: ")
 
     def warn(self, msg):
         """ log a warning condition """
         self.warnings += 1
-        lines = msg.splitlines()
-        self.logger.warning("WARNING: %s" % lines.pop())
-        [self.logger.warning("  %s" % l) for l in lines]
+        self._log(msg, self.logger.warning, prefix="WARNING: ")
 
     def debug(self, msg):
         """ log a silent/debug condition """
-        lines = msg.splitlines()
-        [self.logger.debug("%s" % l) for l in lines]
+        self._log(msg, self.logger.debug)
+
+    def _log(self, msg, logfunc, prefix=""):
+        # a message may itself consist of multiple lines.  wrap() will
+        # elide them all into a single paragraph, which we don't want.
+        # so we split the message into its paragraphs and wrap each
+        # paragraph individually.  this means, unfortunately, that we
+        # lose textwrap's built-in initial indent functionality,
+        # because we want to only treat the very first line of the
+        # first paragraph specially.  so we do some silliness.
+        rawlines = msg.splitlines()
+        firstline = True
+        for rawline in rawlines:
+            lines = self._wrapper.wrap(rawline)
+            for line in lines:
+                if firstline:
+                    logfunc("%s%s" % (prefix, line.lstrip()))
+                    firstline = False
+                else:
+                    logfunc(line)
 
 
 class ServerlessPlugin (Plugin):