Merge branch 'maint'

Conflicts:
	doc/appendix/guides/centos.txt
	doc/server/plugins/grouping/metadata.txt
	setup.py
	src/lib/Bcfg2/Client/Frame.py
	src/lib/Bcfg2/Client/Proxy.py
	src/lib/Bcfg2/Server/Lint/Genshi.py
	src/lib/Bcfg2/Server/Lint/Validate.py
	src/lib/Bcfg2/Server/Plugins/Bundler.py
	src/lib/Bcfg2/Server/Plugins/SSHbase.py
	src/sbin/bcfg2-lint

7 files changed, 78 insertions, 53 deletions

diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py
index 3cd3786f9..93314f1e3 100755
--- a/tools/bcfg2-profile-templates.py
+++ b/tools/bcfg2-profile-templates.py
@@ -1,25 +1,35 @@
 #!/usr/bin/python -Ott
+# -*- coding: utf-8 -*-
 """ Benchmark template rendering times """
 
-import os
 import sys
 import time
+import math
 import logging
 import operator
 import Bcfg2.Logger
+import Bcfg2.Options
 import Bcfg2.Server.Core
 
-LOGGER = None
+
+def stdev(nums):
+    mean = float(sum(nums)) / len(nums)
+    return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums)))
 
 
 def main():
-    optinfo = \
-        dict(client=Bcfg2.Options.Option("Benchmark templates for one client",
-                                         cmd="--client",
-                                         odesc="<client>",
-                                         long_arg=True,
-                                         default=None),
-             )
+    optinfo = dict(
+        client=Bcfg2.Options.Option("Benchmark templates for one client",
+                                    cmd="--client",
+                                    odesc="<client>",
+                                    long_arg=True,
+                                    default=None),
+        runs=Bcfg2.Options.Option("Number of rendering passes per template",
+                                  cmd="--runs",
+                                  odesc="<runs>",
+                                  long_arg=True,
+                                  default=5,
+                                  cook=int))
     optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS)
     optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS)
     setup = Bcfg2.Options.OptionParser(optinfo)
@@ -40,12 +50,11 @@ def main():
 
     core = Bcfg2.Server.Core.BaseCore(setup)
     logger.info("Bcfg2 server core loaded")
+    core.load_plugins()
+    logger.debug("Plugins loaded")
     core.fam.handle_events_in_interval(0.1)
     logger.debug("Repository events processed")
 
-    # how many times to render each template for each client
-    runs = 5
-
     if setup['args']:
         templates = setup['args']
     else:
@@ -57,41 +66,57 @@ def main():
         clients = [core.build_metadata(setup['client'])]
 
     times = dict()
+    client_count = 0
     for metadata in clients:
-        for struct in core.GetStructures(metadata):
-            logger.info("Rendering templates from structure %s:%s" %
-                        (struct.tag, struct.get("name")))
-            for entry in struct.xpath("//Path"):
-                path = entry.get("name")
-                logger.info("Rendering %s..." % path)
-                times[path] = dict()
-                avg = 0.0
-                for i in range(runs):
+        client_count += 1
+        logger.info("Rendering templates for client %s (%s/%s)" %
+                    (metadata.hostname, client_count, len(clients)))
+        structs = core.GetStructures(metadata)
+        struct_count = 0
+        for struct in structs:
+            struct_count += 1
+            logger.info("Rendering templates from structure %s:%s (%s/%s)" %
+                        (struct.tag, struct.get("name"), struct_count,
+                         len(structs)))
+            entries = struct.xpath("//Path")
+            entry_count = 0
+            for entry in entries:
+                entry_count += 1
+                if templates and entry.get("name") not in templates:
+                    continue
+                logger.info("Rendering Path:%s (%s/%s)..." %
+                            (entry.get("name"), entry_count, len(entries)))
+                ptimes = times.setdefault(entry.get("name"), [])
+                for i in range(setup['runs']):
                     start = time.time()
                     try:
                         core.Bind(entry, metadata)
-                        avg += (time.time() - start) / runs
+                        ptimes.append(time.time() - start)
                     except:
                         break
-                if avg:
-                    logger.debug("   %s: %.02f sec" % (metadata.hostname, avg))
-                    times[path][metadata.hostname] = avg
+                if ptimes:
+                    avg = sum(ptimes) / len(ptimes)
+                    if avg:
+                        logger.debug("   %s: %.02f sec" %
+                                     (metadata.hostname, avg))
 
     # print out per-file results
     tmpltimes = []
-    for tmpl, clients in times.items():
+    for tmpl, ptimes in times.items():
         try:
-            avg = sum(clients.values()) / len(clients)
+            mean = float(sum(ptimes)) / len(ptimes)
         except ZeroDivisionError:
             continue
-        if avg > 0.01 or templates:
-            tmpltimes.append((tmpl, avg))
-    print("%-50s %s" % ("Template", "Average Render Time"))
-    for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))):
-        print("%-50s %.02f" % (tmpl, avg))
-
-    # TODO: complain about templates that on average were quick but
-    # for which some clients were slow
+        ptimes.sort()
+        median = ptimes[len(ptimes) / 2]
+        std = stdev(ptimes)
+        if mean > 0.01 or median > 0.01 or std > 1 or templates:
+            tmpltimes.append((tmpl, mean, median, std))
+    print("%-50s %-9s  %-11s  %6s" %
+          ("Template", "Mean Time", "Median Time", "σ"))
+    for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))):
+        print("%-50s %9.02f  %11.02f  %6.02f" % info)
+    core.shutdown()
 
 
 if __name__ == "__main__":
diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py
index edb5a7101..8c164e52e 100755
--- a/tools/bcfg2_local.py
+++ b/tools/bcfg2_local.py
@@ -19,7 +19,8 @@ class LocalCore(BaseCore):
         setup['logging'] = None
         Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup)
         setup['syslog'], setup['logging'] = saved
-        self.fam.handle_events_in_interval(4)
+        self.load_plugins()
+        self.fam.handle_events_in_interval(0.1)
 
     def _daemonize(self):
         return True
diff --git a/tools/export.py b/tools/export.py
index 716c831d9..0f4724e6b 100755
--- a/tools/export.py
+++ b/tools/export.py
@@ -227,9 +227,6 @@ E.G. 1.2.0pre1 is a valid version.
                      'VERSION="%s"\n' % version,
                      startswith=True,
                      dryrun=options.dryrun)
-    # set new version in setup.py
-    find_and_replace('setup.py', 'version=', '      version="%s",\n' % version,
-                     dryrun=options.dryrun)
     # set new version in Bcfg2/version.py
     find_and_replace('src/lib/Bcfg2/version.py',
                      '__version__ =',
diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py
index a4abca42d..c45e54f1a 100755
--- a/tools/posixusers_baseline.py
+++ b/tools/posixusers_baseline.py
@@ -61,8 +61,8 @@ def main():
         if entry.tag == 'POSIXUser':
             entry.set("group", grp.getgrgid(data[3])[0])
             for group in users.user_supplementary_groups(entry):
-                memberof = lxml.etree.SubElement(entry, "MemberOf")
-                memberof.text = group[0]
+                memberof = lxml.etree.SubElement(entry, "MemberOf",
+                                                 group=group[0])
 
         entry.tag = "Bound" + entry.tag
         baseline.append(entry)
diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py
index 06f6e6b98..507a16f43 100755
--- a/tools/selinux_baseline.py
+++ b/tools/selinux_baseline.py
@@ -42,7 +42,10 @@ def main():
         baseline.append(lxml.etree.Comment("%s entries" % etype))
         extra = handler.FindExtra()
         for entry in extra:
-            entry.tag = "Bound%s" % etype
+            if etype != "SEModule":
+                entry.tag = "Bound%s" % etype
+            else:
+                entry.tag = "%s" % etype
         baseline.extend(extra)
 
     print(lxml.etree.tostring(baseline, pretty_print=True))
diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py
index cbd2a6099..07def2ac8 100755
--- a/tools/upgrade/1.3/migrate_dbstats.py
+++ b/tools/upgrade/1.3/migrate_dbstats.py
@@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
 
 _our_backend = None
 
+
 def _quote(value):
     """
     Quote a string to use as a table name or column
@@ -44,12 +45,12 @@ def _migrate_perms():
     fperms = {}
 
     logger.info("Creating FilePerms objects")
-    for data in ( ('owner', 'group', 'perms'), 
+    for data in (('owner', 'group', 'perms'),
             ('current_owner', 'current_group', 'current_perms')):
         for grp in legacy_models.Reason.objects.values_list(*data).distinct():
             if grp in fperms:
                 continue
-            fp =  new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2])
+            fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2])
             fp.save()
             fperms[grp] = fp
 
@@ -60,7 +61,7 @@ def _migrate_perms():
 def _migrate_transaction(inter, entries, fperms):
     """helper"""
 
-    logger.debug("Migrating interaction %s for %s" % 
+    logger.debug("Migrating interaction %s for %s" %
         (inter.id, inter.client.name))
 
     newint = new_models.Interaction(id=inter.id,
@@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms):
         elif ent.kind == 'Package':
             act_dict['target_version'] = ei.reason.version
             act_dict['current_version'] = ei.reason.current_version
-            logger.debug("Adding package %s %s" % 
+            logger.debug("Adding package %s %s" %
                 (name, act_dict['target_version']))
             updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict))
         elif ent.kind == 'Path':
@@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms):
 
             act_dict['target_perms'] = fperms[(
                 ei.reason.owner,
-                ei.reason.group, 
+                ei.reason.group,
                 ei.reason.perms
             )]
 
@@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms):
                 act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED
                 act_dict['details'] = ei.reason.unpruned
 
-            
             if ei.reason.is_sensitive:
                 act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE
             elif ei.reason.is_binary:
@@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms):
     for entry_type in updates.keys():
         i = 0
         while(i < len(updates[entry_type])):
-            getattr(newint, entry_type).add(*updates[entry_type][i:i+100])
+            getattr(newint, entry_type).add(*updates[entry_type][i:i + 100])
             i += 100
 
     for perf in inter.performance_items.all():
@@ -220,8 +220,8 @@ def _restructure():
 
     # run any migrations from the previous schema
     try:
-      from Bcfg2.Server.Reports.updatefix import update_database
-      update_database()
+        from Bcfg2.Server.Reports.updatefix import update_database
+        update_database()
     except:
         logger.error("Failed to run legacy schema updates", exc_info=1)
         return False
@@ -295,4 +295,3 @@ if __name__ == '__main__':
     Reports(setup).__call__(['update'])
 
     _restructure()
-
diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py
index a052e75af..b4c5f6589 100755
--- a/tools/yum-listpkgs-xml.py
+++ b/tools/yum-listpkgs-xml.py
@@ -39,5 +39,5 @@ try:
     sys.argv = [sys.argv[0], '-d', '0', 'list']
     yummain.main(sys.argv[1:])
 except KeyboardInterrupt:
-    print("\n\nExiting on user cancel.", file=sys.stderr)
+    sys.stderr.write("\n\nExiting on user cancel.")
     sys.exit(1)