Merge branch 'master' into 1.4.x

Conflicts:
	doc/appendix/contributors.txt
	schemas/bundle.xsd
	src/lib/Bcfg2/Client/Tools/__init__.py
	src/lib/Bcfg2/Server/Encryption.py
	src/lib/Bcfg2/Server/Lint/Genshi.py
	src/lib/Bcfg2/Server/Plugins/Bundler.py
	src/lib/Bcfg2/Server/Plugins/Decisions.py
	src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
	src/sbin/bcfg2-test
	testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Test__init.py
	testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py
	tools/bcfg2-profile-templates.py

4 files changed, 63 insertions, 225 deletions

diff --git a/tools/README b/tools/README
index dc21426b8..ad6d5eac1 100644
--- a/tools/README
+++ b/tools/README
@@ -1,9 +1,5 @@
 This directory contains repository maintenance tools.
 
-accounts2xml.py
-    - Generate an XML description of accounts on a machine from
-      /etc/passwd
-
 basebuilder.py <image directory>
     - builds v2 base.xml from bcfg1 repo
 
diff --git a/tools/accounts2xml.py b/tools/accounts2xml.py
deleted file mode 100755
index 749f3b68c..000000000
--- a/tools/accounts2xml.py
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/usr/bin/env python
-#===============================================================================
-#
-#         FILE: accounts2xml.py
-#
-#        USAGE: ./accounts2xml.py filename node_name
-#
-#  DESCRIPTION: A python script to generate accounts.xml containing only the login
-#               users from the given /etc/passwd file
-#
-#      OPTIONS: ---
-# REQUIREMENTS: ---
-#         BUGS: ---
-#        NOTES: ---
-#       AUTHOR: DongInn Kim (), dikim@cs.indiana.edu
-# ORGANIZATION: Center for Research in Extreme Scale Technologies
-#      VERSION: 1.0
-#      CREATED: 05/13/2012 01:44:43 PM
-#     REVISION: ---
-#===============================================================================
-
-# encoding: utf-8
-
-"""
-accounts2xml.py
-
-This script coverts a csv file to an XML.
-The script takes 1 paramenters
-* filename
-
-e.g., ./accounts2xml.py /etc/passwd
-
-Created by Giovanni Collazo on 2011-02-19.
-Copyright (c) 2011 24veces.com. All rights reserved.
-
-Modified by DongInn Kim on 2012-05-13
-Copyright (c) 2012 Indiana University. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
-"""
-
-import sys
-import csv
-import os
-import re
-import grp
-from xml.dom.minidom import Document
-
-def main(args):
-  
-  try:
-    filename = "./copied_passwd"
-    with file(args[1], 'r') as original: data = original.read()
-    with file(filename, 'w') as modified: modified.write("name:pass:uid:gid:gecos:home:shell\n" + data); modified.close()
-    safe_filename = "Properties"
-  except IndexError:
-    print("ERROR: Please provide a filename.csv as the first argument")
-    sys.exit()
-    
-  node_user = "UnixUser"
-  node_group = "UnixGroup"
-
-  f = csv.reader(open(filename, 'rb'), delimiter=':')
-  
-  doc = Document()
-  root_element = doc.createElement(safe_filename)
-  doc.appendChild(root_element)
-  
-  columns = f.next()
-  
-  groups = dict()
-  for row in f:
-    match = re.search(r'/bin/\w*sh', row[6])  # adjust this line to match the right shell path
-    if match:
-        item = doc.createElement(node_user)
-        root_element.appendChild(item)
-        extra_groups = os.popen("groups %s" % row[0]).readline()[:-1] 
-        p_group = os.popen("id -gn %s" % row[0]).readline()[:-1]
-        extra_groups_str = extra_groups.split(' : ')[1]
-        populate_groups(groups, extra_groups_str)
-        item.setAttribute('extra_groups', get_extra_group_str(extra_groups_str, p_group))
-        create_col_nodes(columns, item, doc, row)
-    
-  for gkey, gval in groups.items():
-    item = doc.createElement(node_group)
-    root_element.appendChild(item)
-    item.setAttribute('name', gkey)
-    (gid,extra) = gval.split(':')
-    item.setAttribute('gid', gid)
-
-  output_file = "accounts.xml"
-  doc.writexml(open(output_file, 'w'), addindent='    ', newl='\n') # Write file
-  
-  print("Done: Created %s" % output_file)
-  os.remove(filename)
-
-def get_extra_group_str(group_str, p_group):
-    groups = group_str.split(' ')
-    groups = [x for x in groups  if p_group != x]
-    return ' '.join(groups)
-
-  
-def create_col_nodes(cols, item, doc, row): 
-  for col in cols:
-    if col == "gid":
-        att = doc.createAttribute("group")
-        att.nodeValue = grp.getgrgid(int(row.pop(0)))[0]
-    else:
-        att = doc.createAttribute(str.replace(col, " ", "_").lower())
-        att.nodeValue = row.pop(0)
-
-    if col != "pass":
-        item.setAttributeNode(att)
-
-def populate_groups(group_dic, group_str):
-    for g in group_str.split(' '):
-        if not group_dic.has_key(g):
-            group_ent = os.popen("getent group %s" % g).readline()[:-1].split(':')
-            gid = group_ent[2]
-            extra = group_ent[3]
-            extra_list = list(extra)
-            for e in extra_list:
-                if e == ',':
-                    loc = extra_list.index(e)
-                    extra_list[loc] = ' '
-            extra = "".join(extra_list)
-            group_dic[g] = gid + ":" + extra
-
-
-if __name__ == "__main__":
-  sys.exit(main(sys.argv))
-
-# vim:set sr et ts=4 sw=4 ft=python fenc=utf-8: // See Vim, :help 'modeline'
-# Created: Sun, 13 May 2012 13:44:43 -0400
-
-
diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py
index 54f8e87f3..cc7a1a3d8 100755
--- a/tools/bcfg2-profile-templates.py
+++ b/tools/bcfg2-profile-templates.py
@@ -1,62 +1,61 @@
 #!/usr/bin/python -Ott
 """ Benchmark template rendering times """
 
+import os
 import sys
 import time
+import signal
 import logging
-import logging.handlers
 import operator
-import lxml.etree
+import Bcfg2.Logger
 import Bcfg2.Server.Core
 
 LOGGER = None
 
-def get_logger(setup=None):
-    """ set up logging according to the verbose level given on the
-    command line """
-    global LOGGER
-    if LOGGER is None:
-        if setup is None:
-            setup = dict()
-        LOGGER = logging.getLogger(sys.argv[0])
-        stderr = logging.StreamHandler()
-        level = logging.WARNING
-        lformat = "%(message)s"
-        if setup.get("debug", False):
-            stderr.setFormatter(logging.Formatter("%(asctime)s: %(levelname)s: %(message)s"))
-            level = logging.DEBUG
-        elif setup.get("verbose", False):
-            level = logging.INFO
-        LOGGER.setLevel(level)
-        LOGGER.addHandler(stderr)
-        syslog = logging.handlers.SysLogHandler("/dev/log")
-        syslog.setFormatter(logging.Formatter("%(name)s: %(message)s"))
-        LOGGER.addHandler(syslog)
-    return LOGGER
+
+def get_sigint_handler(core):
+    """ Get a function that handles SIGINT/Ctrl-C by shutting down the
+    core and exiting properly."""
+
+    def hdlr(sig, frame):  # pylint: disable=W0613
+        """ Handle SIGINT/Ctrl-C by shutting down the core and exiting
+        properly. """
+        core.shutdown()
+        os._exit(1)  # pylint: disable=W0212
+
+    return hdlr
+
 
 def main():
     optinfo = \
-        dict(configfile=Bcfg2.Options.CFILE,
-             help=Bcfg2.Options.HELP,
-             encoding=Bcfg2.Options.ENCODING,
-             repo=Bcfg2.Options.SERVER_REPOSITORY,
-             plugins=Bcfg2.Options.SERVER_PLUGINS,
-             password=Bcfg2.Options.SERVER_PASSWORD,
-             debug=Bcfg2.Options.DEBUG,
-             verbose=Bcfg2.Options.VERBOSE,
-             client=Bcfg2.Options.Option("Benchmark templates for one client",
+        dict(client=Bcfg2.Options.Option("Benchmark templates for one client",
                                          cmd="--client",
                                          odesc="<client>",
                                          long_arg=True,
                                          default=None),
              )
+    optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS)
+    optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS)
     setup = Bcfg2.Options.OptionParser(optinfo)
     setup.parse(sys.argv[1:])
-    logger = get_logger(setup)
+
+    if setup['debug']:
+        level = logging.DEBUG
+    elif setup['verbose']:
+        level = logging.INFO
+    else:
+        level = logging.WARNING
+    Bcfg2.Logger.setup_logging("bcfg2-test",
+                               to_console=setup['verbose'] or setup['debug'],
+                               to_syslog=False,
+                               to_file=setup['logging'],
+                               level=level)
+    logger = logging.getLogger(sys.argv[0])
 
     core = Bcfg2.Server.Core.BaseCore(setup)
+    signal.signal(signal.SIGINT, get_sigint_handler(core))
     logger.info("Bcfg2 server core loaded")
-    core.fam.handle_events_in_interval(4)
+    core.fam.handle_events_in_interval(0.1)
     logger.debug("Repository events processed")
 
     # how many times to render each template for each client
@@ -73,40 +72,27 @@ def main():
         clients = [core.build_metadata(setup['client'])]
 
     times = dict()
-    if 'Cfg' not in core.plugins:
-        logger.error("Cfg is not enabled")
-        return 1
+    for metadata in clients:
+        for struct in core.GetStructures(metadata):
+            logger.info("Rendering templates from structure %s:%s" %
+                        (struct.tag, struct.get("name")))
+            for entry in struct.xpath("//Path"):
+                path = entry.get("name")
+                logger.info("Rendering %s..." % path)
+                times[path] = dict()
+                avg = 0.0
+                for i in range(runs):
+                    start = time.time()
+                    try:
+                        core.Bind(entry, metadata)
+                        avg += (time.time() - start) / runs
+                    except:
+                        break
+                if avg:
+                    logger.debug("   %s: %.02f sec" % (metadata.hostname, avg))
+                    times[path][metadata.hostname] = avg
 
-    cfg = core.plugins['Cfg']
-    entrysets = []
-    for template in templates:
-        try:
-            entrysets.append(cfg.entries[template])
-        except KeyError:
-            logger.debug("Template %s not found" % template)
-    if not entrysets:
-        logger.debug("Using all entrysets")
-        entrysets = cfg.entries.values()
-
-    for eset in entrysets:
-        path = eset.path.replace(setup['repo'], '')
-        logger.info("Rendering %s..." % path)
-        times[path] = dict()
-        for metadata in clients:
-            avg = 0.0
-            for i in range(runs):
-                entry = lxml.etree.Element("Path")
-                start = time.time()
-                try:
-                    eset.bind_entry(entry, metadata)
-                    avg += (time.time() - start) / runs
-                except:
-                    break
-            if avg:
-                logger.debug("   %s: %.02f sec" % (metadata.hostname, avg))
-                times[path][metadata.hostname] = avg
-
-    # print out per-template results
+    # print out per-file results
     tmpltimes = []
     for tmpl, clients in times.items():
         try:
diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py
index 15bd328f9..69d9514df 100755
--- a/tools/upgrade/1.3/migrate_dbstats.py
+++ b/tools/upgrade/1.3/migrate_dbstats.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import os
+os.environ['BCFG2_LEGACY_MODELS'] = '1'
 os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings'
 
 import sys
@@ -217,6 +218,14 @@ def migrate_stage1():
 def _restructure():
     """major restructure of reporting data"""
 
+    # run any migrations from the previous schema
+    try:
+      from Bcfg2.Server.Reports.updatefix import update_database
+      update_database()
+    except:
+        logger.error("Failed to run legacy schema updates", exc_info=1)
+        return False
+
     # try to avoid dangling transactions
     if not migrate_stage1():
         return