Merge branch '1.3.1' into 1.4.x

7 files changed, 264 insertions, 24 deletions

diff --git a/doc/client/tools/posixusers.txt b/doc/client/tools/posixusers.txt
index 5fa2feb9c..45536632f 100644
--- a/doc/client/tools/posixusers.txt
+++ b/doc/client/tools/posixusers.txt
@@ -40,6 +40,52 @@ entry on the fly; this has a few repercussions:
   specify a particular GID number, you must explicitly define a
   ``POSIXGroup`` entry for the group.
 
+Managed UID/GID Ranges
+======================
+
+In many cases, there will be users on a system that you do not want to
+manage with Bcfg2, nor do you want them to be flagged as extra
+entries.  For example, users from an LDAP directory.  In this case,
+you may want to manage the local users on a machine with Bcfg2, while
+leaving the LDAP users to be managed by the LDAP directory.  To do
+this, you can configure the UID and GID ranges that are to be managed
+by Bcfg2 by setting the following options in the ``[POSIXUsers]``
+section of ``bcfg2.conf`` on the *client*:
+
+* ``uid_whitelist``
+* ``uid_blacklist``
+* ``gid_whitelist``
+* ``gid_blacklist``
+
+Each option takes a comma-delimited list of numeric ranges, inclusive
+at both bounds, one of which may be open-ended on the upper bound,
+e.g.::
+
+    [POSIXUsers]
+    uid_blacklist=1000-
+    gid_whitelist=0-500,700-999
+
+This would tell Bcfg2 to manage all users whose uid numbers were *not*
+greater than or equal to 1000, and all groups whose gid numbers were 0
+<= ``gid`` <= 500 or 700 <= ``gid`` <= 999.
+
+If a whitelist is provided, it will be used; otherwise, the blacklist
+will be used.  (I.e., if you provide both, the blacklist will be
+ignored.)
+
+If a user or group is added to the specification with a uid or gid in
+an unmanaged range, it will produce an error.
+
+.. note::
+
+    If you specify POSIXUser or POSIXGroup tags without an explicit
+    uid or gid, this will **not** prevent the users/groups from being
+    created with a uid/gid in an unmanaged range.  If you want that to
+    happen, you will need to configure your ``useradd``/``groupadd``
+    defaults appropriately.  Note also, however, that this will not
+    cause Bcfg2 errors; it is only an error if a POSIXUser or
+    POSIXGroup has an *explicit* uid/gid in an unmanaged range.
+
 Creating a baseline configuration
 =================================
 
@@ -50,4 +96,3 @@ packaging system.)  The often-tedious task of creating a baseline that
 defines all users and groups can be simplified by use of the
 ``tools/posixusers_baseline.py`` script, which outputs a bundle
 containing all users and groups on the machine it's run on.
-
diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
index 78734f5c2..7c8a4d578 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
@@ -4,9 +4,45 @@ and groupadd/mod/del """
 import sys
 import pwd
 import grp
-import Bcfg2.Client.XML
 import subprocess
+import Bcfg2.Client.XML
 import Bcfg2.Client.Tools
+from Bcfg2.Compat import any  # pylint: disable=W0622
+
+
+class IDRangeSet(object):
+    """ Representation of a set of integer ranges.  Used to describe
+    which UID/GID ranges are managed or unmanaged. """
+
+    def __init__(self, *ranges):
+        self.ranges = []
+        self.ints = []
+        self.str = ",".join(str(r) for r in ranges)
+        for item in ranges:
+            item = str(item).strip()
+            if item.endswith("-"):
+                self.ranges.append((int(item[:-1]), None))
+            elif '-' in str(item):
+                self.ranges.append(tuple(int(x) for x in item.split('-')))
+            else:
+                self.ints.append(int(item))
+
+    def __contains__(self, other):
+        other = int(other)
+        if other in self.ints:
+            return True
+        return any((end is None and other >= start) or
+                   (end is not None and other >= start and other <= end)
+                   for start, end in self.ranges)
+
+    def __repr__(self):
+        return "%s:%s" % (self.__class__.__name__, str(self))
+
+    def __str__(self):
+        return "[%s]" % self.str
+
+    def __len__(self):
+        return len(self.ranges) + len(self.ints)
 
 
 class ExecutionError(Exception):
@@ -68,18 +104,36 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
                    POSIXGroup=['name'])
     experimental = True
 
-    # A mapping of XML entry attributes to the indexes of
-    # corresponding values in the get*ent data structures
+    #: A mapping of XML entry attributes to the indexes of
+    #: corresponding values in the get{pw|gr}all data structures
     attr_mapping = dict(POSIXUser=dict(name=0, uid=2, gecos=4, home=5,
                                        shell=6),
                         POSIXGroup=dict(name=0, gid=2))
 
+    #: A mapping that describes the attribute name of the id of a given
+    #: user or group
+    id_mapping = dict(POSIXUser="uid", POSIXGroup="gid")
+
     def __init__(self, logger, setup, config):
         Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config)
         self.set_defaults = dict(POSIXUser=self.populate_user_entry,
                                  POSIXGroup=lambda g: g)
         self.cmd = Executor(logger)
         self._existing = None
+        self._whitelist = dict(POSIXUser=None, POSIXGroup=None)
+        self._blacklist = dict(POSIXUser=None, POSIXGroup=None)
+        if self.setup['posix_uid_whitelist']:
+            self._whitelist['POSIXUser'] = \
+                IDRangeSet(*self.setup['posix_uid_whitelist'])
+        else:
+            self._blacklist['POSIXUser'] = \
+                IDRangeSet(*self.setup['posix_uid_blacklist'])
+        if self.setup['posix_gid_whitelist']:
+            self._whitelist['POSIXGroup'] = \
+                IDRangeSet(*self.setup['posix_gid_whitelist'])
+        else:
+            self._blacklist['POSIXGroup'] = \
+                IDRangeSet(*self.setup['posix_gid_blacklist'])
 
     @property
     def existing(self):
@@ -91,6 +145,33 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
                                                    for g in grp.getgrall()]))
         return self._existing
 
+    def _in_managed_range(self, tag, eid):
+        """ Check if the given uid or gid is in the appropriate
+        managed range.  This means that either a) a whitelist is
+        defined, and the uid/gid is in that whitelist; or b) no
+        whitelist is defined, and the uid/gid is not in the
+        blacklist. """
+        if self._whitelist[tag] is None:
+            return eid not in self._blacklist[tag]
+        else:
+            return eid in self._whitelist[tag]
+
+    def canInstall(self, entry):
+        if not Bcfg2.Client.Tools.Tool.canInstall(self, entry):
+            return False
+        eid = entry.get(self.id_mapping[entry.tag])
+        if eid is not None and not self._in_managed_range(entry.tag, eid):
+            if self._whitelist[entry.tag] is not None:
+                err = "not in whitelist"
+            else:  # blacklisted
+                err = "in blacklist"
+            self.logger.debug("%s: %s %s %s: %s" %
+                              (self.primarykey(entry), err,
+                               self.id_mapping[entry.tag], eid,
+                               self._blacklist[entry.tag]))
+            return False
+        return True
+
     def Inventory(self, states, structures=None):
         if not structures:
             structures = self.config.getchildren()
@@ -121,9 +202,11 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
             for entry in self.getSupportedEntries():
                 if entry.tag == tag:
                     specified.append(entry.get("name"))
-            extra.extend([Bcfg2.Client.XML.Element(tag, name=e)
-                          for e in self.existing[tag].keys()
-                          if e not in specified])
+            for name, data in self.existing[tag].items():
+                eid = data[self.attr_mapping[tag][self.id_mapping[tag]]]
+                if name not in specified and self._in_managed_range(tag, eid):
+                    extra.append(Bcfg2.Client.XML.Element(tag, name=name))
+
         return extra
 
     def populate_user_entry(self, entry):
@@ -201,7 +284,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
                     errors.append("%s for %s %s is incorrect.  Current %s is "
                                   "%s, but should be %s" %
                                   (attr.title(), entry.tag, entry.get("name"),
-                                   attr, entry.get(attr), val))
+                                   attr, val, entry.get(attr)))
 
         if errors:
             for error in errors:
diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py
index 1f5a8f689..decb726d0 100644
--- a/src/lib/Bcfg2/Options.py
+++ b/src/lib/Bcfg2/Options.py
@@ -922,6 +922,26 @@ CLIENT_YUM_VERIFY_FLAGS = \
            default=[],
            cf=('YUM', 'verify_flags'),
            cook=list_split)
+CLIENT_POSIX_UID_WHITELIST = \
+    Option("UID ranges the POSIXUsers tool will manage",
+           default=[],
+           cf=('POSIXUsers', 'uid_whitelist'),
+           cook=list_split)
+CLIENT_POSIX_GID_WHITELIST = \
+    Option("GID ranges the POSIXUsers tool will manage",
+           default=[],
+           cf=('POSIXUsers', 'gid_whitelist'),
+           cook=list_split)
+CLIENT_POSIX_UID_BLACKLIST = \
+    Option("UID ranges the POSIXUsers tool will not manage",
+           default=[],
+           cf=('POSIXUsers', 'uid_blacklist'),
+           cook=list_split)
+CLIENT_POSIX_GID_BLACKLIST = \
+    Option("GID ranges the POSIXUsers tool will not manage",
+           default=[],
+           cf=('POSIXUsers', 'gid_blacklist'),
+           cook=list_split)
 
 # Logging options
 LOGGING_FILE_PATH = \
@@ -1065,7 +1085,11 @@ DRIVER_OPTIONS = \
          yum_installed_action=CLIENT_YUM_INSTALLED_ACTION,
          yum_version_fail_action=CLIENT_YUM_VERSION_FAIL_ACTION,
          yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION,
-         yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS)
+         yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS,
+         posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST,
+         posix_gid_whitelist=CLIENT_POSIX_UID_WHITELIST,
+         posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST,
+         posix_gid_blacklist=CLIENT_POSIX_UID_BLACKLIST)
 
 CLIENT_COMMON_OPTIONS = \
     dict(extra=CLIENT_EXTRA_DISPLAY,
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 57edcb938..59796a556 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -577,15 +577,12 @@ class StructFile(XMLFileBacked):
 
     def Index(self):
         XMLFileBacked.Index(self)
-        if self.encryption:
+        if self.encryption and HAS_CRYPTO:
             strict = self.xdata.get(
                 "decrypt",
                 self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt",
                                    default="strict")) == "strict"
             for el in self.xdata.xpath("//*[@encrypted]"):
-                if not HAS_CRYPTO:
-                    raise PluginExecutionError("%s: M2Crypto is not available"
-                                               % self.name)
                 try:
                     el.text = self._decrypt(el).encode('ascii',
                                                        'xmlcharrefreplace')
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 0ad5dd788..42d860b89 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -6,6 +6,7 @@ import sys
 import logging
 import Bcfg2.Server.Lint
 import Bcfg2.Server.Plugin
+from Bcfg2.Compat import any  # pylint: disable=W0622
 
 
 class PackedDigitRange(object):
@@ -25,10 +26,8 @@ class PackedDigitRange(object):
         iother = int(other)
         if iother in self.sparse:
             return True
-        for (start, end) in self.ranges:
-            if iother in range(start, end + 1):
-                return True
-        return False
+        return any(iother in range(start, end + 1)
+                   for start, end in self.ranges)
 
 
 class PatternMap(object):
diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
index 46ae4e47b..bcf6cf133 100644
--- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
+++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
@@ -19,6 +19,33 @@ while path != "/":
 from common import *
 
 
+class TestIDRangeSet(Bcfg2TestCase):
+    def test_ranges(self):
+        # test cases.  tuples of (ranges, included numbers, excluded
+        # numbers)
+        # tuples of (range description, numbers that are included,
+        # numebrs that are excluded)
+        tests = [(["0-3"], ["0", 1, "2", 3], [4]),
+                 (["1"], [1], [0, "2"]),
+                 (["10-11"], [10, 11], [0, 1]),
+                 (["9-9"], [9], [8, 10]),
+                 (["0-100"], [0, 10, 99, 100], []),
+                 (["1", "3", "5"], [1, 3, 5], [0, 2, 4, 6]),
+                 (["1-5", "7"], [1, 3, 5, 7], [0, 6, 8]),
+                 (["1-5", 7, "9-11"], [1, 3, 5, 7, 9, 11], [0, 6, 8, 12]),
+                 (["852-855", "321-497", 763], [852, 855, 321, 400, 497, 763],
+                  [851, 320, 766, 999]),
+                 (["0-"], [0, 1, 100, 100000], []),
+                 ([1, "5-10", "1000-"], [1, 5, 10, 1000, 10000000],
+                  [4, 11, 999])]
+        for ranges, inc, exc in tests:
+            rng = IDRangeSet(*ranges)
+            for test in inc:
+                self.assertIn(test, rng)
+            for test in exc:
+                self.assertNotIn(test, rng)
+
+
 class TestExecutor(Bcfg2TestCase):
     test_obj = Executor
 
@@ -97,7 +124,8 @@ class TestPOSIXUsers(Bcfg2TestCase):
     def get_obj(self, logger=None, setup=None, config=None):
         if config is None:
             config = lxml.etree.Element("Configuration")
-        if not logger:
+
+        if logger is None:
             def print_msg(msg):
                 print(msg)
             logger = Mock()
@@ -105,8 +133,10 @@ class TestPOSIXUsers(Bcfg2TestCase):
             logger.warning = Mock(side_effect=print_msg)
             logger.info = Mock(side_effect=print_msg)
             logger.debug = Mock(side_effect=print_msg)
-        if not setup:
+
+        if setup is None:
             setup = MagicMock()
+            setup.__getitem__.return_value = []
         return self.test_obj(logger, setup, config)
 
     @patch("pwd.getpwall")
@@ -145,6 +175,60 @@ class TestPOSIXUsers(Bcfg2TestCase):
         mock_getgrall.assert_called_with()
         mock_getpwall.assert_called_with()
 
+    def test__in_managed_range(self):
+        users = self.get_obj()
+        users._whitelist = dict(POSIXGroup=IDRangeSet("1-10"))
+        users._blacklist = dict(POSIXGroup=IDRangeSet("8-100"))
+        self.assertTrue(users._in_managed_range("POSIXGroup", "9"))
+
+        users._whitelist = dict(POSIXGroup=None)
+        users._blacklist = dict(POSIXGroup=IDRangeSet("8-100"))
+        self.assertFalse(users._in_managed_range("POSIXGroup", "9"))
+
+        users._whitelist = dict(POSIXGroup=None)
+        users._blacklist = dict(POSIXGroup=IDRangeSet("100-"))
+        self.assertTrue(users._in_managed_range("POSIXGroup", "9"))
+
+        users._whitelist = dict(POSIXGroup=IDRangeSet("1-10"))
+        users._blacklist = dict(POSIXGroup=None)
+        self.assertFalse(users._in_managed_range("POSIXGroup", "25"))
+
+    @patch("Bcfg2.Client.Tools.Tool.canInstall")
+    def test_canInstall(self, mock_canInstall):
+        users = self.get_obj()
+        users._in_managed_range = Mock()
+        users._in_managed_range.return_value = False
+        mock_canInstall.return_value = False
+
+        def reset():
+            users._in_managed_range.reset()
+            mock_canInstall.reset()
+
+        # test failure of inherited method
+        entry = lxml.etree.Element("POSIXUser", name="test")
+        self.assertFalse(users.canInstall(entry))
+        mock_canInstall.assertCalledWith(users, entry)
+
+        # test with no uid specified
+        reset()
+        mock_canInstall.return_value = True
+        self.assertTrue(users.canInstall(entry))
+        mock_canInstall.assertCalledWith(users, entry)
+
+        # test with uid specified, not in managed range
+        reset()
+        entry.set("uid", "1000")
+        self.assertFalse(users.canInstall(entry))
+        mock_canInstall.assertCalledWith(users, entry)
+        users._in_managed_range.assert_called_with(entry.tag, "1000")
+
+        # test with uid specified, in managed range
+        reset()
+        users._in_managed_range.return_value = True
+        self.assertTrue(users.canInstall(entry))
+        mock_canInstall.assertCalledWith(users, entry)
+        users._in_managed_range.assert_called_with(entry.tag, "1000")
+
     @patch("Bcfg2.Client.Tools.Tool.Inventory")
     def test_Inventory(self, mock_Inventory):
         config = lxml.etree.Element("Configuration")
@@ -168,6 +252,8 @@ class TestPOSIXUsers(Bcfg2TestCase):
 
     def test_FindExtra(self):
         users = self.get_obj()
+        users._in_managed_range = Mock()
+        users._in_managed_range.side_effect = lambda t, i: i < 100
 
         def getSupportedEntries():
             return [lxml.etree.Element("POSIXUser", name="test1"),
@@ -176,15 +262,20 @@ class TestPOSIXUsers(Bcfg2TestCase):
         users.getSupportedEntries = Mock()
         users.getSupportedEntries.side_effect = getSupportedEntries
 
-        users._existing = dict(POSIXUser=dict(test1=(),
-                                              test2=()),
-                               POSIXGroup=dict(test2=()))
+        users._existing = dict(POSIXUser=dict(test1=("test1", "x", 15),
+                                              test2=("test2", "x", 25),
+                                              test3=("test3", "x", 115)),
+                               POSIXGroup=dict(test2=("test2", "x", 25)))
         extra = users.FindExtra()
         self.assertEqual(len(extra), 2)
         self.assertItemsEqual([e.tag for e in extra],
                               ["POSIXUser", "POSIXGroup"])
         self.assertItemsEqual([e.get("name") for e in extra],
                               ["test2", "test2"])
+        self.assertItemsEqual(users._in_managed_range.call_args_list,
+                              [call("POSIXUser", 25),
+                               call("POSIXUser", 115),
+                               call("POSIXGroup", 25)])
 
     def test_populate_user_entry(self):
         users = self.get_obj()
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
index 84d35ccc5..a7a6b3d6e 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
@@ -22,13 +22,14 @@ class TestPackedDigitRange(Bcfg2TestCase):
     def test_includes(self):
         # tuples of (range description, numbers that are included,
         # numebrs that are excluded)
-        tests = [("1-3", [1, "2", 3], [4, "0"]),
-                 ("1", [1], [0, 2]),
+        tests = [("1-3", [1, "2", 3], [4]),
+                 ("1", [1], [0, "2"]),
                  ("10-11", [10, 11], [0, 1]),
                  ("9-9", [9], [8, 10]),
                  ("0-100", [0, 10, 99, 100], []),
                  ("1,3,5", [1, 3, 5], [0, 2, 4, 6]),
                  ("1-5,7", [1, 3, 5, 7], [0, 6, 8]),
+                 ("1-5,7,9-11", [1, 3, 5, 7, 9, 11], [0, 6, 8, 12]),
                  ("852-855,321-497,763", [852, 855, 321, 400, 497, 763], [])]
         for rng, inc, exc in tests:
             r = PackedDigitRange(rng)