Merge branch 'maint'

Conflicts:
	debian/changelog
	doc/conf.py
	misc/bcfg2-selinux.spec
	misc/bcfg2.spec
	osx/Makefile
	osx/macports/Portfile
	solaris-ips/MANIFEST.bcfg2-server.header
	solaris-ips/MANIFEST.bcfg2.header
	solaris-ips/Makefile
	solaris-ips/pkginfo.bcfg2
	solaris-ips/pkginfo.bcfg2-server
	solaris/Makefile
	solaris/pkginfo.bcfg2
	solaris/pkginfo.bcfg2-server
	src/lib/Bcfg2/Client/Tools/APT.py
	src/lib/Bcfg2/Client/Tools/FreeBSDInit.py
	src/lib/Bcfg2/Client/Tools/__init__.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Reporting/Collector.py
	src/lib/Bcfg2/Reporting/templates/base.html
	src/lib/Bcfg2/Server/CherrypyCore.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/__init__.py
	src/lib/Bcfg2/Server/Lint/Validate.py
	src/lib/Bcfg2/Server/Lint/__init__.py
	src/lib/Bcfg2/Server/MultiprocessingCore.py
	src/lib/Bcfg2/Server/Plugin/__init__.py
	src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/Plugins/Packages/Source.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/SSHbase.py
	src/lib/Bcfg2/Server/Plugins/SSLCA.py
	src/lib/Bcfg2/version.py
	src/sbin/bcfg2-info
	src/sbin/bcfg2-test
	testsuite/requirements.txt

2 files changed, 211 insertions, 54 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index c9f6ea14a..86f7698f7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -135,22 +135,22 @@ class Source(Debuggable):  # pylint: disable=R0902
 
         #: A list of the text of all 'Component' attributes of this
         #: source from XML
-        self.components = [item.text for item in xsource.findall('Component')]
+        self.components = []
 
         #: A list of the arches supported by this source
-        self.arches = [item.text for item in xsource.findall('Arch')]
+        self.arches = []
 
         #: A list of the the names of packages that are blacklisted
         #: from this source
-        self.blacklist = [item.text for item in xsource.findall('Blacklist')]
+        self.blacklist = []
 
         #: A list of the the names of packages that are whitelisted in
         #: this source
-        self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+        self.whitelist = []
 
         #: Whether or not to include deb-src lines in the generated APT
         #: configuration
-        self.debsrc = xsource.get('debsrc', 'false') == 'true'
+        self.debsrc = False
 
         #: A dict of repository options that will be included in the
         #: configuration generated on the server side (if such is
@@ -162,51 +162,38 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: configuration generated for the client (if that is
         #: supported by the backend)
         self.client_options = dict()
-        opts = xsource.findall("Options")
-        for el in opts:
-            repoopts = dict([(k, v)
-                             for k, v in el.attrib.items()
-                             if k != "clientonly" and k != "serveronly"])
-            if el.get("clientonly", "false").lower() == "false":
-                self.server_options.update(repoopts)
-            if el.get("serveronly", "false").lower() == "false":
-                self.client_options.update(repoopts)
 
         #: A list of URLs to GPG keys that apply to this source
-        self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
+        self.gpgkeys = []
 
         #: Whether or not to include essential packages from this source
-        self.essential = xsource.get('essential', 'true').lower() == 'true'
+        self.essential = True
 
         #: Whether or not to include recommended packages from this source
-        self.recommended = xsource.get('recommended',
-                                       'false').lower() == 'true'
+        self.recommended = False
 
         #: The "rawurl" attribute from :attr:`xsource`, if applicable.
         #: A trailing slash is automatically appended to this if there
         #: wasn't one already present.
-        self.rawurl = xsource.get('rawurl', '')
-        if self.rawurl and not self.rawurl.endswith("/"):
-            self.rawurl += "/"
+        self.rawurl = None
 
         #: The "url" attribute from :attr:`xsource`, if applicable.  A
         #: trailing slash is automatically appended to this if there
         #: wasn't one already present.
-        self.url = xsource.get('url', '')
-        if self.url and not self.url.endswith("/"):
-            self.url += "/"
+        self.url = None
 
         #: The "version" attribute from :attr:`xsource`
-        self.version = xsource.get('version', '')
+        self.version = None
 
         #: The "name" attribute from :attr:`xsource`
-        self.name = xsource.get('name', None)
+        self.name = None
 
         #: A list of predicates that are used to determine if this
         #: source applies to a given
         #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
         #: object.
         self.conditions = []
+
         #: Formerly, :ref:`server-plugins-generators-packages` only
         #: supported applying package sources to groups; that is, they
         #: could not be assigned by more complicated logic like
@@ -214,22 +201,6 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: attribute attempts to provide for some limited backwards
         #: compat with older code that relies on this.
         self.groups = []
-        for el in xsource.iterancestors():
-            if el.tag == "Group":
-                if el.get("negate", "false").lower() == "true":
-                    self.conditions.append(lambda m, el=el:
-                                           el.get("name") not in m.groups)
-                else:
-                    self.groups.append(el.get("name"))
-                    self.conditions.append(lambda m, el=el:
-                                           el.get("name") in m.groups)
-            elif el.tag == "Client":
-                if el.get("negate", "false").lower() == "true":
-                    self.conditions.append(lambda m, el=el:
-                                           el.get("name") != m.hostname)
-                else:
-                    self.conditions.append(lambda m, el=el:
-                                           el.get("name") == m.hostname)
 
         #: A set of all package names in this source.  This will not
         #: necessarily be populated, particularly by backends that
@@ -253,6 +224,8 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: symbols>``.  This will not necessarily be populated.
         self.recommends = dict()
 
+        self._init_attributes(xsource)
+
         #: The file (or directory) used for this source's cache data
         self.cachefile = os.path.join(self.basepath,
                                       "cache-%s" % self.cachekey)
@@ -292,6 +265,69 @@ class Source(Debuggable):  # pylint: disable=R0902
                 setting['name'] = self.get_repo_name(setting)
             self.url_map.extend(usettings)
 
+    def _init_attributes(self, xsource):
+        """
+        This functions evaluates the Source tag and parses all
+        attributes. Override this function in a sub class to
+        parse specific attributes. Do not use ``__init__`` because
+        ``Source.__init__`` may call other functions that already
+        need this specific fields. This functions is called before
+        any other function.
+
+        :param xsource: The XML tag that describes this source
+        :type source: lxml.etree._Element
+        """
+
+        self.components = [item.text for item in xsource.findall('Component')]
+        self.arches = [item.text for item in xsource.findall('Arch')]
+        self.blacklist = [item.text for item in xsource.findall('Blacklist')]
+        self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+        self.debsrc = xsource.get('debsrc', 'false') == 'true'
+
+        opts = xsource.findall("Options")
+        for el in opts:
+            repoopts = dict([(k, v)
+                             for k, v in el.attrib.items()
+                             if k != "clientonly" and k != "serveronly"])
+            if el.get("clientonly", "false").lower() == "false":
+                self.server_options.update(repoopts)
+            if el.get("serveronly", "false").lower() == "false":
+                self.client_options.update(repoopts)
+
+        self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
+
+        self.essential = xsource.get('essential', 'true').lower() == 'true'
+        self.recommended = xsource.get('recommended',
+                                       'false').lower() == 'true'
+
+        self.rawurl = xsource.get('rawurl', '')
+        if self.rawurl and not self.rawurl.endswith("/"):
+            self.rawurl += "/"
+
+        self.url = xsource.get('url', '')
+        if self.url and not self.url.endswith("/"):
+            self.url += "/"
+
+        self.version = xsource.get('version', '')
+        self.name = xsource.get('name', None)
+
+        for el in xsource.iterancestors():
+            if el.tag == "Group":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") not in m.groups)
+                else:
+                    self.groups.append(el.get("name"))
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") in m.groups)
+            elif el.tag == "Client":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") != m.hostname)
+                else:
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") == m.hostname)
+
     @property
     def cachekey(self):
         """ A unique key for this source that will be used to generate
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index dbe3f9ce5..14d6db8a0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -1004,8 +1004,20 @@ class YumSource(Source):
     ptype = 'yum'
 
     def __init__(self, basepath, xsource):
-        Source.__init__(self, basepath, xsource)
+        self.filemap = dict()
+        self.file_to_arch = dict()
+        self.needed_paths = set()
+        self.packages = dict()
+        self.yumgroups = dict()
         self.pulp_id = None
+        self.repo = None
+
+        Source.__init__(self, basepath, xsource)
+    __init__.__doc__ = Source.__init__.__doc__
+
+    def _init_attributes(self, xsource):
+        Source._init_attributes(self, xsource)
+
         if HAS_PULP and xsource.get("pulp_id"):
             self.pulp_id = xsource.get("pulp_id")
 
@@ -1034,15 +1046,11 @@ class YumSource(Source):
                                      self.repo['relative_path'])
             self.arches = [self.repo['arch']]
 
-        self.packages = dict()
         self.deps = dict([('global', dict())])
         self.provides = dict([('global', dict())])
         self.filemap = dict([(x, dict())
                              for x in ['global'] + self.arches])
-        self.needed_paths = set()
-        self.file_to_arch = dict()
-        self.yumgroups = dict()
-    __init__.__doc__ = Source.__init__.__doc__
+    _init_attributes.__doc__ = Source._init_attributes.__doc__
 
     @property
     def use_yum(self):
@@ -1130,6 +1138,94 @@ class YumSource(Source):
                 self.file_to_arch[self.escape_url(fullurl)] = arch
         return urls
 
+    # pylint: disable=R0911,R0912
+    # disabling the pylint errors above because we are interesting in
+    # replicating the flow of the RPM code.
+    def _compare_rpm_versions(self, str1, str2):
+        """ Compare RPM versions.
+
+        This is an attempt to reimplement RPM's rpmvercmp method in python.
+
+        :param str1: package 1 version string
+        :param str2: package 2 version string
+        :return: 1 - str1 is newer than str2
+                 0 - str1 and str2 are the same version
+                -1 - str2 is newer than str1"""
+        if str1 == str2:
+            return 0
+
+        front_strip_re = re.compile('^[^A-Za-z0-9~]+')
+        risdigit = re.compile('(^[0-9]+)')
+        risalpha = re.compile('(^[A-Za-z])')
+        lzeroes = re.compile('^0+')
+
+        while len(str1) > 0 or len(str2) > 0:
+            str1 = front_strip_re.sub('', str1)
+            str2 = front_strip_re.sub('', str2)
+
+            if len(str1) == 0 or len(str2) == 0:
+                break
+
+            # handle the tilde separator
+            if str1[0] == '~' and str2[0] == '~':
+                str1 = str1[1:]
+                str2 = str2[1:]
+            elif str1[0] == '~':
+                return -1
+            elif str2[0] == '~':
+                return 1
+
+            # grab continuous segments from each string
+            isnum = False
+            if risdigit.match(str1):
+                segment1 = risdigit.split(str1)[1]
+                str1 = risdigit.split(str1)[2]
+                if risdigit.match(str2):
+                    segment2 = risdigit.split(str2)[1]
+                    str2 = risdigit.split(str2)[2]
+                else:
+                    segment2 = ''
+                isnum = True
+            else:
+                segment1 = risalpha.split(str1)[1]
+                str1 = risalpha.split(str1)[2]
+                if risalpha.match(str2):
+                    segment2 = risalpha.split(str2)[1]
+                    str2 = risalpha.split(str2)[2]
+                else:
+                    segment2 = ''
+
+            # numeric segments are always newer than alpha segments
+            if len(segment2) == 0:
+                if isnum:
+                    return 1
+                return -1
+
+            if isnum:
+                # discard leading zeroes
+                segment1 = lzeroes.sub('', segment1)
+                segment2 = lzeroes.sub('', segment2)
+                # higher number has more digits
+                if len(segment1) > len(segment2):
+                    return 1
+                elif len(segment2) > len(segment1):
+                    return -1
+            # do a simple string comparison
+            if segment1 > segment2:
+                return 1
+            elif segment2 > segment1:
+                return -1
+
+        # if one of the strings is empty, the version of the longer
+        # string is higher
+        if len(str1) > len(str2):
+            return 1
+        elif len(str2) > len(str1):
+            return -1
+        else:
+            return 0
+    # pylint: enable=R0911,R0912
+
     @track_statistics()
     def read_files(self):
         """ When using the builtin yum parser, read and parse locally
@@ -1198,13 +1294,33 @@ class YumSource(Source):
         if arch not in self.packages:
             self.packages[arch] = set()
         if arch not in self.deps:
-            self.deps[arch] = dict()
+            self.deps[arch] = {}
         if arch not in self.provides:
-            self.provides[arch] = dict()
+            self.provides[arch] = {}
+        versionmap = {}
         for pkg in data.getchildren():
             if not pkg.tag.endswith('package'):
                 continue
             pkgname = pkg.find(XP + 'name').text
+            vtag = pkg.find(XP + 'version')
+            epoch = vtag.get('epoch')
+            version = vtag.get('ver')
+            release = vtag.get('rel')
+            if pkgname in self.packages[arch]:
+                # skip if version older than a previous version
+                if (self._compare_rpm_versions(
+                        epoch, versionmap[pkgname]['epoch']) < 0):
+                    continue
+                elif (self._compare_rpm_versions(
+                        version, versionmap[pkgname]['version']) < 0):
+                    continue
+                elif (self._compare_rpm_versions(
+                        release, versionmap[pkgname]['release']) < 0):
+                    continue
+            versionmap[pkgname] = {}
+            versionmap[pkgname]['epoch'] = epoch
+            versionmap[pkgname]['version'] = version
+            versionmap[pkgname]['release'] = release
             self.packages[arch].add(pkgname)
 
             pdata = pkg.find(XP + 'format')
@@ -1256,10 +1372,15 @@ class YumSource(Source):
         arch = [a for a in self.arches if a in metadata.groups]
         if not arch:
             return False
-        return ((package in self.packages['global'] or
-                 package in self.packages[arch[0]]) and
-                package not in self.blacklist and
-                (len(self.whitelist) == 0 or package in self.whitelist))
+        try:
+            return ((package in self.packages['global'] or
+                     package in self.packages[arch[0]]) and
+                    package not in self.blacklist and
+                    (len(self.whitelist) == 0 or package in self.whitelist))
+        except KeyError:
+            self.logger.debug("Packages: Unable to find %s for arch %s" %
+                              (package, arch[0]))
+            return False
     is_package.__doc__ = Source.is_package.__doc__
 
     def get_vpkgs(self, metadata):