Plugins/Packages/Portage: add Packages Plugin for Portage

3 files changed, 338 insertions, 1 deletions

diff --git a/schemas/packages.xsd b/schemas/packages.xsd
index d358a17b3..aede2f815 100644
--- a/schemas/packages.xsd
+++ b/schemas/packages.xsd
@@ -18,6 +18,7 @@
       <xsd:enumeration value="apt"/>
       <xsd:enumeration value="pac"/>
       <xsd:enumeration value="pkgng"/>
+      <xsd:enumeration value="portage"/>
     </xsd:restriction>
   </xsd:simpleType>
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py
new file mode 100644
index 000000000..e9e787734
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py
@@ -0,0 +1,336 @@
+import re
+import gzip
+import sys
+import os
+import lxml.etree
+import Bcfg2.Options
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+
+_portage_python = '/usr/lib/portage/pym/'
+
+def _import_portage(caller):
+    # generate prefix path
+    caller.prefix = os.path.join(caller.basepath, 'cache', 'portage')
+    if not os.path.isdir(caller.prefix):
+        caller.logger.error("Packages: %s is not a dir. "
+                          "Portage will not work. Please "
+                          "remember to setup a EPREFIX there." %
+                          caller.prefix)
+        # TODO: automatic EPREFIX setup
+        raise Exception('Invalid EPREFIX')
+
+    os.environ['PORTAGE_OVERRIDE_EPREFIX'] = caller.prefix
+
+    if not os.path.isdir(_portage_python):
+        self.logger.error("Packages: %s not found. Have you installed "
+                          "the portage python modules?" % _portage_python)
+        raise Exception('Portage not found')
+
+    sys.path = sys.path + [_portage_python]
+    portage = __import__('portage', globals(), locals(),
+                         ['_sets', 'dbapi.porttree' ])
+    emerge = __import__('_emerge', globals(), locals(),
+                        ['RootConfig', 'depgraph', 'Package', 'actions'])
+
+    # setup profile
+    if '_setup_profile' in dir(caller):
+        caller._setup_profile(portage)
+
+    # fix some default settings
+    portage.settings.unlock()
+    portage.settings['PORTAGE_RSYNC_INITIAL_TIMEOUT'] = '0'
+    portage.settings.lock()
+
+    porttree = portage.db[portage.root]['porttree']
+    caller._import_portage(portage, emerge, porttree)
+
+
+class PortageCollection(Collection):
+    def __init__(self, metadata, sources, cachepath, basepath, debug=False):
+        Collection.__init__(self, metadata, sources, cachepath, basepath,
+                            debug=debug)
+        self.portage = None
+        self.emerge = None
+        self.porttree = None
+
+    @property
+    def cachefiles(self):
+        return []
+
+    def complete(self, packagelist, pinnings=None, recommended=None):
+        if not self.portage:
+            _import_portage(self)
+
+        # get global use flags
+        self.portage.settings.unlock()
+        self.portage.settings['USE'] = ''
+        if 'gentoo-use-flags' in self.metadata.Probes:
+            self.portage.settings['USE'] = \
+                self.metadata.Probes['gentoo-use-flags']
+        self.portage.settings.lock()
+
+
+        # calculate deps
+        setconfig = self.portage._sets.load_default_config(
+            self.portage.settings,
+            self.portage.db[self.portage.root])
+        rconfig = self.emerge.RootConfig.RootConfig(
+            self.portage.settings,
+            self.portage.db[self.portage.root],
+            setconfig)
+        self.portage.db[self.portage.root]['root_config'] = rconfig
+
+        pkgs = ["=" + j.cpv for (i, j) in packagelist if i == 'ok']
+        fail = [j for (i, j) in packagelist if i == 'fail']
+
+        x = self.emerge.depgraph.backtrack_depgraph(
+                self.portage.settings,
+                self.portage.db,
+                {'--pretend': True},
+                {'recurse': True},
+                'merge',
+                pkgs,
+                None)
+
+        g = x[1]._dynamic_config.digraph
+        packages = [i for i in g.all_nodes() \
+                    if isinstance(i, self.emerge.Package.Package)]
+
+        return (set(packages), set(fail))
+
+    def get_additional_data(self):
+        return []
+
+    def get_group(self, group):
+        self.logger.warning("Packages: Package sets are currently not supported")
+        return []
+
+    def packages_from_entry(self, entry):
+        if not self.portage:
+            _import_portage(self)
+
+        try:
+            name = entry.get('name')
+            # TODO: handle package specific accept keywords
+            pkgs = self.porttree.dep_bestmatch(name)
+        except self.portage.exception.AmbiguousPackageName as e:
+            self.logger.error("Packages: AmbiguousPackageName: %s" % e)
+            pkgs = ''
+
+        if pkgs == '':
+            return [('fail', name)]
+
+        return [('ok', pkgs)]
+
+    def packages_to_entry(self, pkgs, entry):
+        for pkg in pkgs:
+            if pkg.slot != '0':
+                name = "%s:%s" % (pkg.cp, pkg.slot)
+            else:
+                name = pkg.cp
+
+            lxml.etree.SubElement(entry, 'BoundPackage', name=name,
+                                  version=Bcfg2.Options.setup.packages_version,
+                                  type=self.ptype, origin='Packages')
+
+    def get_new_packages(self, initial, complete):
+        new = []
+        init = [pkg.cp for status, pkg in initial if status == 'ok']
+
+        for pkg in complete:
+            if pkg.cp not in init:
+                new.append(pkg)
+
+        return new
+
+    def setup_data(self, force_update=False):
+        pass
+
+    def _setup_profile(self, portage):
+        if 'gentoo-profile' not in self.metadata.Probes:
+            raise Exception('Unknown profile.')
+
+        profile = os.path.join(self.prefix, 'usr/portage/profiles/',
+            self.metadata.Probes['gentoo-profile'].strip())
+
+        env = portage.settings.configdict['backupenv']
+
+        portage.settings = portage.package.ebuild.config.config(
+            config_root=portage.settings['PORTAGE_CONFIGROOT'],
+            target_root=portage.settings['ROOT'],
+            env=env,
+            eprefix=portage.settings['EPREFIX'],
+            config_profile_path=profile)
+
+        portage.db[portage.root]['porttree'].settings = portage.settings
+        newdbapi = portage.dbapi.porttree.portdbapi(mysettings=portage.settings)
+        portage.db[portage.root]['porttree'].dbapi = newdbapi
+
+        portage.db[portage.root]['vartree'].settings = portage.settings
+        portage.db[portage.root]['vartree'].dbapi.settings = portage.settings
+
+    def _set_portage_config(self):
+        # get global use flags
+        self.portage.settings.unlock()
+        self.portage.settings['USE'] = ''
+        if 'gentoo-use-flags' in self.metadata.Probes:
+            self.portage.settings['USE'] = \
+                self.metadata.Probes['gentoo-use-flags']
+
+        # add package flags (accept_keywords, use)
+        if hasattr(self.metadata, 'PkgVars'):
+            for k in self.metadata.PkgVars['keywords']:
+                keyword = self.metadata.PkgVars['keywords'][k]
+                self.portage.settings._keywords_manager.pkeywordsdict[k] = \
+                    {k: tuple(keyword)}
+
+
+            for u in self.metadata.PkgVars['use']:
+                use = self.metadata.PkgVars['use'][u]
+                self.portage.settings._use_manager._pusedict[u] = \
+                    {u: tuple(use)}
+
+        self.portage.settings.lock()
+
+    def _import_portage(self, portage, emerge, porttree):
+        self.portage = portage
+        self.emerge = emerge
+        self.porttree = porttree
+
+        for s in self:
+            if isinstance(s, PortageSource):
+                s._import_portage(portage, emerge, porttree)
+
+
+class PortageSource(Bcfg2.Server.Plugin.Debuggable):
+    basegroups = ['portage', 'gentoo', 'emerge']
+    ptype = 'ebuild'
+
+    def __init__(self, basepath, xsource):
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
+        self.basepath = basepath
+        self.xsource = xsource
+
+        self.url = xsource.get('url', '')
+        self.priority = xsource.get('priority', 0)
+        self.cachefile = None
+        self.gpgkeys = []
+        self.recommended = False
+        self.essentialpkgs = set()
+        self.arches = [item.text for item in xsource.findall('Arch')]
+
+        self.url_map = [dict(version=None, component=None, arch=arch,
+                             url=self.url, baseurl=self.url) for arch in self.arches]
+
+        #: A list of the the names of packages that are blacklisted
+        #: from this source
+        self.blacklist = [item.text for item in xsource.findall('Blacklist')]
+
+        #: A list of the the names of packages that are whitelisted in
+        #: this source
+        self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+
+
+        self.portage = None
+        self.emerge = None
+        self.porttree = None
+
+        # build the set of conditions to see if this source applies to
+        # a given set of metadata
+        self.conditions = []
+        self.groups = [] # provided for some limited backwards compat
+        for el in xsource.iterancestors():
+            if el.tag == "Group":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") not in m.groups)
+                else:
+                    self.groups.append(el.get("name"))
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") in m.groups)
+            elif el.tag == "Client":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") != m.hostname)
+                else:
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") == m.hostname)
+    def get_repo_name(self, url_map):
+        return "portage"
+
+    def _import_portage(self, portage, emerge, porttree):
+        self.portage = portage
+        self.emerge = emerge
+        self.porttree = porttree
+
+    def save_state(self):
+        pass
+
+    def load_state(self):
+        pass
+
+    def filter_unknown(self, unknown):
+        filtered = set([u for u in unknown if u.startswith('choice')])
+        unknown.difference_update(filtered)
+
+    def get_urls(self):
+        return self.url
+    urls = property(get_urls)
+
+    def setup_data(self, force_update=False):
+        if not self.porttree:
+            _import_portage(self)
+
+        timestamp = os.path.join(self.porttree.portroot, 'metadata/timestamp.chk')
+        if not os.path.isfile(timestamp):
+            self.logger.error("Packages: Timestamp not found; "
+                              "falling back to sync")
+            force_update = True
+
+        if force_update:
+            # update sync url
+            self.portage.settings.unlock()
+            self.portage.settings['SYNC'] = self.url
+            self.portage.settings.lock()
+
+            # realy force the sync
+            if os.path.isfile(timestamp):
+                os.unlink(timestamp)
+
+            # sync
+            self.logger.info("Packages: Syncing with %s" % self.url)
+            self.emerge.actions.action_sync(self.portage.settings,
+                                            self.portage.db, None,
+                                            {'--quiet': True}, 'sync')
+
+    def applies(self, metadata):
+        """ Return true if this source applies to the given client,
+        i.e., the client is in all necessary groups.
+
+        :param metadata: The client metadata to check to see if this
+                         source applies
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :returns: bool
+        """
+        # check arch groups
+        if not self.arch_groups_match(metadata):
+            return False
+
+        # check Group/Client tags from sources.xml
+        for condition in self.conditions:
+            if not condition(metadata):
+                return False
+
+        return True
+
+    def arch_groups_match(self, metadata):
+        """ Returns True if the client is in an arch group that
+        matches the arch of this source.
+
+        :returns: bool
+        """
+        for arch in self.arches:
+            if arch in metadata.groups:
+                return True
+        return False
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index d1d937deb..92ad64c13 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -103,7 +103,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             help="Packages backends to load",
             type=Bcfg2.Options.Types.comma_list,
             action=PackagesBackendAction,
-            default=['Yum', 'Apt', 'Pac', 'Pkgng']),
+            default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Portage']),
         Bcfg2.Options.PathOption(
             cf=("packages", "cache"), dest="packages_cache",
             help="Path to the Packages cache",