diff options
21 files changed, 788 insertions, 25 deletions
diff --git a/doc/client/tools/vcs.txt b/doc/client/tools/vcs.txt index fb9c33684..f21d097ff 100644 --- a/doc/client/tools/vcs.txt +++ b/doc/client/tools/vcs.txt @@ -8,8 +8,34 @@ VCS Client Tool .. warning: This tool is currently under development. -.. note: Currently, the only supported VCS is git. +.. note: Currently, the only supported VCS is git and svn. The VCS tool allows you to checkout particular revisions from a VCS repository on the client to a specified path. The tool requires the appropriate python libraries for the VCS used to be installed. + +See :ref:`server-plugins-generators-rules-vcs` for possible options. + +Example usage: + +You may want to create a `Rules/paths.xml` with the following: + +.. code-block:: xml + + <Rules priority="1"> + <Path name="/srv/bcfg2" type="vcs" + sourceurl="https://github.com/Bcfg2/bcfg2.git" + vcstype="git" + revision="cf6dfd8ca28e941b1e638ff0fa7e7a0a1ebb6a6f"/> + </Rules> + +Once the rule is created a client can reference the path from a +bundle, this path will then be populated from the repository. To +continue the above example, a file `Bundle/bcfg2.xml` might contain +this: + +.. code-block:: xml + + <Bundle name="bcfg"> + <Path name="/srv/bcfg2"> + <Bundle/> diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt index a5c10777a..8acbda8a7 100644 --- a/doc/releases/1.4.0pre2.txt +++ b/doc/releases/1.4.0pre2.txt @@ -31,6 +31,9 @@ backwards-incompatible user-facing changes This fixes potentially long client runs when comparing files that have diverged significantly. +* Ignore directories containing a .bcfg2-ignore file in various plugins + (Bundler, Defaults, Pkgmgr, Properties, PuppetENC, TemplateHelper, Trigger). + Thanks ------ diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 86478a5ae..e4f47c2bf 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -248,10 +248,13 @@ Manage symlinks. :onlyattrs: to :requiredattrs: to +.. _server-plugins-generators-rules-vcs: + vcs ^^^ -Check out the specified VCS repository to the given path. +Check out the specified VCS repository to the given path. See +:ref:`client-tools-vcs` for more details. .. xml:type:: PathType :nochildren: diff --git a/schemas/packages.xsd b/schemas/packages.xsd index fc5a1356c..b61b97dde 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -18,6 +18,9 @@ <xsd:enumeration value="apt"/> <xsd:enumeration value="pac"/> <xsd:enumeration value="pkgng"/> + <xsd:enumeration value="portage"/> + <xsd:enumeration value="layman"/> + <xsd:enumeration value="dummy"/> </xsd:restriction> </xsd:simpleType> @@ -225,6 +228,25 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:integer" name="priority"> + <xsd:annotation> + <xsd:documentation> + The priority of the source. This is used to order the + sources. After sorting, the first source, that could + deliver the package, is used. If not supplied the default + priority is 500. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="pin"> + <xsd:annotation> + <xsd:documentation> + Extra information for pinning. This information is used + to differ between the sources. Should be used in the + supported format of apt. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgvars.xsd b/schemas/pkgvars.xsd new file mode 100644 index 000000000..dbd02726d --- /dev/null +++ b/schemas/pkgvars.xsd @@ -0,0 +1,43 @@ +<xsd:schema xmlns:xsd='http://www.w3.org/2001/XMLSchema' + xmlns:py="http://genshi.edgewall.org/"> + + <xsd:annotation> + <xsd:documentation> + XML-Schema-Definition für PkgVars/*.xml + Alexander Sulfrian + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name='pkgVarType'> + <xsd:attribute type='xsd:string' name='name'/> + + <xsd:attribute type='xsd:string' name='pin'/> + <xsd:attribute type='xsd:string' name='use'/> + <xsd:attribute type='xsd:string' name='keywords'/> + + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name='containerType'> + <xsd:choice maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string' use='required'/> + <xsd:attribute name='negate' type='xsd:boolean'/> + </xsd:complexType> + + <xsd:complexType name='pkgVarsType'> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + </xsd:complexType> + + <xsd:element name='PkgVars' type='pkgVarsType'/> +</xsd:schema> diff --git a/schemas/types.xsd b/schemas/types.xsd index 0a55f6355..5165d186b 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -28,6 +28,7 @@ <xsd:enumeration value='ebuild' /> <xsd:enumeration value='yum' /> <xsd:enumeration value='freebsdpkg' /> + <xsd:enumeration value='dummy' /> </xsd:restriction> </xsd:simpleType> diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 5a86e8cd4..d2fcaa23d 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -35,6 +35,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.debsums = '%s/bin/debsums' % Bcfg2.Options.setup.apt_install_path self.aptget = '%s/bin/apt-get' % Bcfg2.Options.setup.apt_install_path self.dpkg = '%s/bin/dpkg' % Bcfg2.Options.setup.apt_install_path + self.aptmark = '%s/bin/apt-mark' % Bcfg2.Options.setup.apt_install_path self.__execs__ = [self.debsums, self.aptget, self.dpkg] path_entries = os.environ['PATH'].split(':') @@ -87,6 +88,23 @@ class APT(Bcfg2.Client.Tools.Tool): except apt.cache.FetchFailedException: err = sys.exc_info()[1] self.logger.info("Failed to update APT cache: %s" % err) + # mark dependencies as being automatically installed and vice versa + mark = [] + unmark = [] + try: + installed_pkgs = [p.name for p in self.pkg_cache if p.is_installed] + except AttributeError: + installed_pkgs = [p.name for p in self.pkg_cache if p.isInstalled] + for pkg in self.getSupportedEntries(): + if pkg.get('name') in installed_pkgs: + if pkg.get('origin') == 'Packages': + mark.append(pkg.get('name')) + else: + unmark.append(pkg.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) + if unmark: + self.cmd.run("%s unmarkauto %s" % (self.aptmark, (" ".join(unmark)))) self.pkg_cache = apt.cache.Cache() def FindExtra(self): @@ -166,13 +184,15 @@ class APT(Bcfg2.Client.Tools.Tool): pkg = self.pkg_cache[pkgname] installed_version = pkg.installed.version - if entry.get('version') == 'auto': + if entry.get('version').startswith('auto'): if pkg.is_upgradable: desired_version = pkg.candidate.version else: desired_version = installed_version - elif entry.get('version') == 'any': + entry.set('version', "auto: %s" % desired_version) + elif entry.get('version').startswith('any'): desired_version = installed_version + entry.set('version', "any: %s" % desired_version) else: desired_version = entry.get('version') if desired_version != installed_version: @@ -215,7 +235,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT has no information about package %s" % pkgname) continue - if pkg.get('version') in ['auto', 'any']: + if any([pkg.get('version').startswith(v) for v in ['auto', 'any']]): try: ipkgs.append("%s=%s" % ( pkgname, @@ -241,11 +261,16 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT command failed") self.pkg_cache = apt.cache.Cache() self.extra = self.FindExtra() + mark = [] states = dict() for package in packages: states[package] = self.VerifyPackage(package, [], checksums=False) if states[package]: self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) return states def VerifyPath(self, entry, _): # pylint: disable=W0613 diff --git a/src/lib/Bcfg2/Client/Tools/Dummy.py b/src/lib/Bcfg2/Client/Tools/Dummy.py new file mode 100644 index 000000000..9a96eb904 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Dummy.py @@ -0,0 +1,16 @@ +"""This is the Bcfg2 tool for the Dummy package system.""" + +import re +import Bcfg2.Client.Tools + + +class Dummy(Bcfg2.Client.Tools.PkgTool): + __handles__ = [('Package', 'dummy')] + __req__ = {'Package': []} + pkgtype = 'dummy' + + def RefreshPackages(self): + pass + + def VerifyPackage(self, _entry, _): + return True diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 32d4252c6..0fad2460b 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -10,6 +10,7 @@ import fnmatch import logging import argparse import tempfile +import copy import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Client import XML @@ -949,9 +950,10 @@ class Client(object): if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: - item.set('qtext', '') - container.append(item) - item.text = None + new_item = copy.deepcopy(item) + new_item.set('qtext', '') + container.append(new_item) + new_item.text = None timeinfo = XML.Element("OpStamps") feedback.append(stats) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c294e6be5..b5818b515 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -877,6 +877,7 @@ if HAS_DJANGO: Django management system """ command = None args = [] + kwargs = {} def run(self, _): '''Call a django command''' @@ -885,7 +886,7 @@ if HAS_DJANGO: else: command = self.__class__.__name__.lower() args = [command] + self.args - management.call_command(*args) + management.call_command(*args, **self.kwargs) class DBShell(_DjangoProxyCmd): """ Call the Django 'dbshell' command on the database """ @@ -893,6 +894,11 @@ if HAS_DJANGO: class Shell(_DjangoProxyCmd): """ Call the Django 'shell' command on the database """ + class Schemamigration(_DjangoProxyCmd): + """ Call the South 'schemamigration' command on the database """ + args = ['Bcfg2.Reporting'] + kwargs = {'auto': True} + class ValidateDB(_DjangoProxyCmd): """ Call the Django 'validate' command on the database """ command = "validate" diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index d6f18afbb..146f18b0c 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -61,7 +61,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "GroupLogic/groups.xml": "grouplogic.xsd" + "GroupLogic/groups.xml": "grouplogic.xsd", + "PkgVars/*.xml": "pkgvars.xsd" } self.filelists = {} diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 5cfc8998c..245cfc256 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -456,7 +456,9 @@ class DirectoryBacked(Debuggable): # again without having to add a new monitor. elif os.path.isdir(abspath): # Deal with events for directories - if action in ['exists', 'created']: + if os.path.exists(os.path.join(abspath, '.bcfg2-ignore')): + self.logger.debug("Ignoring directory %s" % abspath) + elif action in ['exists', 'created']: self.add_directory_monitor(relpath) elif action == 'changed': if relpath in self.entries: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 004e27874..e0d6e1fc3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package, recs=None): + def get_deps(self, package, recs=None, pinnings=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -297,16 +297,35 @@ class Collection(list, Debuggable): :param package: The name of the symbol, but see :ref:`pkg-objects` :type package: string + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: list of strings, but see :ref:`pkg-objects` """ recommended = None if recs and package in recs: recommended = recs[package] + pin_found = False + pin_source = None + if pinnings and package in pinnings: + pin_source = pinnings[package] + for source in self: + if pin_source and source.name not in pin_source: + continue + pin_found = True + if source.is_package(self.metadata, package): return source.get_deps(self.metadata, package, recommended) + if not pin_found: + if pin_source: + self.logger.error("Packages: Source '%s' for package '%s' not found" % + (' or '.join(pin_source), package)) + else: + self.logger.error("Packages: No source found for package '%s'" % + package); + return [] def get_essential(self): @@ -471,12 +490,14 @@ class Collection(list, Debuggable): @track_statistics() def complete(self, packagelist, # pylint: disable=R0912,R0914 - recommended=None): + recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the specification. :type packagelist: set of strings, but see :ref:`pkg-objects` + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: tuple of sets - The first element contains a set of strings (but see :ref:`pkg-objects`) describing the complete package list, and the second element is a @@ -535,7 +556,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current, recommended) + deps = self.get_deps(current, recommended, pinnings) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py b/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py new file mode 100644 index 000000000..f47b8f22c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py @@ -0,0 +1,35 @@ +""" Dummy backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class DummyCollection(Collection): + """ Handle collections of Dummy sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class DummySource(Source): + """ Handle Dummy sources """ + + #: DummySource sets the ``type`` on Package entries to "dummy" + ptype = 'dummy' + + def __init__(self, basepath, xsource): + xsource.set('rawurl', 'http://example.com/') + Source.__init__(self, basepath, xsource) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py new file mode 100644 index 000000000..19877d32b --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py @@ -0,0 +1,142 @@ +import os +import layman +import Bcfg2.Server.Plugin + +class LaymanSource(Bcfg2.Server.Plugin.Debuggable): + basegroups = ['portage', 'gentoo', 'emerge'] + ptype = 'layman' + cclass = 'Portage' + + def __init__(self, basepath, xsource): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self.basepath = basepath + self.xsource = xsource + + self.url = xsource.get('url', 'http://www.gentoo.org/proj/en/overlays/repositories.xml') + self.name = xsource.get('name', '') + self.priority = xsource.get('priority', 0) + self.cachefile = None + self.gpgkeys = [] + self.recommended = False + self.essentialpkgs = set() + self.arches = [item.text for item in xsource.findall('Arch')] + + self.url_map = [dict(version=None, component=None, arch=arch, + url=self.url, baseurl=self.url) for arch in self.arches] + + #: A list of the the names of packages that are blacklisted + #: from this source + self.blacklist = [item.text for item in xsource.findall('Blacklist')] + + #: A list of the the names of packages that are whitelisted in + #: this source + self.whitelist = [item.text for item in xsource.findall('Whitelist')] + + + # configure layman + base = os.path.join(basepath, 'layman') + storage = os.path.join(base, 'overlays') + config = layman.config.OptionConfig(options = { + 'storage': os.path.join(base, 'overlays'), + 'cache': os.path.join(base, 'cache'), + 'installed': os.path.join(base, 'installed.xml'), + 'local_list': os.path.join(base, 'overlays.xml'), + 'overlays': [self.url] + }) + self.api = layman.LaymanAPI(config) + + # path + self.dir = os.path.join(storage, self.name) + + # build the set of conditions to see if this source applies to + # a given set of metadata + self.conditions = [] + self.groups = [] # provided for some limited backwards compat + for el in xsource.iterancestors(): + if el.tag == "Group": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") not in m.groups) + else: + self.groups.append(el.get("name")) + self.conditions.append(lambda m, el=el: + el.get("name") in m.groups) + elif el.tag == "Client": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") != m.hostname) + else: + self.conditions.append(lambda m, el=el: + el.get("name") == m.hostname) + + def get_repo_name(self, url_map): + return self.name + + def save_state(self): + pass + + def load_state(self): + pass + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + return self.url + urls = property(get_urls) + + def setup_data(self, force_update=False): + self.api.fetch_remote_list() + if not self.api.is_repo(self.name): + self.logger.error("Packages: Layman overlay '%s' not" + " found" % self.name) + return False + + if not self.api.is_installed(self.name): + self.logger.info("Packages: Adding layman overlay '%s'" % + self.name) + if not self.api.add_repos(self.name): + self.logger.error("Packages: Failed adding layman" + " overlay '%s'" % self.name) + return False + + if force_update: + if not self.api.sync(self.name): + self.logger.error("Packages: Failed syncing layman" + " overlay '%s'" % self.name) + return False + + return True + + + def applies(self, metadata): + """ Return true if this source applies to the given client, + i.e., the client is in all necessary groups. + + :param metadata: The client metadata to check to see if this + source applies + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: bool + """ + # check arch groups + if not self.arch_groups_match(metadata): + return False + + # check Group/Client tags from sources.xml + for condition in self.conditions: + if not condition(metadata): + return False + + return True + + def arch_groups_match(self, metadata): + """ Returns True if the client is in an arch group that + matches the arch of this source. + + :returns: bool + """ + for arch in self.arches: + if arch in metadata.groups: + return True + return False diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 1af046ec0..9db521aae 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -106,6 +106,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile): source = self.source_from_xml(xsource) if source is not None: self.entries.append(source) + sorted(self.entries, key=(lambda source: source.priority), + reverse=True) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py new file mode 100644 index 000000000..9df4467e0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py @@ -0,0 +1,333 @@ +import re +import gzip +import sys +import os +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Layman import LaymanSource + +_portage_python = '/usr/lib/portage/pym/' + +def _import_portage(caller): + # generate prefix path + caller.prefix = os.path.join(caller.basepath, 'cache', 'portage') + if not os.path.isdir(caller.prefix): + caller.logger.error("Packages: %s is not a dir. " + "Portage will not work. Please " + "remember to setup a EPREFIX there." % + caller.prefix) + # TODO: automatic EPREFIX setup + raise Exception('Invalid EPREFIX') + + os.environ['PORTAGE_OVERRIDE_EPREFIX'] = caller.prefix + + if not os.path.isdir(_portage_python): + self.logger.error("Packages: %s not found. Have you installed " + "the portage python modules?" % _portage_python) + raise Exception('Portage not found') + + sys.path = sys.path + [_portage_python] + portage = __import__('portage', globals(), locals(), + ['_sets', 'dbapi.porttree' ]) + emerge = __import__('_emerge', globals(), locals(), + ['RootConfig', 'depgraph', 'Package', 'actions']) + + # setup profile + if '_setup_profile' in dir(caller): + caller._setup_profile(portage) + + # fix some default settings + portage.settings.unlock() + portage.settings['PORTAGE_RSYNC_INITIAL_TIMEOUT'] = '0' + portage.settings.lock() + + porttree = portage.db[portage.root]['porttree'] + caller._import_portage(portage, emerge, porttree) + + +class PortageCollection(Collection): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + self.portage = None + self.emerge = None + self.porttree = None + + @property + def cachefiles(self): + return [] + + def complete(self, packagelist, pinnings=None, recommended=None): + if not self.portage: + _import_portage(self) + + # calculate deps + setconfig = self.portage._sets.load_default_config( + self.portage.settings, + self.portage.db[self.portage.root]) + rconfig = self.emerge.RootConfig.RootConfig( + self.portage.settings, + self.portage.db[self.portage.root], + setconfig) + self.portage.db[self.portage.root]['root_config'] = rconfig + + pkgs = ["=" + j.cpv for (i, j) in packagelist if i == 'ok'] + fail = [j for (i, j) in packagelist if i == 'fail'] + + x = self.emerge.depgraph.backtrack_depgraph( + self.portage.settings, + self.portage.db, + {'--pretend': True}, + {'recurse': True}, + 'merge', + pkgs, + None) + + g = x[1]._dynamic_config.digraph + packages = [i for i in g.all_nodes() \ + if isinstance(i, self.emerge.Package.Package)] + + return (set(packages), set(fail)) + + def get_additional_data(self): + return [] + + def get_group(self, group): + self.logger.warning("Packages: Package sets are currently not supported") + return [] + + def packages_from_entry(self, entry): + if not self.portage: + _import_portage(self) + + try: + name = entry.get('name') + pkgs = self.porttree.dep_bestmatch(name) + except self.portage.exception.AmbiguousPackageName as e: + self.logger.error("Packages: AmbiguousPackageName: %s" % e) + pkgs = '' + + if pkgs == '': + return [('fail', name)] + + return [('ok', pkgs)] + + def packages_to_entry(self, pkgs, entry): + for pkg in pkgs: + if pkg.slot != '0': + name = "%s:%s" % (pkg.cp, pkg.slot) + else: + name = pkg.cp + + lxml.etree.SubElement(entry, 'BoundPackage', name=name, + version=Bcfg2.Options.setup.packages_version, + type=self.ptype, origin='Packages') + + def get_new_packages(self, initial, complete): + new = [] + init = [pkg.cp for status, pkg in initial if status == 'ok'] + + for pkg in complete: + if pkg.cp not in init: + new.append(pkg) + + return new + + def setup_data(self, force_update=False): + pass + + def _setup_profile(self, portage): + if 'gentoo-profile' not in self.metadata.Probes: + raise Exception('Unknown profile.') + + profile = os.path.join(self.prefix, 'usr/portage/profiles/', + self.metadata.Probes['gentoo-profile'].strip()) + + env = portage.settings.configdict['backupenv'] + + # add layman overlays + env['PORTDIR_OVERLAY'] = '' + for overlay in self: + if isinstance(overlay, LaymanSource): + env['PORTDIR_OVERLAY'] += ' ' + env['PORTDIR_OVERLAY'] += overlay.dir + + portage.settings = portage.package.ebuild.config.config( + config_root=portage.settings['PORTAGE_CONFIGROOT'], + target_root=portage.settings['ROOT'], + env=env, + eprefix=portage.settings['EPREFIX'], + config_profile_path=profile) + + portage.db[portage.root]['porttree'] \ + = portage.dbapi.porttree.portagetree(portage.settings) + portage.db[portage.root]['vartree'] \ + = portage.dbapi.vartree.vartree(portage.settings) + + def _set_portage_config(self): + # get global use flags + self.portage.settings.unlock() + self.portage.settings['USE'] = '' + if 'gentoo-use-flags' in self.metadata.Probes: + self.portage.settings['USE'] = \ + self.metadata.Probes['gentoo-use-flags'] + + # add package flags (accept_keywords, use) + if hasattr(self.metadata, 'PkgVars'): + for k in self.metadata.PkgVars['keywords']: + keyword = self.metadata.PkgVars['keywords'][k] + self.portage.settings._keywords_manager.pkeywordsdict[k] = \ + {self.portage.dep.Atom(k): tuple(keyword)} + + + for u in self.metadata.PkgVars['use']: + use = self.metadata.PkgVars['use'][u] + self.portage.settings._use_manager._pusedict[u] = \ + {u: tuple(use)} + + self.portage.settings.lock() + + def _import_portage(self, portage, emerge, porttree): + self.portage = portage + self.emerge = emerge + self.porttree = porttree + self._set_portage_config() + + for s in self: + if isinstance(s, PortageSource): + s._import_portage(portage, emerge, porttree) + + +class PortageSource(Bcfg2.Server.Plugin.Debuggable): + basegroups = ['portage', 'gentoo', 'emerge'] + ptype = 'ebuild' + + def __init__(self, basepath, xsource): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self.basepath = basepath + self.xsource = xsource + + self.url = xsource.get('url', '') + self.priority = xsource.get('priority', 0) + self.cachefile = None + self.gpgkeys = [] + self.recommended = False + self.essentialpkgs = set() + self.arches = [item.text for item in xsource.findall('Arch')] + + self.url_map = [dict(version=None, component=None, arch=arch, + url=self.url, baseurl=self.url) for arch in self.arches] + + #: A list of the the names of packages that are blacklisted + #: from this source + self.blacklist = [item.text for item in xsource.findall('Blacklist')] + + #: A list of the the names of packages that are whitelisted in + #: this source + self.whitelist = [item.text for item in xsource.findall('Whitelist')] + + + self.portage = None + self.emerge = None + self.porttree = None + + # build the set of conditions to see if this source applies to + # a given set of metadata + self.conditions = [] + self.groups = [] # provided for some limited backwards compat + for el in xsource.iterancestors(): + if el.tag == "Group": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") not in m.groups) + else: + self.groups.append(el.get("name")) + self.conditions.append(lambda m, el=el: + el.get("name") in m.groups) + elif el.tag == "Client": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") != m.hostname) + else: + self.conditions.append(lambda m, el=el: + el.get("name") == m.hostname) + def get_repo_name(self, url_map): + return "portage" + + def _import_portage(self, portage, emerge, porttree): + self.portage = portage + self.emerge = emerge + self.porttree = porttree + + def save_state(self): + pass + + def load_state(self): + pass + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + return self.url + urls = property(get_urls) + + def setup_data(self, force_update=False): + if not self.porttree: + _import_portage(self) + + timestamp = os.path.join(self.porttree.portroot, 'metadata/timestamp.chk') + if not os.path.isfile(timestamp): + self.logger.error("Packages: Timestamp not found; " + "falling back to sync") + force_update = True + + if force_update: + # update sync url + self.portage.settings.unlock() + self.portage.settings['SYNC'] = self.url + self.portage.settings.lock() + + # realy force the sync + if os.path.isfile(timestamp): + os.unlink(timestamp) + + # sync + self.logger.info("Packages: Syncing with %s" % self.url) + self.emerge.actions.action_sync(self.portage.settings, + self.portage.db, None, + {'--quiet': True}, 'sync') + + def applies(self, metadata): + """ Return true if this source applies to the given client, + i.e., the client is in all necessary groups. + + :param metadata: The client metadata to check to see if this + source applies + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: bool + """ + # check arch groups + if not self.arch_groups_match(metadata): + return False + + # check Group/Client tags from sources.xml + for condition in self.conditions: + if not condition(metadata): + return False + + return True + + def arch_groups_match(self, metadata): + """ Returns True if the client is in an arch group that + matches the arch of this source. + + :returns: bool + """ + for arch in self.arches: + if arch in metadata.groups: + return True + return False diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index c9f6ea14a..12435c2c8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -202,6 +202,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "name" attribute from :attr:`xsource` self.name = xsource.get('name', None) + #: The "priority" attribute from :attr:`xsource` + self.priority = xsource.get('priority', 500) + + #: The "pin" attribute from :attr:`xsource` + self.pin = xsource.get('pin', '') + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -277,7 +283,8 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch, debsrc=self.debsrc) + arch=arch, debsrc=self.debsrc, + priority=self.priority, pin=self.pin) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index dbe3f9ce5..fab06c97d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -853,7 +853,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist, recommended=None): + def complete(self, packagelist, recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -871,7 +871,8 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist, recommended) + return Collection.complete(self, packagelist, recommended, + pinnings) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 3aa5c415f..799c7d8fc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -103,7 +103,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, - default=['Yum', 'Apt', 'Pac', 'Pkgng']), + default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Portage', 'Layman', + 'Dummy']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -361,6 +362,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups = [] recommended = dict() + pinned_src = dict() + if hasattr(metadata, 'PkgVars'): + pinned_src = metadata.PkgVars['pin'] + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): @@ -402,11 +407,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, base.update(collection.get_essential()) # check for this set of packages in the package cache - pkey = hash(tuple(base)) + pkey = hash((tuple(base), tuple(recommended), tuple(pinned_src))) pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base, recommended) + pcache[pkey] = collection.complete(base, recommended, pinned_src) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) @@ -548,21 +553,23 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for source in self.sources.entries: if source.applies(metadata): relevant.append(source) - sclasses.update([source.__class__]) + if 'cclass' in dir(source): + sclasses.update([source.cclass]) + else: + sclass = source.__class__.__name__.replace("Source", "") + sclasses.update([sclass]) if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % - (metadata.hostname, - ",".join([s.__name__ for s in sclasses]))) + (metadata.hostname, ",".join([sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: - cclass = get_collection_class( - sclasses.pop().__name__.replace("Source", "")) + cclass = get_collection_class(sclasses.pop()) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py new file mode 100644 index 000000000..9a2649d02 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py @@ -0,0 +1,65 @@ +import os +import re +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Server.Plugin + +logger = logging.getLogger('Bcfg2.Plugins.PkgVars') +vars = ['pin', 'use', 'keywords'] + +class PkgVarsFile(Bcfg2.Server.Plugin.StructFile): + def get_additional_data(self, meta): + data = self.Match(meta) + results = {} + for d in data: + name = d.get('name', '') + + for v in vars: + value = d.get(v, None) + if value: + if v not in results: + results[v] = {} + if name not in results[v]: + results[v][name] = set() + + results[v][name].add(value) + + return results + +class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + __child__ = PkgVarsFile + patterns = re.compile(r'.*\.xml$') + + def get_additional_data(self, meta): + results = {} + for v in vars: + results[v] = {} + + for files in self.entries: + new = self.entries[files].get_additional_data(meta) + for x in vars: + if x in new: + results[x].update(new[x]) + + return results + +class PkgVars(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + name = 'PkgVars' + version = '$Revision$' + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + self.store = PkgVarsDirectoryBacked(self.data) + except OSError: + e = sys.exc_info()[1] + self.logger.error("Error while creating PkgVars store: %s %s" % + (e.strerror, e.filename)) + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, meta): + return self.store.get_additional_data(meta) |