diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 4 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py | 5 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 119 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/POSIXCompat.py | 5 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 85 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 23 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Reporting.py | 5 |
7 files changed, 210 insertions, 36 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 5f10879be..b3781e299 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -77,6 +77,10 @@ class CfgGenshiGenerator(CfgGenerator): __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): + if self.template is None: + raise PluginExecutionError("Failed to load template %s" % + self.name) + fname = entry.get('realname', entry.get('name')) stream = self.template.generate( name=fname, diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e0794f019..e6ef50fa1 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -9,7 +9,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) self.logger.error("DBStats has been replaced with Reporting") - self.logger.error("DBStats: Be sure to migrate your data "\ - "before running the report collector") + self.logger.error("DBStats: Be sure to migrate your data " + "before running the report collector") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b053e65d3..7f8db7b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -138,7 +138,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): self.logger.error('Failed to parse %s' % self.basefile) return self.extras = [] - self.basedata = copy.copy(xdata) + self.basedata = copy.deepcopy(xdata) self._follow_xincludes(xdata=xdata) if self.extras: try: @@ -263,31 +263,63 @@ class ClientMetadata(object): # pylint: disable=R0913 def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): + #: The client hostname (as a string) self.hostname = client + + #: The client profile (as a string) self.profile = profile + + #: The set of all bundles this client gets self.bundles = bundles + + #: A list of all client aliases self.aliases = aliases + + #: A list of all addresses this client is known by self.addresses = addresses + + #: A list of groups this client is a member of self.groups = groups + + #: A dict of categories of this client's groups. Keys are + #: category names, values are corresponding group names. self.categories = categories + + #: The UUID identifier for this client self.uuid = uuid + + #: The Bcfg2 password for this client self.password = password + + #: Connector plugins known to this client self.connectors = [] + + #: The version of the Bcfg2 client this client is running, as + #: a string self.version = version try: + #: The version of the Bcfg2 client this client is running, + #: as a :class:`Bcfg2.version.Bcfg2VersionInfo` object. self.version_info = Bcfg2VersionInfo(version) except (ValueError, AttributeError): self.version_info = None + + #: A :class:`Bcfg2.Server.Plugins.Metadata.MetadataQuery` + #: object for this client. self.query = query # pylint: enable=R0913 def inGroup(self, group): - """Test to see if client is a member of group.""" + """Test to see if client is a member of group. + + :returns: bool """ return group in self.groups def group_in_category(self, category): - """ return the group in the given category that the client is - a member of, or the empty string """ + """ Return the group in the given category that the client is + a member of, or an empty string. + + :returns: string """ for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp @@ -295,17 +327,59 @@ class ClientMetadata(object): class MetadataQuery(object): - """ object supplied to client metadata to allow client metadata - objects to query metadata without being able to modify it """ + """ This class provides query methods for the metadata of all + clients known to the Bcfg2 server, without being able to modify + that data. + + Note that ``*by_groups()`` and ``*by_profiles()`` behave + differently; for a client to be included in the return value of a + ``*by_groups()`` method, it must be a member of *all* groups + listed in the argument; for a client to be included in the return + value of a ``*by_profiles()`` method, it must have *any* group + listed as its profile group. """ def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): - # resolver is set later + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + #: object for the given hostname. + #: + #: :returns: Bcfg2.Server.Plugins.Metadata.ClientMetadata self.by_name = by_name + + #: Get a list of hostnames of clients that are in all given + #: groups. + #: + #: :param groups: The groups to check clients for membership in + #: :type groups: list + #: + #: :returns: list of strings self.names_by_groups = self._warn_string(by_groups) + + #: Get a list of hostnames of clients whose profile matches + #: any given profile group. + #: + #: :param profiles: The profiles to check clients for + #: membership in. + #: :type profiles: list + #: :returns: list of strings self.names_by_profiles = self._warn_string(by_profiles) + + #: Get all known client hostnames. + #: + #: :returns: list of strings self.all_clients = get_clients + + #: Get all known group names. + #: + #: :returns: list of strings self.all_groups = all_groups + + #: Get the names of all groups in the given category. + #: + #: :param category: The category to query for groups that + #: belong to it. + #: :type category: string + #: :returns: list of strings self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): @@ -326,22 +400,41 @@ class MetadataQuery(object): return inner def by_groups(self, groups): - """ get a list of ClientMetadata objects that are in all given - groups """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that are in all given groups. + + :param groups: The groups to check clients for membership in. + :type groups: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): - """ get a list of ClientMetadata objects that are in any of - the given profiles """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that have any of the given groups as their profile. + + :param profiles: The profiles to check clients for membership + in. + :type profiles: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_profiles is decorated return [self.by_name(name) for name in self.names_by_profiles(profiles)] def all(self): - """ get a list of all ClientMetadata objects """ + """ Get a list of all + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects. + + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ return [self.by_name(name) for name in self.all_clients()] @@ -1255,7 +1348,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup('authentication')) == 'bootstrap': + self.core.setup['authentication']) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 0dd42c9cb..490ee6f20 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -15,6 +15,11 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" + if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + # do not care about a client that is _any_ 1.3.0 release + # (including prereleases and RCs) + return + for goal in goals: for entry in goal.getchildren(): if entry.tag == 'Path' and 'mode' in entry.keys(): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 3799b1723..4535fb76d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -313,9 +313,7 @@ class YumCollection(Collection): @property def __package_groups__(self): - """ YumCollections support package groups only if - :attr:`use_yum` is True """ - return self.use_yum + return True @property def helper(self): @@ -663,11 +661,6 @@ class YumCollection(Collection): In this implementation the packages may be strings or tuples. See :ref:`yum-pkg-objects` for more information. """ - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported " - "by Bcfg2's internal Yum dependency generator") - return dict() - if not grouplist: return dict() @@ -679,7 +672,15 @@ class YumCollection(Collection): ptype = "default" gdicts.append(dict(group=group, type=ptype)) - return self.call_helper("get_groups", inputdata=gdicts) + if self.use_yum: + return self.call_helper("get_groups", inputdata=gdicts) + else: + pkgs = dict() + for gdict in gdicts: + pkgs[gdict['group']] = Collection.get_group(self, + gdict['group'], + gdict['type']) + return pkgs def _element_to_pkg(self, el, name): """ Convert a Package or Instance element to a package tuple """ @@ -975,6 +976,7 @@ class YumSource(Source): for x in ['global'] + self.arches]) self.needed_paths = set() self.file_to_arch = dict() + self.yumgroups = dict() __init__.__doc__ = Source.__init__.__doc__ @property @@ -992,7 +994,8 @@ class YumSource(Source): if not self.use_yum: cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) + self.filemap, self.url_map, + self.yumgroups), cache, 2) cache.close() def load_state(self): @@ -1002,7 +1005,7 @@ class YumSource(Source): if not self.use_yum: data = open(self.cachefile) (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) + self.filemap, self.url_map, self.yumgroups) = cPickle.load(data) @property def urls(self): @@ -1057,7 +1060,7 @@ class YumSource(Source): urls = [] for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: + if elt.get('type') in ['filelists', 'primary', 'group']: floc = elt.find(RPO + 'location') fullurl = url + floc.get('href') urls.append(fullurl) @@ -1074,11 +1077,14 @@ class YumSource(Source): # we have to read primary.xml first, and filelists.xml afterwards; primaries = list() filelists = list() + groups = list() for fname in self.files: if fname.endswith('primary.xml.gz'): primaries.append(fname) elif fname.endswith('filelists.xml.gz'): filelists.append(fname) + elif fname.find('comps'): + groups.append(fname) for fname in primaries: farch = self.file_to_arch[fname] @@ -1088,6 +1094,9 @@ class YumSource(Source): farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) + for fname in groups: + fdata = lxml.etree.parse(fname).getroot() + self.parse_group(fdata) # merge data sdata = list(self.packages.values()) @@ -1151,6 +1160,35 @@ class YumSource(Source): self.provides[arch][prov] = list() self.provides[arch][prov].append(pkgname) + @Bcfg2.Server.Plugin.track_statistics() + def parse_group(self, data): + """ parse comps.xml.gz data """ + for group in data.getchildren(): + if not group.tag.endswith('group'): + continue + try: + groupid = group.xpath('id')[0].text + self.yumgroups[groupid] = {'mandatory': list(), + 'default': list(), + 'optional': list(), + 'conditional': list()} + except IndexError: + continue + try: + packagelist = group.xpath('packagelist')[0] + except IndexError: + continue + for pkgreq in packagelist.getchildren(): + pkgtype = pkgreq.get('type', None) + if pkgtype == 'mandatory': + self.yumgroups[groupid]['mandatory'].append(pkgreq.text) + elif pkgtype == 'default': + self.yumgroups[groupid]['default'].append(pkgreq.text) + elif pkgtype == 'optional': + self.yumgroups[groupid]['optional'].append(pkgreq.text) + elif pkgtype == 'conditional': + self.yumgroups[groupid]['conditional'].append(pkgreq.text) + def is_package(self, metadata, package): arch = [a for a in self.arches if a in metadata.groups] if not arch: @@ -1230,3 +1268,26 @@ class YumSource(Source): return self.pulp_id else: return Source.get_repo_name(self, url_map) + + def get_group(self, metadata, group, ptype=None): # pylint: disable=W0613 + """ Get the list of packages of the given type in a package + group. + + :param group: The name of the group to query + :type group: string + :param ptype: The type of packages to get, for backends that + support multiple package types in package groups + (e.g., "recommended," "optional," etc.) + :type ptype: string + :returns: list of strings - package names + """ + try: + yumgroup = self.yumgroups[group] + except KeyError: + return [] + packages = yumgroup['conditional'] + yumgroup['mandatory'] + if ptype in ['default', 'optional', 'all']: + packages += yumgroup['default'] + if ptype in ['optional', 'all']: + packages += yumgroup['optional'] + return packages diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5d3fbae2e..2175cf0aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -5,6 +5,7 @@ determine the completeness of the client configuration. """ import os import sys import glob +import copy import shutil import lxml.etree import Bcfg2.Logger @@ -296,20 +297,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if self.disableResolver: # Config requests no resolver + for struct in structures: + for pkg in struct.xpath('//Package | //BoundPackage'): + if pkg.get("group"): + if pkg.get("type"): + pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) - # base is the set of initial packages -- explicitly - # given in the specification, from expanded package groups, - # and essential to the distribution - base = set() + initial = set() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - base.update(collection.packages_from_entry(pkg)) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -321,6 +324,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pkg, xml_declaration=False).decode('UTF-8')) + # base is the set of initial packages explicitly given in the + # specification, packages from expanded package groups, and + # packages essential to the distribution + base = set(initial) + # remove package groups for el in to_remove: el.getparent().remove(el) @@ -336,7 +344,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = collection.get_new_packages(base, packages) + newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() @@ -514,7 +522,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data()) + return dict(sources=collection.get_additional_data(), + allsources=copy.deepcopy(self.sources)) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index d072f1a33..a6dc2c1ef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -65,10 +65,13 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): (self.name, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise PluginInitError(msg) + if self.debug_flag: + self.transport.set_debug(self.debug_flag) def set_debug(self, debug): rv = Debuggable.set_debug(self, debug) - self.transport.set_debug(debug) + if self.transport is not None: + self.transport.set_debug(debug) return rv def process_statistics(self, client, xdata): |