diff options
Diffstat (limited to 'src/lib/Bcfg2')
20 files changed, 186 insertions, 106 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index ee4ef35af..fba946bfb 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -24,8 +24,8 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): '''Verify Package status for entry''' - self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), - entry.get('version'))) + self.logger.debug("VerifyPackage: %s : %s" % (entry.get('name'), + entry.get('version'))) if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % @@ -42,11 +42,10 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True else: entry.set('current_version', self.installed[entry.get('name')]) - self.logger.info("attribname: %s" % (entry.attrib['name'])) - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False entry.set('current_exists', 'false') - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False def Remove(self, packages): diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 0ba775318..dc4dfb983 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -10,6 +10,7 @@ import fnmatch import logging import argparse import tempfile +import copy import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Client import XML @@ -950,9 +951,10 @@ class Client(object): if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: - item.set('qtext', '') - container.append(item) - item.text = None + new_item = copy.deepcopy(item) + new_item.set('qtext', '') + container.append(new_item) + new_item.text = None timeinfo = XML.Element("OpStamps") feedback.append(stats) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 3b5cbbbd8..6409f8b37 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -65,7 +65,12 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.media', 'django.core.context_processors.request'), DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter'], - TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner') + TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner', + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + }) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 6: settings['MIDDLEWARE_CLASSES'] += \ @@ -85,15 +90,6 @@ elif HAS_SOUTH: if 'BCFG2_LEGACY_MODELS' in os.environ: settings['INSTALLED_APPS'] += ('Bcfg2.Server.Reports.reports',) -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: - settings['CACHE_BACKEND'] = 'locmem:///' -else: - settings['CACHES'] = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } - } - def finalize_django_config(opts=None, silent=False): """ Perform final Django configuration """ diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index f6ecfd263..2e9ec888a 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,9 +1,5 @@ {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 6809dcc2d..4258fc11d 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,9 +1,6 @@ {% extends "base.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 130d58ede..56594554c 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index eba83670b..1eaa3ca9b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 03918aad7..2379cfe6a 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,8 +1,5 @@ {% extends "base.html" %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 91f37d7dc..1445182be 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,6 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Common Problems{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index 8a5d93690..160d9c738 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Entry Status{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 2e2fd36fa..91c368bd7 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,11 +1,7 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Details{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 0e4812e85..1ae82dab5 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Listing{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index ffafd52e0..53f504c15 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index 8ac5e49bb..2d24fc1c2 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py new file mode 100644 index 000000000..da4c73745 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py @@ -0,0 +1,14 @@ +from django.template import Library + +try: + from django.templatetags.future import url as django_url +except ImportError: + # future is removed in django 1.9 + from django.template.defaulttags import url as django_url + +register = Library() + + +@register.tag +def url(parser, token): + return django_url(parser, token) diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 0b8ed65cc..7d60e724a 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -186,7 +186,7 @@ def config_item_list(request, item_state, timestamp=None, **kwargs): lists = [] for etype in ENTRY_TYPES: ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ - .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') + .annotate(num_entries=Count('id')).select_related() if len(ldata) > 0: # Property doesn't render properly.. lists.append((etype.ENTRY_TYPE, ldata)) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c32b1989b..a387d53c6 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -1228,7 +1228,7 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7: # this has been introduced in django 1.7, so pylint fails with # older django releases django.setup() # pylint: disable=E1101 diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 2637fadfe..5bcc482af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -6,6 +6,15 @@ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def strip_suffix(pkgname): + """ Remove the ':any' suffix from a dependency name if it is present. + """ + if pkgname.endswith(':any'): + return pkgname[:-4] + else: + return pkgname + + class AptCollection(Collection): """ Handle collections of APT sources. This is a no-op object that simply inherits from @@ -115,6 +124,7 @@ class AptSource(Source): cdeps = [re.sub(r'\s+', '', re.sub(r'\(.*\)', '', cdep)) for cdep in dep.split('|')] + cdeps = [strip_suffix(cdep) for cdep in cdeps] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, vindex) @@ -128,6 +138,7 @@ class AptSource(Source): else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() + raw_dep = strip_suffix(raw_dep) if words[0] == 'Recommends': brecs[barch][pkgname].append(raw_dep) else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 0e15d2e15..2661adf67 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -1,10 +1,62 @@ """ Pacman backend for :mod:`Bcfg2.Server.Plugins.Packages` """ +import os import tarfile +from Bcfg2.Compat import cPickle from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def parse_db_file(pkgfile): + """ Parse a Pacman database file, returning a dictionary with + section headings for keys and lists of strings for values. + (Reference: ``sync_db_read`` in ``lib/libalpm/be_sync.c``) + """ + + pkg = {} + section = None + + for line in pkgfile: + line = line.strip() + + if section is not None: + if not line: + section = None + else: + pkg[section].append(line) + elif len(line) >= 2 and line[0] == line[-1] == '%': + section = line + pkg[section] = [] + + return pkg + + +def parse_dep(dep): + """ Parse a Pacman dependency string, returning the package name, + version restriction (or ``None``), and description (or ``None``). + (Reference: ``alpm_dep_from_string`` in ``lib/libalpm/deps.c``) + """ + + rest_desc = dep.split(': ', 1) + if len(rest_desc) == 1: + rest, desc = rest_desc[0], None + else: + rest, desc = rest_desc + + # Search for '=' last, since '<=' and '>=' are possible. + for symb in ['<', '>', '=']: + idx = rest.find(symb) + if idx >= 0: + name = rest[:idx] + version = rest[idx:] + break + else: + name = rest + version = None + + return name, version, desc + + class PacCollection(Collection): """ Handle collections of Pacman sources. This is a no-op object that simply inherits from @@ -24,6 +76,10 @@ class PacCollection(Collection): debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + @property + def __package_groups__(self): + return True + class PacSource(Source): """ Handle Pacman sources """ @@ -31,6 +87,25 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + def __init__(self, basepath, xsource): + self.pacgroups = {} + + Source.__init__(self, basepath, xsource) + __init__.__doc__ = Source.__init__.__doc__ + + def load_state(self): + data = open(self.cachefile, 'rb') + (self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups) = cPickle.load(data) + load_state.__doc__ = Source.load_state.__doc__ + + def save_state(self): + cache = open(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups), cache, 2) + cache.close() + save_state.__doc__ = Source.save_state.__doc__ + @property def urls(self): """ A list of URLs to the base metadata file for each @@ -45,13 +120,10 @@ class PacSource(Source): else: raise Exception("PacSource : RAWUrl not supported (yet)") - def read_files(self): - bdeps = dict() - bprov = dict() - - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') + def read_files(self): # pylint: disable=R0912 + bdeps = {} + brecs = {} + bprov = {} for fname in self.files: if not self.rawurl: @@ -62,8 +134,9 @@ class PacSource(Source): barch = self.arches[0] if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() + bdeps[barch] = {} + brecs[barch] = {} + bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) tar = tarfile.open(fname, "r") @@ -71,11 +144,52 @@ class PacSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise + packages = {} for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - self.debug_log("Packages: added %s" % - tarinfo.name.rsplit("-", 2)[0]) + if not tarinfo.isfile(): + continue + prefix = os.path.dirname(tarinfo.name) + if prefix not in packages: + packages[prefix] = {} + pkg = parse_db_file(tar.extractfile(tarinfo)) + packages[prefix].update(pkg) + + for pkg in packages.values(): + pkgname = pkg['%NAME%'][0] + self.pkgnames.add(pkgname) + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if '%DEPENDS%' in pkg: + for dep in pkg['%DEPENDS%']: + dname = parse_dep(dep)[0] + bdeps[barch][pkgname].append(dname) + + if '%OPTDEPENDS%' in pkg: + for dep in pkg['%OPTDEPENDS%']: + dname = parse_dep(dep)[0] + brecs[barch][pkgname].append(dname) + + if '%PROVIDES%' in pkg: + for dep in pkg['%PROVIDES%']: + dname = parse_dep(dep)[0] + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + if '%GROUPS%' in pkg: + for group in pkg['%GROUPS%']: + if group not in self.pacgroups: + self.pacgroups[group] = [] + self.pacgroups[group].append(pkgname) + tar.close() - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ + + def get_group(self, metadata, group, ptype=None): + try: + return self.pacgroups[group] + except KeyError: + return [] + get_group.__doc__ = Source.get_group.__doc__ diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7f28fd0d8..9c0153c74 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -4,44 +4,20 @@ import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages -LOGGER = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger(__name__) MODELS = [] INTERNAL_DATABASE_VERSION = None -def _get_all_plugins(): - rv = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if module == 'Reporting': - # Exclude Reporting plugin. The reporting database - # is handled separately in Bcfg2.Reporting. - continue - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - rv.append(module) - return rv - - -_ALL_PLUGINS = _get_all_plugins() - - class _OptionContainer(object): + """Options for Bcfg2 database models.""" + # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - options = [ - Bcfg2.Options.Option( - cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, - default=_ALL_PLUGINS, dest="models_plugins", - action=Bcfg2.Options.PluginsAction)] + options = [Bcfg2.Options.Common.plugins] @staticmethod def options_parsed_hook(): @@ -63,7 +39,7 @@ def load_models(plugins=None): global MODELS if not plugins: - plugins = Bcfg2.Options.setup.models_plugins + plugins = Bcfg2.Options.setup.plugins if MODELS: # load_models() has been called once, so first unload all of |