diff options
29 files changed, 287 insertions, 179 deletions
diff --git a/debian/control b/debian/control index f131bbe05..8d642037a 100644 --- a/debian/control +++ b/debian/control @@ -35,7 +35,7 @@ Package: bcfg2-server Architecture: all Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon, python-genshi (>= 0.4.4) Recommends: graphviz, patch -Suggests: python-cheetah, python-profiler, python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) +Suggests: python-cheetah, python-profiler, python-django (>= 1.3), mail-transport-agent, bcfg2-doc (= ${binary:Version}) Description: Configuration management server Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. @@ -44,7 +44,7 @@ Description: Configuration management server Package: bcfg2-web Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, python-django-south (>= 0.7.5) +Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django (>= 1.3), python-django-south (>= 0.7.5) Suggests: python-mysqldb, python-psycopg2, python-sqlite, libapache2-mod-wsgi Description: Configuration management web interface Bcfg2 is a configuration management system that generates configuration sets diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index d89d44894..d45599955 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -72,7 +72,7 @@ reporting, such as Apache + mod_wsgi or nginx. +-------------------------------+----------+--------------------------------+ | Software | Version | Requires | +===============================+==========+================================+ -| django | 1.2.0+ | | +| django | 1.3.0+ | | +-------------------------------+----------+--------------------------------+ | south | 0.7.5+ | | +-------------------------------+----------+--------------------------------+ diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index eea6c6659..5e14d3be5 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -483,6 +483,59 @@ See :ref:`configuration` for more details on these options. .. _native-yum-libraries: +Package Groups +============== + +Some packaging systems provide package groups. To include a package +group, use the :xml:attribute:`PackageStructure:group` attribute of +the :xml:element:`Package` tag. + +pac +--- + +.. versionadded:: 1.4.0 + +Pacman `groups <https://www.archlinux.org/groups/>`_ are supported: + +.. code-block:: xml + + <Package group="base"/> + +yum +--- + +Yum package groups are supported by both the native Yum libraries and +Bcfg2's internal dependency resolver. You can use either the short +group ID or the long group name: + +.. code-block:: xml + + <Package group="SNMP Support"/> + <Package group="system-management-snmp"/> + +By default, only those packages considered the "default" packages in a +group will be installed. You can change this behavior using the +:xml:attribute:`PackageStructure:type` attribute: + +.. code-block:: xml + + <Package group="development" type="optional"/> + <Package group="Administration Tools" type="mandatory"/> + +Valid values of "type" are: + +* ``mandatory``: Only install mandatory packages in the group. +* ``default``: Install default packages from the group (the default). +* ``optional`` or ``all``: Install all packages in the group, + including mandatory, default, and optional packages. + +See :xml:type:`PackageStructure` for details. + +You can view the packages in a group by category with the ``yum +groupinfo`` command. More information about the different levels can +be found at +http://fedoraproject.org/wiki/How_to_use_and_edit_comps.xml_for_package_groups#Installation + Using Native Yum Libraries ========================== @@ -546,43 +599,6 @@ generally be overridden: * ``reposdir`` is set to ``/dev/null`` to prevent the server's Yum configuration from being read; do not change this. -Package Groups --------------- - -Yum package groups are supported by both the native Yum libraries and -Bcfg2's internal dependency resolver. To include a package group, use -the :xml:attribute:`PackageStructure:group` attribute of the -:xml:element:`Package` tag. You can use either the short group ID or -the long group name: - -.. code-block:: xml - - <Package group="SNMP Support"/> - <Package group="system-management-snmp"/> - -By default, only those packages considered the "default" packages in a -group will be installed. You can change this behavior using the -:xml:attribute:`PackageStructure:type` attribute: - -.. code-block:: xml - - <Package group="development" type="optional"/> - <Package group="Administration Tools" type="mandatory"/> - -Valid values of "type" are: - -* ``mandatory``: Only install mandatory packages in the group. -* ``default``: Install default packages from the group (the default). -* ``optional`` or ``all``: Install all packages in the group, - including mandatory, default, and optional packages. - -See :xml:type:`PackageStructure` for details. - -You can view the packages in a group by category with the ``yum -groupinfo`` command. More information about the different levels can -be found at -http://fedoraproject.org/wiki/How_to_use_and_edit_comps.xml_for_package_groups#Installation - Abstract Package Tags --------------------- diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 5db928c3d..91260c317 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -85,9 +85,9 @@ BuildRequires: mock BuildRequires: m2crypto # EPEL uses the properly-named python-django starting with EPEL7 %if 0%{?rhel} && 0%{?rhel} > 6 -BuildRequires: python-django +BuildRequires: python-django >= 1.3 %else -BuildRequires: Django +BuildRequires: Django >= 1.3 %endif BuildRequires: python-genshi BuildRequires: python-cheetah @@ -303,15 +303,15 @@ Requires: bcfg2-server = %{version}-%{release} Requires: httpd %if 0%{?suse_version} Group: System/Management -Requires: python-django >= 1.2 +Requires: python-django >= 1.3 Requires: python-django-south >= 0.7 %else Group: System Tools # EPEL uses the properly-named python-django starting with EPEL7 %if 0%{?rhel} && 0%{?rhel} > 6 -Requires: python-django +Requires: python-django > 1.3 %else -Requires: Django >= 1.2 +Requires: Django >= 1.3 Requires: Django-south >= 0.7 %endif Requires: bcfg2-server diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 7ad7606b2..993114a46 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -31,10 +31,9 @@ <xsd:annotation> <xsd:documentation> Install the named package group. Package groups are only - supported for Yum :xml:element:`Source` repositories, and - only if the :ref:`yum libraries - <native-yum-libraries>` are in use. Either ``group`` - or :xml:attribute:`PackageStructure:name` must be specified. + supported for Pac and Yum :xml:element:`Source` + repositories. Either ``group`` or + :xml:attribute:`PackageStructure:name` must be specified. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index ee4ef35af..fba946bfb 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -24,8 +24,8 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): '''Verify Package status for entry''' - self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), - entry.get('version'))) + self.logger.debug("VerifyPackage: %s : %s" % (entry.get('name'), + entry.get('version'))) if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % @@ -42,11 +42,10 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True else: entry.set('current_version', self.installed[entry.get('name')]) - self.logger.info("attribname: %s" % (entry.attrib['name'])) - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False entry.set('current_exists', 'false') - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False def Remove(self, packages): diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 0ba775318..dc4dfb983 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -10,6 +10,7 @@ import fnmatch import logging import argparse import tempfile +import copy import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Client import XML @@ -950,9 +951,10 @@ class Client(object): if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: - item.set('qtext', '') - container.append(item) - item.text = None + new_item = copy.deepcopy(item) + new_item.set('qtext', '') + container.append(new_item) + new_item.text = None timeinfo = XML.Element("OpStamps") feedback.append(stats) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 3b5cbbbd8..6409f8b37 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -65,7 +65,12 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.media', 'django.core.context_processors.request'), DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter'], - TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner') + TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner', + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + }) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 6: settings['MIDDLEWARE_CLASSES'] += \ @@ -85,15 +90,6 @@ elif HAS_SOUTH: if 'BCFG2_LEGACY_MODELS' in os.environ: settings['INSTALLED_APPS'] += ('Bcfg2.Server.Reports.reports',) -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: - settings['CACHE_BACKEND'] = 'locmem:///' -else: - settings['CACHES'] = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } - } - def finalize_django_config(opts=None, silent=False): """ Perform final Django configuration """ diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index f6ecfd263..2e9ec888a 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,9 +1,5 @@ {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 6809dcc2d..4258fc11d 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,9 +1,6 @@ {% extends "base.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 130d58ede..56594554c 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index eba83670b..1eaa3ca9b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 03918aad7..2379cfe6a 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,8 +1,5 @@ {% extends "base.html" %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 91f37d7dc..1445182be 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,6 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Common Problems{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index 8a5d93690..160d9c738 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Entry Status{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 2e2fd36fa..91c368bd7 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,11 +1,7 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Details{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 0e4812e85..1ae82dab5 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Listing{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index ffafd52e0..53f504c15 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index 8ac5e49bb..2d24fc1c2 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py new file mode 100644 index 000000000..da4c73745 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py @@ -0,0 +1,14 @@ +from django.template import Library + +try: + from django.templatetags.future import url as django_url +except ImportError: + # future is removed in django 1.9 + from django.template.defaulttags import url as django_url + +register = Library() + + +@register.tag +def url(parser, token): + return django_url(parser, token) diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 0b8ed65cc..7d60e724a 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -186,7 +186,7 @@ def config_item_list(request, item_state, timestamp=None, **kwargs): lists = [] for etype in ENTRY_TYPES: ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ - .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') + .annotate(num_entries=Count('id')).select_related() if len(ldata) > 0: # Property doesn't render properly.. lists.append((etype.ENTRY_TYPE, ldata)) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c32b1989b..a387d53c6 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -1228,7 +1228,7 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7: # this has been introduced in django 1.7, so pylint fails with # older django releases django.setup() # pylint: disable=E1101 diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 2637fadfe..5bcc482af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -6,6 +6,15 @@ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def strip_suffix(pkgname): + """ Remove the ':any' suffix from a dependency name if it is present. + """ + if pkgname.endswith(':any'): + return pkgname[:-4] + else: + return pkgname + + class AptCollection(Collection): """ Handle collections of APT sources. This is a no-op object that simply inherits from @@ -115,6 +124,7 @@ class AptSource(Source): cdeps = [re.sub(r'\s+', '', re.sub(r'\(.*\)', '', cdep)) for cdep in dep.split('|')] + cdeps = [strip_suffix(cdep) for cdep in cdeps] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, vindex) @@ -128,6 +138,7 @@ class AptSource(Source): else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() + raw_dep = strip_suffix(raw_dep) if words[0] == 'Recommends': brecs[barch][pkgname].append(raw_dep) else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 0e15d2e15..2661adf67 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -1,10 +1,62 @@ """ Pacman backend for :mod:`Bcfg2.Server.Plugins.Packages` """ +import os import tarfile +from Bcfg2.Compat import cPickle from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def parse_db_file(pkgfile): + """ Parse a Pacman database file, returning a dictionary with + section headings for keys and lists of strings for values. + (Reference: ``sync_db_read`` in ``lib/libalpm/be_sync.c``) + """ + + pkg = {} + section = None + + for line in pkgfile: + line = line.strip() + + if section is not None: + if not line: + section = None + else: + pkg[section].append(line) + elif len(line) >= 2 and line[0] == line[-1] == '%': + section = line + pkg[section] = [] + + return pkg + + +def parse_dep(dep): + """ Parse a Pacman dependency string, returning the package name, + version restriction (or ``None``), and description (or ``None``). + (Reference: ``alpm_dep_from_string`` in ``lib/libalpm/deps.c``) + """ + + rest_desc = dep.split(': ', 1) + if len(rest_desc) == 1: + rest, desc = rest_desc[0], None + else: + rest, desc = rest_desc + + # Search for '=' last, since '<=' and '>=' are possible. + for symb in ['<', '>', '=']: + idx = rest.find(symb) + if idx >= 0: + name = rest[:idx] + version = rest[idx:] + break + else: + name = rest + version = None + + return name, version, desc + + class PacCollection(Collection): """ Handle collections of Pacman sources. This is a no-op object that simply inherits from @@ -24,6 +76,10 @@ class PacCollection(Collection): debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + @property + def __package_groups__(self): + return True + class PacSource(Source): """ Handle Pacman sources """ @@ -31,6 +87,25 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + def __init__(self, basepath, xsource): + self.pacgroups = {} + + Source.__init__(self, basepath, xsource) + __init__.__doc__ = Source.__init__.__doc__ + + def load_state(self): + data = open(self.cachefile, 'rb') + (self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups) = cPickle.load(data) + load_state.__doc__ = Source.load_state.__doc__ + + def save_state(self): + cache = open(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups), cache, 2) + cache.close() + save_state.__doc__ = Source.save_state.__doc__ + @property def urls(self): """ A list of URLs to the base metadata file for each @@ -45,13 +120,10 @@ class PacSource(Source): else: raise Exception("PacSource : RAWUrl not supported (yet)") - def read_files(self): - bdeps = dict() - bprov = dict() - - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') + def read_files(self): # pylint: disable=R0912 + bdeps = {} + brecs = {} + bprov = {} for fname in self.files: if not self.rawurl: @@ -62,8 +134,9 @@ class PacSource(Source): barch = self.arches[0] if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() + bdeps[barch] = {} + brecs[barch] = {} + bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) tar = tarfile.open(fname, "r") @@ -71,11 +144,52 @@ class PacSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise + packages = {} for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - self.debug_log("Packages: added %s" % - tarinfo.name.rsplit("-", 2)[0]) + if not tarinfo.isfile(): + continue + prefix = os.path.dirname(tarinfo.name) + if prefix not in packages: + packages[prefix] = {} + pkg = parse_db_file(tar.extractfile(tarinfo)) + packages[prefix].update(pkg) + + for pkg in packages.values(): + pkgname = pkg['%NAME%'][0] + self.pkgnames.add(pkgname) + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if '%DEPENDS%' in pkg: + for dep in pkg['%DEPENDS%']: + dname = parse_dep(dep)[0] + bdeps[barch][pkgname].append(dname) + + if '%OPTDEPENDS%' in pkg: + for dep in pkg['%OPTDEPENDS%']: + dname = parse_dep(dep)[0] + brecs[barch][pkgname].append(dname) + + if '%PROVIDES%' in pkg: + for dep in pkg['%PROVIDES%']: + dname = parse_dep(dep)[0] + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + if '%GROUPS%' in pkg: + for group in pkg['%GROUPS%']: + if group not in self.pacgroups: + self.pacgroups[group] = [] + self.pacgroups[group].append(pkgname) + tar.close() - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ + + def get_group(self, metadata, group, ptype=None): + try: + return self.pacgroups[group] + except KeyError: + return [] + get_group.__doc__ = Source.get_group.__doc__ diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7f28fd0d8..9c0153c74 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -4,44 +4,20 @@ import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages -LOGGER = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger(__name__) MODELS = [] INTERNAL_DATABASE_VERSION = None -def _get_all_plugins(): - rv = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if module == 'Reporting': - # Exclude Reporting plugin. The reporting database - # is handled separately in Bcfg2.Reporting. - continue - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - rv.append(module) - return rv - - -_ALL_PLUGINS = _get_all_plugins() - - class _OptionContainer(object): + """Options for Bcfg2 database models.""" + # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - options = [ - Bcfg2.Options.Option( - cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, - default=_ALL_PLUGINS, dest="models_plugins", - action=Bcfg2.Options.PluginsAction)] + options = [Bcfg2.Options.Common.plugins] @staticmethod def options_parsed_hook(): @@ -63,7 +39,7 @@ def load_models(plugins=None): global MODELS if not plugins: - plugins = Bcfg2.Options.setup.models_plugins + plugins = Bcfg2.Options.setup.plugins if MODELS: # load_models() has been called once, so first unload all of diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 5a82100d0..9f6a9f320 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -51,6 +51,7 @@ class TestFunctions(Bcfg2TestCase): class TestDatabaseBacked(TestPlugin): test_obj = DatabaseBacked + synced = False def setUp(self): TestPlugin.setUp(self) @@ -76,6 +77,15 @@ class TestDatabaseBacked(TestPlugin): setattr(Bcfg2.Options.setup, attr, True) self.assertRaises(PluginInitError, self.get_obj, core) + def syncdb(self, modeltest): + """ Given an instance of a :class:`DBModelTestCase` object, sync + and clean the database """ + inst = modeltest(methodName='test_syncdb') + if not self.__class__.synced: + inst.test_syncdb() + self.__class__.synced = True + inst.test_cleandb() + class TestPluginDatabaseModel(Bcfg2TestCase): """ placeholder for future tests """ diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index f2721c9ea..5d7ed50b7 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -123,7 +123,7 @@ class TestClientVersions(TestDatabaseBacked): def setUp(self): TestDatabaseBacked.setUp(self) self.test_obj = ClientVersions - syncdb(TestMetadataDB) + self.syncdb(TestMetadataDB) for client, version in self.test_clients.items(): MetadataClientModel(hostname=client, version=version).save() @@ -1252,7 +1252,7 @@ class TestMetadataBase(TestMetadata): TestClientRunHooks.setUp(self) TestDatabaseBacked.setUp(self) Bcfg2.Options.setup.metadata_db = True - syncdb(TestMetadataDB) + self.syncdb(TestMetadataDB) def load_clients_data(self, metadata=None, xdata=None): if metadata is None: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 32766b5c1..9729a0449 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -201,7 +201,7 @@ group-specific""" assert False, "Strange probe found in get_probe_data() return" -class TestProbes(TestPlugin): +class TestProbes(TestDatabaseBacked): test_obj = Probes test_xdata = lxml.etree.Element("test") @@ -241,7 +241,7 @@ group: group:with:colons self.datastore = None Bcfg2.Options.setup.repository = datastore - def get_obj(self): + def get_obj(self, core=None): if not Bcfg2.Options.setup.probes_db: # actually use a real datastore so we can read and write # probed.xml @@ -251,7 +251,7 @@ group: group:with:colons datadir = os.path.join(self.datastore, self.test_obj.name) if not os.path.exists(datadir): os.makedirs(datadir) - return TestPlugin.get_obj(self) + return TestPlugin.get_obj(self, core) def test__init(self): if Bcfg2.Options.setup.probes_db: @@ -278,7 +278,7 @@ group: group:with:colons def test_probes_db(self): """ Set and retrieve probe data with database enabled """ Bcfg2.Options.setup.probes_db = True - syncdb(TestProbesDB) + self.syncdb(TestProbesDB) self._perform_tests() def test_allowed_cgroups(self): diff --git a/testsuite/common.py b/testsuite/common.py index 9db2cb94a..944471ade 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -223,18 +223,29 @@ class DBModelTestCase(Bcfg2TestCase): import django.core.management from django.core.exceptions import ImproperlyConfigured - if django.VERSION[0] == 1 and django.VERSION[1] < 7: - try: - django.core.management.call_command('syncdb', interactive=False, - verbosity=0) - except ImproperlyConfigured: - pass + dbfile = django.conf.settings.DATABASES['default']['NAME'] + # Close all connections to the old database + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + for connection in django.db.connections.all(): + connection.close() + else: + django.db.close_connection() + + # Remove old database + if os.path.exists(dbfile): + os.unlink(dbfile) + self.assertFalse(os.path.exists(dbfile)) + + # Create new + if django.VERSION[0] == 1 and django.VERSION[1] < 7: + django.core.management.call_command('syncdb', interactive=False, + verbosity=1) django.core.management.call_command('migrate', interactive=False, - verbosity=0) - self.assertTrue( - os.path.exists( - django.conf.settings.DATABASES['default']['NAME'])) + verbosity=1) + + # Check if database exists now + self.assertTrue(os.path.exists(dbfile)) @skipUnless(has_django, "Django not found, skipping") def test_cleandb(self): @@ -245,14 +256,6 @@ class DBModelTestCase(Bcfg2TestCase): self.assertItemsEqual(list(model.objects.all()), []) -def syncdb(modeltest): - """ Given an instance of a :class:`DBModelTestCase` object, sync - and clean the database """ - inst = modeltest(methodName='test_syncdb') - inst.test_syncdb() - inst.test_cleandb() - - # in order for patchIf() to decorate a function in the same way as # patch(), we override the default behavior of __enter__ and __exit__ # on the _patch() object to basically be noops. |