diff options
Diffstat (limited to 'src/lib')
32 files changed, 365 insertions, 230 deletions
diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index f1caa383a..f383911a3 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -1,3 +1,4 @@ +import os.path import re import sys import time @@ -202,6 +203,8 @@ class SSLHTTPConnection(httplib.HTTPConnection): raise Exception("unknown protocol %s" % self.protocol) if self.ca: other_side_required = ssl.CERT_REQUIRED + if not os.path.isfile(self.ca): + self.logger.error("CA specified but none found at %s" % self.ca) else: other_side_required = ssl.CERT_NONE self.logger.warning("No ca is specified. Cannot authenticate the " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index fc4e16904..f4f1ee4bf 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -10,10 +10,10 @@ from Bcfg2.Client.Tools.POSIX.File import POSIXFile class AugeasCommand(object): """ Base class for all Augeas command objects """ - def __init__(self, command, augeas_obj, logger): + def __init__(self, entry, command, augeas_obj, logger): self._augeas = augeas_obj self.command = command - self.entry = self.command.getparent() + self.entry = entry self.logger = logger def get_path(self, attr="path"): @@ -115,8 +115,8 @@ class Remove(AugeasCommand): class Move(AugeasCommand): """ Augeas ``move`` command """ - def __init__(self, command, augeas_obj, logger): - AugeasCommand.__init__(self, command, augeas_obj, logger) + def __init__(self, entry, command, augeas_obj, logger): + AugeasCommand.__init__(self, entry, command, augeas_obj, logger) self.source = self.get_path("source") self.dest = self.get_path("destination") @@ -131,8 +131,8 @@ class Move(AugeasCommand): class Set(AugeasCommand): """ Augeas ``set`` command """ - def __init__(self, command, augeas_obj, logger): - AugeasCommand.__init__(self, command, augeas_obj, logger) + def __init__(self, entry, command, augeas_obj, logger): + AugeasCommand.__init__(self, entry, command, augeas_obj, logger) self.value = self.command.get("value") def verify(self): @@ -146,15 +146,15 @@ class Set(AugeasCommand): class Clear(Set): """ Augeas ``clear`` command """ - def __init__(self, command, augeas_obj, logger): - Set.__init__(self, command, augeas_obj, logger) + def __init__(self, entry, command, augeas_obj, logger): + Set.__init__(self, entry, command, augeas_obj, logger) self.value = None class SetMulti(AugeasCommand): """ Augeas ``setm`` command """ - def __init__(self, command, augeas_obj, logger): - AugeasCommand.__init__(self, command, augeas_obj, logger) + def __init__(self, entry, command, augeas_obj, logger): + AugeasCommand.__init__(self, entry, command, augeas_obj, logger) self.sub = self.command.get("sub") self.value = self.command.get("value") self.base = self.get_path("base") @@ -170,8 +170,8 @@ class SetMulti(AugeasCommand): class Insert(AugeasCommand): """ Augeas ``ins`` command """ - def __init__(self, command, augeas_obj, logger): - AugeasCommand.__init__(self, command, augeas_obj, logger) + def __init__(self, entry, command, augeas_obj, logger): + AugeasCommand.__init__(self, entry, command, augeas_obj, logger) self.label = self.command.get("label") self.where = self.command.get("where", "before") self.before = self.where == "before" @@ -230,11 +230,12 @@ class POSIXAugeas(POSIXTool): objects representing the commands. """ rv = [] - for cmd in entry.iterchildren(): + for cmd in entry: if cmd.tag == "Initial": continue if cmd.tag in globals(): - rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), + rv.append(globals()[cmd.tag](entry, cmd, + self.get_augeas(entry), self.logger)) else: err = "Augeas: Unknown command %s in %s" % (cmd.tag, diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py index 6237ccce2..e90ecd384 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -1,4 +1,4 @@ -""" Handle <Path type='nonexistent' ...> entries """ +""" Handle <Path type='device' ...> entries """ import os import sys @@ -6,7 +6,7 @@ from Bcfg2.Client.Tools.POSIX.base import POSIXTool, device_map class POSIXDevice(POSIXTool): - """ Handle <Path type='nonexistent' ...> entries """ + """ Handle <Path type='device' ...> entries """ __req__ = ['name', 'dev_type', 'mode', 'owner', 'group'] def fully_specified(self, entry): diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index ee4ef35af..fba946bfb 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -24,8 +24,8 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): '''Verify Package status for entry''' - self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), - entry.get('version'))) + self.logger.debug("VerifyPackage: %s : %s" % (entry.get('name'), + entry.get('version'))) if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % @@ -42,11 +42,10 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True else: entry.set('current_version', self.installed[entry.get('name')]) - self.logger.info("attribname: %s" % (entry.attrib['name'])) - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False entry.set('current_exists', 'false') - self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.debug("attribname: %s" % (entry.attrib['name'])) return False def Remove(self, packages): diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 0ba775318..dc4dfb983 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -10,6 +10,7 @@ import fnmatch import logging import argparse import tempfile +import copy import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Client import XML @@ -950,9 +951,10 @@ class Client(object): if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: - item.set('qtext', '') - container.append(item) - item.text = None + new_item = copy.deepcopy(item) + new_item.set('qtext', '') + container.append(new_item) + new_item.text = None timeinfo = XML.Element("OpStamps") feedback.append(stats) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 3b5cbbbd8..254dfa4b8 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -65,7 +65,12 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.media', 'django.core.context_processors.request'), DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter'], - TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner') + TEST_RUNNER='django.test.simple.DjangoTestSuiteRunner', + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + }) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 6: settings['MIDDLEWARE_CLASSES'] += \ @@ -85,15 +90,6 @@ elif HAS_SOUTH: if 'BCFG2_LEGACY_MODELS' in os.environ: settings['INSTALLED_APPS'] += ('Bcfg2.Server.Reports.reports',) -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: - settings['CACHE_BACKEND'] = 'locmem:///' -else: - settings['CACHES'] = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } - } - def finalize_django_config(opts=None, silent=False): """ Perform final Django configuration """ @@ -147,6 +143,8 @@ def finalize_django_config(opts=None, silent=False): setattr(module, name, value) try: django.conf.settings.configure(**settings) + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + django.setup() # pylint: disable=E1101 except RuntimeError: if not silent: logger.warning("Failed to finalize Django settings: %s" % @@ -208,7 +206,6 @@ def migrate_databases(**kwargs): for database in settings['DATABASES']: logger.debug("Migrating database %s" % (database)) if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() # pylint: disable=E1101 if initial_django_migration(database): logger.warning( "No applied django migrations found for database %s. " diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index b72a495f1..af1af976b 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -17,7 +17,7 @@ __all__ = ["setup", "OptionParserException", "Parser", "get_parser", #: circular imports. repository = PathOption( # pylint: disable=C0103 '-Q', '--repository', cf=('server', 'repository'), - default='var/lib/bcfg2', help="Server repository path") + default='/var/lib/bcfg2', help="Server repository path") #: A module-level :class:`argparse.Namespace` object that stores all diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index 7ba0265ae..e60b2e82e 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -327,8 +327,6 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() # pylint: disable=E1101 def run(self): """ Run bcfg2-reports """ diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index ac0cde783..347e5cc81 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -2,26 +2,37 @@ The base for the original DjangoORM (DBStats) """ -from lxml import etree -from datetime import datetime +import difflib import traceback +from datetime import datetime from time import strptime -import Bcfg2.Options -import Bcfg2.DBSettings -from Bcfg2.Compat import md5 -from Bcfg2.Reporting.Storage.base import StorageBase, StorageError -from Bcfg2.Server.Plugin.exceptions import PluginExecutionError +from lxml import etree + import django -from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache -#Used by GetCurrentEntry -import difflib -from Bcfg2.Compat import b64decode -from Bcfg2.Reporting.models import * +import Bcfg2.Options +import Bcfg2.DBSettings +from Bcfg2.Compat import b64decode, md5 from Bcfg2.Reporting.Compat import transaction +from Bcfg2.Reporting.Storage.base import StorageBase, StorageError +from Bcfg2.Server.Plugin.exceptions import PluginExecutionError + + +def load_django_models(): + """ Load models for Django after option parsing has completed """ + # pylint: disable=W0602 + global Interaction, PackageEntry, FilePerms, PathEntry, LinkEntry, \ + Group, Client, Bundle, TYPE_EXTRA, TYPE_BAD, TYPE_MODIFIED, \ + FailureEntry, Performance, BaseEntry + # pylint: enable=W0602 + + from Bcfg2.Reporting.models import \ + Interaction, PackageEntry, FilePerms, PathEntry, LinkEntry, \ + Group, Client, Bundle, TYPE_EXTRA, TYPE_BAD, TYPE_MODIFIED, \ + FailureEntry, Performance, BaseEntry def get_all_field_names(model): @@ -42,6 +53,7 @@ class DjangoORM(StorageBase): type=Bcfg2.Options.Types.size, help='Reporting file size limit', default=1024 * 1024)] + options_parsed_hook = staticmethod(load_django_models) def _import_default(self, entry, state, entrytype=None, defaults=None, mapping=None, boolean=None, xforms=None): diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index f6ecfd263..2e9ec888a 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,9 +1,5 @@ {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 6809dcc2d..4258fc11d 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,9 +1,6 @@ {% extends "base.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 130d58ede..56594554c 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index eba83670b..1eaa3ca9b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 03918aad7..2379cfe6a 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,8 +1,5 @@ {% extends "base.html" %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block extra_header_info %} {% endblock%} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 91f37d7dc..1445182be 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,6 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Common Problems{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index 8a5d93690..160d9c738 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Entry Status{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 2e2fd36fa..91c368bd7 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,11 +1,7 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} - +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Details{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 0e4812e85..1ae82dab5 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Element Listing{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index ffafd52e0..53f504c15 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index 8ac5e49bb..2d24fc1c2 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,9 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} -{% comment %} -This is needed for Django versions less than 1.5 -{% endcomment %} -{% load url from future %} +{% load url from bcfg2_compat %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py new file mode 100644 index 000000000..da4c73745 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_compat.py @@ -0,0 +1,14 @@ +from django.template import Library + +try: + from django.templatetags.future import url as django_url +except ImportError: + # future is removed in django 1.9 + from django.template.defaulttags import url as django_url + +register = Library() + + +@register.tag +def url(parser, token): + return django_url(parser, token) diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 0b8ed65cc..7d60e724a 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -186,7 +186,7 @@ def config_item_list(request, item_state, timestamp=None, **kwargs): lists = [] for etype in ENTRY_TYPES: ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ - .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') + .annotate(num_entries=Count('id')).select_related() if len(ldata) > 0: # Property doesn't render properly.. lists.append((etype.ENTRY_TYPE, ldata)) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c32b1989b..77bca88eb 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -666,7 +666,7 @@ bcfg2 = %s def create_key(self): """Creates a bcfg2.key at the directory specifed by keypath.""" cmd = Executor(timeout=120) - subject = "/C=%s/ST=%s/L=%s/CN=%s'" % ( + subject = "/C=%s/ST=%s/L=%s/CN=%s" % ( self.data['country'], self.data['state'], self.data['location'], self.data['shostname']) key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", @@ -1228,10 +1228,6 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - # this has been introduced in django 1.7, so pylint fails with - # older django releases - django.setup() # pylint: disable=E1101 def run(self): """ Run bcfg2-admin """ diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index ebf4c4954..56b4e7477 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -47,70 +47,118 @@ def is_device_mode(val): return re.match(r'^\d+$', val) +def is_vcs_type(val): + """ Return True if val is a supported vcs type handled by the + current client tool """ + return (val != 'Path' and + hasattr(Bcfg2.Client.Tools.VCS.VCS, 'Install%s' % val)) + + class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): """ Verify attributes for configuration entries that cannot be verified with an XML schema alone. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.required_attrs = dict( - Path=dict( - device=dict(name=is_filename, - owner=is_username, - group=is_username, - dev_type=lambda v: v in device_map), - directory=dict(name=is_filename, owner=is_username, - group=is_username, mode=is_octal_mode), - file=dict(name=is_filename, owner=is_username, - group=is_username, mode=is_octal_mode, - __text__=None), - hardlink=dict(name=is_filename, to=is_filename), - symlink=dict(name=is_filename), - ignore=dict(name=is_filename), - nonexistent=dict(name=is_filename), - permissions=dict(name=is_filename, owner=is_username, - group=is_username, mode=is_octal_mode), - vcs=dict(vcstype=lambda v: (v != 'Path' and - hasattr(Bcfg2.Client.Tools.VCS.VCS, - "Install%s" % v)), - revision=None, sourceurl=None)), - Service={"__any__": dict(name=None), - "smf": dict(name=None, FMRI=None)}, - Action={None: dict(name=None, - timing=lambda v: v in ['pre', 'post', 'both'], - when=lambda v: v in ['modified', 'always'], - status=lambda v: v in ['ignore', 'check'], - command=None)}, - ACL=dict( - default=dict(scope=lambda v: v in ['user', 'group'], - perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', - v)), - access=dict(scope=lambda v: v in ['user', 'group'], - perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', - v)), - mask=dict(perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', - v))), - Package={"__any__": dict(name=None)}, - SEBoolean={None: dict(name=None, - value=lambda v: v in ['on', 'off'])}, - SEModule={None: dict(name=None, __text__=None)}, - SEPort={ - None: dict(name=lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)', - v), - selinuxtype=is_selinux_type)}, - SEFcontext={None: dict(name=None, selinuxtype=is_selinux_type)}, - SENode={None: dict(name=lambda v: "/" in v, - selinuxtype=is_selinux_type, - proto=lambda v: v in ['ipv6', 'ipv4'])}, - SELogin={None: dict(name=is_username, - selinuxuser=is_selinux_user)}, - SEUser={None: dict(name=is_selinux_user, - roles=lambda v: all(is_selinux_user(u) - for u in " ".split(v)), - prefix=None)}, - SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)}, - SEPermissive={None: dict(name=is_selinux_type)}, - POSIXGroup={None: dict(name=is_username)}, - POSIXUser={None: dict(name=is_username)}) + self.required_attrs = { + 'Path': { + '__any__': {'name': is_filename}, + 'augeas': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode}, + 'device': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode, + 'dev_type': lambda v: v in device_map}, + 'directory': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode}, + 'file': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode, '__text__': None}, + 'hardlink': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode, 'to': is_filename}, + 'symlink': {}, + 'ignore': {}, + 'nonexistent': {}, + 'permissions': {'owner': is_username, 'group': is_username, + 'mode': is_octal_mode}, + 'vcs': {'vcstype': is_vcs_type, 'revision': None, + 'sourceurl': None}, + }, + 'Service': { + '__any__': {'name': None}, + 'smf': {'name': None, 'FMRI': None} + }, + 'Action': { + None: { + 'name': None, + 'timing': lambda v: v in ['pre', 'post', 'both'], + 'when': lambda v: v in ['modified', 'always'], + 'status': lambda v: v in ['ignore', 'check'], + 'command': None, + }, + }, + 'ACL': { + 'default': { + 'scope': lambda v: v in ['user', 'group'], + 'perms': lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v), + }, + 'access': { + 'scope': lambda v: v in ['user', 'group'], + 'perms': lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v), + }, + 'mask': { + 'perms': lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v), + }, + }, + 'Package': { + '__any__': {'name': None}, + }, + 'SEBoolean': { + None: { + 'name': None, + 'value': lambda v: v in ['on', 'off'], + }, + }, + 'SEModule': { + None: {'name': None, '__text__': None}, + }, + 'SEPort': { + None: { + 'name': lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)', v), + 'selinuxtype': is_selinux_type, + }, + }, + 'SEFcontext': { + None: {'name': None, 'selinuxtype': is_selinux_type}, + }, + 'SENode': { + None: { + 'name': lambda v: "/" in v, + 'selinuxtype': is_selinux_type, + 'proto': lambda v: v in ['ipv6', 'ipv4'] + }, + }, + 'SELogin': { + None: {'name': is_username, 'selinuxuser': is_selinux_user}, + }, + 'SEUser': { + None: { + 'name': is_selinux_user, + 'roles': lambda v: all(is_selinux_user(u) + for u in " ".split(v)), + 'prefix': None, + }, + }, + 'SEInterface': { + None: {'name': None, 'selinuxtype': is_selinux_type}, + }, + 'SEPermissive': { + None: {'name': is_selinux_type}, + }, + 'POSIXGroup': { + None: {'name': is_username}, + }, + 'POSIXUser': { + None: {'name': is_username}, + }, + } def Run(self): self.check_packages() diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py index a158302be..241bce34c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py @@ -19,8 +19,8 @@ class CfgSSLCAKeyCreator(XMLCfgCreator): self.logger.info("Cfg: Generating new SSL key for %s" % self.name) spec = self.XMLMatch(metadata) key = spec.find("Key") - if not key: - key = dict() + if key is None: + key = {} ktype = key.get('type', 'rsa') bits = key.get('bits', '2048') if ktype == 'rsa': diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 40504e15e..b912d3725 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -21,30 +21,27 @@ from Bcfg2.Compat import MutableMapping, all, any, wraps # pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo +try: + from django.db import models + HAS_DJANGO = True +except ImportError: + HAS_DJANGO = False + # pylint: disable=C0103 ClientVersions = None MetadataClientModel = None # pylint: enable=C0103 -HAS_DJANGO = False def load_django_models(): """ Load models for Django after option parsing has completed """ # pylint: disable=W0602 - global MetadataClientModel, ClientVersions, HAS_DJANGO + global MetadataClientModel, ClientVersions # pylint: enable=W0602 - try: - import django - from django.db import models - HAS_DJANGO = True - except ImportError: - HAS_DJANGO = False + if not HAS_DJANGO: return - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() # pylint: disable=E1101 - class MetadataClientModel(models.Model, # pylint: disable=W0621 Bcfg2.Server.Plugin.PluginDatabaseModel): """ django model for storing clients in the database """ diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index d3c38ef19..067e2faad 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -92,17 +92,15 @@ class NagiosGen(Plugin, Generator): for host in host_configs: host_data.append(open(host, 'r').read()) - group_list = [] + used_groups = set(['default']) for line in "\n".join(host_data).splitlines(): # only include those groups which are actually used if "hostgroup" in line: - group_list += line.split()[1].split(',') - - group_list = list(set(group_list)) + used_groups.update(line.split()[1].split(',')) for group in group_configs: group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) - if group_name in group_list: + if group_name in used_groups: groupfile = open(group, 'r') group_data.append(groupfile.read()) groupfile.close() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 2637fadfe..956cb9f51 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -6,6 +6,15 @@ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def strip_suffix(pkgname): + """ Remove the ':any' suffix from a dependency name if it is present. + """ + if pkgname.endswith(':any'): + return pkgname[:-4] + else: + return pkgname + + class AptCollection(Collection): """ Handle collections of APT sources. This is a no-op object that simply inherits from @@ -77,6 +86,7 @@ class AptSource(Source): bdeps = dict() brecs = dict() bprov = dict() + self.pkgnames = set() self.essentialpkgs = set() for fname in self.files: if not self.rawurl: @@ -115,6 +125,7 @@ class AptSource(Source): cdeps = [re.sub(r'\s+', '', re.sub(r'\(.*\)', '', cdep)) for cdep in dep.split('|')] + cdeps = [strip_suffix(cdep) for cdep in cdeps] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, vindex) @@ -128,6 +139,7 @@ class AptSource(Source): else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() + raw_dep = strip_suffix(raw_dep) if words[0] == 'Recommends': brecs[barch][pkgname].append(raw_dep) else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 0e15d2e15..6fc084cc4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -1,10 +1,62 @@ """ Pacman backend for :mod:`Bcfg2.Server.Plugins.Packages` """ +import os import tarfile +from Bcfg2.Compat import cPickle from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source +def parse_db_file(pkgfile): + """ Parse a Pacman database file, returning a dictionary with + section headings for keys and lists of strings for values. + (Reference: ``sync_db_read`` in ``lib/libalpm/be_sync.c``) + """ + + pkg = {} + section = None + + for line in pkgfile: + line = line.strip() + + if section is not None: + if not line: + section = None + else: + pkg[section].append(line) + elif len(line) >= 2 and line[0] == line[-1] == '%': + section = line + pkg[section] = [] + + return pkg + + +def parse_dep(dep): + """ Parse a Pacman dependency string, returning the package name, + version restriction (or ``None``), and description (or ``None``). + (Reference: ``alpm_dep_from_string`` in ``lib/libalpm/deps.c``) + """ + + rest_desc = dep.split(': ', 1) + if len(rest_desc) == 1: + rest, desc = rest_desc[0], None + else: + rest, desc = rest_desc + + # Search for '=' last, since '<=' and '>=' are possible. + for symb in ['<', '>', '=']: + idx = rest.find(symb) + if idx >= 0: + name = rest[:idx] + version = rest[idx:] + break + else: + name = rest + version = None + + return name, version, desc + + class PacCollection(Collection): """ Handle collections of Pacman sources. This is a no-op object that simply inherits from @@ -24,6 +76,10 @@ class PacCollection(Collection): debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + @property + def __package_groups__(self): + return True + class PacSource(Source): """ Handle Pacman sources """ @@ -31,6 +87,25 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + def __init__(self, basepath, xsource): + self.pacgroups = {} + + Source.__init__(self, basepath, xsource) + __init__.__doc__ = Source.__init__.__doc__ + + def load_state(self): + data = open(self.cachefile, 'rb') + (self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups) = cPickle.load(data) + load_state.__doc__ = Source.load_state.__doc__ + + def save_state(self): + cache = open(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides, + self.recommends, self.pacgroups), cache, 2) + cache.close() + save_state.__doc__ = Source.save_state.__doc__ + @property def urls(self): """ A list of URLs to the base metadata file for each @@ -45,14 +120,12 @@ class PacSource(Source): else: raise Exception("PacSource : RAWUrl not supported (yet)") - def read_files(self): - bdeps = dict() - bprov = dict() - - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') - + def read_files(self): # pylint: disable=R0912 + bdeps = {} + brecs = {} + bprov = {} + self.pkgnames = set() + self.pacgroups = {} for fname in self.files: if not self.rawurl: barch = [x for x in fname.split('@') if x in self.arches][0] @@ -62,8 +135,9 @@ class PacSource(Source): barch = self.arches[0] if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() + bdeps[barch] = {} + brecs[barch] = {} + bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) tar = tarfile.open(fname, "r") @@ -71,11 +145,52 @@ class PacSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise + packages = {} for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - self.debug_log("Packages: added %s" % - tarinfo.name.rsplit("-", 2)[0]) + if not tarinfo.isfile(): + continue + prefix = os.path.dirname(tarinfo.name) + if prefix not in packages: + packages[prefix] = {} + pkg = parse_db_file(tar.extractfile(tarinfo)) + packages[prefix].update(pkg) + + for pkg in packages.values(): + pkgname = pkg['%NAME%'][0] + self.pkgnames.add(pkgname) + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if '%DEPENDS%' in pkg: + for dep in pkg['%DEPENDS%']: + dname = parse_dep(dep)[0] + bdeps[barch][pkgname].append(dname) + + if '%OPTDEPENDS%' in pkg: + for dep in pkg['%OPTDEPENDS%']: + dname = parse_dep(dep)[0] + brecs[barch][pkgname].append(dname) + + if '%PROVIDES%' in pkg: + for dep in pkg['%PROVIDES%']: + dname = parse_dep(dep)[0] + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + if '%GROUPS%' in pkg: + for group in pkg['%GROUPS%']: + if group not in self.pacgroups: + self.pacgroups[group] = [] + self.pacgroups[group].append(pkgname) + tar.close() - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ + + def get_group(self, metadata, group, ptype=None): + try: + return self.pacgroups[group] + except KeyError: + return [] + get_group.__doc__ = Source.get_group.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py index 736cdcdd4..4938efb94 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -56,6 +56,7 @@ class PkgngSource(Source): def read_files(self): bdeps = dict() + self.pkgnames = set() for fname in self.files: if not self.rawurl: abi = [x diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 33b0d4284..ae4ea4cb1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -15,6 +15,12 @@ import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics +try: + from django.db import models + HAS_DJANGO = True +except ImportError: + HAS_DJANGO = False + HAS_DJANGO = False # pylint: disable=C0103 ProbesDataModel = None @@ -25,20 +31,12 @@ ProbesGroupsModel = None def load_django_models(): """ Load models for Django after option parsing has completed """ # pylint: disable=W0602 - global ProbesDataModel, ProbesGroupsModel, HAS_DJANGO + global ProbesDataModel, ProbesGroupsModel # pylint: enable=W0602 - try: - import django - from django.db import models - HAS_DJANGO = True - except ImportError: - HAS_DJANGO = False + if not HAS_DJANGO: return - if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() # pylint: disable=E1101 - class ProbesDataModel(models.Model, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.PluginDatabaseModel): """ The database model for storing probe data """ diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7f28fd0d8..9c0153c74 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -4,44 +4,20 @@ import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages -LOGGER = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger(__name__) MODELS = [] INTERNAL_DATABASE_VERSION = None -def _get_all_plugins(): - rv = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if module == 'Reporting': - # Exclude Reporting plugin. The reporting database - # is handled separately in Bcfg2.Reporting. - continue - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - rv.append(module) - return rv - - -_ALL_PLUGINS = _get_all_plugins() - - class _OptionContainer(object): + """Options for Bcfg2 database models.""" + # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - options = [ - Bcfg2.Options.Option( - cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, - default=_ALL_PLUGINS, dest="models_plugins", - action=Bcfg2.Options.PluginsAction)] + options = [Bcfg2.Options.Common.plugins] @staticmethod def options_parsed_hook(): @@ -63,7 +39,7 @@ def load_models(plugins=None): global MODELS if not plugins: - plugins = Bcfg2.Options.setup.models_plugins + plugins = Bcfg2.Options.setup.plugins if MODELS: # load_models() has been called once, so first unload all of |