From 111193ef96adb711b1b1b4859291c77197eb8ea8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 31 Jul 2012 09:12:01 -0400 Subject: unified Metadata/DBMetadata plugins made django optional --- doc/server/database.txt | 2 +- doc/server/plugins/grouping/dbmetadata.txt | 39 -- doc/server/plugins/grouping/metadata.txt | 42 +- src/lib/Bcfg2/Server/Plugin.py | 21 +- src/lib/Bcfg2/Server/Plugins/DBMetadata.py | 128 ------ src/lib/Bcfg2/Server/Plugins/Metadata.py | 179 ++++++-- src/lib/Bcfg2/Server/Plugins/Probes.py | 34 +- src/lib/Bcfg2/settings.py | 11 +- .../TestServer/TestPlugins/TestDBMetadata.py | 407 ------------------ .../Testlib/TestServer/TestPlugins/TestMetadata.py | 478 +++++++++++++++++++-- .../Testlib/TestServer/TestPlugins/TestProbes.py | 45 +- 11 files changed, 695 insertions(+), 691 deletions(-) delete mode 100644 doc/server/plugins/grouping/dbmetadata.txt delete mode 100644 src/lib/Bcfg2/Server/Plugins/DBMetadata.py delete mode 100644 testsuite/Testlib/TestServer/TestPlugins/TestDBMetadata.py diff --git a/doc/server/database.txt b/doc/server/database.txt index 61d065854..70dc43319 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -9,7 +9,7 @@ Global Database Settings .. versionadded:: 1.3.0 Several Bcfg2 plugins, including -:ref:`server-plugins-grouping-dbmetadata` and +:ref:`server-plugins-grouping-metadata` and :ref:`server-plugins-probes-index`, can connect use a relational database to store data. They use the global database settings in ``bcfg2.conf``, described in this document, to connect. diff --git a/doc/server/plugins/grouping/dbmetadata.txt b/doc/server/plugins/grouping/dbmetadata.txt deleted file mode 100644 index 292367f0c..000000000 --- a/doc/server/plugins/grouping/dbmetadata.txt +++ /dev/null @@ -1,39 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-grouping-dbmetadata: - -========== -DBMetadata -========== - -.. versionadded:: 1.3.0 - -The DBMetadata plugin is an alternative to the -:ref:`server-plugins-grouping-metadata` plugin that stores client -records in a database rather than writing back to ``clients.xml``. -This provides several advantages: - -* ``clients.xml`` will never be written by the server, removing an - area of contention between the user and server. -* ``clients.xml`` can be removed entirely for many sites. -* The Bcfg2 client list can be queried by other machines without - obtaining and parsing ``clients.xml``. -* A single client list can be shared amongst multiple Bcfg2 servers. - -In general, DBMetadata works almost the same as Metadata. -``groups.xml`` is parsed identically. If ``clients.xml`` is present, -it is parsed, but ```` tags in ``clients.xml`` *do not* assert -client existence; they are only used to set client options *if* the -client exists (in the database). That is, the two purposes of -``clients.xml`` -- to track which clients exist, and to set client -options -- have been separated. - -With the improvements in ``groups.xml`` parsing in 1.3, client groups -can now be set directly in ``groups.xml`` with ```` tags. (See -:ref:`metadata-client-tag` for more details.) As a result, -``clients.xml`` is only necessary with DBMetadata if you need to set -options (e.g., aliases, floating clients, per-client passwords, etc.) -on clients. - -DBMetadata uses the :ref:`Global Server Database Settings -` to connect to its database. diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index 5a437756a..88bb0c460 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -107,6 +107,44 @@ but that is deprecated. For detailed information on client authentication see :ref:`appendix-guides-authentication` +================ +Clients Database +================ + +.. versionadded:: 1.3.0 + +It is also possible to store client records in a database rather than +writing back to ``clients.xml``. This provides several advantages: + +* ``clients.xml`` will never be written by the server, removing an + area of contention between the user and server. +* ``clients.xml`` can be removed entirely for many sites. +* The Bcfg2 client list can be queried by other machines without + obtaining and parsing ``clients.xml``. +* A single client list can be shared amongst multiple Bcfg2 servers. + +In general, storing clients in the database works almost the same as +``clients.xml``. ``groups.xml`` is parsed identically. If +``clients.xml`` is present, it is parsed, but ```` tags in +``clients.xml`` *do not* assert client existence; they are only used +to set client options *if* the client exists (in the database). That +is, the two purposes of ``clients.xml`` -- to track which clients +exist, and to set client options -- have been separated. + +With the improvements in ``groups.xml`` parsing in 1.3, client groups +can now be set directly in ``groups.xml`` with ```` tags. (See +:ref:`metadata-client-tag` for more details.) As a result, +``clients.xml`` is only necessary if you need to set +options (e.g., aliases, floating clients, per-client passwords, etc.) +on clients. + +To use the database backend instead of ``clients.xml``, set +``use_database`` in the ``[metadata]`` section of ``bcfg2.conf`` to +``true``. You will also need to configure the :ref:`Global Server +Database Settings `. + +The ``clients.xml``-based model remains the default. + Metadata/groups.xml =================== @@ -161,8 +199,8 @@ profile group would be a member of the ``apache-server``, Client tags in ``groups.xml`` allow you to supplement the profile group declarations in ``clients.xml`` and/or client group assignments with the :ref:`server-plugins-grouping-grouppatterns` plugin. They -should be used sparingly. (They are more useful with the -:ref:`server-plugins-grouping-dbmetadata` plugin.) +should be used sparingly. (They are more useful when you are using +the database backend for client records.) You can also declare that a group should be negated; this allows you to set defaults and override them efficiently. Negation is applied diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 696dacc06..2a68ea3b7 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -11,9 +11,14 @@ import sys import threading import Bcfg2.Server from Bcfg2.Bcfg2Py3k import ConfigParser - import Bcfg2.Options +try: + import django + has_django = True +except ImportError: + has_django = False + # py3k compatibility if sys.hexversion >= 0x03000000: from functools import reduce @@ -106,6 +111,20 @@ class DatabaseBacked(object): def __init__(self): pass + @property + def _use_db(self): + use_db = self.core.setup.cfp.getboolean(self.name.lower(), + "use_database", + default=False) + if use_db and has_django: + return True + elif not use_db: + return False + else: + self.logger.error("use_database is true but django not found") + return False + + class PluginDatabaseModel(object): class Meta: diff --git a/src/lib/Bcfg2/Server/Plugins/DBMetadata.py b/src/lib/Bcfg2/Server/Plugins/DBMetadata.py deleted file mode 100644 index 16a6e0dcc..000000000 --- a/src/lib/Bcfg2/Server/Plugins/DBMetadata.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import sys -from UserDict import DictMixin -from django.db import models -import Bcfg2.Server.Lint -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Metadata import * - -class MetadataClientModel(models.Model, - Bcfg2.Server.Plugin.PluginDatabaseModel): - hostname = models.CharField(max_length=255, primary_key=True) - version = models.CharField(max_length=31, null=True) - - -class ClientVersions(DictMixin): - def __getitem__(self, key): - try: - return MetadataClientModel.objects.get(hostname=key).version - except MetadataClientModel.DoesNotExist: - raise KeyError(key) - - def __setitem__(self, key, value): - client = MetadataClientModel.objects.get_or_create(hostname=key)[0] - client.version = value - client.save() - - def keys(self): - return [c.hostname for c in MetadataClientModel.objects.all()] - - def __contains__(self, key): - try: - client = MetadataClientModel.objects.get(hostname=key) - return True - except MetadataClientModel.DoesNotExist: - return False - - -class DBMetadata(Metadata, Bcfg2.Server.Plugin.DatabaseBacked): - __files__ = ["groups.xml"] - experimental = True - conflicts = ['Metadata'] - - def __init__(self, core, datastore, watch_clients=True): - Metadata.__init__(self, core, datastore, watch_clients=watch_clients) - Bcfg2.Server.Plugin.DatabaseBacked.__init__(self) - if os.path.exists(os.path.join(self.data, "clients.xml")): - self.logger.warning("DBMetadata: clients.xml found, parsing in " - "compatibility mode") - self._handle_file("clients.xml") - self.versions = ClientVersions() - - def add_group(self, group_name, attribs): - msg = "DBMetadata does not support adding groups" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def add_bundle(self, bundle_name): - msg = "DBMetadata does not support adding bundles" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def add_client(self, client_name): - """Add client to clients database.""" - client = MetadataClientModel(hostname=client_name) - client.save() - self.clients = self.list_clients() - return client - - def update_group(self, group_name, attribs): - msg = "DBMetadata does not support updating groups" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def update_bundle(self, bundle_name): - msg = "DBMetadata does not support updating bundles" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def update_client(self, client_name, attribs): - msg = "DBMetadata does not support updating clients" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def list_clients(self): - """ List all clients in client database """ - return set([c.hostname for c in MetadataClientModel.objects.all()]) - - def remove_group(self, group_name, attribs): - msg = "DBMetadata does not support removing groups" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def remove_bundle(self, bundle_name): - msg = "DBMetadata does not support removing bundles" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def remove_client(self, client_name): - """Remove a client""" - try: - client = MetadataClientModel.objects.get(hostname=client_name) - except MetadataClientModel.DoesNotExist: - msg = "Client %s does not exist" % client_name - self.logger.warning(msg) - raise MetadataConsistencyError(msg) - client.delete() - self.clients = self.list_clients() - - def _set_profile(self, client, profile, addresspair): - if client not in self.clients: - # adding a new client - self.add_client(client) - if client not in self.clientgroups: - self.clientgroups[client] = [profile] - else: - msg = "DBMetadata does not support asserting client profiles" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - def _handle_clients_xml_event(self, event): - # clients.xml is parsed and the options specified in it are - # understood, but it does _not_ assert client existence. - Metadata._handle_clients_xml_event(self, event) - self.clients = self.list_clients() - - -class DBMetadataLint(MetadataLint): - pass diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 5dcaa8bdb..07659b62a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -3,19 +3,27 @@ This file stores persistent metadata for the Bcfg2 Configuration Repository. """ import re -import copy -import fcntl -import lxml.etree import os -import socket import sys import time +import copy +import fcntl +import socket +import lxml.etree import Bcfg2.Server import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor +from UserDict import DictMixin from Bcfg2.version import Bcfg2VersionInfo +try: + from django.db import models + has_django = True +except ImportError: + has_django = False + + def locked(fd): """Aquire a lock on a file""" try: @@ -25,6 +33,35 @@ def locked(fd): return False +if has_django: + class MetadataClientModel(models.Model, + Bcfg2.Server.Plugin.PluginDatabaseModel): + hostname = models.CharField(max_length=255, primary_key=True) + version = models.CharField(max_length=31, null=True) + + class ClientVersions(DictMixin): + def __getitem__(self, key): + try: + return MetadataClientModel.objects.get(hostname=key).version + except MetadataClientModel.DoesNotExist: + raise KeyError(key) + + def __setitem__(self, key, value): + client = MetadataClientModel.objects.get_or_create(hostname=key)[0] + client.version = value + client.save() + + def keys(self): + return [c.hostname for c in MetadataClientModel.objects.all()] + + def __contains__(self, key): + try: + client = MetadataClientModel.objects.get(hostname=key) + return True + except MetadataClientModel.DoesNotExist: + return False + + class MetadataConsistencyError(Exception): """This error gets raised when metadata is internally inconsistent.""" pass @@ -264,23 +301,30 @@ class MetadataGroup(tuple): class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Statistics): + Bcfg2.Server.Plugin.Statistics, + Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" sort_order = 500 - __files__ = ["groups.xml", "clients.xml"] def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Metadata.__init__(self) Bcfg2.Server.Plugin.Statistics.__init__(self) + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self) self.watch_clients = watch_clients self.states = dict() self.extra = dict() self.handlers = [] - for fname in self.__files__: - self._handle_file(fname) + self._handle_file("groups.xml") + if (self._use_db and + os.path.exists(os.path.join(self.data, "clients.xml"))): + self.logger.warning("Metadata: database enabled but clients.xml" + "found, parsing in compatibility mode") + self._handle_file("clients.xml") + elif not self._use_db: + self._handle_file("clients.xml") # mapping of clientname -> authtype self.auth = dict() @@ -304,7 +348,10 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, self.group_membership = dict() self.negated_groups = dict() # mapping of hostname -> version string - self.versions = dict() + if self._use_db: + self.versions = ClientVersions() + else: + self.versions = dict() self.uuid = {} self.session_cache = {} self.default = None @@ -322,7 +369,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, # must use super here; inheritance works funny with class methods super(Metadata, cls).init_repo(repo) - for fname in cls.__files__: + for fname in ["clients.xml", "groups.xml"]: aname = re.sub(r'[^A-z0-9_]', '_', fname) if aname in kwargs: open(os.path.join(repo, cls.name, fname), @@ -380,17 +427,36 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def add_group(self, group_name, attribs): """Add group to groups.xml.""" - return self._add_xdata(self.groups_xml, "Group", group_name, - attribs=attribs) + if self._use_db: + msg = "Metadata does not support adding groups with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._add_xdata(self.groups_xml, "Group", group_name, + attribs=attribs) def add_bundle(self, bundle_name): """Add bundle to groups.xml.""" - return self._add_xdata(self.groups_xml, "Bundle", bundle_name) + if self._use_db: + msg = "Metadata does not support adding bundles with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._add_xdata(self.groups_xml, "Bundle", bundle_name) - def add_client(self, client_name, attribs): + def add_client(self, client_name, attribs=None): """Add client to clients.xml.""" - return self._add_xdata(self.clients_xml, "Client", client_name, - attribs=attribs, alias=True) + print "add_client(%s, attribs=%s)" % (client_name, attribs) + if attribs is None: + attribs = dict() + if self._use_db: + client = MetadataClientModel(hostname=client_name) + client.save() + self.clients = self.list_clients() + return client + else: + return self._add_xdata(self.clients_xml, "Client", client_name, + attribs=attribs, alias=True) def _update_xdata(self, config, tag, name, attribs, alias=False): node = self._search_xdata(tag, name, config.xdata, alias=alias) @@ -409,12 +475,30 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def update_group(self, group_name, attribs): """Update a groups attributes.""" - return self._update_xdata(self.groups_xml, "Group", group_name, attribs) + if self._use_db: + msg = "Metadata does not support updating groups with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._update_xdata(self.groups_xml, "Group", group_name, + attribs) def update_client(self, client_name, attribs): """Update a clients attributes.""" - return self._update_xdata(self.clients_xml, "Client", client_name, - attribs, alias=True) + if self._use_db: + msg = "Metadata does not support updating clients with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._update_xdata(self.clients_xml, "Client", client_name, + attribs, alias=True) + + def list_clients(self): + """ List all clients in client database """ + if self._use_db: + return set([c.hostname for c in MetadataClientModel.objects.all()]) + else: + return self.clients def _remove_xdata(self, config, tag, name, alias=False): node = self._search_xdata(tag, name, config.xdata) @@ -432,15 +516,35 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def remove_group(self, group_name): """Remove a group.""" - return self._remove_xdata(self.groups_xml, "Group", group_name) + if self._use_db: + msg = "Metadata does not support removing groups with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" - return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) + if self._use_db: + msg = "Metadata does not support removing bundles with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + else: + return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) def remove_client(self, client_name): """Remove a bundle.""" - return self._remove_xdata(self.clients_xml, "Client", client_name) + if self._use_db: + try: + client = MetadataClientModel.objects.get(hostname=client_name) + except MetadataClientModel.DoesNotExist: + msg = "Client %s does not exist" % client_name + self.logger.warning(msg) + raise MetadataConsistencyError(msg) + client.delete() + self.clients = self.list_clients() + else: + return self._remove_xdata(self.clients_xml, "Client", client_name) def _handle_clients_xml_event(self, event): xdata = self.clients_xml.xdata @@ -497,6 +601,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, except KeyError: self.clientgroups[clname] = [client.get('profile')] self.states['clients.xml'] = True + if self._use_db: + self.clients = self.list_clients() def _handle_groups_xml_event(self, event): self.groups = {} @@ -627,10 +733,13 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) raise MetadataConsistencyError(msg) - self._set_profile(client, profile, addresspair) - def _set_profile(self, client, profile, addresspair): if client in self.clients: + if self._use_db: + msg = "DBMetadata does not support asserting client profiles" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + profiles = [g for g in self.clientgroups[client] if g in self.groups and self.groups[g].is_profile] self.logger.info("Changing %s profile from %s to %s" % @@ -645,16 +754,20 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) - if addresspair in self.session_cache: - # we are working with a uuid'd client - self.add_client(self.session_cache[addresspair][1], - dict(uuid=client, profile=profile, - address=addresspair[0])) + if self._use_db: + self.add_client(client) else: - self.add_client(client, dict(profile=profile)) - self.clients.append(client) - self.clientgroups[client] = [profile] - self.clients_xml.write() + if addresspair in self.session_cache: + # we are working with a uuid'd client + self.add_client(self.session_cache[addresspair][1], + dict(uuid=client, profile=profile, + address=addresspair[0])) + else: + self.add_client(client, dict(profile=profile)) + self.clients.append(client) + self.clientgroups[client] = [profile] + if not self._use_db: + self.clients_xml.write() def set_version(self, client, version): """Set group parameter for provided client.""" diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 3932c44d1..114a9bbd8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -5,9 +5,14 @@ import time import operator import lxml.etree import Bcfg2.Server -from django.db import models import Bcfg2.Server.Plugin +try: + from django.db import models + has_django = True +except ImportError: + has_django = False + try: import json has_json = True @@ -32,18 +37,18 @@ except ImportError: import Bcfg2.Server.Plugin -class ProbesDataModel(models.Model, - Bcfg2.Server.Plugin.PluginDatabaseModel): - hostname = models.CharField(max_length=255) - probe = models.CharField(max_length=255) - timestamp = models.DateTimeField(auto_now=True) - data = models.TextField(null=True) +if has_django: + class ProbesDataModel(models.Model, + Bcfg2.Server.Plugin.PluginDatabaseModel): + hostname = models.CharField(max_length=255) + probe = models.CharField(max_length=255) + timestamp = models.DateTimeField(auto_now=True) + data = models.TextField(null=True) - -class ProbesGroupsModel(models.Model, - Bcfg2.Server.Plugin.PluginDatabaseModel): - hostname = models.CharField(max_length=255) - group = models.CharField(max_length=255) + class ProbesGroupsModel(models.Model, + Bcfg2.Server.Plugin.PluginDatabaseModel): + hostname = models.CharField(max_length=255) + group = models.CharField(max_length=255) class ClientProbeDataSet(dict): @@ -172,11 +177,6 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups = dict() self.load_data() - @property - def _use_db(self): - return self.core.setup.cfp.getboolean("probes", "use_database", - default=False) - def write_data(self, client): """Write probe data out for use with bcfg2-info.""" if self._use_db: diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 5de590fec..c72ef0ebe 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -1,7 +1,12 @@ import sys -import django import Bcfg2.Options +try: + import django + has_django = True +except: + has_django = False + DATABASES = dict() # Django < 1.2 compat @@ -39,7 +44,7 @@ def read_config(cfile='/etc/bcfg2.conf', repo=None, quiet=False): HOST=setup['db_host'], PORT=setup['db_port']) - if django.VERSION[0] == 1 and django.VERSION[1] < 2: + if has_django and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] DATABASE_NAME = DATABASES['default']['NAME'] DATABASE_USER = DATABASES['default']['USER'] @@ -51,7 +56,7 @@ def read_config(cfile='/etc/bcfg2.conf', repo=None, quiet=False): # this lets manage.py work in all cases read_config(quiet=True) -if django.VERSION[0] == 1 and django.VERSION[1] > 2: +if has_django and django.VERSION[0] == 1 and django.VERSION[1] > 2: TIME_ZONE = None DEBUG = False diff --git a/testsuite/Testlib/TestServer/TestPlugins/TestDBMetadata.py b/testsuite/Testlib/TestServer/TestPlugins/TestDBMetadata.py deleted file mode 100644 index 99cbf1962..000000000 --- a/testsuite/Testlib/TestServer/TestPlugins/TestDBMetadata.py +++ /dev/null @@ -1,407 +0,0 @@ -import os -import sys -import unittest -import lxml.etree -from mock import Mock, patch -from django.core.management import setup_environ - -os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" - -import Bcfg2.settings -Bcfg2.settings.DATABASE_NAME = \ - os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") -Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME - -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.DBMetadata import * - -from TestMetadata import datastore, groups_test_tree, clients_test_tree, \ - TestMetadata - -def test_syncdb(): - # create the test database - setup_environ(Bcfg2.settings) - from django.core.management.commands import syncdb - cmd = syncdb.Command() - cmd.handle_noargs(interactive=False) - assert os.path.exists(Bcfg2.settings.DATABASE_NAME) - - # ensure that we a) can connect to the database; b) start with a - # clean database - MetadataClientModel.objects.all().delete() - assert list(MetadataClientModel.objects.all()) == [] - - -class TestClientVersions(unittest.TestCase): - test_clients = dict(client1="1.2.0", - client2="1.2.2", - client3="1.3.0pre1", - client4="1.1.0", - client5=None, - client6=None) - - def setUp(self): - test_syncdb() - for client, version in self.test_clients.items(): - MetadataClientModel(hostname=client, version=version).save() - - def test__contains(self): - v = ClientVersions() - self.assertIn("client1", v) - self.assertIn("client5", v) - self.assertNotIn("client__contains", v) - - def test_keys(self): - v = ClientVersions() - self.assertItemsEqual(self.test_clients.keys(), v.keys()) - - def test__setitem(self): - v = ClientVersions() - - # test setting version of existing client - v["client1"] = "1.2.3" - self.assertIn("client1", v) - self.assertEqual(v['client1'], "1.2.3") - client = MetadataClientModel.objects.get(hostname="client1") - self.assertEqual(client.version, "1.2.3") - - # test adding new client - new = "client__setitem" - v[new] = "1.3.0" - self.assertIn(new, v) - self.assertEqual(v[new], "1.3.0") - client = MetadataClientModel.objects.get(hostname=new) - self.assertEqual(client.version, "1.3.0") - - # test adding new client with no version - new2 = "client__setitem_2" - v[new2] = None - self.assertIn(new2, v) - self.assertEqual(v[new2], None) - client = MetadataClientModel.objects.get(hostname=new2) - self.assertEqual(client.version, None) - - def test__getitem(self): - v = ClientVersions() - - # test getting existing client - self.assertEqual(v['client2'], "1.2.2") - self.assertIsNone(v['client5']) - - # test exception on nonexistent client. can't use assertRaises - # for this because assertRaises requires a callable - try: - v['clients__getitem'] - assert False - except KeyError: - assert True - except: - assert False - - -class TestDBMetadataBase(TestMetadata): - __test__ = False - - def __init__(self, *args, **kwargs): - TestMetadata.__init__(self, *args, **kwargs) - test_syncdb() - - def load_clients_data(self, metadata=None, xdata=None): - if metadata is None: - metadata = get_metadata_object() - for client in clients_test_tree.findall("Client"): - metadata.add_client(client.get("name")) - return metadata - - def get_metadata_object(self, core=None, watch_clients=False): - if core is None: - core = Mock() - metadata = DBMetadata(core, datastore, watch_clients=watch_clients) - return metadata - - def get_nonexistent_client(self, _, prefix="client"): - clients = [o.hostname for o in MetadataClientModel.objects.all()] - i = 0 - client_name = "%s%s" % (prefix, i) - while client_name in clients: - i += 1 - client_name = "%s%s" % (prefix, i) - return client_name - - @patch('os.path.exists') - def test__init(self, mock_exists): - core = Mock() - core.fam = Mock() - mock_exists.return_value = False - metadata = self.get_metadata_object(core=core, watch_clients=True) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.DatabaseBacked) - core.fam.AddMonitor.assert_called_once_with(os.path.join(metadata.data, - "groups.xml"), - metadata) - - mock_exists.return_value = True - core.fam.reset_mock() - metadata = self.get_metadata_object(core=core, watch_clients=True) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "groups.xml"), - metadata) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "clients.xml"), - metadata) - - def test_add_group(self): - pass - - def test_add_bundle(self): - pass - - def test_add_client(self): - metadata = self.get_metadata_object() - hostname = self.get_nonexistent_client(metadata) - client = metadata.add_client(hostname) - self.assertIsInstance(client, MetadataClientModel) - self.assertEqual(client.hostname, hostname) - self.assertIn(hostname, metadata.clients) - self.assertIn(hostname, metadata.list_clients()) - self.assertItemsEqual(metadata.clients, - [c.hostname - for c in MetadataClientModel.objects.all()]) - - def test_update_group(self): - pass - - def test_update_bundle(self): - pass - - def test_update_client(self): - pass - - def test_list_clients(self): - metadata = self.get_metadata_object() - self.assertItemsEqual(metadata.list_clients(), - [c.hostname - for c in MetadataClientModel.objects.all()]) - - def test_remove_group(self): - pass - - def test_remove_bundle(self): - pass - - def test_remove_client(self): - metadata = self.get_metadata_object() - client_name = self.get_nonexistent_client(metadata) - - self.assertRaises(MetadataConsistencyError, - metadata.remove_client, - client_name) - - metadata.add_client(client_name) - metadata.remove_client(client_name) - self.assertNotIn(client_name, metadata.clients) - self.assertNotIn(client_name, metadata.list_clients()) - self.assertItemsEqual(metadata.clients, - [c.hostname - for c in MetadataClientModel.objects.all()]) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("Bcfg2.Server.Plugins.DBMetadata.DBMetadata._set_profile") - def test_set_profile(self, mock_set_profile): - TestMetadata.test_set_profile(self, - inherited_set_profile=mock_set_profile) - - def test__set_profile(self): - metadata = self.get_metadata_object() - profile = "group1" - client_name = self.get_nonexistent_client(metadata) - metadata._set_profile(client_name, profile, None) - self.assertIn(client_name, metadata.list_clients()) - self.assertIn(client_name, metadata.clientgroups) - self.assertItemsEqual(metadata.clientgroups[client_name], [profile]) - - self.assertRaises(Bcfg2.Server.Plugin.PluginExecutionError, - metadata._set_profile, - client_name, profile, None) - - def test_process_statistics(self): - pass - - -class TestDBMetadata_NoClientsXML(TestDBMetadataBase): - """ test DBMetadata without a clients.xml. we have to disable or - override tests that rely on client options """ - __test__ = True - - def __init__(self, *args, **kwargs): - TestMetadata.__init__(self, *args, **kwargs) - - for client in self.clients_test_tree.findall("Client"): - newclient = lxml.etree.SubElement(self.groups_test_tree.getroot(), - "Client", name=client.get("name")) - lxml.etree.SubElement(newclient, "Group", - name=client.get("profile")) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.write_xml", Mock()) - @patch("Bcfg2.Server.Plugins.Metadata.ClientMetadata") - def test_get_initial_metadata(self, mock_clientmetadata): - metadata = self.get_metadata_object() - if 'clients.xml' in metadata.states: - metadata.states['clients.xml'] = False - self.assertRaises(MetadataRuntimeError, - metadata.get_initial_metadata, None) - - self.load_groups_data(metadata=metadata) - self.load_clients_data(metadata=metadata) - - # test basic client metadata - metadata.get_initial_metadata("client1") - self.assertEqual(mock_clientmetadata.call_args[0][:9], - ("client1", "group1", set(["group1"]), set(), set(), - set(), dict(category1='group1'), None, None)) - - # test bundles, category suppression - metadata.get_initial_metadata("client2") - self.assertEqual(mock_clientmetadata.call_args[0][:9], - ("client2", "group2", set(["group2"]), - set(["bundle1", "bundle2"]), set(), set(), - dict(category1="group2"), None, None)) - - # test new client creation - new1 = self.get_nonexistent_client(metadata) - imd = metadata.get_initial_metadata(new1) - self.assertEqual(mock_clientmetadata.call_args[0][:9], - (new1, "group1", set(["group1"]), set(), set(), set(), - dict(category1="group1"), None, None)) - - # test nested groups, per-client groups - imd = metadata.get_initial_metadata("client8") - self.assertEqual(mock_clientmetadata.call_args[0][:9], - ("client8", "group1", - set(["group1", "group8", "group9", "group10"]), set(), - set(), set(), dict(category1="group1"), None, None)) - - # test per-client groups, group negation, nested groups - imd = metadata.get_initial_metadata("client9") - self.assertEqual(mock_clientmetadata.call_args[0][:9], - ("client9", "group2", - set(["group2", "group8", "group11"]), - set(["bundle1", "bundle2"]), set(), set(), - dict(category1="group2"), None, None)) - - # test exception on new client with no default profile - metadata.default = None - new2 = self.get_nonexistent_client(metadata) - self.assertRaises(MetadataConsistencyError, - metadata.get_initial_metadata, - new2) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("Bcfg2.Server.Plugins.Metadata.Metadata.resolve_client") - def test_validate_client_address(self, mock_resolve_client): - metadata = self.load_clients_data(metadata=self.load_groups_data()) - # this is upper case to ensure that case is folded properly in - # validate_client_address() - mock_resolve_client.return_value = "CLIENT4" - self.assertTrue(metadata.validate_client_address("client4", - ("1.2.3.7", None))) - mock_resolve_client.assert_called_with(("1.2.3.7", None)) - - mock_resolve_client.reset_mock() - self.assertFalse(metadata.validate_client_address("client5", - ("1.2.3.5", None))) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("Bcfg2.Server.Plugins.Metadata.Metadata.validate_client_address") - @patch("Bcfg2.Server.Plugins.Metadata.Metadata.resolve_client") - def test_AuthenticateConnection(self, mock_resolve_client, - mock_validate_client_address): - metadata = self.load_clients_data(metadata=self.load_groups_data()) - metadata.password = "password1" - - cert = dict(subject=[[("commonName", "client1")]]) - mock_validate_client_address.return_value = False - self.assertFalse(metadata.AuthenticateConnection(cert, "root", None, - "1.2.3.1")) - mock_validate_client_address.return_value = True - self.assertTrue(metadata.AuthenticateConnection(cert, "root", - metadata.password, - "1.2.3.1")) - - cert = dict(subject=[[("commonName", "client8")]]) - - mock_resolve_client.return_value = "client5" - self.assertTrue(metadata.AuthenticateConnection(None, "root", - "password1", "1.2.3.8")) - - mock_resolve_client.side_effect = MetadataConsistencyError - self.assertFalse(metadata.AuthenticateConnection(None, "root", - "password1", - "1.2.3.8")) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("socket.gethostbyaddr") - def test_resolve_client(self, mock_gethostbyaddr): - metadata = self.load_clients_data(metadata=self.load_groups_data()) - metadata.session_cache[('1.2.3.3', None)] = (time.time(), 'client3') - self.assertEqual(metadata.resolve_client(('1.2.3.3', None)), 'client3') - - metadata.session_cache[('1.2.3.3', None)] = (time.time() - 100, - 'client3') - mock_gethostbyaddr.return_value = ("client3", [], ['1.2.3.3']) - self.assertEqual(metadata.resolve_client(('1.2.3.3', None), - cleanup_cache=True), 'client3') - self.assertEqual(metadata.session_cache, dict()) - - mock_gethostbyaddr.return_value = ('client6', [], ['1.2.3.6']) - self.assertEqual(metadata.resolve_client(('1.2.3.6', None)), 'client6') - mock_gethostbyaddr.assert_called_with('1.2.3.6') - - mock_gethostbyaddr.reset_mock() - mock_gethostbyaddr.return_value = None - mock_gethostbyaddr.side_effect = socket.herror - self.assertRaises(MetadataConsistencyError, - metadata.resolve_client, - ('1.2.3.8', None)) - mock_gethostbyaddr.assert_called_with('1.2.3.8') - - def test_clients_xml_event(self): - pass - - -class TestDBMetadata_ClientsXML(TestDBMetadataBase): - """ test DBMetadata with a clients.xml. """ - __test__ = True - - def load_clients_data(self, metadata=None, xdata=None): - if metadata is None: - metadata = self.get_metadata_object() - metadata.core.fam = Mock() - metadata._handle_file("clients.xml") - metadata = TestMetadata.load_clients_data(self, metadata=metadata, - xdata=xdata) - return TestDBMetadataBase.load_clients_data(self, metadata=metadata, - xdata=xdata) - - @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml") - @patch("Bcfg2.Server.Plugins.Metadata.Metadata._handle_clients_xml_event") - @patch("Bcfg2.Server.Plugins.DBMetadata.DBMetadata.list_clients") - def test_clients_xml_event(self, mock_list_clients, mock_handle_event, - mock_load_xml): - metadata = self.get_metadata_object() - metadata.profiles = ["group1", "group2"] - evt = Mock() - evt.filename = os.path.join(datastore, "DBMetadata", "clients.xml") - evt.code2str = Mock(return_value="changed") - metadata.HandleEvent(evt) - self.assertFalse(mock_handle_event.called) - self.assertFalse(mock_load_xml.called) - - mock_load_xml.reset_mock() - mock_handle_event.reset_mock() - mock_list_clients.reset_mock() - metadata._handle_file("clients.xml") - metadata.HandleEvent(evt) - mock_handle_event.assert_called_with(metadata, evt) - mock_list_clients.assert_any_call() - mock_load_xml.assert_any_call() diff --git a/testsuite/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testlib/TestServer/TestPlugins/TestMetadata.py index 9aacbe5cc..83da19bad 100644 --- a/testsuite/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -1,10 +1,25 @@ import os +import sys import copy import time import socket import unittest import lxml.etree from mock import Mock, patch + +try: + from django.core.management import setup_environ + has_django = True + + os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" + + import Bcfg2.settings + Bcfg2.settings.DATABASE_NAME = \ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") + Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME +except ImportError: + has_django = False + import Bcfg2.Server import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Metadata import * @@ -74,11 +89,95 @@ groups_test_tree = lxml.etree.XML(''' datastore = "/" -def get_metadata_object(core=None, watch_clients=False): +def test_syncdb(): + if not has_django: + raise unittest.SkipTest("Django not found, skipping") + + # create the test database + setup_environ(Bcfg2.settings) + from django.core.management.commands import syncdb + cmd = syncdb.Command() + cmd.handle_noargs(interactive=False) + assert os.path.exists(Bcfg2.settings.DATABASE_NAME) + + # ensure that we a) can connect to the database; b) start with a + # clean database + MetadataClientModel.objects.all().delete() + assert list(MetadataClientModel.objects.all()) == [] + +def get_metadata_object(core=None, watch_clients=False, use_db=False): if core is None: core = Mock() - metadata = Metadata(core, datastore, watch_clients=watch_clients) - return metadata + core.setup.cfp.getboolean = Mock() + core.setup.cfp.getboolean.return_value = use_db + return Metadata(core, datastore, watch_clients=watch_clients) + + +class TestClientVersions(unittest.TestCase): + test_clients = dict(client1="1.2.0", + client2="1.2.2", + client3="1.3.0pre1", + client4="1.1.0", + client5=None, + client6=None) + + def setUp(self): + test_syncdb() + for client, version in self.test_clients.items(): + MetadataClientModel(hostname=client, version=version).save() + + def test__contains(self): + v = ClientVersions() + self.assertIn("client1", v) + self.assertIn("client5", v) + self.assertNotIn("client__contains", v) + + def test_keys(self): + v = ClientVersions() + self.assertItemsEqual(self.test_clients.keys(), v.keys()) + + def test__setitem(self): + v = ClientVersions() + + # test setting version of existing client + v["client1"] = "1.2.3" + self.assertIn("client1", v) + self.assertEqual(v['client1'], "1.2.3") + client = MetadataClientModel.objects.get(hostname="client1") + self.assertEqual(client.version, "1.2.3") + + # test adding new client + new = "client__setitem" + v[new] = "1.3.0" + self.assertIn(new, v) + self.assertEqual(v[new], "1.3.0") + client = MetadataClientModel.objects.get(hostname=new) + self.assertEqual(client.version, "1.3.0") + + # test adding new client with no version + new2 = "client__setitem_2" + v[new2] = None + self.assertIn(new2, v) + self.assertEqual(v[new2], None) + client = MetadataClientModel.objects.get(hostname=new2) + self.assertEqual(client.version, None) + + def test__getitem(self): + v = ClientVersions() + + # test getting existing client + self.assertEqual(v['client2'], "1.2.2") + self.assertIsNone(v['client5']) + + # test exception on nonexistent client. can't use assertRaises + # for this because assertRaises requires a callable + try: + v['clients__getitem'] + assert False + except KeyError: + assert True + except: + assert False class TestXMLMetadataConfig(unittest.TestCase): @@ -268,9 +367,11 @@ class TestClientMetadata(unittest.TestCase): class TestMetadata(unittest.TestCase): groups_test_tree = groups_test_tree clients_test_tree = clients_test_tree + use_db = False def get_metadata_object(self, core=None, watch_clients=False): - return get_metadata_object(core=core, watch_clients=watch_clients) + return get_metadata_object(core=core, watch_clients=watch_clients, + use_db=self.use_db) def get_nonexistent_client(self, metadata, prefix="client"): if metadata is None: @@ -631,13 +732,7 @@ class TestMetadata(unittest.TestCase): negated_groups) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) - @patch("Bcfg2.Server.Plugins.Metadata.Metadata._set_profile") - def test_set_profile(self, mock_set_profile, inherited_set_profile=None): - if inherited_set_profile: - # allow a subclass of TestMetadata to patch a different - # _set_profile object and pass it in. this probably isn't - # the best way to accomplish that, but it seems to work. - mock_set_profile = inherited_set_profile + def test_set_profile(self): metadata = self.get_metadata_object() if 'clients.xml' in metadata.states: metadata.states['clients.xml'] = False @@ -656,43 +751,48 @@ class TestMetadata(unittest.TestCase): metadata.set_profile, "client1", "group3", None) - metadata.set_profile("client1", "group5", None, force=True) - mock_set_profile.assert_called_with("client1", "group5", None) - - metadata.set_profile("client1", "group3", None, force=True) - mock_set_profile.assert_called_with("client1", "group3", None) + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) + def test_set_profile_db(self): + metadata = self.load_clients_data(metadata=self.load_groups_data()) + if metadata._use_db: + profile = "group1" + client_name = self.get_nonexistent_client(metadata) + metadata.set_profile(client_name, profile, None) + self.assertIn(client_name, metadata.clients) + self.assertRaises(Bcfg2.Server.Plugin.PluginExecutionError, + metadata.set_profile, + client_name, profile, None) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) @patch("Bcfg2.Server.Plugins.Metadata.Metadata.add_client") @patch("Bcfg2.Server.Plugins.Metadata.Metadata.update_client") - def test__set_profile(self, mock_update_client, mock_add_client): - metadata = self.get_metadata_object() - self.load_groups_data(metadata=metadata) - self.load_clients_data(metadata=metadata) - - metadata.clients_xml.write = Mock() - metadata._set_profile("client1", "group2", None) - mock_update_client.assert_called_with("client1", dict(profile="group2")) - metadata.clients_xml.write.assert_any_call() - self.assertEqual(metadata.clientgroups["client1"], ["group2"]) - - metadata.clients_xml.write.reset_mock() - new1 = self.get_nonexistent_client(metadata) - metadata._set_profile(new1, "group1", None) - mock_add_client.assert_called_with(new1, dict(profile="group1")) - metadata.clients_xml.write.assert_any_call() - self.assertEqual(metadata.clientgroups[new1], ["group1"]) - - metadata.clients_xml.write.reset_mock() - new2 = self.get_nonexistent_client(metadata) - metadata.session_cache[('1.2.3.6', None)] = (None, new2) - metadata._set_profile("uuid_new", "group1", ('1.2.3.6', None)) - mock_add_client.assert_called_with(new2, - dict(uuid='uuid_new', - profile="group1", - address='1.2.3.6')) - metadata.clients_xml.write.assert_any_call() - self.assertEqual(metadata.clientgroups["uuid_new"], ["group1"]) + def test_set_profile_xml(self, mock_update_client, mock_add_client): + metadata = self.load_clients_data(metadata=self.load_groups_data()) + if not metadata._use_db: + metadata.clients_xml.write = Mock() + metadata.set_profile("client1", "group2", None) + mock_update_client.assert_called_with("client1", + dict(profile="group2")) + metadata.clients_xml.write.assert_any_call() + self.assertEqual(metadata.clientgroups["client1"], ["group2"]) + + metadata.clients_xml.write.reset_mock() + new1 = self.get_nonexistent_client(metadata) + metadata.set_profile(new1, "group1", None) + mock_add_client.assert_called_with(new1, dict(profile="group1")) + metadata.clients_xml.write.assert_any_call() + self.assertEqual(metadata.clientgroups[new1], ["group1"]) + + metadata.clients_xml.write.reset_mock() + new2 = self.get_nonexistent_client(metadata) + metadata.session_cache[('1.2.3.6', None)] = (None, new2) + metadata.set_profile("uuid_new", "group1", ('1.2.3.6', None)) + mock_add_client.assert_called_with(new2, + dict(uuid='uuid_new', + profile="group1", + address='1.2.3.6')) + metadata.clients_xml.write.assert_any_call() + self.assertEqual(metadata.clientgroups["uuid_new"], ["group1"]) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) @patch("socket.gethostbyaddr") @@ -1004,3 +1104,293 @@ class TestMetadata(unittest.TestCase): def test_viz(self): pass + + + +class TestMetadataBase(TestMetadata): + """ base test object for testing Metadata with database enabled """ + __test__ = False + use_db = True + + def __init__(self, *args, **kwargs): + TestMetadata.__init__(self, *args, **kwargs) + test_syncdb() + + def setUp(self): + if not has_django: + self.skipTest("Django not found, skipping") + + def load_clients_data(self, metadata=None, xdata=None): + if metadata is None: + metadata = get_metadata_object() + for client in clients_test_tree.findall("Client"): + metadata.add_client(client.get("name")) + return metadata + + def get_nonexistent_client(self, _, prefix="client"): + clients = [o.hostname for o in MetadataClientModel.objects.all()] + i = 0 + client_name = "%s%s" % (prefix, i) + while client_name in clients: + i += 1 + client_name = "%s%s" % (prefix, i) + return client_name + + @patch('os.path.exists') + def test__init(self, mock_exists): + core = Mock() + core.fam = Mock() + mock_exists.return_value = False + metadata = self.get_metadata_object(core=core, watch_clients=True) + self.assertIsInstance(metadata, Bcfg2.Server.Plugin.DatabaseBacked) + core.fam.AddMonitor.assert_called_once_with(os.path.join(metadata.data, + "groups.xml"), + metadata) + + mock_exists.return_value = True + core.fam.reset_mock() + metadata = self.get_metadata_object(core=core, watch_clients=True) + core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, + "groups.xml"), + metadata) + core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, + "clients.xml"), + metadata) + + def test_add_group(self): + pass + + def test_add_bundle(self): + pass + + def test_add_client(self): + metadata = self.get_metadata_object() + hostname = self.get_nonexistent_client(metadata) + client = metadata.add_client(hostname) + self.assertIsInstance(client, MetadataClientModel) + self.assertEqual(client.hostname, hostname) + self.assertIn(hostname, metadata.clients) + self.assertIn(hostname, metadata.list_clients()) + self.assertItemsEqual(metadata.clients, + [c.hostname + for c in MetadataClientModel.objects.all()]) + + def test_update_group(self): + pass + + def test_update_bundle(self): + pass + + def test_update_client(self): + pass + + def test_list_clients(self): + metadata = self.get_metadata_object() + self.assertItemsEqual(metadata.list_clients(), + [c.hostname + for c in MetadataClientModel.objects.all()]) + + def test_remove_group(self): + pass + + def test_remove_bundle(self): + pass + + def test_remove_client(self): + metadata = self.get_metadata_object() + client_name = self.get_nonexistent_client(metadata) + + self.assertRaises(MetadataConsistencyError, + metadata.remove_client, + client_name) + + metadata.add_client(client_name) + metadata.remove_client(client_name) + self.assertNotIn(client_name, metadata.clients) + self.assertNotIn(client_name, metadata.list_clients()) + self.assertItemsEqual(metadata.clients, + [c.hostname + for c in MetadataClientModel.objects.all()]) + + def test_process_statistics(self): + pass + + +class TestMetadata_NoClientsXML(TestMetadataBase): + """ test Metadata without a clients.xml. we have to disable or + override tests that rely on client options """ + __test__ = True + + def __init__(self, *args, **kwargs): + TestMetadata.__init__(self, *args, **kwargs) + + for client in self.clients_test_tree.findall("Client"): + newclient = lxml.etree.SubElement(self.groups_test_tree.getroot(), + "Client", name=client.get("name")) + lxml.etree.SubElement(newclient, "Group", + name=client.get("profile")) + + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.write_xml", Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.ClientMetadata") + def test_get_initial_metadata(self, mock_clientmetadata): + metadata = self.get_metadata_object() + if 'clients.xml' in metadata.states: + metadata.states['clients.xml'] = False + self.assertRaises(MetadataRuntimeError, + metadata.get_initial_metadata, None) + + self.load_groups_data(metadata=metadata) + self.load_clients_data(metadata=metadata) + + # test basic client metadata + metadata.get_initial_metadata("client1") + self.assertEqual(mock_clientmetadata.call_args[0][:9], + ("client1", "group1", set(["group1"]), set(), set(), + set(), dict(category1='group1'), None, None)) + + # test bundles, category suppression + metadata.get_initial_metadata("client2") + self.assertEqual(mock_clientmetadata.call_args[0][:9], + ("client2", "group2", set(["group2"]), + set(["bundle1", "bundle2"]), set(), set(), + dict(category1="group2"), None, None)) + + # test new client creation + new1 = self.get_nonexistent_client(metadata) + imd = metadata.get_initial_metadata(new1) + self.assertEqual(mock_clientmetadata.call_args[0][:9], + (new1, "group1", set(["group1"]), set(), set(), set(), + dict(category1="group1"), None, None)) + + # test nested groups, per-client groups + imd = metadata.get_initial_metadata("client8") + self.assertEqual(mock_clientmetadata.call_args[0][:9], + ("client8", "group1", + set(["group1", "group8", "group9", "group10"]), set(), + set(), set(), dict(category1="group1"), None, None)) + + # test per-client groups, group negation, nested groups + imd = metadata.get_initial_metadata("client9") + self.assertEqual(mock_clientmetadata.call_args[0][:9], + ("client9", "group2", + set(["group2", "group8", "group11"]), + set(["bundle1", "bundle2"]), set(), set(), + dict(category1="group2"), None, None)) + + # test exception on new client with no default profile + metadata.default = None + new2 = self.get_nonexistent_client(metadata) + self.assertRaises(MetadataConsistencyError, + metadata.get_initial_metadata, + new2) + + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.Metadata.resolve_client") + def test_validate_client_address(self, mock_resolve_client): + metadata = self.load_clients_data(metadata=self.load_groups_data()) + # this is upper case to ensure that case is folded properly in + # validate_client_address() + mock_resolve_client.return_value = "CLIENT4" + self.assertTrue(metadata.validate_client_address("client4", + ("1.2.3.7", None))) + mock_resolve_client.assert_called_with(("1.2.3.7", None)) + + mock_resolve_client.reset_mock() + self.assertFalse(metadata.validate_client_address("client5", + ("1.2.3.5", None))) + + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.Metadata.validate_client_address") + @patch("Bcfg2.Server.Plugins.Metadata.Metadata.resolve_client") + def test_AuthenticateConnection(self, mock_resolve_client, + mock_validate_client_address): + metadata = self.load_clients_data(metadata=self.load_groups_data()) + metadata.password = "password1" + + cert = dict(subject=[[("commonName", "client1")]]) + mock_validate_client_address.return_value = False + self.assertFalse(metadata.AuthenticateConnection(cert, "root", None, + "1.2.3.1")) + mock_validate_client_address.return_value = True + self.assertTrue(metadata.AuthenticateConnection(cert, "root", + metadata.password, + "1.2.3.1")) + + cert = dict(subject=[[("commonName", "client8")]]) + + mock_resolve_client.return_value = "client5" + self.assertTrue(metadata.AuthenticateConnection(None, "root", + "password1", "1.2.3.8")) + + mock_resolve_client.side_effect = MetadataConsistencyError + self.assertFalse(metadata.AuthenticateConnection(None, "root", + "password1", + "1.2.3.8")) + + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) + @patch("socket.gethostbyaddr") + def test_resolve_client(self, mock_gethostbyaddr): + metadata = self.load_clients_data(metadata=self.load_groups_data()) + metadata.session_cache[('1.2.3.3', None)] = (time.time(), 'client3') + self.assertEqual(metadata.resolve_client(('1.2.3.3', None)), 'client3') + + metadata.session_cache[('1.2.3.3', None)] = (time.time() - 100, + 'client3') + mock_gethostbyaddr.return_value = ("client3", [], ['1.2.3.3']) + self.assertEqual(metadata.resolve_client(('1.2.3.3', None), + cleanup_cache=True), 'client3') + self.assertEqual(metadata.session_cache, dict()) + + mock_gethostbyaddr.return_value = ('client6', [], ['1.2.3.6']) + self.assertEqual(metadata.resolve_client(('1.2.3.6', None)), 'client6') + mock_gethostbyaddr.assert_called_with('1.2.3.6') + + mock_gethostbyaddr.reset_mock() + mock_gethostbyaddr.return_value = None + mock_gethostbyaddr.side_effect = socket.herror + self.assertRaises(MetadataConsistencyError, + metadata.resolve_client, + ('1.2.3.8', None)) + mock_gethostbyaddr.assert_called_with('1.2.3.8') + + def test_clients_xml_event(self): + pass + + +class TestMetadata_ClientsXML(TestMetadataBase): + """ test Metadata with a clients.xml. """ + __test__ = True + + def load_clients_data(self, metadata=None, xdata=None): + if metadata is None: + metadata = self.get_metadata_object() + metadata.core.fam = Mock() + metadata._handle_file("clients.xml") + metadata = TestMetadata.load_clients_data(self, metadata=metadata, + xdata=xdata) + return TestMetadataBase.load_clients_data(self, metadata=metadata, + xdata=xdata) + + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml") + @patch("Bcfg2.Server.Plugins.Metadata.Metadata._handle_clients_xml_event") + @patch("Bcfg2.Server.Plugins.Metadata.Metadata.list_clients") + def test_clients_xml_event(self, mock_list_clients, mock_handle_event, + mock_load_xml): + metadata = self.get_metadata_object() + metadata.profiles = ["group1", "group2"] + evt = Mock() + evt.filename = os.path.join(datastore, "Metadata", "clients.xml") + evt.code2str = Mock(return_value="changed") + metadata.HandleEvent(evt) + self.assertFalse(mock_handle_event.called) + self.assertFalse(mock_load_xml.called) + + mock_load_xml.reset_mock() + mock_handle_event.reset_mock() + mock_list_clients.reset_mock() + metadata._handle_file("clients.xml") + metadata.HandleEvent(evt) + mock_handle_event.assert_called_with(evt) + mock_list_clients.assert_any_call() + mock_load_xml.assert_any_call() diff --git a/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py index b05f8c146..92e0037f3 100644 --- a/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py @@ -4,14 +4,19 @@ import time import unittest import lxml.etree from mock import Mock, patch -from django.core.management import setup_environ -os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" +try: + from django.core.management import setup_environ + has_django = True -import Bcfg2.settings -Bcfg2.settings.DATABASE_NAME = \ - os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") -Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME + os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" + + import Bcfg2.settings + Bcfg2.settings.DATABASE_NAME = \ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") + Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME +except ImportError: + has_django = False import Bcfg2.Server import Bcfg2.Server.Plugin @@ -23,6 +28,9 @@ datastore = "/" test_data = dict(a=1, b=[1, 2, 3], c="test") def test_syncdb(): + if not has_django: + raise unittest.SkipTest("Django not found, skipping") + # create the test database setup_environ(Bcfg2.settings) from django.core.management.commands import syncdb @@ -216,16 +224,12 @@ text "group-with-dashes"], "bar.example.com": []} + @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data", Mock()) def get_probes_object(self, use_db=False): - p = Probes(Mock(), datastore) - p.core.setup = Mock() - p.core.setup.cfp = Mock() - p.core.setup.cfp.getboolean = Mock() - if use_db: - p.core.setup.cfp.getboolean.return_value = True - else: - p.core.setup.cfp.getboolean.return_value = False - return p + core = Mock() + core.setup.cfp.getboolean = Mock() + core.setup.cfp.getboolean.return_value = use_db + return Probes(core, datastore) @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data") def test__init(self, mock_load_data): @@ -253,6 +257,8 @@ text probes._write_data_xml.assert_called_with("test") self.assertFalse(probes._write_data_db.called) + if not has_django: + self.skipTest("Django not found, skipping") probes = self.get_probes_object(use_db=True) probes._write_data_xml.reset_mock() probes._write_data_db.reset_mock() @@ -317,6 +323,8 @@ text self.assertItemsEqual(test_data, json.loads(jdata.get("value"))) def test__write_data_db(self): + if not has_django: + self.skipTest("Django not found, skipping") test_syncdb() probes = self.get_probes_object(use_db=True) probes.probedata = self.get_test_probedata() @@ -331,7 +339,6 @@ text self.assertEqual(len(pdata), len(probes.probedata[cname])) for probe in pdata: - print "probe: %s" % probe.probe self.assertEqual(probe.hostname, client.hostname) self.assertIsNotNone(probe.data) if probe.probe == "xml": @@ -379,6 +386,9 @@ text probes._load_data_xml.assert_any_call() self.assertFalse(probes._load_data_db.called) + if not has_django: + self.skipTest("Django not found, skipping") + probes = self.get_probes_object(use_db=True) probes._load_data_xml.reset_mock() probes._load_data_db.reset_mock() @@ -398,6 +408,7 @@ text probes._write_data_xml(None) xdata = \ lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0])) + print "rv = %s" % lxml.etree.tostring(xdata) mock_parse.return_value = xdata.getroottree() probes.probedata = dict() probes.cgroups = dict() @@ -410,6 +421,8 @@ text self.assertItemsEqual(probes.cgroups, self.get_test_cgroups()) def test__load_data_db(self): + if not has_django: + self.skipTest("Django not found, skipping") test_syncdb() probes = self.get_probes_object(use_db=True) probes.probedata = self.get_test_probedata() -- cgit v1.2.3-1-g7c22