From b476aabce0097d9e4ef81182de5dc30025edceb1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 30 Jul 2012 11:30:28 -0400 Subject: Added ability to store probe data in database instead of probed.xml --- doc/server/database.txt | 8 + doc/server/plugins/grouping/metadata.txt | 1 - doc/server/plugins/probes/fileprobes.txt | 56 +++ doc/server/plugins/probes/index.txt | 95 ++-- src/lib/Bcfg2/Server/Plugins/Probes.py | 171 ++++--- .../Testlib/TestServer/TestPlugins/TestProbes.py | 517 +++++++++++++++++++++ 6 files changed, 728 insertions(+), 120 deletions(-) create mode 100644 doc/server/plugins/probes/fileprobes.txt create mode 100644 testsuite/Testlib/TestServer/TestPlugins/TestProbes.py diff --git a/doc/server/database.txt b/doc/server/database.txt index 8094e9c5e..61d065854 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -43,3 +43,11 @@ of ``/etc/bcfg2.conf``. +-------------+------------------------------------------------------------+-------------------------------+ | port | The port to connect to | None | +-------------+------------------------------------------------------------+-------------------------------+ + +Database Schema Sync +==================== + +After making changes to the configuration options or adding a plugin +that uses the global database, you should run ``bcfg2-admin syncdb`` +to resync the database schema. + diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index 1ab3b9c05..5a437756a 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -241,7 +241,6 @@ The Group Tag has the following possible attributes: | | group in any one category. This provides the | | | | basis for representing groups which are | | | | conjugates of one another in a rigorous way. | | -| | way. | +----------+----------------------------------------------+--------------+ | default | Set as the profile to use for clients that | True|False | | | are not associated with a profile in | | diff --git a/doc/server/plugins/probes/fileprobes.txt b/doc/server/plugins/probes/fileprobes.txt new file mode 100644 index 000000000..67ed5047e --- /dev/null +++ b/doc/server/plugins/probes/fileprobes.txt @@ -0,0 +1,56 @@ +.. _server-plugins-probes-fileprobes: + +FileProbes +========== + +The FileProbes plugin allows you to probe a client for a file, +which is then added to the :ref:`server-plugins-generators-cfg` +specification. If the file changes on the client, FileProbes can +either update it in the specification or allow Cfg to replace it. + +FileProbes will not probe a file if there's already a file in Cfg that +will apply to the client. So if, for instance, you have a generic +file in ``Cfg/etc/foo.conf/foo.conf`` that applies to all hosts, +FileProbes will not retrieve ``/etc/foo.conf`` from the client (unless +``update`` is enabled; see Configuration_ below). + +When a new config file is first probed, an ``info.xml`` file is also +written to enforce the permissions from that client. Subsequent +probes from other clients will not modify or overwrite the data in +``info.xml``. (This ensures that any manual changes you make to +``info.xml`` for that file are not circumvented.) + +Configuration +------------- + +FileProbes is configured in ``FileProbes/config.xml``, which might +look something like: + +.. code-block:: xml + + + + + + + + + + + +This will result in ``/etc/foo.conf`` being retrieved from all +clients; if it changes on a client, it will be overwritten by the +version that was retrieved initially. + +Clients in the ``blah-servers`` group will be probed for +``/etc/blah.conf``; if it changes on a client, those changes will be +written into the Bcfg2 specification. If the file is deleted from a +client, it will be rewritten from Bcfg2. + +``bar.example.com`` will be probed for ``/var/lib/bar.gz``, which +contains non-ASCII characters and so needs to use base64 encoding when +transferring the file. + +The paths probed by FileProbes must also be included as Path entries +in your bundles in order to be handled properly by Cfg. Permissions +are handled as usual, with ``info.xml`` files in Cfg. diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index cacc42bc1..26c656374 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -155,6 +155,39 @@ the client-specific one will be used. If you want to to detect information about the client operating system, the :ref:`server-plugins-probes-ohai` plugin can help. +Data Storage +============ + +.. versionadded:: 1.3.0 + +The Probes plugin stores the output of client probes locally on the +Bcfg2 server in order to ensure that probe data and groups are +available on server startup (rather than having to wait until all +probes have run every time the server is restarted) and to +:ref:`bcfg2-info ` and related tools. There are +two options for storing this data: ``Probes/probed.xml``, a plain XML +file stored in the Bcfg2 specification; or in a database. + +Advantages and disadvantages of using the database: + +* The database is easier to query from other machines, for instance if + you run ``bcfg2-info`` or ``bcfg2-test`` on a machine that is not + your Bcfg2 server. +* The database allows multiple Bcfg2 servers to share probe data. +* The database is likely to handle probe data writes (which happen on + every client run) more quickly, since it can only write the probes + whose data has changed. +* The database is likely to handle probe data reads (which happen only + on server startup) more slowly, since it must query a database + rather than the local filesystem. Once the data has been read in + initially (from XML file or from the database) it is kept in memory. + +To use the database-backed storage model, set ``use_database`` in the +``[probes]`` section of ``bcfg2.conf`` to ``true``. You will also +need to configure the :ref:`server-database`. + +The file-based storage model is the default, although that is likely +to change in future versions of Bcfg2. Other examples ============== @@ -170,64 +203,10 @@ Other examples producttype serial-console-speed +Other Probing plugins +===================== + .. toctree:: - :hidden: ohai - -.. _server-plugins-probes-fileprobes: - -FileProbes -========== - -The FileProbes plugin allows you to probe a client for a file, -which is then added to the :ref:`server-plugins-generators-cfg` -specification. If the file changes on the client, FileProbes can -either update it in the specification or allow Cfg to replace it. - -FileProbes will not probe a file if there's already a file in Cfg that -will apply to the client. So if, for instance, you have a generic -file in ``Cfg/etc/foo.conf/foo.conf`` that applies to all hosts, -FileProbes will not retrieve ``/etc/foo.conf`` from the client (unless -``update`` is enabled; see Configuration_ below). - -When a new config file is first probed, an ``info.xml`` file is also -written to enforce the permissions from that client. Subsequent -probes from other clients will not modify or overwrite the data in -``info.xml``. (This ensures that any manual changes you make to -``info.xml`` for that file are not circumvented.) - -Configuration -------------- - -FileProbes is configured in ``FileProbes/config.xml``, which might -look something like: - -.. code-block:: xml - - - - - - - - - - - -This will result in ``/etc/foo.conf`` being retrieved from all -clients; if it changes on a client, it will be overwritten by the -version that was retrieved initially. - -Clients in the ``blah-servers`` group will be probed for -``/etc/blah.conf``; if it changes on a client, those changes will be -written into the Bcfg2 specification. If the file is deleted from a -client, it will be rewritten from Bcfg2. - -``bar.example.com`` will be probed for ``/var/lib/bar.gz``, which -contains non-ASCII characters and so needs to use base64 encoding when -transferring the file. - -The paths probed by FileProbes must also be included as Path entries -in your bundles in order to be handled properly by Cfg. Permissions -are handled as usual, with ``info.xml`` files in Cfg. + fileprobes diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 8ef6c8737..3932c44d1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -1,9 +1,12 @@ -import time -import lxml.etree -import operator import re import os +import sys +import time +import operator +import lxml.etree import Bcfg2.Server +from django.db import models +import Bcfg2.Server.Plugin try: import json @@ -16,20 +19,32 @@ except ImportError: has_json = False try: - import syck - has_syck = True + import syck as yaml + has_yaml = True + yaml_error = syck.error except ImportError: - has_syck = False try: import yaml + yaml_error = yaml.YAMLError has_yaml = True except ImportError: has_yaml = False import Bcfg2.Server.Plugin -specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") -probe_matcher = re.compile("(.*/)?(?P\S+)") +class ProbesDataModel(models.Model, + Bcfg2.Server.Plugin.PluginDatabaseModel): + hostname = models.CharField(max_length=255) + probe = models.CharField(max_length=255) + timestamp = models.DateTimeField(auto_now=True) + data = models.TextField(null=True) + + +class ProbesGroupsModel(models.Model, + Bcfg2.Server.Plugin.PluginDatabaseModel): + hostname = models.CharField(max_length=255) + group = models.CharField(max_length=255) + class ClientProbeDataSet(dict): """ dict of probe => [probe data] that records a for each host """ @@ -42,9 +57,9 @@ class ClientProbeDataSet(dict): class ProbeData(str): - """ a ProbeData object emulates a str object, but also has .xdata - and .json properties to provide convenient ways to use ProbeData - objects as XML or JSON data """ + """ a ProbeData object emulates a str object, but also has .xdata, + .json, and .yaml properties to provide convenient ways to use + ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): return str.__new__(cls, data) @@ -81,22 +96,18 @@ class ProbeData(str): @property def yaml(self): - if self._yaml is None: - if has_yaml: - try: - self._yaml = yaml.load(self.data) - except yaml.YAMLError: - pass - elif has_syck: - try: - self._yaml = syck.load(self.data) - except syck.error: - pass + if self._yaml is None and has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml_error: + pass return self._yaml class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") + probename = re.compile("(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$") + bangline = re.compile('^#!\s*(?P.*)$') def __init__(self, path, fam, encoding, plugin_name): fpattern = '[0-9A-Za-z_\-]+' @@ -105,20 +116,10 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.SpecificData, encoding) fam.AddMonitor(path, self) - self.bangline = re.compile('^#!(?P.*)$') def HandleEvent(self, event): - if event.filename != self.path: - if (event.code2str == 'changed' and - event.filename.endswith("probed.xml") and - event.filename not in self.entries): - # for some reason, probed.xml is particularly prone to - # getting changed events before created events, - # because gamin is the worst ever. anyhow, we - # specifically handle it here to avoid a warning on - # every single server startup. - self.entry_init(event) - return + if (event.filename != self.path and + not event.filename.endswith("probed.xml")): return self.handle_event(event) def get_probe_data(self, metadata): @@ -127,9 +128,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): candidates = self.get_matching(metadata) candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: - rem = specific_probe_matcher.match(entry.name) - if not rem: - rem = probe_matcher.match(entry.name) + rem = self.probename.match(entry.name) pname = rem.group('basename') if pname not in build: build[pname] = entry @@ -150,7 +149,8 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Connector): + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.DatabaseBacked): """A plugin to gather information from a client machine.""" name = 'Probes' __author__ = 'bcfg-dev@mcs.anl.gov' @@ -159,19 +159,32 @@ class Probes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self) try: self.probes = ProbeSet(self.data, core.fam, core.encoding, self.name) except: - raise Bcfg2.Server.Plugin.PluginInitError + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError(err) self.probedata = dict() self.cgroups = dict() self.load_data() - def write_data(self): + @property + def _use_db(self): + return self.core.setup.cfp.getboolean("probes", "use_database", + default=False) + + def write_data(self, client): """Write probe data out for use with bcfg2-info.""" + if self._use_db: + return self._write_data_db(client) + else: + return self._write_data_xml(client) + + def _write_data_xml(self, _): top = lxml.etree.Element("Probed") for client, probed in sorted(self.probedata.items()): cx = lxml.etree.SubElement(top, 'Client', name=client, @@ -185,17 +198,45 @@ class Probes(Bcfg2.Server.Plugin.Plugin, xml_declaration=True, pretty_print='true') try: - datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') + datafile = open(os.path.join(self.data, 'probed.xml'), 'w') datafile.write(data.decode('utf-8')) except IOError: - self.logger.error("Failed to write probed.xml") + err = sys.exc_info()[1] + self.logger.error("Failed to write probed.xml: %s" % err) + + def _write_data_db(self, client): + for probe, data in self.probedata[client.hostname].items(): + pdata = \ + ProbesDataModel.objects.get_or_create(hostname=client.hostname, + probe=probe)[0] + if pdata.data != data: + pdata.data = data + pdata.save() + ProbesDataModel.objects.filter(hostname=client.hostname).exclude(probe__in=self.probedata[client.hostname]).delete() + + for group in self.cgroups[client.hostname]: + try: + ProbesGroupsModel.objects.get(hostname=client.hostname, + group=group) + except ProbesGroupsModel.DoesNotExist: + grp = ProbesGroupsModel(hostname=client.hostname, + group=group) + grp.save() + ProbesGroupsModel.objects.filter(hostname=client.hostname).exclude(group__in=self.cgroups[client.hostname]).delete() def load_data(self): + if self._use_db: + return self._load_data_db() + else: + return self._load_data_xml() + + def _load_data_xml(self): try: data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), parser=Bcfg2.Server.XMLParser).getroot() except: - self.logger.error("Failed to read file probed.xml") + err = sys.exc_info()[1] + self.logger.error("Failed to read file probed.xml: %s" % err) return self.probedata = {} self.cgroups = {} @@ -204,12 +245,25 @@ class Probes(Bcfg2.Server.Plugin.Plugin, ClientProbeDataSet(timestamp=client.get("timestamp")) self.cgroups[client.get('name')] = [] for pdata in client: - if (pdata.tag == 'Probe'): + if pdata.tag == 'Probe': self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get('value')) - elif (pdata.tag == 'Group'): + ProbeData(pdata.get("value")) + elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) + def _load_data_db(self): + self.probedata = {} + self.cgroups = {} + for pdata in ProbesDataModel.objects.all(): + if pdata.hostname not in self.probedata: + self.probedata[pdata.hostname] = \ + ClientProbeDataSet(timestamp=time.mktime(pdata.timestamp.timetuple())) + self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) + for pgroup in ProbesGroupsModel.objects.all(): + if pgroup.hostname not in self.cgroups: + self.cgroups[pgroup.hostname] = [] + self.cgroups[pgroup.hostname].append(pgroup.group) + def GetProbes(self, meta, force=False): """Return a set of probes for execution on client.""" return self.probes.get_probe_data(meta) @@ -219,25 +273,24 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.probedata[client.hostname] = ClientProbeDataSet() for data in datalist: self.ReceiveDataItem(client, data) - self.write_data() + self.write_data(client) def ReceiveDataItem(self, client, data): """Receive probe results pertaining to client.""" if client.hostname not in self.cgroups: self.cgroups[client.hostname] = [] + if client.hostname not in self.probedata: + self.probedata[client.hostname] = ClientProbeDataSet() if data.text == None: - self.logger.error("Got null response to probe %s from %s" % \ - (data.get('name'), client.hostname)) - try: - self.probedata[client.hostname].update({data.get('name'): + self.logger.info("Got null response to probe %s from %s" % + (data.get('name'), client.hostname)) + self.probedata[client.hostname].update({data.get('name'): ProbeData('')}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), ProbeData(''))]) return dlines = data.text.split('\n') - self.logger.debug("%s:probe:%s:%s" % (client.hostname, - data.get('name'), [line.strip() for line in dlines])) + self.logger.debug("%s:probe:%s:%s" % + (client.hostname, data.get('name'), + [line.strip() for line in dlines])) for line in dlines[:]: if line.split(':')[0] == 'group': newgroup = line.split(':')[1].strip() @@ -245,11 +298,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.hostname].append(newgroup) dlines.remove(line) dobj = ProbeData("\n".join(dlines)) - try: - self.probedata[client.hostname].update({data.get('name'): dobj}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), dobj)]) + self.probedata[client.hostname].update({data.get('name'): dobj}) def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py new file mode 100644 index 000000000..b05f8c146 --- /dev/null +++ b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py @@ -0,0 +1,517 @@ +import os +import sys +import time +import unittest +import lxml.etree +from mock import Mock, patch +from django.core.management import setup_environ + +os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" + +import Bcfg2.settings +Bcfg2.settings.DATABASE_NAME = \ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") +Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME + +import Bcfg2.Server +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Probes import * + +datastore = "/" + +# test data for JSON and YAML tests +test_data = dict(a=1, b=[1, 2, 3], c="test") + +def test_syncdb(): + # create the test database + setup_environ(Bcfg2.settings) + from django.core.management.commands import syncdb + cmd = syncdb.Command() + cmd.handle_noargs(interactive=False) + assert os.path.exists(Bcfg2.settings.DATABASE_NAME) + + # ensure that we a) can connect to the database; b) start with a + # clean database + ProbesDataModel.objects.all().delete() + ProbesGroupsModel.objects.all().delete() + assert list(ProbesDataModel.objects.all()) == [] + + +class FakeList(list): + sort = Mock() + + +class TestClientProbeDataSet(unittest.TestCase): + def test__init(self): + ds = ClientProbeDataSet() + self.assertLessEqual(ds.timestamp, time.time()) + self.assertIsInstance(ds, dict) + self.assertNotIn("timestamp", ds) + + ds = ClientProbeDataSet(timestamp=123) + self.assertEqual(ds.timestamp, 123) + self.assertNotIn("timestamp", ds) + +class TestProbeData(unittest.TestCase): + def test_str(self): + # a value that is not valid XML, JSON, or YAML + val = "'test" + + # test string behavior + data = ProbeData(val) + self.assertIsInstance(data, str) + self.assertEqual(data, val) + # test 1.2.0-1.2.2 broken behavior + self.assertEqual(data.data, val) + # test that formatted data accessors return None + self.assertIsNone(data.xdata) + self.assertIsNone(data.yaml) + self.assertIsNone(data.json) + + def test_xdata(self): + xdata = lxml.etree.Element("test") + lxml.etree.SubElement(xdata, "test2") + data = ProbeData(lxml.etree.tostring(xdata)) + self.assertIsNotNone(data.xdata) + self.assertIsNotNone(data.xdata.find("test2")) + + def test_json(self): + if not has_json: + self.skipTest("JSON libraries not found, skipping JSON tests") + jdata = json.dumps(test_data) + data = ProbeData(jdata) + self.assertIsNotNone(data.json) + self.assertItemsEqual(test_data, data.json) + + def test_yaml(self): + if not has_yaml: + self.skipTest("YAML libraries not found, skipping YAML tests") + jdata = yaml.dump(test_data) + data = ProbeData(jdata) + self.assertIsNotNone(data.yaml) + self.assertItemsEqual(test_data, data.yaml) + + +class TestProbeSet(unittest.TestCase): + def get_probeset_object(self, fam=None): + if fam is None: + fam = Mock() + return ProbeSet(datastore, fam, None, "Probes") + + def test__init(self): + fam = Mock() + ps = self.get_probeset_object(fam) + self.assertEqual(ps.plugin_name, "Probes") + fam.AddMonitor.assert_called_with(datastore, ps) + + def test_HandleEvent(self): + ps = self.get_probeset_object() + ps.handle_event = Mock() + + # test that events on the data store itself are skipped + evt = Mock() + evt.filename = datastore + ps.HandleEvent(evt) + self.assertFalse(ps.handle_event.called) + + # test that events on probed.xml are skipped + evt.reset_mock() + evt.filename = "probed.xml" + ps.HandleEvent(evt) + self.assertFalse(ps.handle_event.called) + + # test that other events are processed appropriately + evt.reset_mock() + evt.filename = "fooprobe" + ps.HandleEvent(evt) + ps.handle_event.assert_called_with(evt) + + @patch("__builtin__.list", FakeList) + def test_get_probe_data(self): + ps = self.get_probeset_object() + + # build some fairly complex test data for this. in the end, + # we want the probe data to include only the most specific + # version of a given probe, and by basename only, not full + # (specific) name. We don't fully test the specificity stuff, + # we just check to make sure sort() is called and trust that + # sort() does the right thing on Specificity objects. (I.e., + # trust that Specificity is well-tested. Hah!) We also test + # to make sure the interpreter is determined correctly. + ps.get_matching = Mock() + matching = [] + + p1 = Mock() + p1.specific = Bcfg2.Server.Plugin.Specificity(group=True, prio=10) + p1.name = "fooprobe.G10_foogroup" + p1.data = """#!/bin/bash +group-specific""" + matching.append(p1) + + p2 = Mock() + p2.specific = Bcfg2.Server.Plugin.Specificity(all=True) + p2.name = "fooprobe" + p2.data = "#!/bin/bash" + matching.append(p2) + + p3 = Mock() + p3.specific = Bcfg2.Server.Plugin.Specificity(all=True) + p3.name = "barprobe" + p3.data = "#! /usr/bin/env python" + matching.append(p3) + + p4 = Mock() + p4.specific = Bcfg2.Server.Plugin.Specificity(all=True) + p4.name = "bazprobe" + p4.data = "" + matching.append(p4) + + ps.get_matching.return_value = matching + + metadata = Mock() + pdata = ps.get_probe_data(metadata) + ps.get_matching.assert_called_with(metadata) + FakeList.sort.assert_any_call() + + self.assertEqual(len(pdata), 3, + "Found: %s" % [p.get("name") for p in pdata]) + for probe in pdata: + if probe.get("name") == "fooprobe": + self.assertIn("group-specific", probe.text) + self.assertEqual(probe.get("interpreter"), "/bin/bash") + elif probe.get("name") == "barprobe": + self.assertEqual(probe.get("interpreter"), + "/usr/bin/env python") + elif probe.get("name") == "bazprobe": + self.assertIsNotNone(probe.get("interpreter")) + else: + assert False, "Strange probe found in get_probe_data() return" + + +class TestProbes(unittest.TestCase): + def get_test_probedata(self): + test_xdata = lxml.etree.Element("test") + lxml.etree.SubElement(test_xdata, "test", foo="foo") + rv = dict() + rv["foo.example.com"] = ClientProbeDataSet(timestamp=time.time()) + rv["foo.example.com"]["xml"] = \ + ProbeData(lxml.etree.tostring(test_xdata)) + rv["foo.example.com"]["text"] = ProbeData("freeform text") + rv["foo.example.com"]["multiline"] = ProbeData("""multiple +lines +of +freeform +text +""") + rv["bar.example.com"] = ClientProbeDataSet(timestamp=time.time()) + rv["bar.example.com"]["empty"] = ProbeData("") + if has_yaml: + rv["bar.example.com"]["yaml"] = ProbeData(yaml.dump(test_data)) + if has_json: + rv["bar.example.com"]["json"] = ProbeData(json.dumps(test_data)) + return rv + + def get_test_cgroups(self): + return {"foo.example.com": ["group", "group with spaces", + "group-with-dashes"], + "bar.example.com": []} + + def get_probes_object(self, use_db=False): + p = Probes(Mock(), datastore) + p.core.setup = Mock() + p.core.setup.cfp = Mock() + p.core.setup.cfp.getboolean = Mock() + if use_db: + p.core.setup.cfp.getboolean.return_value = True + else: + p.core.setup.cfp.getboolean.return_value = False + return p + + @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data") + def test__init(self, mock_load_data): + probes = self.get_probes_object() + probes.core.fam.AddMonitor.assert_called_with(os.path.join(datastore, + probes.name), + probes.probes) + mock_load_data.assert_any_call() + self.assertEqual(probes.probedata, ClientProbeDataSet()) + self.assertEqual(probes.cgroups, dict()) + + @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data", Mock()) + def test__use_db(self): + probes = self.get_probes_object() + self.assertFalse(probes._use_db) + probes.core.setup.cfp.getboolean.assert_called_with("probes", + "use_database", + default=False) + + @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_db", Mock()) + @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_xml", Mock()) + def test_write_data(self): + probes = self.get_probes_object(use_db=False) + probes.write_data("test") + probes._write_data_xml.assert_called_with("test") + self.assertFalse(probes._write_data_db.called) + + probes = self.get_probes_object(use_db=True) + probes._write_data_xml.reset_mock() + probes._write_data_db.reset_mock() + probes.write_data("test") + probes._write_data_db.assert_called_with("test") + self.assertFalse(probes._write_data_xml.called) + + @patch("__builtin__.open") + def test__write_data_xml(self, mock_open): + probes = self.get_probes_object(use_db=False) + probes.probedata = self.get_test_probedata() + probes.cgroups = self.get_test_cgroups() + probes._write_data_xml(None) + + mock_open.assert_called_with(os.path.join(datastore, probes.name, + "probed.xml"), "w") + data = lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0])) + self.assertEqual(len(data.xpath("//Client")), 2) + + foodata = data.find("Client[@name='foo.example.com']") + self.assertIsNotNone(foodata) + self.assertIsNotNone(foodata.get("timestamp")) + self.assertEqual(len(foodata.findall("Probe")), + len(probes.probedata['foo.example.com'])) + self.assertEqual(len(foodata.findall("Group")), + len(probes.cgroups['foo.example.com'])) + xml = foodata.find("Probe[@name='xml']") + self.assertIsNotNone(xml) + self.assertIsNotNone(xml.get("value")) + xdata = lxml.etree.XML(xml.get("value")) + self.assertIsNotNone(xdata) + self.assertIsNotNone(xdata.find("test")) + self.assertEqual(xdata.find("test").get("foo"), "foo") + text = foodata.find("Probe[@name='text']") + self.assertIsNotNone(text) + self.assertIsNotNone(text.get("value")) + multiline = foodata.find("Probe[@name='multiline']") + self.assertIsNotNone(multiline) + self.assertIsNotNone(multiline.get("value")) + self.assertGreater(len(multiline.get("value").splitlines()), 1) + + bardata = data.find("Client[@name='bar.example.com']") + self.assertIsNotNone(bardata) + self.assertIsNotNone(bardata.get("timestamp")) + self.assertEqual(len(bardata.findall("Probe")), + len(probes.probedata['bar.example.com'])) + self.assertEqual(len(bardata.findall("Group")), + len(probes.cgroups['bar.example.com'])) + empty = bardata.find("Probe[@name='empty']") + self.assertIsNotNone(empty) + self.assertIsNotNone(empty.get("value")) + self.assertEqual(empty.get("value"), "") + if has_yaml: + ydata = bardata.find("Probe[@name='yaml']") + self.assertIsNotNone(ydata) + self.assertIsNotNone(ydata.get("value")) + self.assertItemsEqual(test_data, yaml.load(ydata.get("value"))) + if has_json: + jdata = bardata.find("Probe[@name='json']") + self.assertIsNotNone(jdata) + self.assertIsNotNone(jdata.get("value")) + self.assertItemsEqual(test_data, json.loads(jdata.get("value"))) + + def test__write_data_db(self): + test_syncdb() + probes = self.get_probes_object(use_db=True) + probes.probedata = self.get_test_probedata() + probes.cgroups = self.get_test_cgroups() + + for cname in ["foo.example.com", "bar.example.com"]: + client = Mock() + client.hostname = cname + probes._write_data_db(client) + + pdata = ProbesDataModel.objects.filter(hostname=cname).all() + self.assertEqual(len(pdata), len(probes.probedata[cname])) + + for probe in pdata: + print "probe: %s" % probe.probe + self.assertEqual(probe.hostname, client.hostname) + self.assertIsNotNone(probe.data) + if probe.probe == "xml": + xdata = lxml.etree.XML(probe.data) + self.assertIsNotNone(xdata) + self.assertIsNotNone(xdata.find("test")) + self.assertEqual(xdata.find("test").get("foo"), "foo") + elif probe.probe == "text": + pass + elif probe.probe == "multiline": + self.assertGreater(len(probe.data.splitlines()), 1) + elif probe.probe == "empty": + self.assertEqual(probe.data, "") + elif probe.probe == "yaml": + self.assertItemsEqual(test_data, yaml.load(probe.data)) + elif probe.probe == "json": + self.assertItemsEqual(test_data, json.loads(probe.data)) + else: + assert False, "Strange probe found in _write_data_db data" + + pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() + self.assertEqual(len(pgroups), len(probes.cgroups[cname])) + + # test that old probe data is removed properly + cname = 'foo.example.com' + del probes.probedata[cname]['text'] + probes.cgroups[cname].pop() + client = Mock() + client.hostname = cname + probes._write_data_db(client) + + pdata = ProbesDataModel.objects.filter(hostname=cname).all() + self.assertEqual(len(pdata), len(probes.probedata[cname])) + pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() + self.assertEqual(len(pgroups), len(probes.cgroups[cname])) + + @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_db", Mock()) + @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_xml", Mock()) + def test_load_data(self): + probes = self.get_probes_object(use_db=False) + probes._load_data_xml.reset_mock() + probes._load_data_db.reset_mock() + + probes.load_data() + probes._load_data_xml.assert_any_call() + self.assertFalse(probes._load_data_db.called) + + probes = self.get_probes_object(use_db=True) + probes._load_data_xml.reset_mock() + probes._load_data_db.reset_mock() + probes.load_data() + probes._load_data_db.assert_any_call() + self.assertFalse(probes._load_data_xml.called) + + @patch("__builtin__.open") + @patch("lxml.etree.parse") + def test__load_data_xml(self, mock_parse, mock_open): + probes = self.get_probes_object(use_db=False) + # to get the value for lxml.etree.parse to parse, we call + # _write_data_xml, mock the open() call, and grab the data + # that gets "written" to probed.xml + probes.probedata = self.get_test_probedata() + probes.cgroups = self.get_test_cgroups() + probes._write_data_xml(None) + xdata = \ + lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0])) + mock_parse.return_value = xdata.getroottree() + probes.probedata = dict() + probes.cgroups = dict() + + probes._load_data_xml() + mock_parse.assert_called_with(os.path.join(datastore, probes.name, + 'probed.xml'), + parser=Bcfg2.Server.XMLParser) + self.assertItemsEqual(probes.probedata, self.get_test_probedata()) + self.assertItemsEqual(probes.cgroups, self.get_test_cgroups()) + + def test__load_data_db(self): + test_syncdb() + probes = self.get_probes_object(use_db=True) + probes.probedata = self.get_test_probedata() + probes.cgroups = self.get_test_cgroups() + for cname in probes.probedata.keys(): + client = Mock() + client.hostname = cname + probes._write_data_db(client) + + probes.probedata = dict() + probes.cgroups = dict() + probes._load_data_db() + self.assertItemsEqual(probes.probedata, self.get_test_probedata()) + # the db backend does not store groups at all if a client has + # no groups set, so we can't just use assertItemsEqual here, + # because loading saved data may _not_ result in the original + # data if some clients had no groups set. + test_cgroups = self.get_test_cgroups() + for cname, groups in test_cgroups.items(): + if cname in probes.cgroups: + self.assertEqual(groups, probes.cgroups[cname]) + else: + self.assertEqual(groups, []) + + @patch("Bcfg2.Server.Plugins.Probes.ProbeSet.get_probe_data") + def test_GetProbes(self, mock_get_probe_data): + probes = self.get_probes_object() + metadata = Mock() + probes.GetProbes(metadata) + mock_get_probe_data.assert_called_with(metadata) + + @patch("Bcfg2.Server.Plugins.Probes.Probes.write_data") + @patch("Bcfg2.Server.Plugins.Probes.Probes.ReceiveDataItem") + def test_ReceiveData(self, mock_ReceiveDataItem, mock_write_data): + # we use a simple (read: bogus) datalist here to make this + # easy to test + datalist = ["a", "b", "c"] + + probes = self.get_probes_object() + client = Mock() + client.hostname = "foo.example.com" + probes.ReceiveData(client, datalist) + + self.assertItemsEqual(mock_ReceiveDataItem.call_args_list, + [((client, "a"), {}), ((client, "b"), {}), + ((client, "c"), {})]) + mock_write_data.assert_called_with(client) + + def test_ReceiveDataItem(self): + probes = self.get_probes_object() + for cname, cdata in self.get_test_probedata().items(): + client = Mock() + client.hostname = cname + for pname, pdata in cdata.items(): + dataitem = lxml.etree.Element("Probe", name=pname) + if pname == "text": + # add some groups to the plaintext test to test + # group parsing + data = [pdata] + for group in self.get_test_cgroups()[cname]: + data.append("group:%s" % group) + dataitem.text = "\n".join(data) + else: + dataitem.text = str(pdata) + + probes.ReceiveDataItem(client, dataitem) + + self.assertIn(client.hostname, probes.probedata) + self.assertIn(pname, probes.probedata[cname]) + self.assertEqual(pdata, probes.probedata[cname][pname]) + self.assertIn(client.hostname, probes.cgroups) + self.assertEqual(probes.cgroups[cname], + self.get_test_cgroups()[cname]) + + def test_get_additional_groups(self): + probes = self.get_probes_object() + test_cgroups = self.get_test_cgroups() + probes.cgroups = self.get_test_cgroups() + for cname in test_cgroups.keys(): + metadata = Mock() + metadata.hostname = cname + self.assertEqual(test_cgroups[cname], + probes.get_additional_groups(metadata)) + # test a non-existent client + metadata = Mock() + metadata.hostname = "nonexistent" + self.assertEqual(probes.get_additional_groups(metadata), + list()) + + def test_get_additional_data(self): + probes = self.get_probes_object() + test_probedata = self.get_test_probedata() + probes.probedata = self.get_test_probedata() + for cname in test_probedata.keys(): + metadata = Mock() + metadata.hostname = cname + self.assertEqual(test_probedata[cname], + probes.get_additional_data(metadata)) + # test a non-existent client + metadata = Mock() + metadata.hostname = "nonexistent" + self.assertEqual(probes.get_additional_data(metadata), + ClientProbeDataSet()) + + -- cgit v1.2.3-1-g7c22