diff options
| author | Jason Kincl <kincljc@ornl.gov> | 2012-11-27 14:28:01 -0500 |
|---|---|---|
| committer | Jason Kincl <kincljc@ornl.gov> | 2012-11-27 14:28:01 -0500 |
| commit | 5e0265f837f0eb72123be0b5150451aebdf8b031 (patch) | |
| tree | dbd5fdbe4ec93c48cbba6fec3f608ffefb26eac5 | |
| parent | 894299b01b6138c54a99fd41f166554d175d6106 (diff) | |
| parent | 4c70626094248495bf2c11c09bf2f2f60917187d (diff) | |
| download | bcfg2-5e0265f837f0eb72123be0b5150451aebdf8b031.tar.gz bcfg2-5e0265f837f0eb72123be0b5150451aebdf8b031.tar.bz2 bcfg2-5e0265f837f0eb72123be0b5150451aebdf8b031.zip | |
Merge remote branch 'upstream/master' into jasons-hacking
31 files changed, 480 insertions, 257 deletions
diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index a680e7d04..91a4e6868 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -67,6 +67,7 @@ provide interfaces that a given plugin may or must implement. Interfaces ^^^^^^^^^^ +.. class:: Bcfg2.Server.Plugin.interfaces .. automodule:: Bcfg2.Server.Plugin.interfaces Exposing XML-RPC Functions diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index b8e252cc4..d8f2bc3df 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -434,6 +434,28 @@ Trigger Plugin The Trigger plugin provides a method for calling external scripts when clients are configured. +Caching options +--------------- + +These options are specified in the **[caching]** section. + + client_metadata + The following four caching modes are available for client + metadata: + + * off: No caching of client metadata objects is performed. This + is the default. + * initial: Only initial metadata objects are cached. Initial + metadata objects are created only from the data in the + Metadata plugin, before additional groups from other plugins + are merged in. + * cautious: Final metadata objects are cached, but each client’s + cache is cleared at the start of each client run, immediately + after probe data is received. Cache is also cleared as in + aggressive mode. *on* is a synonym for cautious. + * aggressive: Final metadata objects are cached. Each plugin is + responsible for clearing cache when appropriate. + Client options -------------- diff --git a/doc/server/caching.txt b/doc/server/caching.txt index ab98e9902..51245bd08 100644 --- a/doc/server/caching.txt +++ b/doc/server/caching.txt @@ -20,7 +20,8 @@ can be generated anywhere from 7 to dozens of times per client run was made more complex and powerful in 1.3.0, caching these objects provides the easiest performance gain. -There are four caching modes available: +To enable caching, add a ``[caching]`` section to bcfg2.conf with a +client_metadata option containing one of the following modes: * ``off``: No caching of client metadata objects is performed. This is the default. diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 25986a413..7d0e0acff 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -106,26 +106,13 @@ Genshi templates allow you to use the `Genshi the deprecated :ref:`server-plugins-generators-tgenshi-index` plugin. Genshi templates should be named with a ``.genshi`` extension, e.g.:: - % ls Cfg/etc/motd + % ls Cfg/etc/motd info.xml motd.genshi See the genshi `documentation <http://genshi.edgewall.org/wiki/Documentation>`_ for examples of Genshi syntax. -Inside Genshi Templates -~~~~~~~~~~~~~~~~~~~~~~~ - -Several variables are pre-defined inside templates: - -* **metadata** is the client's :ref:`metadata - <server-plugins-grouping-metadata-clientmetadata>` -* **name** is the path name specified in Bcfg2 -* **path** is the path to the TGenshi template. It starts with a - leading slash, and is relative to the Bcfg2 specification root. - E.g., ``/Cfg/etc/foo.conf/foo.conf.genshi`` or - ``/TGenshi/etc/foo.conf/template.newtxt.H_foo.example.com`` - Troubleshooting ~~~~~~~~~~~~~~~ @@ -138,7 +125,8 @@ E.g.:: bcfg2-info buildfile /etc/foo.conf foo.example.com -To generate a file with an altsrc attribute, you can run:: +To generate a file with an :ref:`altsrc +<server-plugins-structures-altsrc>` attribute, you can run:: bcfg2-info buildfile /etc/foo/foo.conf --altsrc=/etc/foo.conf \ foo.example.com @@ -149,13 +137,13 @@ debug``, and, once in the Python interpreter, run:: metadata = self.build_metadata("<hostname>") path = "<relative path to template (see note below)>" - + ``path`` should be set to the path to the template file with a leading slash, relative to the Bcfg2 specification root. See `Inside Genshi Templates`_ for examples. Then, run:: - + import os, Bcfg2.Options from genshi.template import TemplateLoader, NewTextTemplate name = os.path.dirname(path[path.find('/', 1):]) @@ -230,19 +218,9 @@ Cheetah templates allow you to use the `cheetah templating system the deprecated :ref:`server-plugins-generators-tcheetah` plugin. Cheetah templates should be named with a ``.cheetah`` extension, e.g.:: - % ls Cfg/etc/motd + % ls Cfg/etc/motd info.xml motd.cheetah -Inside Cheetah Templates -~~~~~~~~~~~~~~~~~~~~~~~~ - -Several variables are pre-defined inside templates: - -* **self.metadata** is the client's :ref:`metadata - <server-plugins-grouping-metadata-clientmetadata>` -* **self.path** is the path name specified in Bcfg2 -* **self.source_path** is the path to the Genshi template on the filesystem. - Examples ~~~~~~~~ @@ -265,12 +243,48 @@ comment to appear in the final config file.:: # This is a comment in my template which will be stripped when it's processed through Cheetah \# This comment will appear in the generated config file. +Inside Templates +---------------- + +Several variables are pre-defined inside templates: + ++-------------+--------------------------------------------------------+ +| Name | Description | ++=============+========================================================+ +| metadata | :ref:`Client metadata | +| | <server-plugins-grouping-metadata-clientmetadata>` | ++-------------+--------------------------------------------------------+ +| name | The value of the ``name`` attribute as specified in | +| | the Path entry in Bcfg2. If an :ref:`altsrc | +| | <server-plugins-structures-altsrc>` attribute is used, | +| | then ``name`` will be the value of that attribute. | ++-------------+--------------------------------------------------------+ +| source_path | The path to the template file on the filesystem | ++-------------+--------------------------------------------------------+ +| repo | The path to the Bcfg2 repository on the filesystem | ++-------------+--------------------------------------------------------+ +| path | In Genshi templates, ``path`` is a synonym for | +| | ``source_path``. In Cheetah templates, it's a synonym | +| | for ``name``. For this reason, use of ``path`` is | +| | discouraged, and it may be deprecated in a future | +| | release. | ++-------------+--------------------------------------------------------+ + +To access these variables in a Genshi template, you can simply use the +name, e.g.:: + + Path to this file: ${name} + +In a Cheetah template, the variables are properties of ``self``, +e.g.:: + + Path to this file: $self.name Notes on Using Templates ------------------------ Templates can be host and group specific as well. Deltas will not be -processed for any genshi or cheetah base file. +processed for any Genshi or Cheetah base file. .. note:: @@ -278,19 +292,22 @@ processed for any genshi or cheetah base file. or group-specific files, you will need to ensure that the ``.genshi`` or ``.cheetah`` extension is at the **end** of the filename. Using the examples from above for *host.example.com* and group *server* you would - have the following (using genshi only):: + have the following:: Cfg/etc/fstab/fstab.H_host.example.com.genshi - Cfg/etc/fstab/fstab.G50_server.genshi + Cfg/etc/fstab/fstab.G50_server.cheetah Genshi templates take precence over cheetah templates. For example, if -two files exist named +two files exist named:: Cfg/etc/fstab/fstab.genshi Cfg/etc/fstab/fstab.cheetah -the cheetah template is ignored. But you can mix genshi and cheetah when -using different host-specific or group-specific files. For example: +The Cheetah template is ignored. Exploiting this fact is probably a +pretty bad idea in practice. + +You can mix Genshi and Cheetah when using different host-specific or +group-specific files. For example: Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 079911238..542b38f01 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -300,24 +300,26 @@ nonexistent +===========+====================+=============+ | type | Type of file | nonexistent | +-----------+--------------------+-------------+ -| recursive | Recursively remove | true | +| recursive | Recursively remove | Boolean | | | directory contents | | +-----------+--------------------+-------------+ permissions ^^^^^^^^^^^ -+-----------+--------------------------+--------+ -| Name | Description | Values | -+===========+==========================+========+ -| mode | Mode of the file. | String | -+-----------+--------------------------+--------+ -| owner | Owner of the file. | String | -+-----------+--------------------------+--------+ -| group | Group of the file. | String | -+-----------+--------------------------+--------+ -| secontext | SELinux context | String | -+-----------+--------------------------+--------+ ++-----------+-----------------------------+---------+ +| Name | Description | Values | ++===========+=============================+=========+ +| mode | Mode of the file | String | ++-----------+-----------------------------+---------+ +| owner | Owner of the file | String | ++-----------+-----------------------------+---------+ +| group | Group of the file | String | ++-----------+-----------------------------+---------+ +| secontext | SELinux context | String | ++-----------+-----------------------------+---------+ +| recursive | Recursively set permissions | Boolean | ++-----------+-----------------------------+---------+ symlink ^^^^^^^ diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 528df79db..1cc287ebd 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -113,40 +113,31 @@ demonstrates how group entries can be used in bundles) Genshi templates ================ -Genshi xml templates can be specified one of two ways: +Genshi XML templates allow you to use the `Genshi +<http://genshi.edgewall.org>`_ templating system to dynamically +generate a bundle. Genshi templates can be specified one of two ways: -1. Add an xml-style genshi template to the Bundler directory with a +1. Add an XML-style genshi template to the Bundler directory with a ``.genshi`` and the associated namespace attribute. -2. Simply add the appropriate namespace attribute to your existing xml +2. Simply add the appropriate namespace attribute to your existing XML bundle. -The namespace attribute in this case should look like the following:: +The top-level Bundle tag should look like the following:: - xmlns:py="http://genshi.edgewall.org/" + <Bundle name="foo" xmlns:py="http://genshi.edgewall.org/"> -Motivation ----------- +Several variables are pre-defined inside templates: -Static Bundles have served us relatively well, but have a relatively -weak set of interal per-client differentiation mechanisms. In static -Bundles, the group hierarchy (from the perspective of the current -client) is available for use via boolean constraints with negation. This -notion does not mesh well with the use of Probes, which can result in -freeform data. In the presence of probe results, clients can have a wide -array of data, and rendering down to a boolean logic may not always -be desirable. Moreover, while static Bundles allow entry inclusion or -exclusion based on group memberships, they do not allow programatic entry -rendering. Hence, Genshi templates not only provide more control options, -but it also provide the ability to perform new entry rendering operations. ++-------------+--------------------------------------------------------+ +| Name | Description | ++=============+========================================================+ +| metadata | :ref:`Client metadata | +| | <server-plugins-grouping-metadata-clientmetadata>` | ++-------------+--------------------------------------------------------+ +| repo | The path to the Bcfg2 repository on the filesystem | ++-------------+--------------------------------------------------------+ -The `Genshi templating system`_ is used internally. - -.. _Genshi templating system: http://genshi.edgewall.org/ - -Use ---- - -.. warning:: +.. note:: ``<Group>`` and ``<Client>`` tags are allowed inside of Genshi templates as of Bcfg2 1.2. However, they do not behave the same @@ -154,7 +145,7 @@ Use <py:if test="'groupname' in metadata.groups"> </py:if> - + The conditional is evaluated when the template is rendered, so code inside the conditional is not executed if the conditional fails. A ``<Group>`` tag is evaluated *after* the template is @@ -163,16 +154,6 @@ Use groups, you *must* use a Genshi conditional, not a ``<Group>`` tag. The same caveats apply to ``<Client>`` tags. -Bcfg2 uses the Genshi API for templates, and performs a XML format -stream rendering of the template into an lxml entry, which is included -in the client configuration. :ref:`Client metadata <client-metadata>` -is available inside of the template using the 'metadata' name. Note that -only the markup Genshi template format can be used, as the target output -format is XML. - -A Genshi template looks much like a Bundler file, except the Bundle tag -has an additional `xmlns:py` attribute. See the examples. - Troubleshooting --------------- diff --git a/examples/TemplateHelper/include.py b/examples/TemplateHelper/include.py index 5fba75558..be0034f52 100644 --- a/examples/TemplateHelper/include.py +++ b/examples/TemplateHelper/include.py @@ -1,17 +1,18 @@ -"""IncludeHelper makes it easier to include group- and host-specific files in a template. +""" IncludeHelper makes it easier to include group- and host-specific +files in a template. Synopsis: {% python import os - include = metadata.TemplateHelper['include'].IncludeHelper - custom = include(metadata, path).files(os.path.basename(name)) + include = metadata.TemplateHelper['include'] + custom = include.IncludeHelper(metadata, path).files(os.path.basename(name)) %}\ {% for file in custom %}\ - - ########## Start ${include.specificity(file)} ########## + + ########## Start ${include.describe_specificity(file)} ########## {% include ${file} %} - ########## End ${include.specificity(file)} ########## + ########## End ${include.describe_specificity(file)} ########## {% end %}\ This would let you include files with the same base name; e.g. in a @@ -27,71 +28,86 @@ This would result in two different sets of custom files being used, one drawn from ''bar.conf.G_<group>.genshi_include'' and the other from ''baz.conf.G_<group>.genshi_include''. -==== Methods ==== - - -=== files === - -Usage: - - - """ import os import re -import Bcfg2.Options -__export__ = ["IncludeHelper"] +__export__ = ["IncludeHelper", "get_specificity", "describe_specificity"] + -class IncludeHelper (object): +class IncludeHelper(object): def __init__(self, metadata, path): """ Constructor. - The template path can be found in the ''path'' variable that is set for all Genshi templates.""" + The template path can be found in the ''path'' variable that + is set for all Genshi templates. """ self.metadata = metadata self.path = path - - def _get_basedir(self): - setup = Bcfg2.Options.OptionParser({'repo': - Bcfg2.Options.SERVER_REPOSITORY}) - setup.parse('--') - return os.path.join(setup['repo'], os.path.dirname(self.path)) - - def files(self, fname): + + def get_basedir(self): + return os.path.dirname(self.path) + + def files(self, fname, groups=None): """ Return a list of files to include for this host. Files are found in the template directory based on the following patterns: * ''<prefix>.H_<hostname>.genshi_include'': Host-specific files * ''<prefix>.G_<group>.genshi_include'': Group-specific files + * ''<prefix>.genshi_include'': Non-specific includes Note that there is no numeric priority on the group-specific - files. All matching files are returned by - ''IncludeHelper.files()''. """ + files; all matching files are returned by + ``IncludeHelper.files()``. If you wish to only include files + for a subset of groups, pass the ``groups`` keyword argument. + Host-specific files are always included in the return + value. """ files = [] - hostfile = os.path.join(self._get_basedir(), + hostfile = os.path.join(self.get_basedir(), "%s.H_%s.genshi_include" % (fname, self.metadata.hostname)) if os.path.isfile(hostfile): files.append(hostfile) - - for group in self.metadata.groups: - filename = os.path.join(self._get_basedir(), + + allfile = os.path.join(self.get_basedir(), "%s.genshi_include" % fname) + if os.path.isfile(allfile): + files.append(allfile) + + if groups is None: + groups = sorted(self.metadata.groups) + + for group in groups: + filename = os.path.join(self.get_basedir(), "%s.G_%s.genshi_include" % (fname, group)) if os.path.isfile(filename): files.append(filename) - return sorted(files) - - @staticmethod - def specificity(fname): - """ Get a string describing the specificity of the given file """ - match = re.search(r'(G|H)_(.*)\.genshi_include', fname) - if match: - if match.group(1) == "G": - stype = "group" - else: - stype = "host" - return "%s-specific configs for %s" % (stype, match.group(2)) - return "Unknown specificity" + return files + + +SPECIFICITY_RE = re.compile(r'(G|H)_(.*)\.genshi_include') + + +def get_specificity(fname): + """ Get a tuple of (<type>, <parameter>) describing the + specificity of the given file. Specificity types are "host", + "group", or "all". The parameter will be either a hostname, a + group name, or None (for "all"). """ + match = SPECIFICITY_RE.search(fname) + if match: + if match.group(1) == "G": + stype = "group" + else: + stype = "host" + return (stype, match.group(2)) + return ("all", None) + + +def describe_specificity(fname): + """ Get a string describing the specificity of the given file """ + (stype, param) = get_specificity(fname) + if stype != "all": + return "%s-specific configs for %s" % (stype, param) + else: + return "Generic configs for all clients" diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 49aa5369f..dfcb42a24 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "November 14, 2012" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "November 26, 2012" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -394,6 +394,37 @@ executed on the client in the created files. .sp The Trigger plugin provides a method for calling external scripts when clients are configured. +.SH CACHING OPTIONS +.sp +These options are specified in the \fB[caching]\fP section. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.B client_metadata +The following four caching modes are available for client +metadata: +.INDENT 7.0 +.IP \(bu 2 +off: No caching of client metadata objects is performed. This +is the default. +.IP \(bu 2 +initial: Only initial metadata objects are cached. Initial +metadata objects are created only from the data in the +Metadata plugin, before additional groups from other plugins +are merged in. +.IP \(bu 2 +cautious: Final metadata objects are cached, but each client’s +cache is cleared at the start of each client run, immediately +after probe data is received. Cache is also cleared as in +aggressive mode. \fIon\fP is a synonym for cautious. +.IP \(bu 2 +aggressive: Final metadata objects are cached. Each plugin is +responsible for clearing cache when appropriate. +.UNINDENT +.UNINDENT +.UNINDENT +.UNINDENT .SH CLIENT OPTIONS .sp These options only affect client functionality. They can be specified in diff --git a/redhat/bcfg2.spec.in b/redhat/bcfg2.spec.in index ede45a05e..70396f900 100644 --- a/redhat/bcfg2.spec.in +++ b/redhat/bcfg2.spec.in @@ -111,9 +111,6 @@ Configuration management system documentation %{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' debian/bcfg2-server.init %{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' tools/bcfg2-cron -%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.daily -%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.hourly - # don't start servers by default %{__perl} -pi -e 's@chkconfig: (\d+)@chkconfig: -@' debian/bcfg2.init %{__perl} -pi -e 's@chkconfig: (\d+)@chkconfig: -@' debian/bcfg2-server.init diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 5d20c0462..928aba1e1 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -751,7 +751,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): rescode, restring = self.yumbase.buildTransaction() except yum.Errors.YumBaseError: err = sys.exc_info()[1] - self.logger.error("Yum transaction error: %s" % err) + self.logger.error("Error building Yum transaction: %s" % err) cleanup() return @@ -767,7 +767,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.info("Single Pass for Install Succeeded") except yum.Errors.YumBaseError: err = sys.exc_info()[1] - self.logger.error("Yum transaction error: %s" % err) + self.logger.error("Error processing Yum transaction: %s" % err) cleanup() return else: @@ -788,7 +788,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.debug(" %s" % restring) except yum.Errors.YumBaseError: err = sys.exc_info()[1] - self.logger.error("Yum transaction error: %s" % err) + self.logger.error("Error rerunning Yum transaction: %s" % err) self.yumbase.conf.skip_broken = skip_broken @@ -850,7 +850,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if inst not in self.instance_status: self.logger.warning( " Asked to install/update package never " - "verified: %s" % + "verified: %s" % nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index a06d6e79e..24028f71c 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -158,7 +158,7 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): logging.Formatter('%(name)s[%(process)d]: %(message)s')) logging.root.addHandler(syslog) except socket.error: - logging.root.error("failed to activate syslogging") + logging.root.error("Failed to activate syslogging") except: print("Failed to activate syslogging") @@ -178,17 +178,22 @@ def setup_logging(procname, to_console=True, to_syslog=True, if hasattr(logging, 'already_setup'): return + params = [] + if to_console: if to_console == True: - clvl = min(logging.WARNING, level) - else: - clvl = min(to_console, level) + to_console = logging.WARNING + clvl = min(to_console, level) + params.append("%s to console" % logging.getLevelName(clvl)) add_console_handler(clvl) if to_syslog: slvl = min(level, logging.INFO) + params.append("%s to syslog" % logging.getLevelName(slvl)) add_syslog_handler(procname, syslog_facility, level=slvl) if to_file is not None: + params.append("%s to %s" % (logging.getLevelName(level), to_file)) add_file_handler(to_file, level=level) logging.root.setLevel(logging.DEBUG) + logging.root.debug("Configured logging: %s" % "; ".join(params)) logging.already_setup = True diff --git a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py index 8677efb5f..79d1b5aba 100644 --- a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py +++ b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py @@ -15,9 +15,14 @@ class DirectStore(TransportBase, threading.Thread): TransportBase.__init__(self, setup) threading.Thread.__init__(self) self.save_file = os.path.join(self.data, ".saved") + self.storage = load_storage_from_config(setup) + self.storage.validate() + self.queue = Queue(100000) self.terminate = threading.Event() + self.debug_log("Reporting: Starting %s thread" % + self.__class__.__name__) self.start() def shutdown(self): @@ -35,6 +40,8 @@ class DirectStore(TransportBase, threading.Thread): def run(self): if not self._load(): + self.logger.warning("Reporting: Failed to load saved data, " + "DirectStore thread exiting") return while not self.terminate.isSet() and self.queue is not None: try: @@ -42,16 +49,19 @@ class DirectStore(TransportBase, threading.Thread): timeout=self.timeout) start = time.time() self.storage.import_interaction(interaction) - self.logger.info("Imported data for %s in %s seconds" \ - % (interaction.get('hostname', '<unknown>'), \ - time.time() - start)) + self.logger.info("Imported data for %s in %s seconds" % + (interaction.get('hostname', '<unknown>'), + time.time() - start)) except Empty: + self.debug_log("Reporting: Queue is empty") continue except: err = sys.exc_info()[1] self.logger.error("Reporting: Could not import interaction: %s" % err) continue + self.debug_log("Reporting: Stopping %s thread" % + self.__class__.__name__) if self.queue is not None and not self.queue.empty(): self._save() @@ -74,6 +84,8 @@ class DirectStore(TransportBase, threading.Thread): def _save(self): """ Save any saved data to a file """ + self.debug_log("Reporting: Saving pending data to %s" % + self.save_file) saved_data = [] try: while not self.queue.empty(): @@ -93,6 +105,7 @@ class DirectStore(TransportBase, threading.Thread): def _load(self): """ Load any saved data from a file """ if not os.path.exists(self.save_file): + self.debug_log("Reporting: No saved data to load") return True saved_data = [] try: @@ -106,6 +119,8 @@ class DirectStore(TransportBase, threading.Thread): for interaction in saved_data: # check that shutdown wasnt called early if self.terminate.isSet(): + self.logger.warning("Reporting: Shutdown called while loading " + " saved data") return False try: diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index 8ccb9ed56..30ea39263 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -20,7 +20,7 @@ class LocalFilesystem(TransportBase): super(LocalFilesystem, self).__init__(setup) self.work_path = "%s/work" % self.data - self.logger.debug("LocalFilesystem: work path %s" % self.work_path) + self.debug_log("LocalFilesystem: work path %s" % self.work_path) self.fmon = None self._phony_collector = None @@ -34,6 +34,11 @@ class LocalFilesystem(TransportBase): traceback.format_exc().splitlines()[-1])) raise TransportError + def set_debug(self, debug): + rv = TransportBase.set_debug(self, debug) + self.fmon.set_debug(debug) + return rv + def start_monitor(self, collector): """Start the file monitor. Most of this comes from BaseCore""" setup = self.setup @@ -44,12 +49,13 @@ class LocalFilesystem(TransportBase): "forcing to default" % setup['filemonitor']) fmon = Bcfg2.Server.FileMonitor.available['default'] - fmdebug = setup.get('debug', False) try: - self.fmon = fmon(debug=fmdebug) - self.logger.info("Using the %s file monitor" % self.fmon.__class__.__name__) + self.fmon = fmon(debug=self.debug_flag) + self.logger.info("Using the %s file monitor" % + self.fmon.__class__.__name__) except IOError: - msg = "Failed to instantiate file monitor %s" % setup['filemonitor'] + msg = "Failed to instantiate file monitor %s" % \ + setup['filemonitor'] self.logger.error(msg, exc_info=1) raise TransportError(msg) self.fmon.start() @@ -108,11 +114,11 @@ class LocalFilesystem(TransportBase): #deviate from the normal routines here we only want one event etype = event.code2str() - self.logger.debug("Recieved event %s for %s" % (etype, event.filename)) + self.debug_log("Recieved event %s for %s" % (etype, event.filename)) if os.path.basename(event.filename)[0] == '.': return None if etype in ('created', 'exists'): - self.logger.debug("Handling event %s" % event.filename) + self.debug_log("Handling event %s" % event.filename) payload = os.path.join(self.work_path, event.filename) try: payloadfd = open(payload, "r") @@ -150,7 +156,7 @@ class LocalFilesystem(TransportBase): except ReportingError: raise TransportError except: - self.logger.error("Failed to load collector: %s" % + self.logger.error("Failed to load collector: %s" % traceback.format_exc().splitlines()[-1]) raise TransportError diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index cca7beda0..530011e47 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -2,26 +2,38 @@ The base for all server -> collector Transports """ -import os.path -import logging +import os +import sys +from Bcfg2.Server.Plugin import Debuggable + class TransportError(Exception): """Generic TransportError""" pass + class TransportImportError(TransportError): """Raised when a transport fails to import""" pass -class TransportBase(object): + +class TransportBase(Debuggable): """The base for all transports""" def __init__(self, setup): """Do something here""" clsname = self.__class__.__name__ - self.logger = logging.getLogger(clsname) - self.logger.debug("Loading %s transport" % clsname) + Debuggable.__init__(self, name=clsname) + self.debug_log("Loading %s transport" % clsname) self.data = os.path.join(setup['repo'], 'Reporting', clsname) + if not os.path.exists(self.data): + self.logger.info("%s does not exist, creating" % self.data) + try: + os.makedirs(self.data) + except OSError: + self.logger.warning("Could not create %s: %s" % + (self.data, sys.exc_info()[1])) + self.logger.warning("The transport may not function properly") self.setup = setup self.timeout = 2 diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 63149c15e..4d7453840 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -9,6 +9,7 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer +from lockfile import LockFailed # pylint: disable=E0611 try: from daemon.pidfile import PIDLockFile @@ -80,9 +81,14 @@ class Core(BaseCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + try: + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True + except LockFailed: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) + return False def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index ee875a7e8..040036fb2 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -10,7 +10,6 @@ import threading import time import inspect import lxml.etree -from traceback import format_exc import Bcfg2.settings import Bcfg2.Server import Bcfg2.Logger @@ -181,6 +180,19 @@ class BaseCore(object): #: backend for plugins that have that capability self._database_available = False if Bcfg2.settings.HAS_DJANGO: + db_settings = Bcfg2.settings.DATABASES['default'] + if ('daemon' in self.setup and 'daemon_uid' in self.setup and + self.setup['daemon'] and self.setup['daemon_uid'] and + db_settings['ENGINE'].endswith(".sqlite3") and + not os.path.exists(db_settings['NAME'])): + # syncdb will create the sqlite database, and we're + # going to daemonize, dropping privs to a non-root + # user, so we need to chown the database after + # creating it + do_chown = True + else: + do_chown = False + from django.core.exceptions import ImproperlyConfigured from django.core import management try: @@ -188,11 +200,21 @@ class BaseCore(object): verbosity=0) self._database_available = True except ImproperlyConfigured: - self.logger.error("Django configuration problem: %s" % - format_exc().splitlines()[-1]) + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) except: - self.logger.error("Database update failed: %s" % - format_exc().splitlines()[-1]) + err = sys.exc_info()[1] + self.logger.error("Database update failed: %s" % err) + + if do_chown and self._database_available: + try: + os.chown(db_settings['NAME'], + self.setup['daemon_uid'], + self.setup['daemon_gid']) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to set ownership of database " + "at %s: %s" % (db_settings['NAME'], err)) if '' in setup['plugins']: setup['plugins'].remove('') @@ -207,20 +229,23 @@ class BaseCore(object): "Unloading %s" % (plugin, blacklist)) for plug in blacklist: del self.plugins[plug] - # This section logs the experimental plugins + + # Log experimental plugins expl = [plug for plug in list(self.plugins.values()) if plug.experimental] if expl: self.logger.info("Loading experimental plugin(s): %s" % (" ".join([x.name for x in expl]))) self.logger.info("NOTE: Interfaces subject to change") - # This section logs the deprecated plugins + + # Log deprecated plugins depr = [plug for plug in list(self.plugins.values()) if plug.deprecated] if depr: self.logger.info("Loading deprecated plugin(s): %s" % (" ".join([x.name for x in depr]))) + # Find the metadata plugin and set self.metadata mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) if len(mlist) >= 1: #: The Metadata plugin @@ -522,7 +547,7 @@ class BaseCore(object): except Exception: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % format_exc()) + entry.set('failure', 'bind error: %s' % exc) self.logger.error("Unexpected failure in BindStructure: %s %s" % (entry.tag, entry.get('name')), exc_info=1) @@ -599,7 +624,7 @@ class BaseCore(object): try: structures = self.GetStructures(meta) except: - self.logger.error("error in GetStructures", exc_info=1) + self.logger.error("Error in GetStructures", exc_info=1) return lxml.etree.Element("error", type='structure error') self.validate_structures(meta, structures) @@ -662,7 +687,7 @@ class BaseCore(object): os.chown(piddir, self.setup['daemon_uid'], self.setup['daemon_gid']) - os.chmod(piddir, 420) # 0644 + os.chmod(piddir, 493) # 0775 if not self._daemonize(): return False else: @@ -676,6 +701,9 @@ class BaseCore(object): self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) + + for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): + plug.start_threads() except: self.shutdown() raise diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index cba3e8145..f42ada773 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -299,12 +299,27 @@ class Statistics(Plugin): raise NotImplementedError -class ThreadedStatistics(Statistics, threading.Thread): +class Threaded(object): + """ Threaded plugins use threads in any way. The thread must be + started after daemonization, so this class implements a single + method, :func:`start_threads`, that can be used to start threads + after daemonization of the server core. """ + + def start_threads(self): + """ Start this plugin's threads after daemonization. + + :return: None + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` + """ + raise NotImplementedError + +class ThreadedStatistics(Statistics, Threaded, threading.Thread): """ ThreadedStatistics plugins process client statistics in a separate thread. """ def __init__(self, core, datastore): Statistics.__init__(self, core, datastore) + Threaded.__init__(self) threading.Thread.__init__(self) # Event from the core signaling an exit self.terminate = core.terminate @@ -312,6 +327,8 @@ class ThreadedStatistics(Statistics, threading.Thread): self.pending_file = os.path.join(datastore, "etc", "%s.pending" % self.name) self.daemon = False + + def start_threads(self): self.start() def _save(self): @@ -517,11 +534,11 @@ class Version(Plugin): def __init__(self, core, datastore): Plugin.__init__(self, core, datastore) + if core.setup['vcs_root']: + self.vcs_root = core.setup['vcs_root'] + else: + self.vcs_root = datastore if self.__vcs_metadata_path__: - if core.setup['vcs_root']: - self.vcs_root = core.setup['vcs_root'] - else: - self.vcs_root = datastore self.vcs_path = os.path.join(self.vcs_root, self.__vcs_metadata_path__) diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 7933fe9be..b200346bc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -19,6 +19,9 @@ except ImportError: HAS_GENSHI = False +SETUP = None + + class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): @@ -49,7 +52,8 @@ if HAS_GENSHI: msg = "No parsed template information for %s" % self.name self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate(metadata=metadata).filter( + stream = self.template.generate(metadata=metadata, + repo=SETUP['repo']).filter( Bcfg2.Server.Plugins.TGenshi.removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), @@ -93,8 +97,13 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.data, self.core.fam) except OSError: - self.logger.error("Failed to load Bundle repository") - raise Bcfg2.Server.Plugin.PluginInitError + err = sys.exc_info()[1] + msg = "Failed to load Bundle repository %s: %s" % (self.data, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + + global SETUP + SETUP = core.setup def template_dispatch(self, name, _): """ Add the correct child entry type to Bundler depending on diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 8ebd8d921..724164cf5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -3,7 +3,7 @@ :ref:`server-plugins-generators-cfg` files. """ from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: from Cheetah.Template import Template @@ -37,7 +37,9 @@ class CfgCheetahGenerator(CfgGenerator): template = Template(self.data.decode(self.encoding), compilerSettings=self.settings) template.metadata = metadata + template.name = entry.get('realname', entry.get('name')) template.path = entry.get('realname', entry.get('name')) template.source_path = self.name + template.repo = SETUP['repo'] return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index df0c30c09..3a78b4847 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -6,7 +6,7 @@ import re import sys import traceback from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: import genshi.core @@ -74,7 +74,9 @@ class CfgGenshiGenerator(CfgGenerator): stream = \ self.template.generate(name=fname, metadata=metadata, - path=self.name).filter(removecomment) + path=self.name, + source_path=self.name, + repo=SETUP['repo']).filter(removecomment) try: try: return stream.render('text', encoding=self.encoding, @@ -135,9 +137,13 @@ class CfgGenshiGenerator(CfgGenerator): # single line break) real_lineno = lineno - contents.code.co_firstlineno src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - raise PluginExecutionError("%s: %s at '%s'" % - (err.__class__.__name__, err, - src[real_lineno])) + try: + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + src[real_lineno])) + except IndexError: + raise PluginExecutionError("%s: %s" % + (err.__class__.__name__, err)) raise def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 220146100..37171e1b1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -536,22 +536,37 @@ class YumCollection(Collection): consumerapi = ConsumerAPI() consumer = self._get_pulp_consumer(consumerapi=consumerapi) if consumer is None: - consumer = consumerapi.create(self.metadata.hostname, - self.metadata.hostname, - capabilities=dict(bind=False)) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-update", timing="pre", - when="always", status="check", - command="pulp-consumer consumer update") - self.pulp_cert_set.write_data(consumer['certificate'], - self.metadata) + try: + consumer = \ + consumerapi.create(self.metadata.hostname, + self.metadata.hostname, + capabilities=dict(bind=False)) + lxml.etree.SubElement( + independent, "BoundAction", name="pulp-update", + timing="pre", when="always", status="check", + command="pulp-consumer consumer update") + self.pulp_cert_set.write_data(consumer['certificate'], + self.metadata) + except server.ServerRequestError: + err = sys.exc_info()[1] + self.logger.error("Packages: Could not create Pulp " + "consumer %s: %s" % + (self.metadata.hostname, err)) for source in self: # each pulp source can only have one arch, so we don't # have to check the arch in url_map if (source.pulp_id and source.pulp_id not in consumer['repoids']): - consumerapi.bind(self.metadata.hostname, source.pulp_id) + try: + consumerapi.bind(self.metadata.hostname, + source.pulp_id) + except server.ServerRequestError: + err = sys.exc_info()[1] + self.logger.error("Packages: Could not bind %s to " + "Pulp repo %s: %s" % + (self.metadata.hostname, + source.pulp_id, err)) crt = lxml.etree.SubElement(independent, "BoundPath", name=self.pulp_cert_set.certpath) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 90dff4a66..f106b75a4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -162,6 +162,9 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret.append(probe) return ret + def __str__(self): + return "ProbeSet for %s" % self.plugin_name + class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector, diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 1a8c3d941..60f5b1e09 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -7,8 +7,8 @@ import lxml.etree from Bcfg2.Reporting.Transport import load_transport_from_config, \ TransportError from Bcfg2.Options import REPORTING_COMMON_OPTIONS -from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \ - PluginExecutionError +from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ + Debuggable, PluginInitError, PluginExecutionError # required for reporting try: @@ -31,9 +31,10 @@ def _rpc_call(method): return _real_rpc_call -class Reporting(Statistics, PullSource): # pylint: disable=W0223 +# pylint: disable=W0223 +class Reporting(Statistics, Threaded, PullSource, Debuggable): """ Unified statistics and reporting plugin """ - __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') @@ -41,7 +42,8 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 def __init__(self, core, datastore): Statistics.__init__(self, core, datastore) PullSource.__init__(self) - self.core = core + Threaded.__init__(self) + Debuggable.__init__(self) self.whoami = platform.node() self.transport = None @@ -54,14 +56,20 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 self.logger.error(msg) raise PluginInitError(msg) + def start_threads(self): try: - self.transport = load_transport_from_config(core.setup) + self.transport = load_transport_from_config(self.core.setup) except TransportError: msg = "%s: Failed to load transport: %s" % \ (self.name, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise PluginInitError(msg) + def set_debug(self, debug): + rv = Debuggable.set_debug(self, debug) + self.transport.set_debug(debug) + return rv + def process_statistics(self, client, xdata): stats = xdata.find("Statistics") stats.set('time', time.asctime(time.localtime())) @@ -84,8 +92,8 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 lxml.etree.tostring( stats, xml_declaration=False).decode('UTF-8')) - self.logger.debug("%s: Queued statistics data for %s" % - (self.__class__.__name__, client.hostname)) + self.debug_log("%s: Queued statistics data for %s" % + (self.__class__.__name__, client.hostname)) return except TransportError: continue @@ -94,7 +102,7 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 % (self.__class__.__name__, i, traceback.format_exc().splitlines()[-1])) self.logger.error("%s: Retry limit reached for %s" % - (self.__class__.__name__, client.hostname)) + (self.__class__.__name__, client.hostname)) def shutdown(self): super(Reporting, self).shutdown() diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 62396f860..b3a49c047 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -73,9 +73,8 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cert_spec.get('append_chain', 'false').lower() == 'true', } - cfp = ConfigParser.ConfigParser() - cfp.read(self.core.cfile) - self.CAs[ca] = dict(cfp.items('sslca_' + ca)) + self.CAs[ca] = dict(self.core.setup.cfp.items('sslca_%s' % + ca)) self.Entries['Path'][ident] = self.get_cert elif event.filename.endswith("info.xml"): self.infoxml[ident] = Bcfg2.Server.Plugin.InfoXML(epath) diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py index 1956af4ad..cc5946bb2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py +++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py @@ -65,6 +65,8 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics): self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) self.work_queue = Queue() self.loader = threading.Thread(target=self.load_snapshot) + + def start_threads(self): self.loader.start() def load_snapshot(self): diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 17a275340..bc585570d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -17,10 +17,9 @@ except ImportError: class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".svn" if HAS_SVN: __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update', 'Commit'] - else: - __vcs_metadata_path__ = ".svn" def callback_conflict_resolver(self): """PySvn callback function to resolve conflicts""" diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index acb9e4f44..fa8c89b46 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -27,12 +27,6 @@ try: except ImportError: HAS_PROFILE = False -try: - from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False - class MockLog(object): """ Fake logger that just discards all messages in order to mask @@ -401,28 +395,24 @@ Bcfg2 client itself.""") def do_buildbundle(self, args): """ buildbundle <bundle> <hostname> - Render a templated bundle for hostname (not written to disk) """ - if len(args.split()) == 2: - bname, client = args.split() - try: - metadata = self.build_metadata(client) - if bname in self.plugins['Bundler'].entries: - bundle = self.plugins['Bundler'].entries[bname] - if (HAS_GENSHI and - isinstance(bundle, - BundleTemplateFile)): - stream = bundle.template.generate(metadata=metadata) - print(stream.render("xml")) - else: - print(bundle.data) - else: - print("No such bundle %s" % bname) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Failed to render bundle %s for host %s: %s" % (bname, - client, - err)) - else: + if len(args.split()) != 2: print(self._get_usage(self.do_buildbundle)) + return 1 + + bname, client = args.split() + try: + metadata = self.build_metadata(client) + bundle = self.plugins['Bundler'].entries[bname] + print(lxml.etree.tostring(bundle.get_xml_value(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + except KeyError: + print("No such bundle %s" % bname) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + print("Failed to render bundle %s for host %s: %s" % (bname, + client, + err)) def do_automatch(self, args): """ automatch [-f] <propertyfile> <hostname> - Perform automatch on diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index 2eda38cdc..a1e624824 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -71,10 +71,12 @@ class TestPlugin(TestDebuggable): def get_obj(self, core=None): if core is None: core = Mock() + core.setup = MagicMock() return self.test_obj(core, datastore) def test__init(self): core = Mock() + core.setup = MagicMock() p = self.get_obj(core=core) self.assertEqual(p.data, os.path.join(datastore, p.name)) self.assertEqual(p.core, core) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py index 9a064663e..343f088b3 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py @@ -108,15 +108,30 @@ class TestStatistics(TestPlugin): s.process_statistics, None, None) -class TestThreadedStatistics(TestStatistics): +class TestThreaded(Bcfg2TestCase): + test_obj = Threaded + + def get_obj(self): + return self.test_obj() + + def test_start_threads(self): + s = self.get_obj() + self.assertRaises(NotImplementedError, + s.start_threads) + + +class TestThreadedStatistics(TestStatistics, TestThreaded): test_obj = ThreadedStatistics data = [("foo.example.com", "<foo/>"), ("bar.example.com", "<bar/>")] + def get_obj(self, core=None): + return TestStatistics.get_obj(self, core=core) + @patch("threading.Thread.start") - def test__init(self, mock_start): - core = Mock() - ts = self.get_obj(core) + def test_start_threads(self, mock_start): + ts = self.get_obj() + ts.start_threads() mock_start.assert_any_call() @patch("%s.open" % builtins) @@ -157,7 +172,7 @@ class TestThreadedStatistics(TestStatistics): # verify this call in an ugly way self.assertItemsEqual(mock_dump.call_args[0][0], self.data) self.assertEqual(mock_dump.call_args[0][1], mock_open.return_value) - + @patch("os.unlink") @patch("os.path.exists") @patch("%s.open" % builtins) @@ -169,7 +184,7 @@ class TestThreadedStatistics(TestStatistics): core = Mock() core.terminate.isSet.return_value = False ts = self.get_obj(core) - + ts.work_queue = Mock() ts.work_queue.data = [] def reset(): @@ -279,7 +294,7 @@ class TestThreadedStatistics(TestStatistics): ts = self.get_obj() self.assertRaises(NotImplementedError, ts.handle_statistic, None, None) - + class TestPullSource(Bcfg2TestCase): def test_GetCurrentEntry(self): @@ -302,7 +317,7 @@ class TestPullTarget(Bcfg2TestCase): class TestDecision(Bcfg2TestCase): test_obj = Decision - + def get_obj(self): return self.test_obj() @@ -332,6 +347,7 @@ class TestVersion(TestPlugin): def get_obj(self, core=None): if core is None: core = Mock() + core.setup = MagicMock() return self.test_obj(core, datastore) def test_get_revision(self): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py index 1832e5e03..fc5d5e53d 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py @@ -31,9 +31,18 @@ if HAS_CHEETAH or can_skip: ccg.data = "data" entry = lxml.etree.Element("Path", name="/test.txt") metadata = Mock() + Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP = MagicMock() self.assertEqual(ccg.get_data(entry, metadata), mock_Template.return_value.respond.return_value) + Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.assert_called_with("repo") mock_Template.assert_called_with("data".decode(ccg.encoding), compilerSettings=ccg.settings) - mock_Template.return_value.respond.assert_called_with() + tmpl = mock_Template.return_value + tmpl.respond.assert_called_with() + self.assertEqual(tmpl.metadata, metadata) + self.assertEqual(tmpl.name, entry.get("name")) + self.assertEqual(tmpl.path, entry.get("name")) + self.assertEqual(tmpl.source_path, ccg.name) + self.assertEqual(tmpl.repo, + Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.return_value) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py index 4a849c11a..385f8df77 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py @@ -51,15 +51,24 @@ if can_skip or HAS_GENSHI: entry = lxml.etree.Element("Path", name="/test.txt") metadata = Mock() + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP = MagicMock() + def reset(): cgg.template.reset_mock() cgg._handle_genshi_exception.reset_mock() + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.reset_mock() + + template_vars = dict( + name=entry.get("name"), + metadata=metadata, + path=cgg.name, + source_path=cgg.name, + repo=Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.return_value) self.assertEqual(cgg.get_data(entry, metadata), stream.render.return_value) - cgg.template.generate.assert_called_with(name=entry.get("name"), - metadata=metadata, - path=cgg.name) + cgg.template.generate.assert_called_with(**template_vars) + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") fltr.filter.assert_called_with(removecomment) stream.render.assert_called_with("text", encoding=cgg.encoding, strip_whitespace=False) @@ -71,9 +80,8 @@ if can_skip or HAS_GENSHI: stream.render.side_effect = render self.assertEqual(cgg.get_data(entry, metadata), stream.render.return_value) - cgg.template.generate.assert_called_with(name=entry.get("name"), - metadata=metadata, - path=cgg.name) + cgg.template.generate.assert_called_with(**template_vars) + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") fltr.filter.assert_called_with(removecomment) self.assertEqual(stream.render.call_args_list, [call("text", encoding=cgg.encoding, @@ -84,9 +92,8 @@ if can_skip or HAS_GENSHI: stream.render.side_effect = UndefinedError("test") self.assertRaises(UndefinedError, cgg.get_data, entry, metadata) - cgg.template.generate.assert_called_with(name=entry.get("name"), - metadata=metadata, - path=cgg.name) + cgg.template.generate.assert_called_with(**template_vars) + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") fltr.filter.assert_called_with(removecomment) stream.render.assert_called_with("text", encoding=cgg.encoding, strip_whitespace=False) @@ -96,9 +103,8 @@ if can_skip or HAS_GENSHI: cgg._handle_genshi_exception.side_effect = ValueError self.assertRaises(ValueError, cgg.get_data, entry, metadata) - cgg.template.generate.assert_called_with(name=entry.get("name"), - metadata=metadata, - path=cgg.name) + cgg.template.generate.assert_called_with(**template_vars) + Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") fltr.filter.assert_called_with(removecomment) stream.render.assert_called_with("text", encoding=cgg.encoding, strip_whitespace=False) |