diff options
| author | Jason Kincl <kincljc@ornl.gov> | 2012-11-19 10:37:34 -0500 |
|---|---|---|
| committer | Jason Kincl <kincljc@ornl.gov> | 2012-11-19 10:37:34 -0500 |
| commit | 894299b01b6138c54a99fd41f166554d175d6106 (patch) | |
| tree | 88e11cb8c49d3f933cf5f622a93dfa123922960f | |
| parent | de0ae51b6dc635a3acd2491d4ca3fd021aa55873 (diff) | |
| parent | f4da37aa0a360add3f5c40f37cd3cc010ef8788f (diff) | |
| download | bcfg2-894299b01b6138c54a99fd41f166554d175d6106.tar.gz bcfg2-894299b01b6138c54a99fd41f166554d175d6106.tar.bz2 bcfg2-894299b01b6138c54a99fd41f166554d175d6106.zip | |
Merge remote branch 'upstream/master' into jasons-hacking
69 files changed, 769 insertions, 632 deletions
diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 000000000..7f99a8eb7 --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,16 @@ +bcfg2 (1.1.0-1) unstable; urgency=low + + Due to repository changes made to support Path entries it is recommended + to upgrade the clients before upgrading the server. After that, you can + use the posixunified.py script (included as an example in the server + package) to convert your repository. + + -- Arto Jantunen <viiru@debian.org> Fri, 05 Nov 2010 19:07:59 +0200 + +bcfg2 (1.0.1-1) unstable; urgency=low + + Since version 1.0, Bcfg2 no longer supports agent mode. Please update your + configuration accordingly if you were using it. + + -- Arto Jantunen <viiru@debian.org> Tue, 03 Aug 2010 12:28:54 +0300 + diff --git a/debian/bcfg2-doc.docs b/debian/bcfg2-doc.docs new file mode 100644 index 000000000..6f7511e54 --- /dev/null +++ b/debian/bcfg2-doc.docs @@ -0,0 +1 @@ +build/sphinx/html diff --git a/debian/bcfg2-doc.links b/debian/bcfg2-doc.links deleted file mode 100644 index 133a58e2e..000000000 --- a/debian/bcfg2-doc.links +++ /dev/null @@ -1 +0,0 @@ -usr/share/doc/bcfg2/html/_sources usr/share/doc/bcfg2/rst diff --git a/debian/bcfg2-server.examples b/debian/bcfg2-server.examples new file mode 100644 index 000000000..98ec4b785 --- /dev/null +++ b/debian/bcfg2-server.examples @@ -0,0 +1 @@ +tools/upgrade diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index a1fa65d14..8de16b9b5 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -7,13 +7,13 @@ # ### BEGIN INIT INFO # Provides: bcfg2-server -# Required-Start: $network $remote_fs $named -# Required-Stop: $network $remote_fs $named +# Required-Start: $network $remote_fs $named $syslog +# Required-Stop: $network $remote_fs $named $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Configuration management Server -# Description: Bcfg2 is a configuration management system that builds -# installs configuration files served by bcfg2-server +# Description: The server component of the Bcfg2 configuration management +# system ### END INIT INFO # Include lsb functions diff --git a/debian/bcfg2.init b/debian/bcfg2.init index add840c90..4f83adbf6 100644..100755 --- a/debian/bcfg2.init +++ b/debian/bcfg2.init @@ -70,16 +70,17 @@ start () { case "$1" in start) start - ;; - stop) + ;; + stop|status) exit 0 - ;; + ;; restart|force-reload) start - ;; - *) - echo "Usage: $0 {start|stop|restart|force-reload}" - exit 1 + ;; + *) + echo "Usage: $0 {start|stop|status|restart|force-reload}" + exit 1 + ;; esac exit 0 diff --git a/debian/control b/debian/control index 28ddbf9aa..be03f4942 100644 --- a/debian/control +++ b/debian/control @@ -42,9 +42,9 @@ Description: Configuration management web interface bcfg2-web is the reporting server for bcfg2. Package: bcfg2-doc +Section: doc Architecture: all Depends: ${sphinxdoc:Depends}, ${misc:Depends} -XB-Python-Version: >= 2.4 Description: Configuration management system documentation Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. diff --git a/debian/rules b/debian/rules index 30fd64a43..fcbf6346c 100755 --- a/debian/rules +++ b/debian/rules @@ -16,9 +16,9 @@ override_dh_installinit: # Install bcfg2-server initscript without starting it on postinst dh_installinit --package=bcfg2-server --no-start -override_dh_installdocs: +override_dh_auto_build: + dh_auto_build python setup.py build_sphinx - dh_installdocs build/sphinx/html override_dh_auto_clean: dh_auto_clean diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 942ead40d..b8e252cc4 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -143,6 +143,9 @@ vcs_root E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. +umask + The umask to set for the server. Default is *0077*. + Server Plugins -------------- diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 57e80ba25..19c947a71 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -266,6 +266,7 @@ are available: * LocalFilesystem: Statistics are written to the local file system and collected on the local machine. +* RedisTransport: Statistics are sent through a list in redis. * DirectStore: DBStats style threaded imports in the main server process. Future transports will allow multiple servers to pass data to a single or multiple @@ -277,6 +278,30 @@ release. If DirectStore is used, the bcfg2-report-collector process will refuse to run since this method is not compatible with an external process. +RedisTransport +^^^^^^^^^^^^^^ + +This transport uses a single redis instance for communication between bcfg2-server and +bcfg2-report-collector. Multiple servers can write to a single redis instance and multiple +report collectors may be run as well. + +An example configuration with the default values:: + + [reporting] + transport = RedisTransport + redis_host = 127.0.0.1 + redis_port = 6379 + redis_db = 0 + +bcfg2-admin commands operate slightly differently in this mode. Instead of querying the +database directly, rpc commands are issued to the report collectors. This only affects +the minestruct and pull commands. + +.. warning:: + + At the time of this writing the version of python-redis in EPEL is too old to use with + this transport. Current versions of the python-redis package require python >= 2.5. + Usage ===== diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 7b8b7a6c9..079911238 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -474,8 +474,6 @@ node +-------------+------------------------------------+------------------+----------+ | proto | Protocol | (ipv4|ipv6) | Yes | +-------------+------------------------------------+------------------+----------+ -| netmask | Netmask | String | Yes | -+-------------+------------------------------------+------------------+----------+ login ^^^^^ @@ -496,9 +494,9 @@ user +=============+===============================+===========+==========+ | name | SELinux username | String | Yes | +-------------+-------------------------------+-----------+----------+ -| roles | Space-separated list of roles | String | No | +| roles | Space-separated list of roles | String | Yes | +-------------+-------------------------------+-----------+----------+ -| prefix | Home directory context prefix | String | No | +| prefix | Home directory context prefix | String | Yes | +-------------+-------------------------------+-----------+----------+ interface diff --git a/doc/server/plugins/version/git.txt b/doc/server/plugins/version/git.txt index d4b9187fa..3f7ab9d9b 100644 --- a/doc/server/plugins/version/git.txt +++ b/doc/server/plugins/version/git.txt @@ -13,19 +13,45 @@ reporting purposes. Once the plugin is enabled, every time a client checks in, it will include the current repository revision in the reports/statistics. -If the ``dulwich`` library is installed, the Git plugin will use -that. If ``dulwich`` is not installed, but ``GitPython`` is, that -will be used instead. If neither is installed, then calls will be -made to the git command. - -If you plan to edit your git repository in-place on the Bcfg2 server -(which is probably not recommended), then you may want to avoid using -``dulwich``; it's sufficiently low-level that it may not present a -user-friendly git repository at all times. - -Additionally, the Git plugin exposes one XML-RPC method calls, -``Git.Update``, which updates the working copy to the latest version -in the remote origin. +As with the other Version plugins, the Git plugin enables you to get +revision information out of your repository for reporting +purposes. Once the plugin is enabled, every time a client checks in, +it will include the current repository revision in the +reports/statistics. + +Additionally, if the ``GitPython`` library is installed, the Git +plugin exposes an additional XML-RPC method call, ``Git.Update``. +With no arguments, ``Git.Update`` updates the working copy to the +latest version in the remote tracking branch. If the current working +copy doesn't have a remote tracking branch, then nothing is done. + +``Git.Update`` can also be given a single argument, the name of a git +tree-ish (branch, tag, ref, commit, etc.) to check out. When this is +done, the new working is updated as well. + +For example:: + + bcfg2-admin xcmd Git.Update master + +This checks out the ``master`` branch and updates it to the latest +data from the remote ``master`` (if applicable). If you then run:: + + bcfg2-admin xcmd Git.Update + +This updates to the latest remote data without changing branches. +Then:: + + bcfg2-admin xcmd Git.Update dd0bb776c + +This checks out the specified commit. Subsequently:: + + bcfg2-admin xcmd Git.Update + +This does nothing, because the working copy is now in "detached HEAD" +state, and there can be no remote tracking branch to update from. + +To put it another way, once you tell ``Git.Update`` which tree-ish to +checkout, it stays on that tree-ish until you tell it otherwise. Enabling the Git plugin ======================= diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 2cfff35af..008f56fa2 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-ADMIN" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-ADMIN" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-admin \- Perform repository administration tasks . diff --git a/man/bcfg2-build-reports.8 b/man/bcfg2-build-reports.8 index 6030e8b6b..1639adc74 100644 --- a/man/bcfg2-build-reports.8 +++ b/man/bcfg2-build-reports.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-BUILD-REPORTS" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-BUILD-REPORTS" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-build-reports \- Generate state reports for Bcfg2 clients . diff --git a/man/bcfg2-crypt.8 b/man/bcfg2-crypt.8 index 1e161c099..ab428c266 100644 --- a/man/bcfg2-crypt.8 +++ b/man/bcfg2-crypt.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-CRYPT" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-CRYPT" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-crypt \- Bcfg2 encryption and decryption utility . diff --git a/man/bcfg2-info.8 b/man/bcfg2-info.8 index 1ea428865..57c9e012c 100644 --- a/man/bcfg2-info.8 +++ b/man/bcfg2-info.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-INFO" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-INFO" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-info \- Creates a local version of the Bcfg2 server core for state observation . diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8 index a908f5877..01ba87a51 100644 --- a/man/bcfg2-lint.8 +++ b/man/bcfg2-lint.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-LINT" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-LINT" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-lint \- Check Bcfg2 specification for validity, common mistakes, and style . diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5 index e99ac1bb6..d02b4e380 100644 --- a/man/bcfg2-lint.conf.5 +++ b/man/bcfg2-lint.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2-LINT.CONF" "5" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-LINT.CONF" "5" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-lint.conf \- Configuration parameters for bcfg2-lint . diff --git a/man/bcfg2-reports.8 b/man/bcfg2-reports.8 index 4841d9e7a..3b9e549e7 100644 --- a/man/bcfg2-reports.8 +++ b/man/bcfg2-reports.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-REPORTS" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-REPORTS" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-reports \- Query reporting system for client status . diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index b717ba797..1fbbb0ec7 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-SERVER" "8" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2-SERVER" "8" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2-server \- Server for client configuration specifications . diff --git a/man/bcfg2.1 b/man/bcfg2.1 index adf7d1d42..6ee34831f 100644 --- a/man/bcfg2.1 +++ b/man/bcfg2.1 @@ -1,4 +1,4 @@ -.TH "BCFG2" "1" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2" "1" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2 \- Bcfg2 client tool . diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 6f5771af7..49aa5369f 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "November 07, 2012" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "November 14, 2012" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -180,6 +180,9 @@ Specifies the path to the root of the VCS working copy that holds your Bcfg2 specification, if it is different from \fIrepository\fP. E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. +.TP +.B umask +The umask to set for the server. Default is \fI0077\fP. .UNINDENT .SH SERVER PLUGINS .sp diff --git a/redhat/scripts/bcfg2.init b/redhat/scripts/bcfg2.init index b4ea37763..5cfdf47bc 100755 --- a/redhat/scripts/bcfg2.init +++ b/redhat/scripts/bcfg2.init @@ -54,15 +54,15 @@ start () { case "$1" in start) start - ;; - stop) + ;; + stop|status) exit 0 - ;; + ;; restart|force-reload) start - ;; + ;; *) - echo "Usage: $0 {start|stop|restart|force-reload}" + echo "Usage: $0 {start|stop|status|restart|force-reload}" RETVAL=3 esac diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 0400e3ff7..f197a9074 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -106,7 +106,9 @@ class Client(object): self.logger.info(ret.text) finally: os.unlink(scriptname) - except: # pylint: disable=W0702 + except SystemExit: + raise + except: self._probe_failure(name, sys.exc_info()[1]) return ret @@ -258,8 +260,7 @@ class Client(object): except Bcfg2.Client.XML.ParseError: syntax_error = sys.exc_info()[1] self.fatal_error("The configuration could not be parsed: %s" % - (syntax_error)) - return(1) + syntax_error) times['config_parse'] = time.time() @@ -292,14 +293,17 @@ class Client(object): fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: # otherwise exit and give a warning to the user - self.fatal_error("An other instance of Bcfg2 is running. " - "If you what to bypass the check, run " - "with %s option" % + self.fatal_error("Another instance of Bcfg2 is running. " + "If you want to bypass the check, run " + "with the %s option" % Bcfg2.Options.OMIT_LOCK_CHECK.cmd) - except: # pylint: disable=W0702 + except SystemExit: + raise + except: lockfile = None self.logger.error("Failed to open lockfile") - # execute the said configuration + + # execute the configuration self.tools.Execute() if not self.setup['omit_lock_check']: diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 64460ea66..53180ab68 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,8 +1,10 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import os import sys import time +import select import fnmatch import logging import Bcfg2.Client.Tools @@ -160,6 +162,9 @@ class Frame(object): iprompt = entry.get('qtext') else: iprompt = prompt % (entry.tag, entry.get('name')) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) try: ans = input(iprompt.encode(sys.stdout.encoding, 'replace')) if ans in ['y', 'Y']: diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 7726da94c..b1a897c81 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -1,5 +1,8 @@ """Action driver""" +import os +import sys +import select import Bcfg2.Client.Tools from Bcfg2.Client.Frame import matches_white_list, passes_black_list from Bcfg2.Compat import input # pylint: disable=W0622 @@ -33,6 +36,10 @@ class Action(Bcfg2.Client.Tools.Tool): if self.setup['interactive']: prompt = ('Run Action %s, %s: (y/N): ' % (entry.get('name'), entry.get('command'))) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input(prompt) if ans not in ['y', 'Y']: return False diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index 9b0b998bb..9d0fe05e0 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -3,7 +3,6 @@ import os import sys import stat -import shutil import Bcfg2.Client.XML from Bcfg2.Client.Tools.POSIX.base import POSIXTool @@ -67,25 +66,14 @@ class POSIXDirectory(POSIXTool): rv &= self._makedirs(entry) if entry.get('prune', 'false') == 'true': - ulfailed = False for pent in entry.findall('Prune'): pname = pent.get('path') - ulfailed = False - if os.path.isdir(pname): - remove = shutil.rmtree - else: - remove = os.unlink try: self.logger.debug("POSIX: Removing %s" % pname) - remove(pname) + self._remove(pent) except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to unlink %s: %s" % (pname, err)) - ulfailed = True - if ulfailed: - # even if prune failed, we still want to install the - # entry to make sure that we get permissions and - # whatnot set - rv = False + rv = False return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py index 5f1fbbe7c..f7251ca50 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -2,7 +2,6 @@ import os import sys -import shutil from Bcfg2.Client.Tools.POSIX.base import POSIXTool @@ -19,25 +18,21 @@ class POSIXNonexistent(POSIXTool): def install(self, entry): ename = entry.get('name') - if entry.get('recursive', '').lower() == 'true': + recursive = entry.get('recursive', '').lower() == 'true' + if recursive: # ensure that configuration spec is consistent first for struct in self.config.getchildren(): - for entry in struct.getchildren(): - if (entry.tag == 'Path' and - entry.get('type') != 'nonexistent' and - entry.get('name').startswith(ename)): + for el in struct.getchildren(): + if (el.tag == 'Path' and + el.get('type') != 'nonexistent' and + el.get('name').startswith(ename)): self.logger.error('POSIX: Not removing %s. One or ' 'more files in this directory are ' 'specified in your configuration.' % ename) return False - remove = shutil.rmtree - elif os.path.isdir(ename): - remove = os.rmdir - else: - remove = os.remove try: - remove(ename) + self._remove(entry, recursive=recursive) return True except OSError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index a9566b698..6388f6731 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -66,18 +66,26 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): rv &= self._set_perms(entry, path=os.path.join(root, path)) return rv + def _remove(self, entry, recursive=True): + """ Remove a Path entry, whatever that takes """ + if os.path.islink(entry.get('name')): + os.unlink(entry.get('name')) + elif os.path.isdir(entry.get('name')): + if recursive: + shutil.rmtree(entry.get('name')) + else: + os.rmdir(entry.get('name')) + else: + os.unlink(entry.get('name')) + def _exists(self, entry, remove=False): """ check for existing paths and optionally remove them. if the path exists, return the lstat of it """ try: ondisk = os.lstat(entry.get('name')) if remove: - if os.path.isdir(entry.get('name')): - remove = shutil.rmtree - else: - remove = os.unlink try: - remove(entry.get('name')) + self._remove(entry) return None except OSError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 5ac96f999..fc47883c9 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -41,7 +41,7 @@ def netmask_itoa(netmask, proto="ipv4"): size = 128 family = socket.AF_INET6 try: - int(netmask) + netmask = int(netmask) except ValueError: return netmask diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 4022692be..927b25ba8 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,7 +1,9 @@ """This contains all Bcfg2 Tool modules""" import os +import sys import stat +import select from subprocess import Popen, PIPE import Bcfg2.Client.XML from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 @@ -373,6 +375,10 @@ class SvcTool(Tool): if self.setup['interactive']: prompt = ('Restart service %s?: (y/N): ' % entry.get('name')) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input(prompt) if ans not in ['y', 'Y']: continue diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index f3765a5ec..b418d57b0 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -577,6 +577,11 @@ SERVER_VCS_ROOT = \ default=None, odesc='<VCS repository root>', cf=('server', 'vcs_root')) +SERVER_UMASK = \ + Option('Server umask', + default='0077', + odesc='<Server umask>', + cf=('server', 'umask')) # database options DB_ENGINE = \ @@ -1068,6 +1073,7 @@ CLI_COMMON_OPTIONS = dict(configfile=CFILE, syslog=LOGGING_SYSLOG) DAEMON_COMMON_OPTIONS = dict(daemon=DAEMON, + umask=SERVER_UMASK, listen_all=SERVER_LISTEN_ALL, daemon_uid=SERVER_DAEMON_USER, daemon_gid=SERVER_DAEMON_GROUP) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 869dc1aca..14065980d 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -1,11 +1,13 @@ """ Interactively initialize a new repository. """ -import getpass + import os +import sys +import stat +import select import random import socket -import stat import string -import sys +import getpass import subprocess import Bcfg2.Server.Admin @@ -85,6 +87,14 @@ OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), ('Arch', 'arch')] +def safe_input(prompt): + """ input() that flushes the input buffer before accepting input """ + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + return input(prompt) + + def gen_password(length): """Generates a random alphanumeric password with length characters.""" chars = string.letters + string.digits @@ -116,8 +126,8 @@ def create_conf(confpath, confdata): """ create the config file """ # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): - result = input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) + result = safe_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) if result not in ['Y', 'y']: print("Leaving %s unchanged" % confpath) return @@ -186,8 +196,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_hostname(self): """Ask for the server hostname.""" - data = input("What is the server's hostname [%s]: " % - socket.getfqdn()) + data = safe_input("What is the server's hostname [%s]: " % + socket.getfqdn()) if data != '': self.data['shostname'] = data else: @@ -195,21 +205,21 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_config(self): """Ask for the configuration file path.""" - newconfig = input("Store Bcfg2 configuration in [%s]: " % - self.configfile) + newconfig = safe_input("Store Bcfg2 configuration in [%s]: " % + self.configfile) if newconfig != '': self.data['configfile'] = os.path.abspath(newconfig) def _prompt_repopath(self): """Ask for the repository path.""" while True: - newrepo = input("Location of Bcfg2 repository [%s]: " % + newrepo = safe_input("Location of Bcfg2 repository [%s]: " % self.data['repopath']) if newrepo != '': self.data['repopath'] = os.path.abspath(newrepo) if os.path.isdir(self.data['repopath']): - response = input("Directory %s exists. Overwrite? [y/N]:" % - self.data['repopath']) + response = safe_input("Directory %s exists. Overwrite? [y/N]:" + % self.data['repopath']) if response.lower().strip() == 'y': break else: @@ -225,8 +235,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = input("Input the server location [%s]: " % - self.data['server_uri']) + newserver = safe_input("Input the server location [%s]: " % + self.data['server_uri']) if newserver != '': self.data['server_uri'] = newserver @@ -238,7 +248,7 @@ class Init(Bcfg2.Server.Admin.Mode): prompt += ': ' while True: try: - osidx = int(input(prompt)) + osidx = int(safe_input(prompt)) self.data['os_sel'] = OS_LIST[osidx - 1][1] break except ValueError: @@ -248,27 +258,28 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the key details (country, state, and location).""" print("The following questions affect SSL certificate generation.") print("If no data is provided, the default values are used.") - newcountry = input("Country name (2 letter code) for certificate: ") + newcountry = safe_input("Country name (2 letter code) for " + "certificate: ") if newcountry != '': if len(newcountry) == 2: self.data['country'] = newcountry else: while len(newcountry) != 2: - newcountry = input("2 letter country code (eg. US): ") + newcountry = safe_input("2 letter country code (eg. US): ") if len(newcountry) == 2: self.data['country'] = newcountry break else: self.data['country'] = 'US' - newstate = input("State or Province Name (full name) for " - "certificate: ") + newstate = safe_input("State or Province Name (full name) for " + "certificate: ") if newstate != '': self.data['state'] = newstate else: self.data['state'] = 'Illinois' - newlocation = input("Locality Name (eg, city) for certificate: ") + newlocation = safe_input("Locality Name (eg, city) for certificate: ") if newlocation != '': self.data['location'] = newlocation else: @@ -277,12 +288,12 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_keypath(self): """ Ask for the key pair location. Try to use sensible defaults depending on the OS """ - keypath = input("Path where Bcfg2 server private key will be created " - "[%s]: " % self.data['keypath']) + keypath = safe_input("Path where Bcfg2 server private key will be " + "created [%s]: " % self.data['keypath']) if keypath: self.data['keypath'] = keypath - certpath = input("Path where Bcfg2 server cert will be created" - "[%s]: " % self.data['certpath']) + certpath = safe_input("Path where Bcfg2 server cert will be created" + "[%s]: " % self.data['certpath']) if certpath: self.data['certpath'] = certpath diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index e41652205..130e85b67 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -1,8 +1,10 @@ """ Retrieves entries from clients and integrates the information into the repository """ -import getopt +import os import sys +import getopt +import select import Bcfg2.Server.Admin from Bcfg2.Compat import input # pylint: disable=W0622 @@ -99,6 +101,10 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): else: print(" => host entry: %s" % (choice.hostname)) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input("Use this entry? [yN]: ") in ['y', 'Y'] if ans: return choice diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 15ff79a35..6e313e84b 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -74,7 +74,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: import south except ImportError: - print "Django south is required for Reporting" + print("Django south is required for Reporting") raise SystemExit(-3) def __call__(self, args): diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 69fb8d0cb..63149c15e 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -28,17 +28,15 @@ class Core(BaseCore): #: this server core self.server = None + daemon_args = dict(uid=self.setup['daemon_uid'], + gid=self.setup['daemon_gid'], + umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - #: The :class:`daemon.DaemonContext` used to drop - #: privileges, write the PID file (with :class:`PidFile`), - #: and daemonize this core. - self.context = \ - daemon.DaemonContext(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - pidfile=PIDLockFile(self.setup['daemon'])) - else: - self.context = daemon.DaemonContext(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid']) + daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + #: The :class:`daemon.DaemonContext` used to drop + #: privileges, write the PID file (with :class:`PidFile`), + #: and daemonize this core. + self.context = daemon.DaemonContext(**daemon_args) __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] def _dispatch(self, method, args, dispatch_dict): diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index 4ddcd7bdf..d097fd08f 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -107,8 +107,10 @@ class Core(BaseCore): :class:`cherrypy.process.plugins.DropPrivileges`, daemonize with :class:`cherrypy.process.plugins.Daemonizer`, and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ - DropPrivileges(cherrypy.engine, uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid']).subscribe() + DropPrivileges(cherrypy.engine, + uid=self.setup['daemon_uid'], + gid=self.setup['daemon_gid'], + umask=int(self.setup['umask'], 8)).subscribe() Daemonizer(cherrypy.engine).subscribe() PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() return True diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index a5fda6f0d..ee875a7e8 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -516,7 +516,7 @@ class BaseCore(object): except PluginExecutionError: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % format_exc()) + entry.set('failure', 'bind error: %s' % exc) self.logger.error("Failed to bind entry %s:%s: %s" % (entry.tag, entry.get('name'), exc)) except Exception: @@ -665,6 +665,8 @@ class BaseCore(object): os.chmod(piddir, 420) # 0644 if not self._daemonize(): return False + else: + os.umask(int(self.setup['umask'], 8)) if not self._run(): self.shutdown() @@ -1065,5 +1067,59 @@ class BaseCore(object): @exposed def get_statistics(self, _): """ Get current statistics about component execution from - :attr:`Bcfg2.Statistics.stats`. """ + :attr:`Bcfg2.Statistics.stats`. + + :returns: dict - The statistics data as returned by + :func:`Bcfg2.Statistics.Statistics.display` """ return Bcfg2.Statistics.stats.display() + + @exposed + def toggle_debug(self, address): + """ Toggle debug status of the FAM and all plugins + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - The new debug state of the FAM + """ + for plugin in self.plugins.values(): + plugin.toggle_debug() + return self.toggle_fam_debug(address) + + @exposed + def toggle_fam_debug(self, _): + """ Toggle debug status of the FAM + + :returns: bool - The new debug state of the FAM + """ + return self.fam.toggle_debug() + + @exposed + def set_debug(self, address, debug): + """ Explicitly set debug status of the FAM and all plugins + + :param debug: The new debug status. This can either be a + boolean, or a string describing the state (e.g., + "true" or "false"; case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + for plugin in self.plugins.values(): + plugin.set_debug(debug) + return self.set_fam_debug(address, debug) + + @exposed + def set_fam_debug(self, _, debug): + """ Explicitly set debug status of the FAM + + :param debug: The new debug status of the FAM. This can + either be a boolean, or a string describing the + state (e.g., "true" or "false"; + case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state of the FAM + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + return self.fam.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index d5aa8e4ad..178a47b1a 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -110,11 +110,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if ievent.mask & amask: action = aname break + else: + # event action is not in the mask, and thus is not + # something we care about + self.debug_log("Ignoring event %s for %s" % (action, + ievent.pathname)) + return + try: watch = self.watchmgr.watches[ievent.wd] except KeyError: - LOGGER.error("Error handling event for %s: Watch %s not found" % - (ievent.pathname, ievent.wd)) + LOGGER.error("Error handling event %s for %s: Watch %s not found" % + (action, ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 72b1d2dd7..42ad4c041 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -50,6 +50,7 @@ import sys import fnmatch import logging from time import sleep, time +from Bcfg2.Server.Plugin import Debuggable LOGGER = logging.getLogger(__name__) @@ -104,7 +105,7 @@ class Event(object): return "%s (request ID %s)" % (str(self), self.requestID) -class FileMonitor(object): +class FileMonitor(Debuggable): """ The base class that all FAM implementions must inherit. The simplest instance of a FileMonitor subclass needs only to add @@ -128,8 +129,8 @@ class FileMonitor(object): .. ----- .. autoattribute:: __priority__ """ - #: Whether or not to produce debug logging - self.debug = debug + Debuggable.__init__(self) + self.debug_flag = debug #: A dict that records which objects handle which events. #: Keys are monitor handle IDs and values are objects whose @@ -168,13 +169,6 @@ class FileMonitor(object): example of this. """ self.started = True - def debug_log(self, msg): - """ Log a debug message. - - :param msg: The message to log iff :attr:`debug` is set.""" - if self.debug: - LOGGER.info(msg) - def should_ignore(self, event): """ Returns True if an event should be ignored, False otherwise. For events that include the full path, both the diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 8d288f835..e74909ee9 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -9,7 +9,7 @@ class Debuggable(object): via XML-RPC on :class:`Bcfg2.Server.Plugin.base.Plugin` objects """ #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug'] + __rmi__ = ['toggle_debug', 'set_debug'] def __init__(self, name=None): """ @@ -26,17 +26,25 @@ class Debuggable(object): self.debug_flag = False self.logger = logging.getLogger(name) - def toggle_debug(self): - """ Turn debugging output on or off. This method is exposed + def set_debug(self, debug): + """ Explicitly enable or disable debugging. This method is exposed via XML-RPC. :returns: bool - The new value of the debug flag """ - self.debug_flag = not self.debug_flag + self.debug_flag = debug self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, self.debug_flag), flag=True) - return self.debug_flag + return debug + + def toggle_debug(self): + """ Turn debugging output on or off. This method is exposed + via XML-RPC. + + :returns: bool - The new value of the debug flag + """ + return self.set_debug(not self.debug_flag) def debug_log(self, message, flag=None): """ Log a message at the debug level. diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 894ed9851..318bf03f1 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1526,12 +1526,12 @@ class GroupSpool(Plugin, Generator): else: return self.handles[event.requestID].rstrip("/") - def toggle_debug(self): + def set_debug(self, debug): for entry in self.entries.values(): - if hasattr(entry, "toggle_debug"): - entry.toggle_debug() - return Plugin.toggle_debug(self) - toggle_debug.__doc__ = Plugin.toggle_debug.__doc__ + if hasattr(entry, "set_debug"): + entry.set_debug(debug) + return Plugin.set_debug(self, debug) + set_debug.__doc__ = Plugin.set_debug.__doc__ def HandleEvent(self, event): """ HandleEvent is the event dispatcher for GroupSpool diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 73c70901b..8ebd8d921 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,12 +2,9 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator -LOGGER = logging.getLogger(__name__) - try: from Cheetah.Template import Template HAS_CHEETAH = True @@ -33,9 +30,7 @@ class CfgCheetahGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_CHEETAH: - msg = "Cfg: Cheetah is not available: %s" % self.name - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py index 00b95c970..da506a195 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -1,14 +1,11 @@ """ Handle .diff files, which apply diffs to plaintext files """ import os -import logging import tempfile -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgFilter -LOGGER = logging.getLogger(__name__) - class CfgDiffFilter(CfgFilter): """ CfgDiffFilter applies diffs to plaintext @@ -32,8 +29,7 @@ class CfgDiffFilter(CfgFilter): output = open(basename, 'r').read() os.unlink(basename) if ret != 0: - msg = "Error applying diff %s: %s" % (self.name, stderr) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Error applying diff %s: %s" % + (self.name, stderr)) return output modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 26faf6e2c..3b4703ddb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,7 +1,6 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ -import logging from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: @@ -11,8 +10,6 @@ try: except ImportError: HAS_CRYPTO = False -LOGGER = logging.getLogger(__name__) - class CfgEncryptedGenerator(CfgGenerator): """ CfgEncryptedGenerator lets you encrypt your plaintext @@ -28,9 +25,7 @@ class CfgEncryptedGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_CRYPTO: - msg = "Cfg: M2Crypto is not available" - LOGGER.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("M2Crypto is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index feedbdb1b..130652aef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -1,7 +1,6 @@ """ Handle encrypted Genshi templates (.crypt.genshi or .genshi.crypt files) """ -import logging from Bcfg2.Compat import StringIO from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import SETUP @@ -19,10 +18,6 @@ except ImportError: # CfgGenshiGenerator will raise errors if genshi doesn't exist TemplateLoader = object # pylint: disable=C0103 -LOGGER = logging.getLogger(__name__) - -LOGGER = logging.getLogger(__name__) - class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt @@ -53,6 +48,4 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): def __init__(self, fname, spec, encoding): CfgGenshiGenerator.__init__(self, fname, spec, encoding) if not HAS_CRYPTO: - msg = "Cfg: M2Crypto is not available" - LOGGER.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 023af7d4e..b702ac899 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -3,13 +3,10 @@ import os import sys import shlex -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError -LOGGER = logging.getLogger(__name__) - class CfgExternalCommandVerifier(CfgVerifier): """ Invoke an external script to verify @@ -46,9 +43,6 @@ class CfgExternalCommandVerifier(CfgVerifier): if bangpath.startswith("#!"): self.cmd.extend(shlex.split(bangpath[2:].strip())) else: - msg = "%s: Cannot execute %s" % (self.__class__.__name__, - self.name) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Cannot execute %s" % self.name) self.cmd.append(self.name) handle_event.__doc__ = CfgVerifier.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index ce77717da..df0c30c09 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -4,13 +4,10 @@ import re import sys -import logging import traceback -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator -LOGGER = logging.getLogger(__name__) - try: import genshi.core from genshi.template import TemplateLoader, NewTextTemplate @@ -67,11 +64,9 @@ class CfgGenshiGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_GENSHI: - msg = "Cfg: Genshi is not available: %s" % fname - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - self.loader = self.__loader_cls__() + raise PluginExecutionError("Genshi is not available") self.template = None + self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): @@ -92,15 +87,15 @@ class CfgGenshiGenerator(CfgGenerator): stack = traceback.extract_tb(sys.exc_info()[2]) for quad in stack: if quad[0] == self.name: - LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, quad[2], err.__class__.__name__, err)) - break + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + quad[2])) raise except: - self._handle_genshi_exception(fname, sys.exc_info()) + self._handle_genshi_exception(sys.exc_info()) get_data.__doc__ = CfgGenerator.get_data.__doc__ - def _handle_genshi_exception(self, fname, exc): + def _handle_genshi_exception(self, exc): """ this is horrible, and I deeply apologize to whoever gets to maintain this after I go to the Great Beer Garden in the Sky. genshi is incredibly opaque about what's being executed, @@ -140,21 +135,16 @@ class CfgGenshiGenerator(CfgGenerator): # single line break) real_lineno = lineno - contents.code.co_firstlineno src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, src[real_lineno], err.__class__.__name__, - err)) + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + src[real_lineno])) raise def handle_event(self, event): - CfgGenerator.handle_event(self, event) - if self.data is None: - return try: self.template = self.loader.load(self.name, cls=NewTextTemplate, encoding=self.encoding) except: - msg = "Cfg: Could not load template %s: %s" % (self.name, - sys.exc_info()[1]) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Failed to load template: %s" % + sys.exc_info()[1]) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index e5ba0a51b..3b6fc8fa0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -1,11 +1,8 @@ """ Handle info.xml files """ -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError, InfoXML from Bcfg2.Server.Plugins.Cfg import CfgInfo -LOGGER = logging.getLogger(__name__) - class CfgInfoXML(CfgInfo): """ CfgInfoXML handles :file:`info.xml` files for @@ -16,16 +13,15 @@ class CfgInfoXML(CfgInfo): def __init__(self, path): CfgInfo.__init__(self, path) - self.infoxml = Bcfg2.Server.Plugin.InfoXML(path) + self.infoxml = InfoXML(path) __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): mdata = dict() self.infoxml.pnode.Match(metadata, mdata, entry=entry) if 'Info' not in mdata: - msg = "Failed to set metadata for file %s" % entry.get('name') - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Failed to set metadata for file %s" % + entry.get('name')) self._set_info(entry, mdata['Info'][None]) bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 58f6e1e42..db6810e7c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -542,8 +542,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): try: return generator.get_data(entry, metadata) except: - msg = "Cfg: exception rendering %s with %s: %s" % \ - (entry.get("name"), generator, sys.exc_info()[1]) + msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), + sys.exc_info()[1]) LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 5faa6c018..8cc63a46f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -1,119 +1,15 @@ """ The Git plugin provides a revision interface for Bcfg2 repos using git. """ -import os import sys import Bcfg2.Server.Plugin - - -class GitAPIBase(object): - """ Base class for the various Git APIs (dulwich, GitPython, - subprocesses) """ - def __init__(self, path): - self.path = path - - def revision(self): - """ Get the current revision of the git repo as a string """ - raise NotImplementedError - - def pull(self): - """ Pull the latest version of the upstream git repo and - rebase against it. """ - raise NotImplementedError - +from subprocess import Popen, PIPE try: - from dulwich.client import get_transport_and_path - from dulwich.repo import Repo - from dulwich.file import GitFile, ensure_dir_exists - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using :mod:`dulwich` """ - def __init__(self, path): - GitAPIBase.__init__(self, path) - self.repo = Repo(self.path) - self.client, self.origin_path = get_transport_and_path( - self.repo.get_config().get(("remote", "origin"), - "url")) - - def revision(self): - return self.repo.head() - - def pull(self): - try: - remote_refs = self.client.fetch( - self.origin_path, self.repo, - determine_wants=self.repo.object_store.determine_wants_all) - except KeyError: - etype, err = sys.exc_info()[:2] - # try to work around bug - # https://bugs.launchpad.net/dulwich/+bug/1025886 - try: - # pylint: disable=W0212 - self.client._fetch_capabilities.remove('thin-pack') - # pylint: enable=W0212 - except KeyError: - raise etype(err) - remote_refs = self.client.fetch( - self.origin_path, self.repo, - determine_wants=self.repo.object_store.determine_wants_all) - - tree_id = self.repo[remote_refs['HEAD']].tree - # iterate over tree content, giving path and blob sha. - for entry in self.repo.object_store.iter_tree_contents(tree_id): - entry_in_path = entry.in_path(self.repo.path) - ensure_dir_exists(os.path.split(entry_in_path.path)[0]) - GitFile(entry_in_path.path, - 'wb').write(self.repo[entry.sha].data) - + import git + HAS_GITPYTHON = True except ImportError: - try: - import git - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using :mod:`git` (GitPython) """ - def __init__(self, path): - GitAPIBase.__init__(self, path) - self.repo = git.Repo(path) - - def revision(self): - return self.repo.head.commit.hexsha - - def pull(self): - self.repo.git.pull("--rebase") - - except ImportError: - from subprocess import Popen, PIPE - - try: - Popen(["git"], stdout=PIPE, stderr=PIPE).wait() - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using subprocess to run git - commands """ - def revision(self): - proc = Popen(["git", "--work-tree", - os.path.join(self.path, ".git"), - "rev-parse", "HEAD"], stdout=PIPE, - stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - raise Exception("Git: Error getting revision from %s: " - "%s" % (self.path, err)) - return rv.strip() # pylint: disable=E1103 - - def pull(self): - proc = Popen(["git", "--work-tree", - os.path.join(self.path, ".git"), - "pull", "--rebase"], stdout=PIPE, - stderr=PIPE) - err = proc.communicate()[1].strip() - if proc.wait(): - raise Exception("Git: Error pulling: %s" % err) - - except OSError: - raise ImportError("Could not import dulwich or GitPython " - "libraries, and no 'git' command found in PATH") + HAS_GITPYTHON = False class Git(Bcfg2.Server.Plugin.Version): @@ -121,35 +17,83 @@ class Git(Bcfg2.Server.Plugin.Version): using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".git" - __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] + if HAS_GITPYTHON: + __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) - self.repo = GitAPI(self.vcs_root) + if HAS_GITPYTHON: + self.repo = git.Repo(self.vcs_root) + else: + self.logger.debug("Git: GitPython not found, using CLI interface " + "to Git") + self.repo = None self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: - return self.repo.revision() + if HAS_GITPYTHON: + return self.repo.head.commit.hexsha + else: + cmd = ["git", "--git-dir", self.vcs_path, + "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + self.debug_log("Git: Running cmd") + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + rv, err = proc.communicate() + if proc.wait(): + raise Exception(err) + return rv except: err = sys.exc_info()[1] - msg = "Failed to read git repository: %s" % err + msg = "Git: Error getting revision from %s: %s" % (self.vcs_root, + err) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - def Update(self): + def Update(self, ref=None): """ Git.Update() => True|False Update the working copy against the upstream repository """ + self.logger.info("Git: Git.Update(ref='%s')" % ref) + self.debug_log("Git: Performing garbage collection on repo at %s" % + self.vcs_root) try: - self.repo.pull() - self.logger.info("Git repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) - return True - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - msg = "Failed to pull from git repository: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.repo.git.gc('--auto') + except git.GitCommandError: + self.logger.warning("Git: Failed to perform garbage collection: %s" + % sys.exc_info()[1]) + + if ref: + self.debug_log("Git: Checking out %s" % ref) + try: + self.repo.git.checkout('-f', ref) + except git.GitCommandError: + err = sys.exc_info()[1] + msg = "Git: Failed to checkout %s: %s" % (ref, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + # determine if we should try to pull to get the latest commit + # on this head + tracking = None + if not self.repo.head.is_detached: + self.debug_log("Git: Determining if %s is a tracking branch" % + self.repo.head.ref.name) + tracking = self.repo.head.ref.tracking_branch() + + if tracking is not None: + self.debug_log("Git: %s is a tracking branch, pulling from %s" % + (self.repo.head.ref.name, tracking)) + try: + self.repo.git.pull("--rebase") + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + msg = "Git: Failed to pull from upstream: %s" % err + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + self.logger.info("Git: Repo at %s updated to %s" % + (self.vcs_root, self.get_revision())) + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8b0fc16ce..0ab72f2c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -967,9 +967,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self.aliases[cname] return cname except socket.herror: - warning = "address resolution error for %s" % address - self.logger.warning(warning) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(warning) + err = "Address resolution error for %s: %s" % (address, + sys.exc_info()[1]) + self.logger.error(err) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) def _merge_groups(self, client, groups, categories=None): """ set group membership based on the contents of groups.xml diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index fbad0a37b..023547b7e 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -81,7 +81,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, if xtra: host_config.extend([self.line_fmt % (opt, val) for opt, val in list(xtra.items())]) - else: + if 'use' not in xtra: host_config.append(self.line_fmt % ('use', 'default')) host_config.append('}') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 94dc6d2fd..2735e389a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -75,11 +75,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, #: should be told to reload its data. self.parsed = set() - def toggle_debug(self): - Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) + def set_debug(self, debug): + Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) for source in self.entries: - source.toggle_debug() - toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ + source.set_debug(debug) + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -121,8 +121,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ -``Index`` is responsible for calling :func:`source_from_xml` for each -``Source`` tag in each file. """ + ``Index`` is responsible for calling :func:`source_from_xml` + for each ``Source`` tag in each file. """ @Bcfg2.Server.Plugin.track_statistics() def source_from_xml(self, xsource): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 59e7a206e..220146100 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -102,6 +102,9 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None +#: The path to bcfg2-yum-helper +HELPER = None + def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -308,8 +311,6 @@ class YumCollection(Collection): (certdir, err)) self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) - self._helper = None - @property def __package_groups__(self): """ YumCollections support package groups only if @@ -324,20 +325,20 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - try: - return self.setup.cfp.get("packages:yum", "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - pass - - if not self._helper: - # first see if bcfg2-yum-helper is in PATH + global HELPER + if not HELPER: try: - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - self._helper = 'bcfg2-yum-helper' - except OSError: - self._helper = "/usr/sbin/bcfg2-yum-helper" - return self._helper + HELPER = self.setup.cfp.get("packages:yum", "helper") + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + # first see if bcfg2-yum-helper is in PATH + try: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + Popen(['bcfg2-yum-helper'], + stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() + HELPER = 'bcfg2-yum-helper' + except OSError: + HELPER = "/usr/sbin/bcfg2-yum-helper" + return HELPER @property def use_yum(self): @@ -357,7 +358,7 @@ class YumCollection(Collection): def cachefiles(self): """ A list of the full path to all cachefiles used by this collection.""" - cachefiles = set(Collection.cachefiles(self)) + cachefiles = set(Collection.cachefiles.fget(self)) if self.cachefile: cachefiles.add(self.cachefile) return list(cachefiles) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5a193219c..f30e060bd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -113,13 +113,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def toggle_debug(self): - rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) - self.sources.toggle_debug() + def set_debug(self, debug): + rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) + self.sources.set_debug(debug) for collection in self.collections.values(): - collection.toggle_debug() + collection.set_debug(debug) return rv - toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ @property def disableResolver(self): diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index bab7c4a4a..feb76aa57 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -9,7 +9,8 @@ import logging import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -from Bcfg2.Compat import u_str, reduce, b64encode # pylint: disable=W0622 +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Compat import any, u_str, reduce, b64encode # pylint: disable=W0622 LOGGER = logging.getLogger(__name__) @@ -111,9 +112,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, is regenerated each time a new key is generated. """ - name = 'SSHbase' __author__ = 'bcfg-dev@mcs.anl.gov' - keypatterns = ["ssh_host_dsa_key", "ssh_host_ecdsa_key", "ssh_host_rsa_key", @@ -250,9 +249,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for entry in list(self.entries.values()): if entry.specific.match(event.filename): entry.handle_event(event) - if event.filename.endswith(".pub"): - self.logger.info("New public key %s; invalidating " - "ssh_known_hosts cache" % event.filename) + if any(event.filename.startswith(kp) + for kp in self.keypatterns + if kp.endswith(".pub")): + self.debug_log("New public key %s; invalidating " + "ssh_known_hosts cache" % event.filename) self.skn = False return @@ -365,8 +366,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, is_bound = False while not is_bound: if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "%s still not registered" % filename + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.core.fam.handle_events_in_interval(1) tries += 1 try: @@ -385,26 +387,30 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: keytype = 'rsa1' else: - self.logger.error("Unknown key filename: %s" % filename) - return + raise PluginExecutionError("Unknown key filename: %s" % filename) - fileloc = "%s/%s" % (self.data, hostkey) - publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', - "H_%s" % client]) + fileloc = os.path.join(self.data, hostkey) + publoc = os.path.join(self.data, + ".".join([hostkey.split('.')[0], 'pub', + "H_%s" % client])) tempdir = tempfile.mkdtemp() - temploc = "%s/%s" % (tempdir, hostkey) + temploc = os.path.join(tempdir, hostkey) cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", "-t", keytype, "-C", "root@%s" % client] + self.debug_log("SSHbase: Running: %s" % " ".join(cmd)) proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - proc.communicate() - proc.wait() + err = proc.communicate()[1] + if proc.wait(): + raise PluginExecutionError("SSHbase: Error running ssh-keygen: %s" + % err) try: shutil.copy(temploc, fileloc) shutil.copy("%s.pub" % temploc, publoc) except IOError: err = sys.exc_info()[1] - self.logger.error("Temporary SSH keys not found: %s" % err) + raise PluginExecutionError("Temporary SSH keys not found: %s" % + err) try: os.unlink(temploc) @@ -412,7 +418,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, os.rmdir(tempdir) except OSError: err = sys.exc_info()[1] - self.logger.error("Failed to unlink temporary ssh keys: %s" % err) + raise PluginExecutionError("Failed to unlink temporary ssh keys: " + "%s" % err) def AcceptChoices(self, _, metadata): return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)] @@ -420,8 +427,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def AcceptPullData(self, specific, entry, log): """Per-plugin bcfg2-admin pull support.""" # specific will always be host specific - filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1], - specific.hostname) + filename = os.path.join(self.data, + "%s.H_%s" % (entry['name'].split('/')[-1], + specific.hostname)) try: open(filename, 'w').write(entry['text']) if log: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index ab55425a6..62396f860 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -1,13 +1,15 @@ """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ +import os +import sys import Bcfg2.Server.Plugin import Bcfg2.Options import lxml.etree import tempfile -import os from subprocess import Popen, PIPE, STDOUT from Bcfg2.Compat import ConfigParser, md5 +from Bcfg2.Server.Plugin import PluginExecutionError class SSLCA(Bcfg2.Server.Plugin.GroupSpool): @@ -107,6 +109,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): filename = os.path.join(path, "%s.H_%s" % (os.path.basename(path), metadata.hostname)) if filename not in list(self.entries.keys()): + self.logger.info("SSLCA: Generating new key %s" % filename) key = self.build_key(entry) open(self.data + filename, 'w').write(key) entry.text = key @@ -130,6 +133,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cmd = ["openssl", "genrsa", bits] elif ktype == 'dsa': cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) return Popen(cmd, stdout=PIPE).stdout.read() def get_cert(self, entry, metadata): @@ -151,10 +155,11 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): self.core.Bind(el, metadata) # check if we have a valid hostfile - if (filename in list(self.entries.keys()) and + if (filename in self.entries.keys() and self.verify_cert(filename, key_filename, entry)): entry.text = self.entries[filename].data else: + self.logger.info("SSLCA: Generating new cert %s" % filename) cert = self.build_cert(key_filename, entry, metadata) open(self.data + filename, 'w').write(cert) self.entries[filename] = self.__child__(self.data + filename) @@ -231,22 +236,37 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ creates a new certificate according to the specification """ - req_config = self.build_req_config(entry, metadata) - req = self.build_request(key_filename, req_config, entry) - ca = self.cert_specs[entry.get('name')]['ca'] - ca_config = self.CAs[ca]['config'] - days = self.cert_specs[entry.get('name')]['days'] - passphrase = self.CAs[ca].get('passphrase') - cmd = ["openssl", "ca", "-config", ca_config, "-in", req, - "-days", days, "-batch"] - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - cert = Popen(cmd, stdout=PIPE).stdout.read() + req_config = None + req = None try: - os.unlink(req_config) - os.unlink(req) - except OSError: - self.logger.error("Failed to unlink temporary files") + req_config = self.build_req_config(entry, metadata) + req = self.build_request(key_filename, req_config, entry) + ca = self.cert_specs[entry.get('name')]['ca'] + ca_config = self.CAs[ca]['config'] + days = self.cert_specs[entry.get('name')]['days'] + passphrase = self.CAs[ca].get('passphrase') + cmd = ["openssl", "ca", "-config", ca_config, "-in", req, + "-days", days, "-batch"] + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + self.debug_log("SSLCA: Generating new certificate: %s" % + " ".join(cmd)) + proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + (cert, err) = proc.communicate() + if proc.wait(): + # pylint: disable=E1103 + raise PluginExecutionError("SSLCA: Failed to generate cert: %s" + % err.splitlines()[-1]) + # pylint: enable=E1103 + finally: + try: + if req_config and os.path.exists(req_config): + os.unlink(req_config) + if req and os.path.exists(req): + os.unlink(req) + except OSError: + self.logger.error("SSLCA: Failed to unlink temporary files: %s" + % sys.exc_info()[1]) if (self.cert_specs[entry.get('name')]['append_chain'] and self.CAs[ca]['chaincert']): cert += open(self.CAs[ca]['chaincert']).read() @@ -258,7 +278,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): used to generate the required certificate request """ # create temp request config file - conffile = open(tempfile.mkstemp()[1], 'w') + fd, fname = tempfile.mkstemp() cfp = ConfigParser.ConfigParser({}) cfp.optionxform = str defaults = { @@ -290,18 +310,28 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cfp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item]) cfp.set('req_distinguished_name', 'CN', metadata.hostname) - cfp.write(conffile) - conffile.close() - return conffile.name + self.debug_log("SSLCA: Writing temporary request config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise PluginExecutionError("SSLCA: Failed to write temporary CSR " + "config file: %s" % sys.exc_info()[1]) + return fname def build_request(self, key_filename, req_config, entry): """ creates the certificate request """ - req = tempfile.mkstemp()[1] + fd, req = tempfile.mkstemp() + os.close(fd) days = self.cert_specs[entry.get('name')]['days'] key = self.data + key_filename cmd = ["openssl", "req", "-new", "-config", req_config, "-days", days, "-key", key, "-text", "-out", req] - Popen(cmd, stdout=PIPE).wait() + self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + err = proc.communicate()[1] + if proc.wait(): + raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % + err) return req diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 627c82f25..f09d4839e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -13,15 +13,25 @@ LOGGER = logging.getLogger(__name__) MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') -class HelperModule(Bcfg2.Server.Plugin.FileBacked): +class HelperModule(object): """ Representation of a TemplateHelper module """ def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + self.name = name + self.fam = fam self._module_name = MODULE_RE.search(self.name).group('module') self._attrs = [] - def Index(self): + def HandleEvent(self, event=None): + """ HandleEvent is called whenever the FAM registers an event. + + :param event: The event object + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ + if event and event.code2str() not in ['exists', 'changed', 'created']: + return + try: module = imp.load_source(self._module_name, self.name) except: # pylint: disable=W0702 @@ -54,27 +64,23 @@ class HelperModule(Bcfg2.Server.Plugin.FileBacked): self._attrs = newattrs -class HelperSet(Bcfg2.Server.Plugin.DirectoryBacked): - """ A set of template helper modules """ - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") - patterns = MODULE_RE - __child__ = HelperModule - - class TemplateHelper(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to provide helper classes and functions to templates """ - name = 'TemplateHelper' __author__ = 'chris.a.st.pierre@gmail.com' + ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + patterns = MODULE_RE + __child__ = HelperModule def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) - self.helpers = HelperSet(self.data, core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 - for h in self.helpers.entries.values()]) + for h in self.entries.values()]) class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): @@ -130,9 +136,9 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): @classmethod def Errors(cls): - return {"templatehelper-import-error":"error", - "templatehelper-no-export":"error", - "templatehelper-nonlist-export":"error", - "templatehelper-nonexistent-export":"error", - "templatehelper-reserved-export":"error", - "templatehelper-underscore-export":"warning"} + return {"templatehelper-import-error": "error", + "templatehelper-no-export": "error", + "templatehelper-nonlist-export": "error", + "templatehelper-nonexistent-export": "error", + "templatehelper-reserved-export": "error", + "templatehelper-underscore-export": "warning"} diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 0693b430c..961a8dc58 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -4,6 +4,7 @@ import os import sys import copy +import select import logging import lxml.etree import Bcfg2.Logger @@ -31,7 +32,7 @@ class Encryptor(object): self.passphrase = None self.pname = None self.logger = logging.getLogger(self.__class__.__name__) - + def get_encrypted_filename(self, plaintext_filename): """ get the name of the file encrypted data should be written to """ return plaintext_filename @@ -67,7 +68,7 @@ class Encryptor(object): if self.setup['passphrase']: self.pname = self.setup['passphrase'] - + if self.pname: if self.setup.cfp.has_option("encryption", self.pname): self.passphrase = self.setup.cfp.get("encryption", @@ -182,7 +183,7 @@ class Encryptor(object): self.logger.error("Error getting encrypted data from %s: %s" % (fname, err)) return False - + try: return self.unchunk(plaintext, crypted) except EncryptionChunkingError: @@ -317,10 +318,14 @@ class PropertiesEncryptor(Encryptor): print(lxml.etree.tostring( elt, xml_declaration=False).decode("UTF-8").strip()) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input("Encrypt this element? [y/N] ") if not ans.lower().startswith("y"): elements.remove(element) - + # this is not a good use of a generator, but we need to # generate the full list of elements in order to ensure that # some exist before we know what to return @@ -386,11 +391,11 @@ def main(): # pylint: disable=R0912,R0915 elif setup['interactive']: logger.error("Cannot decrypt interactively") setup['interactive'] = False - + if setup['cfg']: if setup['properties']: logger.error("You cannot specify both --cfg and --properties") - raise SystemExit(1) + raise SystemExit(1) if setup['xpath']: logger.error("Specifying --xpath with --cfg is nonsensical, " "ignoring --xpath") @@ -411,7 +416,7 @@ def main(): # pylint: disable=R0912,R0915 if not os.path.exists(fname): logger.error("%s does not exist, skipping" % fname) continue - + # figure out if we need to encrypt this as a Properties file # or as a Cfg file props = False diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 7277fa765..acb9e4f44 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -94,7 +94,7 @@ def load_interpreters(): best = "bpython" except ImportError: pass - + try: # whether ipython is actually better than bpython is # up for debate, but this is the behavior that existed @@ -454,7 +454,7 @@ Bcfg2 client itself.""") print(lxml.etree.tostring(pfile.XMLMatch(metadata), xml_declaration=False, pretty_print=True).decode('UTF-8')) - + def do_bundles(self, _): """ bundles - Print out group/bundle info """ data = [('Group', 'Bundles')] @@ -503,7 +503,6 @@ Bcfg2 client itself.""") pretty = True alist.remove('-p') if len(alist) != 1: - print 'alist=%s' % alist print(self._get_usage(self.do_probes)) return hostname = alist[0] @@ -736,7 +735,7 @@ Bcfg2 client itself.""") def _block(self): pass - + def build_usage(): diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py index 6dd130bee..16490808e 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py @@ -83,34 +83,35 @@ class TestPOSIXDirectory(TestPOSIXTool): mock_verify.assert_called_with(self.ptool, entry, modlist) mock_listdir.assert_called_with(entry.get("name")) self.assertEqual(len(entry.findall("Prune")), 0) - + @patch("os.unlink") - @patch("os.path.isdir") - @patch("shutil.rmtree") @patch("Bcfg2.Client.Tools.POSIX.base.POSIXTool.install") @patch("Bcfg2.Client.Tools.POSIX.Directory.%s._exists" % test_obj.__name__) @patch("Bcfg2.Client.Tools.POSIX.Directory.%s._makedirs" % test_obj.__name__) def test_install(self, mock_makedirs, mock_exists, mock_install, - mock_rmtree, mock_isdir, mock_unlink): + mock_unlink): entry = lxml.etree.Element("Path", name="/test/foo/bar", type="directory", mode='0644', owner='root', group='root') - + + self.ptool._makedirs = Mock() + self.ptool._remove = Mock() + def reset(): mock_exists.reset_mock() mock_install.reset_mock() mock_unlink.reset_mock() - mock_rmtree.reset_mock() - mock_rmtree.mock_makedirs() + self.ptool._makedirs.reset_mock() + self.ptool._remove.reset_mock() - mock_makedirs.return_value = True + self.ptool._makedirs.return_value = True mock_exists.return_value = False mock_install.return_value = True self.assertTrue(self.ptool.install(entry)) mock_exists.assert_called_with(entry) mock_install.assert_called_with(self.ptool, entry) - mock_makedirs.assert_called_with(entry) + self.ptool._makedirs.assert_called_with(entry) reset() exists_rv = MagicMock() @@ -119,7 +120,7 @@ class TestPOSIXDirectory(TestPOSIXTool): self.assertTrue(self.ptool.install(entry)) mock_unlink.assert_called_with(entry.get("name")) mock_exists.assert_called_with(entry) - mock_makedirs.assert_called_with(entry) + self.ptool._makedirs.assert_called_with(entry) mock_install.assert_called_with(self.ptool, entry) reset() @@ -138,20 +139,13 @@ class TestPOSIXDirectory(TestPOSIXTool): prune = ["/test/foo/bar/prune1", "/test/foo/bar/prune2"] for path in prune: lxml.etree.SubElement(entry, "Prune", path=path) - + reset() mock_install.return_value = True - def isdir_rv(path): - if path.endswith("prune2"): - return True - else: - return False - mock_isdir.side_effect = isdir_rv self.assertTrue(self.ptool.install(entry)) mock_exists.assert_called_with(entry) mock_install.assert_called_with(self.ptool, entry) - self.assertItemsEqual(mock_isdir.call_args_list, - [call(p) for p in prune]) - mock_unlink.assert_called_with("/test/foo/bar/prune1") - mock_rmtree.assert_called_with("/test/foo/bar/prune2") + self.assertItemsEqual([c[0][0].get("path") + for c in self.ptool._remove.call_args_list], + prune) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py index 676b18f5d..583d17e32 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py @@ -18,6 +18,7 @@ from Test__init import get_config, get_posix_object from Testbase import TestPOSIXTool from common import * + class TestPOSIXNonexistent(TestPOSIXTool): test_obj = POSIXNonexistent @@ -31,59 +32,40 @@ class TestPOSIXNonexistent(TestPOSIXTool): self.assertEqual(self.ptool.verify(entry, []), not val) mock_lexists.assert_called_with(entry.get("name")) - @patch("os.rmdir") - @patch("os.remove") - @patch("os.path.isdir") - @patch("shutil.rmtree") - def test_install(self, mock_rmtree, mock_isdir, mock_remove, mock_rmdir): + def test_install(self): entry = lxml.etree.Element("Path", name="/test", type="nonexistent") - def reset(): - mock_isdir.reset_mock() - mock_remove.reset_mock() - mock_rmdir.reset_mock() - mock_rmtree.reset_mock() + self.ptool._remove = Mock() - mock_isdir.return_value = False - self.assertTrue(self.ptool.install(entry)) - mock_remove.assert_called_with(entry.get("name")) - - reset() - mock_remove.side_effect = OSError - self.assertFalse(self.ptool.install(entry)) - mock_remove.assert_called_with(entry.get("name")) + def reset(): + self.ptool._remove.reset_mock() - reset() - mock_isdir.return_value = True self.assertTrue(self.ptool.install(entry)) - mock_rmdir.assert_called_with(entry.get("name")) - - reset() - mock_rmdir.side_effect = OSError - self.assertFalse(self.ptool.install(entry)) - mock_rmdir.assert_called_with(entry.get("name")) + self.ptool._remove.assert_called_with(entry, recursive=False) reset() entry.set("recursive", "true") self.assertTrue(self.ptool.install(entry)) - mock_rmtree.assert_called_with(entry.get("name")) - - reset() - mock_rmtree.side_effect = OSError - self.assertFalse(self.ptool.install(entry)) - mock_rmtree.assert_called_with(entry.get("name")) + self.ptool._remove.assert_called_with(entry, recursive=True) reset() child_entry = lxml.etree.Element("Path", name="/test/foo", type="nonexistent") ptool = self.get_obj(posix=get_posix_object(config=get_config([child_entry]))) - mock_rmtree.side_effect = None + ptool._remove = Mock() self.assertTrue(ptool.install(entry)) - mock_rmtree.assert_called_with(entry.get("name")) + ptool._remove.assert_called_with(entry, recursive=True) reset() child_entry = lxml.etree.Element("Path", name="/test/foo", type="file") ptool = self.get_obj(posix=get_posix_object(config=get_config([child_entry]))) - mock_rmtree.side_effect = None + ptool._remove = Mock() self.assertFalse(ptool.install(entry)) + self.assertFalse(ptool._remove.called) + + reset() + entry.set("recursive", "false") + self.ptool._remove.side_effect = OSError + self.assertFalse(self.ptool.install(entry)) + self.ptool._remove.assert_called_with(entry, recursive=False) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index ec194d401..b3599db83 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -123,57 +123,84 @@ class TestPOSIXTool(Bcfg2TestCase): mock_walk.assert_called_with(entry.get("name")) self.assertItemsEqual(mock_set_perms.call_args_list, all_set_perms) + @patch('os.rmdir') + @patch('os.unlink') + @patch('shutil.rmtree') + @patch('os.path.isdir') + @patch('os.path.islink') + def test_remove(self, mock_islink, mock_isdir, mock_rmtree, mock_unlink, + mock_rmdir): + entry = lxml.etree.Element("Path", name="/etc/foo") + + def reset(): + mock_islink.reset_mock() + mock_isdir.reset_mock() + mock_rmtree.reset_mock() + mock_unlink.reset_mock() + mock_rmdir.reset_mock() + + mock_islink.return_value = True + mock_isdir.return_value = False + self.ptool._remove(entry) + mock_unlink.assert_called_with(entry.get('name')) + self.assertFalse(mock_rmtree.called) + self.assertFalse(mock_rmdir.called) + + reset() + mock_islink.return_value = False + mock_isdir.return_value = True + self.ptool._remove(entry) + mock_rmtree.assert_called_with(entry.get('name')) + self.assertFalse(mock_unlink.called) + self.assertFalse(mock_rmdir.called) + + reset() + self.ptool._remove(entry, recursive=False) + mock_rmdir.assert_called_with(entry.get('name')) + self.assertFalse(mock_unlink.called) + self.assertFalse(mock_rmtree.called) + + reset() + mock_islink.return_value = False + mock_isdir.return_value = False + self.ptool._remove(entry, recursive=False) + mock_unlink.assert_called_with(entry.get('name')) + self.assertFalse(mock_rmtree.called) + self.assertFalse(mock_rmdir.called) + @patch('os.lstat') - @patch("os.unlink") - @patch("os.path.isdir") - @patch("shutil.rmtree") - def test_exists(self, mock_rmtree, mock_isdir, mock_unlink, mock_lstat): + def test_exists(self, mock_lstat): entry = lxml.etree.Element("Path", name="/etc/foo", type="file") + self.ptool._remove = Mock() + + def reset(): + mock_lstat.reset_mock() + self.ptool._remove.reset_mock() + mock_lstat.side_effect = OSError self.assertFalse(self.ptool._exists(entry)) mock_lstat.assert_called_with(entry.get('name')) - self.assertFalse(mock_unlink.called) + self.assertFalse(self.ptool._remove.called) - mock_lstat.reset_mock() - mock_unlink.reset_mock() + reset() rv = MagicMock() mock_lstat.return_value = rv mock_lstat.side_effect = None self.assertEqual(self.ptool._exists(entry), rv) mock_lstat.assert_called_with(entry.get('name')) - self.assertFalse(mock_unlink.called) - - mock_lstat.reset_mock() - mock_unlink.reset_mock() - mock_isdir.return_value = False - self.assertFalse(self.ptool._exists(entry, remove=True)) - mock_isdir.assert_called_with(entry.get('name')) - mock_lstat.assert_called_with(entry.get('name')) - mock_unlink.assert_called_with(entry.get('name')) - self.assertFalse(mock_rmtree.called) + self.assertFalse(self.ptool._remove.called) - mock_lstat.reset_mock() - mock_isdir.reset_mock() - mock_unlink.reset_mock() - mock_rmtree.reset_mock() - mock_isdir.return_value = True - self.assertFalse(self.ptool._exists(entry, remove=True)) - mock_isdir.assert_called_with(entry.get('name')) + reset() + self.assertEqual(self.ptool._exists(entry, remove=True), None) mock_lstat.assert_called_with(entry.get('name')) - mock_rmtree.assert_called_with(entry.get('name')) - self.assertFalse(mock_unlink.called) + self.ptool._remove.assert_called_with(entry) - mock_isdir.reset_mock() - mock_lstat.reset_mock() - mock_unlink.reset_mock() - mock_rmtree.reset_mock() - mock_rmtree.side_effect = OSError + reset() + self.ptool._remove.side_effect = OSError self.assertEqual(self.ptool._exists(entry, remove=True), rv) - mock_isdir.assert_called_with(entry.get('name')) mock_lstat.assert_called_with(entry.get('name')) - mock_rmtree.assert_called_with(entry.get('name')) - self.assertFalse(mock_unlink.called) + self.ptool._remove.assert_called_with(entry) @patch("os.chown") @patch("os.chmod") diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index f2cd39142..2eda38cdc 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -27,22 +27,27 @@ class TestDebuggable(Bcfg2TestCase): self.assertIsInstance(d.logger, logging.Logger) self.assertFalse(d.debug_flag) - def test_toggle_debug(self): + def test_set_debug(self): d = self.get_obj() d.debug_log = Mock() - orig = d.debug_flag - d.toggle_debug() - self.assertNotEqual(orig, d.debug_flag) + self.assertEqual(True, d.set_debug(True)) + self.assertEqual(d.debug_flag, True) self.assertTrue(d.debug_log.called) d.debug_log.reset_mock() - changed = d.debug_flag - d.toggle_debug() - self.assertNotEqual(changed, d.debug_flag) - self.assertEqual(orig, d.debug_flag) + self.assertEqual(False, d.set_debug(False)) + self.assertEqual(d.debug_flag, False) self.assertTrue(d.debug_log.called) + def test_toggle_debug(self): + d = self.get_obj() + d.set_debug = Mock() + orig = d.debug_flag + self.assertEqual(d.toggle_debug(), + d.set_debug.return_value) + d.set_debug.assert_called_with(not orig) + def test_debug_log(self): d = self.get_obj() d.logger = Mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index e48507a57..d3e97df8d 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -210,7 +210,7 @@ class TestDirectoryBacked(Bcfg2TestCase): db = self.get_obj() db.fam = Mock() db.fam.rv = 0 - + def reset(): db.fam.rv += 1 db.fam.AddMonitor.return_value = db.fam.rv @@ -242,6 +242,7 @@ class TestDirectoryBacked(Bcfg2TestCase): def test_add_entry(self): db = self.get_obj() db.fam = Mock() + class MockChild(Mock): def __init__(self, path, fam, **kwargs): Mock.__init__(self, **kwargs) @@ -346,7 +347,7 @@ class TestDirectoryBacked(Bcfg2TestCase): event = get_event(fname, "deleted", req_id) db.HandleEvent(event) self.assertNotIn(relpath, db.entries) - + # test that changing a file that doesn't exist works reset() event = get_event(fname, "changed", req_id) @@ -354,7 +355,7 @@ class TestDirectoryBacked(Bcfg2TestCase): db.add_entry.assert_called_with(relpath, event) self.assertFalse(db.add_directory_monitor.called) db.entries[relpath] = MagicMock() - + # test that deleting a directory works. this is a little # strange because the _parent_ directory has to handle the # deletion @@ -383,7 +384,7 @@ class TestDirectoryBacked(Bcfg2TestCase): msg="Failed to ignore %s" % fname) self.assertFalse(db.add_directory_monitor.called, msg="Failed to ignore %s" % fname) - + class TestXMLFileBacked(TestFileBacked): test_obj = XMLFileBacked @@ -508,7 +509,7 @@ class TestXMLFileBacked(TestFileBacked): test_obj.__name__) def test_Index(self, mock_follow): xfb = self.get_obj() - + def reset(): mock_follow.reset_mock() FakeElementTree.xinclude.reset_mock() @@ -594,7 +595,7 @@ class TestStructFile(TestXMLFileBacked): def _get_test_data(self): """ build a very complex set of test data """ - # top-level group and client elements + # top-level group and client elements groups = dict() # group and client elements that are descendents of other group or # client elements @@ -698,7 +699,7 @@ class TestStructFile(TestXMLFileBacked): def test__match(self, mock_include): sf = self.get_obj() metadata = Mock() - + (xdata, groups, subgroups, children, subchildren, standalone) = \ self._get_test_data() @@ -757,7 +758,7 @@ class TestStructFile(TestXMLFileBacked): def test__xml_match(self, mock_include): sf = self.get_obj() metadata = Mock() - + (xdata, groups, subgroups, children, subchildren, standalone) = \ self._get_test_data() @@ -943,7 +944,7 @@ class TestINode(Bcfg2TestCase): self.assertItemsEqual(inode.contents, dict()) inner() - + data = lxml.etree.Element("Parent") child1 = lxml.etree.SubElement(data, "Data", name="child1", attr="some attr") @@ -973,7 +974,7 @@ class TestINode(Bcfg2TestCase): __children__=[subchild1])) inner2() - + # test ignore. no ignore is set on INode by default, so we # have to set one old_ignore = copy.copy(self.test_obj.ignore) @@ -1029,7 +1030,7 @@ class TestINode(Bcfg2TestCase): inode.Match(metadata, data, entry=child) self.assertEqual(data, inode.contents) inode.predicate.assert_called_with(metadata, child) - + class TestInfoNode(TestINode): __test__ = True @@ -1107,7 +1108,7 @@ class TestXMLSrc(TestXMLFileBacked): mock_open.reset_mock() xsrc = self.get_obj("/test/foo.xml") - xsrc.__node__ = Mock() + xsrc.__node__ = Mock() xsrc.HandleEvent(Mock()) mock_open.assert_called_with("/test/foo.xml") mock_open.return_value.read.assert_any_call() @@ -1115,14 +1116,14 @@ class TestXMLSrc(TestXMLFileBacked): self.assertEqual(xsrc.__node__.call_args[0][1], dict()) self.assertEqual(xsrc.pnode, xsrc.__node__.return_value) self.assertEqual(xsrc.cache, None) - + @patch("Bcfg2.Server.Plugin.helpers.XMLSrc.HandleEvent") def test_Cache(self, mock_HandleEvent): xsrc = self.get_obj("/test/foo.xml") metadata = Mock() xsrc.Cache(metadata) mock_HandleEvent.assert_any_call() - + xsrc.pnode = Mock() xsrc.Cache(metadata) xsrc.pnode.Match.assert_called_with(metadata, xsrc.__cacheobj__()) @@ -1181,7 +1182,7 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): "/etc/baz.conf": pd.BindEntry}, Package={"quux": pd.BindEntry, "xyzzy": pd.BindEntry})) - + inner() def test__matches(self): @@ -1207,7 +1208,7 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): self.assertItemsEqual(entry.attrib, dict(name="/etc/foo.conf", test1="test1", test2="test2")) - + def test_get_attrs(self): pd = self.get_obj() entry = lxml.etree.Element("Path", name="/etc/foo.conf") @@ -1271,7 +1272,7 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): entry = lxml.etree.Element("Package", name="xyzzy") self.assertRaises(PluginExecutionError, pd.get_attrs, entry, metadata) - + class TestSpecificity(Bcfg2TestCase): test_obj = Specificity @@ -1307,7 +1308,7 @@ class TestSpecificity(Bcfg2TestCase): elif i < j: self.assertEqual(1, specs[i].__cmp__(specs[j])) self.assertEqual(-1, specs[j].__cmp__(specs[i])) - + def test_cmp(self): """ test __lt__/__gt__/__eq__ """ specs = [self.get_obj(all=True), @@ -1384,7 +1385,7 @@ class TestEntrySet(TestDebuggable): # filenames that should be ignored ignore = ["foo~", ".#foo", ".foo.swp", ".foo.swx", "test.txt.genshi_include", "test.G_foo.genshi_include"] - + def get_obj(self, basename="test", path=datastore, entry_type=MagicMock(), encoding=None): return self.test_obj(basename, path, entry_type, encoding) @@ -1526,7 +1527,7 @@ class TestEntrySet(TestDebuggable): eset.update_metadata.assert_called_with(event) self.assertFalse(eset.entry_init.called) self.assertFalse(eset.reset_metadata.called) - + reset() event = Mock() event.code2str.return_value = "deleted" @@ -1535,7 +1536,7 @@ class TestEntrySet(TestDebuggable): eset.reset_metadata.assert_called_with(event) self.assertFalse(eset.entry_init.called) self.assertFalse(eset.update_metadata.called) - + for evt in ["exists", "created", "changed"]: reset() event = Mock() @@ -1592,7 +1593,7 @@ class TestEntrySet(TestDebuggable): self.assertFalse(eset.specificity_from_filename.called) self.assertFalse(eset.entry_type.called) eset.entries["test.txt"].handle_event.assert_called_with(event) - + # test keyword args etype = Mock() specific = Mock() @@ -1617,7 +1618,7 @@ class TestEntrySet(TestDebuggable): eset.specificity_from_filename.assert_called_with("test3.txt", specific=None) self.assertFalse(eset.entry_type.called) - + @patch("Bcfg2.Server.Plugin.helpers.Specificity") def test_specificity_from_filename(self, mock_spec): # There's a strange scoping issue in py3k that prevents this @@ -1658,7 +1659,7 @@ class TestEntrySet(TestDebuggable): hostname="fqdn.subdomain.example.com") test(eset, ppath + ".G20_group_with_underscores", group="group_with_underscores", prio=20) - + for bogus in self.bogus_names: fails(eset, bogus) fails(eset, ppath + ".G_group with spaces") @@ -1692,11 +1693,11 @@ class TestEntrySet(TestDebuggable): eset.update_metadata(event) self.assertFalse(mock_InfoXML.called) eset.infoxml.HandleEvent.assert_called_with(event) - + for fname in [':info', 'info']: event = Mock() event.filename = fname - + idata = ["owner:owner", "group: GROUP", "mode: 775", @@ -1711,7 +1712,7 @@ class TestEntrySet(TestDebuggable): expected['important'] = 'true' self.assertItemsEqual(eset.metadata, expected) - + def test_reset_metadata(self): eset = self.get_obj() @@ -1825,7 +1826,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): event.filename)) self.assertNotIn(ident, gs.entries) mock_isdir.assert_called_with(epath) - + # file that is not in self.entries reset() event = Mock() @@ -1850,7 +1851,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): gs.es_cls.return_value.bind_entry) gs.entries[ident].handle_event.assert_called_with(event) mock_isfile.assert_called_with(epath) - + # file that is in self.entries reset() gs.add_entry(event) @@ -1867,7 +1868,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): event.filename = "foo" for i in range(1, 4): event.requestID = i - self.assertEqual(gs.event_path(event), + self.assertEqual(gs.event_path(event), os.path.join(datastore, gs.name, gs.handles[event.requestID].lstrip('/'), event.filename)) @@ -1876,7 +1877,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): def test_event_id(self, mock_isdir): gs = self.get_obj() gs.event_path = Mock() - + def reset(): gs.event_path.reset_mock() mock_isdir.reset_mock() @@ -1894,29 +1895,29 @@ class TestGroupSpool(TestPlugin, TestGenerator): os.path.join(gs.handles[event.requestID].lstrip('/'), event.filename)) mock_isdir.assert_called_with(gs.event_path.return_value) - + reset() mock_isdir.return_value = False self.assertEqual(gs.event_id(event), gs.handles[event.requestID].rstrip('/')) mock_isdir.assert_called_with(gs.event_path.return_value) - def test_toggle_debug(self): + def test_set_debug(self): gs = self.get_obj() gs.entries = {"/foo": Mock(), "/bar": Mock(), "/baz/quux": Mock()} - - @patch("Bcfg2.Server.Plugin.helpers.Plugin.toggle_debug") + + @patch("Bcfg2.Server.Plugin.helpers.Plugin.set_debug") def inner(mock_debug): - gs.toggle_debug() - mock_debug.assert_called_with(gs) + gs.set_debug(True) + mock_debug.assert_called_with(gs, True) for entry in gs.entries.values(): - entry.toggle_debug.assert_any_call() - + entry.set_debug.assert_called_with(True) + inner() - - TestPlugin.test_toggle_debug(self) + + TestPlugin.test_set_debug(self) def test_HandleEvent(self): gs = self.get_obj() @@ -1950,7 +1951,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): gs.HandleEvent(event) gs.event_id.assert_called_with(event) gs.add_entry.assert_called_with(event) - + # test deleting entry, changing entry that does exist for evt in ["changed", "deleted"]: reset() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py index baad10933..4a849c11a 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py @@ -105,27 +105,18 @@ if can_skip or HAS_GENSHI: self.assertTrue(cgg._handle_genshi_exception.called) def test_handle_event(self): - @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") - def inner(mock_handle_event): - cgg = self.get_obj() - cgg.loader = Mock() - cgg.data = "template data" - event = Mock() - cgg.handle_event(event) - cgg.loader.load.assert_called_with(cgg.name, - cls=NewTextTemplate, - encoding=cgg.encoding) - - cgg.loader.reset_mock() - cgg.loader.load.side_effect = OSError - self.assertRaises(PluginExecutionError, - cgg.handle_event, event) - cgg.loader.load.assert_called_with(cgg.name, - cls=NewTextTemplate, - encoding=cgg.encoding) + cgg = self.get_obj() + cgg.loader = Mock() + event = Mock() + cgg.handle_event(event) + cgg.loader.load.assert_called_with(cgg.name, + cls=NewTextTemplate, + encoding=cgg.encoding) - inner() - loader_cls = self.test_obj.__loader_cls__ - self.test_obj.__loader_cls__ = Mock - TestCfgGenerator.test_handle_event(self) - self.test_obj.__loader_cls__ = loader_cls + cgg.loader.reset_mock() + cgg.loader.load.side_effect = OSError + self.assertRaises(PluginExecutionError, + cgg.handle_event, event) + cgg.loader.load.assert_called_with(cgg.name, + cls=NewTextTemplate, + encoding=cgg.encoding) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py index 832857601..43d594482 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py @@ -18,54 +18,59 @@ from TestPlugin import TestDirectoryBacked, TestConnector, TestPlugin, \ TestFileBacked -class TestHelperModule(TestFileBacked): +class TestHelperModule(Bcfg2TestCase): test_obj = HelperModule path = os.path.join(datastore, "test.py") + def get_obj(self, path=None): + if path is None: + path = self.path + return self.test_obj(path, fam=Mock()) + def test__init(self): hm = self.get_obj() self.assertEqual(hm._module_name, "test") self.assertEqual(hm._attrs, []) @patch("imp.load_source") - def test_Index(self, mock_load_source): + def test_HandleEvent(self, mock_load_source): hm = self.get_obj() mock_load_source.side_effect = ImportError attrs = dir(hm) - hm.Index() + hm.HandleEvent() mock_load_source.assert_called_with(hm._module_name, hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) - + mock_load_source.reset() mock_load_source.side_effect = None # a regular Mock (not a MagicMock) won't automatically create - # __export__, so this triggers a failure condition in Index + # __export__, so this triggers a failure condition in HandleEvent mock_load_source.return_value = Mock() attrs = dir(hm) - hm.Index() + hm.HandleEvent() mock_load_source.assert_called_with(hm._module_name, hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) # test reserved attributes module = Mock() - module.__export__ = ["_attrs", "Index", "__init__"] + module.__export__ = ["_attrs", "HandleEvent", "__init__"] mock_load_source.reset() mock_load_source.return_value = module attrs = dir(hm) - hm.Index() + hm.HandleEvent() mock_load_source.assert_called_with(hm._module_name, hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) # test adding attributes module = Mock() - module.__export__ = ["foo", "bar", "baz", "Index"] + module.__export__ = ["foo", "bar", "baz", "HandleEvent"] mock_load_source.reset() mock_load_source.return_value = module - hm.Index() + hm.HandleEvent() mock_load_source.assert_called_with(hm._module_name, hm.name) self.assertTrue(hasattr(hm, "foo")) self.assertTrue(hasattr(hm, "bar")) @@ -74,34 +79,36 @@ class TestHelperModule(TestFileBacked): # test removing attributes module = Mock() - module.__export__ = ["foo", "bar", "quux", "Index"] + module.__export__ = ["foo", "bar", "quux", "HandleEvent"] mock_load_source.reset() mock_load_source.return_value = module - hm.Index() + hm.HandleEvent() mock_load_source.assert_called_with(hm._module_name, hm.name) self.assertTrue(hasattr(hm, "foo")) self.assertTrue(hasattr(hm, "bar")) self.assertTrue(hasattr(hm, "quux")) self.assertFalse(hasattr(hm, "baz")) self.assertEqual(hm._attrs, ["foo", "bar", "quux"]) - -class TestHelperSet(TestDirectoryBacked): - test_obj = HelperSet +class TestTemplateHelper(TestPlugin, TestConnector, TestDirectoryBacked): + test_obj = TemplateHelper testfiles = ['foo.py', 'foo_bar.py', 'foo.bar.py'] ignore = ['fooo.py~', 'fooo.pyc', 'fooo.pyo'] badevents = ['foo'] + def get_obj(self, core=None, fam=None): + if core is None: + core = Mock() + if fam is not None: + core.fam = fam -class TestTemplateHelper(TestPlugin, TestConnector): - test_obj = TemplateHelper - - def test__init(self): - TestPlugin.test__init(self) - - th = self.get_obj() - self.assertIsInstance(th.helpers, HelperSet) + @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__, + self.test_obj.__name__), + Mock()) + def inner(): + return TestPlugin.get_obj(self, core=core) + return inner() def test_get_additional_data(self): TestConnector.test_get_additional_data(self) @@ -113,6 +120,6 @@ class TestTemplateHelper(TestPlugin, TestConnector): module = Mock() module._module_name = mname rv[mname] = module - th.helpers.entries['%s.py' % mname] = module + th.entries['%s.py' % mname] = module actual = th.get_additional_data(Mock()) self.assertItemsEqual(actual, rv) diff --git a/tools/README b/tools/README index 5b1fc0baf..400cfc55c 100644 --- a/tools/README +++ b/tools/README @@ -62,6 +62,9 @@ export.py export.sh - Export a tagged version of the Bcfg2 source +generate-manpages.bash + - Generate man pages from the Sphinx source + hostbasepush.py - Call the Hostbase.rebuildState XML-RPC method |