diff options
51 files changed, 514 insertions, 203 deletions
diff --git a/debian/changelog b/debian/changelog index 01e4478d0..cf46d1507 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +bcfg2 (1.2.0pre3-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Sat, 18 Jun 2011 22:41:29 -0500 + bcfg2 (1.2.0-0.0pre2) unstable; urgency=low * New upstream release diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt index 0b5b83211..ce8e49703 100644 --- a/doc/appendix/guides/centos.txt +++ b/doc/appendix/guides/centos.txt @@ -497,6 +497,18 @@ Then add the files to Cfg:: mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL cp /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL !$/RPM-GPG-KEY-EPEL +You will also want to add an *important* attribute to these files so +that they are installed on the client prior to any attempts to install +the **gpg-pubkey** rpm packages. This is especially important during the +bootstrapping phase and can be accomplished using an :ref:`server-info` +file that looks like the following: + +.. code-block:: xml + + <FileInfo> + <Info owner='root' group='root' perms='0644' important='true'/> + </FileInfo> + Now, running the client shows only unmanaged Service entries. Woohoo! Manage services diff --git a/doc/conf.py b/doc/conf.py index 77ab6fd94..4b13579d7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -40,8 +40,13 @@ source_suffix = '.txt' master_doc = 'index' # General information about the project. -project = u'Bcfg2' -copyright = u'2009-%s, Narayan Desai' % time.strftime('%Y') +# py3k compatibility +if sys.hexversion >= 0x03000000: + project = 'Bcfg2' + copyright = '2009-%s, Narayan Desai' % time.strftime('%Y') +else: + project = u'Bcfg2' + copyright = u'2009-%s, Narayan Desai' % time.strftime('%Y') # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -174,10 +179,17 @@ latex_font_size = '11pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('contents', 'bcfg2.tex', u'Bcfg2 Documentation', - u'Narayan Desai et al.', 'manual', True), -] +# py3k compatibility +if sys.hexversion >= 0x03000000: + latex_documents = [ + ('contents', 'bcfg2.tex', 'Bcfg2 Documentation', + 'Narayan Desai et al.', 'manual', True), + ] +else: + latex_documents = [ + ('contents', 'bcfg2.tex', u'Bcfg2 Documentation', + u'Narayan Desai et al.', 'manual', True), + ] # The name of an image file (relative to this directory) to place at the top of # the title page. diff --git a/doc/server/configurationentries.txt b/doc/server/configurationentries.txt index 10eccf6be..0c12ce9c9 100644 --- a/doc/server/configurationentries.txt +++ b/doc/server/configurationentries.txt @@ -107,8 +107,8 @@ Path type specified. | | | that should not | | | | | exist | | +-------------+----------------------+-----------------+--------------------------+ -| permissions | Replaces Permissions | Permissions of | name, owner, | -| | entries | POSIX entities | group, perms | +| permissions | Replaces Permissions | Permissions of | name, owner, group, | +| | entries | POSIX entities | perms, recursive | | | | | | +-------------+----------------------+-----------------+--------------------------+ | vcs | New | Create version | vcstype (git), | @@ -119,6 +119,19 @@ Path type specified. Keep in mind that permissions for files served up by Cfg/TGenshi/TCheetah are still handled via the traditional :ref:`server-info` mechanisms. +Additional information +---------------------- + +This section describes some additional behavior relating to POSIX entry +attributes. + +Recursive permissions +^^^^^^^^^^^^^^^^^^^^^ + +As per the request in ticket 871, Path type='permissions' entries allow you to +set a recursive attribute which allows the owner/group to be set recursively +for a directory. + .. _boundentries: Bound Entries diff --git a/doc/server/info.txt b/doc/server/info.txt index 231a83a52..f1154e665 100644 --- a/doc/server/info.txt +++ b/doc/server/info.txt @@ -41,6 +41,8 @@ possible fields in an info file are: | paranoid: | yes | no | Backup file before replacement? | no | +------------+-------------------+----------------------------------+---------+ | perms: | Numeric file mode | Sets the permissions of the file | 0644 | +| | | 'inherit' | (or inherits from the files on | | +| | | disk if set to inherit) | | +------------+-------------------+----------------------------------+---------+ A sample info file for CGI script on a web server might look like:: diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index 1cbc4cf65..3329f48bd 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -38,7 +38,7 @@ Usage Specific property files can be referred to in templates as ``metadata.Properties[<filename>]``. The -data attribute is an LXML element object. (Documented +``xdata`` attribute is an LXML element object. (Documented `here <http://codespeak.net/lxml/tutorial.html#the-element-class>`_) Currently, only one access method is defined for this data, ``Match``. @@ -56,10 +56,21 @@ more details on how Group and Client tags are parsed.) For instance:: As we formulate more common use cases, we will add them to the !PropertyFile class as methods. This will simplify templates. +You can also access the XML data that comprises a property file +directly in one of several ways: + +* ``metadata.Properties['prop-file'].xdata`` is an lxml.etree._Element + object representing the top-level element in the file. +* ``metadata.Properties['prop-file'].data`` is the raw contents of the + property file as a string. +* ``metadata.Properties['prop-file'].entries`` is a list of + lxml.etree._Element objects representing the direct children of the + top-level element. (I.e., everything directly under the + ``<Properties>`` tag.) + Accessing Properties contents from TGenshi ========================================== Access contents of ``Properties/auth.xml``:: - - ${metadata.Properties['auth.xml'].data.find('file').find('bcfg2.key').text} + ${metadata.Properties['auth.xml'].xdata.find('file').find('bcfg2.key').text} diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt index 8ef3d5af3..ca1cf459a 100644 --- a/doc/server/plugins/index.txt +++ b/doc/server/plugins/index.txt @@ -65,8 +65,8 @@ Literal Configuration (Generators) :maxdepth: 1 :glob: - generators/tgenshi/index generators/* + generators/tgenshi/index Each of these plugins has a corresponding subdirectory with the same name in the Bcfg2 repository. diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index b9f698a0f..5b1f9e259 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -85,10 +85,12 @@ file looks like:: #end if Any Probe script you run will store its output in -``$self.metadata.Probes["scriptname"]``, so we get to our `scratchlocal` -script's output as seen above. Note that we had to wrap the output in an -`int()` call; the script output is treated as a string, so it needs to -be converted before it can be tested numerically. +``$self.metadata.Probes["scriptname"]``, so we get to our +`scratchlocal` script's output as seen above. (See `Handling Probe +Output`_, below, for more information on how this is done.) Note that +we had to wrap the output in an `int()` call; the script output is +treated as a string, so it needs to be converted before it can be +tested numerically. With all of these pieces in place, the following series of events will happen when the client is run: @@ -110,8 +112,37 @@ is to add the ``/etc/auto.master`` to a Bundle: <Path name='/etc/auto.master'/> -Host and Group Specific probes -============================== +Handling Probe Output +===================== + +Bcfg2 stores output from probes in the ``Probes`` property of a +client's metadata object. To access this data in TGenshi, for +instance, you could do:: + + ${metadata.Probes['script-name']} + +This is not the full output of the probe; any lines that start with +"group:" have been stripped from the output. The data is a +string-like object that has some interesting and salient features: + +* If the data is a valid XML document, then + ``metadata.Probes['script-name'].xdata`` will be an + ``lxml.etree._Element`` object representing the XML data. +* If the data is a valid JSON document, and either the Python ``json`` + or ``simplejson`` module is installed, then + ``metadata.Probes['script-name'].json`` will be a data structure + representing the JSON data. +* If the data is a valid YAML document, and either the Python ``yaml`` + or ``syck`` module is installed, then + ``metadata.Probes['script-name'].yaml`` will be a data structure + representing the YAML data. + +If these conditions are not met, then the named properties will be +``None``. In all other fashions, the probe data objects should act +like strings. + +Host- and Group-Specific probes +=============================== Bcfg2 has the ability to alter probes based on client hostname and group membership. These files work similarly to files in Cfg. diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 6b5c246aa..c74e13e9d 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -146,6 +146,16 @@ The `Genshi templating system`_ is used internally. Use --- +.. warning:: + + Group tags are not used inside of Genshi templates. You can get the + same logic (and more) using Genshi conditionals. + + .. code-xml:: + + <py:if test="groupname in metadata.groups"> + </py:if> + Bcfg uses the Genshi API for templates, and performs a XML format stream rendering of the template into an lxml entry, which is included in the client configuration. :ref:`Client metadata <client-metadata>` diff --git a/doc/server/plugins/generators/deps.txt b/doc/server/plugins/structures/deps.txt index 7c5861d06..7c5861d06 100644 --- a/doc/server/plugins/generators/deps.txt +++ b/doc/server/plugins/structures/deps.txt diff --git a/gentoo/bcfg2-1.1.1.ebuild b/gentoo/bcfg2-1.1.2.ebuild index 1992ddaa4..4da67d865 100644 --- a/gentoo/bcfg2-1.1.1.ebuild +++ b/gentoo/bcfg2-1.1.2.ebuild @@ -33,7 +33,7 @@ PYTHON_MODNAME="Bcfg2" distutils_src_install_post_hook() { if ! use server; then - rm -f "${T}/images/${PYTHON_ABI}${EPREFIX}/usr/sbin/bcfg2-"* + rm -f "$(distutils_get_intermediate_installation_image)${EPREFIX}/usr/sbin/bcfg2-"* fi } diff --git a/man/bcfg2.1 b/man/bcfg2.1 index 49fd1e208..0ace97e8a 100644 --- a/man/bcfg2.1 +++ b/man/bcfg2.1 @@ -3,7 +3,7 @@ bcfg2 \- reconfigure machine based on settings in Bcfg2 .SH SYNOPSIS .B bcfg2 -.I [\-d] [\-v] [\-p] [\-c cache file] [\-e] [\-f config file] [\-I] [\-q] [\-b bundle] [\-r removal mode] +.I [\-d] [\-v] [\-p] [\-c cache file] [\-e] [\-f config file] [\-I] [\-q] [\-z] [\-b bundle] [\-r removal mode] [\-\-ca\-cert=file] [\-\-ssl\-cns=list] [\-\-ssl\-cert=file] [\-\-ssl\-key=file] .SH DESCRIPTION .TP .BR bcfg2 @@ -62,6 +62,10 @@ Run bcfg2 against one or multiple bundles in the configuration. Cache a copy of the configuration in cachefile. .TP +.BR "\-\-ca\-cert=<ca cert>" +Specifiy the path to the SSL CA certificate. + +.TP .BR "\-d" Run bcfg2 in debug mode. @@ -118,7 +122,7 @@ should only be used in safe conditions. .TP .BR "\-Q" Run bcfg2 in "bundle quick" mode, where only entries in a bundle are -or installed. This runs much faster than -q, but doesn't provide +verified or installed. This runs much faster than -q, but doesn't provide statistics to the server at all. In order for this option to work, the -b option must also be provided. This option is incompatible with -r. @@ -137,6 +141,14 @@ to stop all services started. disabled suppresses all attempts to modify services. .TP +.BR "\-\-ssl\-cert=<ssl cert>" +Specifiy the path to the SSL certificate. + +.TP +.BR "\-\-ssl\-cns=<CommonName1:CommonName2 ...>" +List of acceptable SSL server Common Names. + +.TP .BR "\-\-ssl\-key=<ssl key>" Specifiy the path to the SSL key. @@ -156,6 +168,10 @@ seconds. .TP .BR "\-v" Run bcfg2 in verbose mode. + +.TP +.BR "\-z" +Only configure independent entries, ignore bundles. .RE .SH "SEE ALSO" .BR bcfg2-server(8), diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 786f69f9a..7cd04a0b7 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -379,6 +379,11 @@ Communication protocol to use. Defaults to xmlrpc/ssl. A client-only option. Number of times to retry network communication. .TP +.B serverCommonNames +A client-only option. A colon-separated list of Common Names the client +will accept in the SSL certificate presented by the server. + +.TP .B user A client-only option. The UUID of the client. diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 1e6b71ea5..d23478c19 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -13,7 +13,7 @@ %define lxmldep %(rpm -q %{alt_lxml} 2>&1 > /dev/null && echo %{alt_lxml} || echo %{dfl_lxml}) Name: bcfg2 -Version: 1.2.0pre2 +Version: 1.2.0pre3 Release: %{release} Summary: Configuration management system diff --git a/redhat/RELEASE b/redhat/RELEASE index a0694f914..08f0fd716 100644 --- a/redhat/RELEASE +++ b/redhat/RELEASE @@ -1 +1 @@ -0.0pre2 +0.0pre3 diff --git a/schemas/pathentry.xsd b/schemas/pathentry.xsd index 0c27f9112..24be22612 100644 --- a/schemas/pathentry.xsd +++ b/schemas/pathentry.xsd @@ -24,6 +24,7 @@ <xsd:attribute type='xsd:string' name='owner' use='optional'/> <xsd:attribute type='xsd:string' name='perms' use='optional'/> <xsd:attribute type='xsd:string' name='prune' use='optional'/> + <xsd:attribute type='xsd:string' name='recursive' use='optional'/> <xsd:attribute type='xsd:string' name='to' use='optional'/> <xsd:attribute type='xsd:string' name='type' use='optional'/> <xsd:attributeGroup ref="py:genshiAttrs"/> diff --git a/schemas/rules.xsd b/schemas/rules.xsd index 101b62384..0a408c35c 100644 --- a/schemas/rules.xsd +++ b/schemas/rules.xsd @@ -38,6 +38,7 @@ <xsd:attribute type='xsd:string' name='perms'/> <xsd:attribute type='xsd:string' name='owner'/> <xsd:attribute type='xsd:string' name='group'/> + <xsd:attribute type='xsd:string' name='recursive'/> <xsd:attribute type='xsd:string' name='prune'/> <xsd:attribute type='xsd:string' name='to'/> <xsd:attributeGroup ref="py:genshiAttrs"/> @@ -121,7 +121,7 @@ if sys.hexversion < 0x03000000 and os.path.exists(py3lib): setup(cmdclass=cmdclass, name="Bcfg2", - version="1.2.0pre2", + version="1.2.0pre3", description="Bcfg2 Server", author="Narayan Desai", author_email="desai@mcs.anl.gov", diff --git a/solaris/Makefile b/solaris/Makefile index 65ab6309e..6be7ed2fb 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -1,7 +1,7 @@ #!/usr/sfw/bin/gmake PYTHON="/opt/csw/bin/python" -VERS=1.2.0pre2-1 +VERS=1.2.0pre3-1 PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2 index cd8215741..293e64632 100644 --- a/solaris/pkginfo.bcfg2 +++ b/solaris/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.2.0pre2" +VERSION="1.2.0pre3" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server index 8bc069d08..49ea4e294 100644 --- a/solaris/pkginfo.bcfg2-server +++ b/solaris/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.2.0pre2" +VERSION="1.2.0pre3" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index e33e1a7df..d8e308ee0 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -200,15 +200,18 @@ class Frame: tl = [t for t in self.tools if t.handlesEntry(cfile) \ and t.canVerify(cfile)] if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tl[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) def Inventory(self): """ diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index d7be44dc0..2b8cc3304 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -73,7 +73,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.cmd.run("%s clean" % APTGET) try: self.pkg_cache = apt.cache.Cache() - except SystemError, e: + except SystemError: + e = sys.exc_info()[1] self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError self.pkg_cache.update() diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 64821f621..faec2e251 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -16,6 +16,10 @@ import shutil import stat import sys import time +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML @@ -145,7 +149,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: content = open(entry.get('name')).read() entry.set('current_bfile', binascii.b2a_base64(content)) - except IOError, error: + except IOError: + error = sys.exc_info()[1] self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror)) @@ -457,12 +462,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): if type(tempdata) == unicode: try: tempdata = tempdata.encode(self.setup['encoding']) - except UnicodeEncodeError, e: + except UnicodeEncodeError: + e = sys.exc_info()[1] self.logger.error("Error encoding file %s:\n %s" % \ (entry.get('name'), e)) try: content = open(entry.get('name')).read() - except IOError, error: + except IOError: + error = sys.exc_info()[1] if error.strerror == "No such file or directory": # print diff for files that don't exist (yet) content = '' @@ -586,7 +593,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): datetime.isoformat(datetime.now()))) self.logger.info("Backup of %s saved to %s" % (entry.get('name'), self.ppath)) - except IOError, e: + except IOError: + e = sys.exc_info()[1] self.logger.error("Failed to create backup file for %s" % \ (entry.get('name'))) self.logger.error(e) @@ -622,7 +630,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): % (entry.get('name'))) return False return True - except (OSError, IOError), err: + except (OSError, IOError): + err = sys.exc_info()[1] if err.errno == errno.EACCES: self.logger.info("Failed to open %s for writing" % (entry.get('name'))) else: @@ -702,7 +711,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): return False try: shutil.rmtree(ename) - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) else: @@ -710,20 +720,63 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: os.rmdir(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False try: os.remove(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False def Verifypermissions(self, entry, _): """Verify Path type='permissions' entry""" + if entry.get('perms') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid: + # owner mismatch for path + entry.set('current_owner', str(pstat.st_uid)) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Owner for path %s is incorrect. " + "Current owner is %s but should be %s\n" % \ + (path, pstat.st_uid, entry.get('owner'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + if group != pstat.st_gid: + # group mismatch for path + entry.set('current_group', str(pstat.st_gid)) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Group for path %s is incorrect. " + "Current group is %s but should be %s\n" % \ + (path, pstat.st_gid, entry.get('group'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False return self.Verifydirectory(entry, _) def Installpermissions(self, entry): @@ -734,9 +787,23 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error('Entry %s not completely specified. ' 'Try running bcfg2-lint.' % (entry.get('name'))) return False + plist = [entry.get('name')] + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid or group != pstat.st_gid: + # owner mismatch for path + plist.append(path) try: - os.chown(entry.get('name'), normUid(entry), normGid(entry)) - os.chmod(entry.get('name'), calcPerms(S_IFDIR, entry.get('perms'))) + for p in plist: + os.chown(p, normUid(entry), normGid(entry)) + os.chmod(p, calcPerms(S_IFDIR, entry.get('perms'))) return True except (OSError, KeyError): self.logger.error('Permission fixup failed for %s' % \ diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py index 082897934..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Client/Tools/Pacman.py @@ -78,5 +78,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): try: self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline)) self.cmd.run("%s -S %s" % (self.pkgtool, pkgline)) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("Error occurred during installation: %s" % e) diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py index a1e14b3a6..5376118c2 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Client/Tools/RPMng.py @@ -2,11 +2,12 @@ __revision__ = '$Revision$' -import ConfigParser import os.path import rpm import rpmtools import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 5387fdd6a..66768fb34 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -1,13 +1,14 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision: $' -import ConfigParser import copy import os.path import sys import yum import Bcfg2.Client.XML import Bcfg2.Client.Tools.RPMng +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -151,7 +152,8 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): try: pkgDict = dict([(i.name, i) for i in \ self.yb.returnPackagesByDep(entry.get('name'))]) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error('Yum Error Depsolving for %s: %s' % \ (entry.get('name'), str(e))) pkgDict = {} diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 0435ca0d7..6aab9817c 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -1,9 +1,9 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision$' -import ConfigParser import copy import os.path +import sys import yum import yum.packages import yum.rpmtrans @@ -13,6 +13,8 @@ import yum.misc import rpmUtils.arch import Bcfg2.Client.XML import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -167,10 +169,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.doConfigSetup() self.yb.doTsSetup() self.yb.doRpmDBSetup() - except yum.Errors.RepoError, e: + except yum.Errors.RepoError: + e = sys.exc_info()[1] self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -505,7 +509,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): try: vResult = self._verifyHelper(_POs[0]) - except Exception, e: + except Exception: + e = sys.exc_info()[1] # Unknown Yum exception self.logger.warning(" Verify Exception: %s" % str(e)) package_fail = True @@ -680,7 +685,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): # Run the Yum Transaction try: rescode, restring = self.yb.buildTransaction() - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -695,7 +701,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.processTransaction(callback=yDisplay, rpmDisplay=rDisplay) self.logger.info("Single Pass for Install Succeeded") - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -822,7 +829,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.install(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error installing some packages: %s" % yume) if len(upgrade_pkgs) > 0: @@ -832,7 +840,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.update(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" % yume) if len(reinstall_pkgs) > 0: @@ -841,7 +850,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.reinstall(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" \ % yume) diff --git a/src/lib/Options.py b/src/lib/Options.py index 0d978c519..619b16787 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -323,7 +323,7 @@ CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[], cmd='-b', odesc='<bundle:bundle>', cook=colon_split) CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, cmd='-Q') -CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, +CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index e4a0f6a3d..4cb0bbe80 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -47,6 +47,9 @@ __all__ = ["ComponentProxy", class CertificateError(Exception): def __init__(self, commonName): self.commonName = commonName + def __str__(self): + return ("Got unallowed commonName %s from server" + % self.commonName) class RetryMethod(xmlrpclib._Method): @@ -193,13 +196,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, ssl_version=ssl_protocol_ver) - try: - self.sock.connect((self.host, self.port)) - except socket.gaierror: - e = sys.exc_info()[1] - self.logger.error("Unable to connect to %s:%s\n%s" % - (self.host, self.port, e.strerror)) - sys.exit(1) + self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index 8cac8a53f..21bf48d3e 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -46,7 +46,12 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): if '.' not in method: params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) - response = (response, ) + # py3k compatibility + if isinstance(response, bool) or isinstance(response, str) \ + or isinstance(response, list): + response = (response, ) + else: + response = (response.decode('utf-8'), ) raw_response = xmlrpclib.dumps(response, methodresponse=1, allow_none=self.allow_none, encoding=self.encoding) @@ -191,9 +196,18 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(auth_content) try: - username, password = auth_content.split(":") + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') except ValueError: username = auth_content password = "" @@ -234,11 +248,13 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): print("got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) + L.append(self.rfile.read(chunk_size).decode('utf-8')) size_remaining -= len(L[-1]) data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') except: try: self.send_response(500) diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 3af25b15a..81fc4a1b1 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val @@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index a9170530e..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,6 +77,7 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', + 'Base', 'Bundler', 'Bzr', 'Cfg', @@ -102,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "<ok/>" diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py index aa822409c..bf774f695 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Server/Hostbase/backends.py @@ -57,12 +57,14 @@ class NISBackend(object): return user - except NISAUTHError, e: + except NISAUTHError: + e = sys.exc_info()[1] return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: + except User.DoesNotExist: + e = sys.exc_info()[1] return None diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index 21b462c86..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -69,7 +69,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -86,7 +87,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -108,7 +110,8 @@ class ldapauth(object): raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] return ('success', distinguishedName[0],) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -134,7 +137,8 @@ class ldapauth(object): self.is_superuser = False return - except KeyError, e: + except KeyError: + e = sys.exc_info()[1] raise LDAPAUTHError("Portions of the LDAP User profile not present") def member_of(self): diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index c44c7bf16..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -1,9 +1,10 @@ -from ConfigParser import ConfigParser, NoSectionError, NoOptionError import os.path +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -c = ConfigParser() +c = ConfigParser.ConfigParser() #This needs to be configurable one day somehow c.read(['./bcfg2.conf']) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3b331b300..30c4f9686 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,53 +456,51 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + if metadata.hostname not in self.matches: + rv = [] + for child in self.entries: + rv.extend(self._match(child, metadata)) + logger.debug("File %s got %d match(es)" % (self.name, len(rv))) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: @@ -693,6 +692,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,9 +4,11 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re +import stat import sys import tempfile @@ -23,9 +25,10 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -95,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -105,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: @@ -119,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,12 +1,100 @@ import lxml.etree +import operator import re +try: + import json + has_json = True +except ImportError: + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P<basename>\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: @@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, @@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index 6ef4c9aff..ec9a17468 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.2.0pre2</span> + <span>Bcfg2 Version 1.2.0pre3</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 291528e2e..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -15,6 +15,7 @@ try: except: colorize = False +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 2aa35f1ec..f30c38a05 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 534ab8238..5ddfd8791 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -7,6 +7,7 @@ import fcntl import logging import os import signal +import socket import stat import sys import tempfile @@ -199,6 +200,13 @@ class Client: self.logger.error("Failed to download probes from bcfg2") self.logger.error(flt.faultString) raise SystemExit(1) + except (Bcfg2.Proxy.CertificateError, + socket.gaierror, + socket.error): + e = sys.exc_info()[1] + self.logger.error("Failed to download probes from bcfg2: %s" + % e) + raise SystemExit(1) times['probe_download'] = time.time() diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index c36e1af42..07953ae69 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -375,21 +375,21 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): except: print("Client %s not defined" % client) continue - print("Hostname:\t", client_meta.hostname) - print("Profile:\t", client_meta.profile) - print("Groups:\t\t", list(client_meta.groups)[0]) + print("Hostname:\t%s" % client_meta.hostname) + print("Profile:\t%s" % client_meta.profile) + print("Groups:\t\t%s" % list(client_meta.groups)[0]) for grp in list(client_meta.groups)[1:]: - print('\t\t%s' % grp) + print("\t\t%s" % grp) if client_meta.bundles: - print("Bundles:\t", list(client_meta.bundles)[0]) + print("Bundles:\t%s" % list(client_meta.bundles)[0]) for bnd in list(client_meta.bundles)[1:]: - print('\t\t%s' % bnd) + print("\t\t%s" % bnd) if client_meta.connectors: print("Connector data") print("=" * 80) for conn in client_meta.connectors: if getattr(client_meta, conn): - print("%s:\t" % (conn), getattr(client_meta, conn)) + print("%s:\t%s" % (conn, getattr(client_meta, conn))) print("=" * 80) def do_mappings(self, args): diff --git a/tools/export.py b/tools/export.py index 47b23469a..2885625d5 100755 --- a/tools/export.py +++ b/tools/export.py @@ -8,8 +8,11 @@ import fileinput from subprocess import Popen, PIPE import sys -# Compatibility import -from Bcfg2.Bcfg2Py3k import formatdate +# py3k compatibility +try: + from email.Utils import formatdate +except ImportError: + from email.utils import formatdate pkgname = 'bcfg2' ftphost = 'terra.mcs.anl.gov' @@ -47,7 +50,7 @@ except NameError: name = input("Your name: ") email = input("Your email: ") newchangelog = \ -"""bcfg2 (%s-0.0%s) unstable; urgency=low +"""bcfg2 (%s%s-0.0) unstable; urgency=low * New upstream release @@ -86,7 +89,7 @@ find_and_replace('src/lib/Server/Reports/reports/templates/base.html', find_and_replace('doc/conf.py', 'version =', 'version = \'%s\'\n' % majorver[0:3], startswith=True) find_and_replace('doc/conf.py', 'release =', - 'release = \'%s\'\n' % (majorver + minorver), startswith=True) + 'release = \'%s\'\n' % (majorver), startswith=True) # tag the release #FIXME: do this using python-dulwich |