summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2011-06-22 17:01:40 -0500
committerNarayan Desai <desai@mcs.anl.gov>2011-06-22 17:01:40 -0500
commit4849378a62ab0dbf72f8ce4e6b2073dc73f2337a (patch)
tree14945dd6045c438f5dbc8e96f023258d31e1adf9
parent8d08d5195fa0b6f503752273183948e5d24f33d6 (diff)
parent2db056574f5f55c3c802da63abe7e3de10db5d76 (diff)
downloadbcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.tar.gz
bcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.tar.bz2
bcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.zip
Merge branch 'master' of git.mcs.anl.gov:bcfg2
-rw-r--r--debian/changelog6
-rw-r--r--doc/appendix/guides/centos.txt22
-rw-r--r--doc/conf.py24
-rw-r--r--doc/help/troubleshooting.txt112
-rw-r--r--doc/reports/dynamic.txt55
-rw-r--r--doc/server/configurationentries.txt35
-rw-r--r--doc/server/info.txt2
-rw-r--r--doc/server/plugins/connectors/properties.txt17
-rw-r--r--doc/server/plugins/generators/cfg.txt6
-rw-r--r--doc/server/plugins/generators/decisions.txt38
-rw-r--r--doc/server/plugins/generators/tgenshi/index.txt61
-rw-r--r--doc/server/plugins/index.txt2
-rw-r--r--doc/server/plugins/probes/index.txt43
-rw-r--r--doc/server/plugins/structures/base.txt7
-rw-r--r--doc/server/plugins/structures/bundler/index.txt27
-rw-r--r--doc/server/plugins/structures/deps.txt (renamed from doc/server/plugins/generators/deps.txt)0
-rw-r--r--examples/bcfg2-lint.conf5
-rw-r--r--gentoo/bcfg2-1.1.2.ebuild (renamed from gentoo/bcfg2-1.1.1.ebuild)2
-rw-r--r--man/bcfg2-lint.87
-rw-r--r--man/bcfg2-lint.conf.516
-rw-r--r--man/bcfg2-reports.821
-rw-r--r--man/bcfg2.135
-rw-r--r--man/bcfg2.conf.519
-rw-r--r--misc/bcfg2.spec2
-rw-r--r--redhat/RELEASE2
-rw-r--r--schemas/base.xsd22
-rw-r--r--schemas/bundle.xsd9
-rw-r--r--schemas/fileprobes.xsd2
-rw-r--r--schemas/genshi.xsd3
-rw-r--r--schemas/info.xsd2
-rw-r--r--schemas/pathentry.xsd1
-rw-r--r--schemas/rules.xsd6
-rw-r--r--setup.py2
-rw-r--r--solaris/Makefile2
-rw-r--r--solaris/pkginfo.bcfg22
-rw-r--r--solaris/pkginfo.bcfg2-server2
-rw-r--r--src/lib/Client/Frame.py92
-rw-r--r--src/lib/Client/Tools/APT.py5
-rw-r--r--src/lib/Client/Tools/POSIX.py132
-rw-r--r--src/lib/Client/Tools/Pacman.py3
-rw-r--r--src/lib/Client/Tools/RPMng.py3
-rw-r--r--src/lib/Client/Tools/YUM24.py8
-rw-r--r--src/lib/Client/Tools/YUMng.py86
-rw-r--r--src/lib/Component.py13
-rw-r--r--src/lib/Options.py35
-rw-r--r--src/lib/Proxy.py9
-rw-r--r--src/lib/SSLServer.py49
-rw-r--r--src/lib/Server/Admin/Client.py6
-rw-r--r--src/lib/Server/Admin/Init.py3
-rw-r--r--src/lib/Server/Admin/Perf.py4
-rw-r--r--src/lib/Server/Admin/Reports.py6
-rw-r--r--src/lib/Server/Admin/Xcmd.py5
-rw-r--r--src/lib/Server/Admin/__init__.py5
-rw-r--r--src/lib/Server/Core.py4
-rw-r--r--src/lib/Server/Hostbase/backends.py6
-rw-r--r--src/lib/Server/Hostbase/ldapauth.py12
-rw-r--r--src/lib/Server/Hostbase/settings.py5
-rw-r--r--src/lib/Server/Lint/Comments.py2
-rw-r--r--src/lib/Server/Lint/MergeFiles.py71
-rw-r--r--src/lib/Server/Lint/Validate.py23
-rw-r--r--src/lib/Server/Lint/__init__.py44
-rw-r--r--src/lib/Server/Plugin.py98
-rw-r--r--src/lib/Server/Plugins/Base.py1
-rw-r--r--src/lib/Server/Plugins/Cfg.py35
-rw-r--r--src/lib/Server/Plugins/DBStats.py1
-rw-r--r--src/lib/Server/Plugins/Metadata.py2
-rw-r--r--src/lib/Server/Plugins/Probes.py109
-rw-r--r--src/lib/Server/Plugins/Properties.py39
-rw-r--r--src/lib/Server/Plugins/SSHbase.py21
-rw-r--r--src/lib/Server/Plugins/Snapshots.py1
-rwxr-xr-xsrc/lib/Server/Reports/importscript.py18
-rw-r--r--src/lib/Server/Reports/reports/templates/base.html2
-rw-r--r--src/lib/Server/Reports/reports/templatetags/syntax_coloring.py1
-rw-r--r--src/lib/Server/Snapshots/model.py1
-rwxr-xr-xsrc/sbin/bcfg222
-rwxr-xr-xsrc/sbin/bcfg2-info19
-rwxr-xr-xsrc/sbin/bcfg2-lint23
-rwxr-xr-xsrc/sbin/bcfg2-reports145
-rwxr-xr-xsrc/sbin/bcfg2-server2
-rwxr-xr-xtools/export.py65
80 files changed, 1369 insertions, 486 deletions
diff --git a/debian/changelog b/debian/changelog
index 01e4478d0..cf46d1507 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+bcfg2 (1.2.0pre3-0.0) unstable; urgency=low
+
+ * New upstream release
+
+ -- Sol Jerome <sol.jerome@gmail.com> Sat, 18 Jun 2011 22:41:29 -0500
+
bcfg2 (1.2.0-0.0pre2) unstable; urgency=low
* New upstream release
diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt
index d788891de..ce8e49703 100644
--- a/doc/appendix/guides/centos.txt
+++ b/doc/appendix/guides/centos.txt
@@ -195,17 +195,17 @@ line of ``bcfg2.conf``. Then create Packages layout (as per
<Sources>
<!-- CentOS (5.4) sources -->
<YUMSource>
- <Group>centos5.4</Group>
+ <Group>centos-5.4</Group>
<RawURL>http://mrepo/centos5-x86_64/RPMS.os</RawURL>
<Arch>x86_64</Arch>
</YUMSource>
<YUMSource>
- <Group>centos5.4</Group>
+ <Group>centos-5.4</Group>
<RawURL>http://mrepo/centos5-x86_64/RPMS.updates</RawURL>
<Arch>x86_64</Arch>
</YUMSource>
<YUMSource>
- <Group>centos5.4</Group>
+ <Group>centos-5.4</Group>
<RawURL>http://mrepo/centos5-x86_64/RPMS.extras</RawURL>
<Arch>x86_64</Arch>
</YUMSource>
@@ -227,9 +227,9 @@ file should look something like this
<Groups version='3.0'>
<Group profile='true' public='true' default='true' name='basic'>
- <Group name='centos5.4'/>
+ <Group name='centos-5.4'/>
</Group>
- <Group name='centos5.4'>
+ <Group name='centos-5.4'>
<Group name='centos'/>
</Group>
<Group name='ubuntu'/>
@@ -497,6 +497,18 @@ Then add the files to Cfg::
mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL
cp /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL !$/RPM-GPG-KEY-EPEL
+You will also want to add an *important* attribute to these files so
+that they are installed on the client prior to any attempts to install
+the **gpg-pubkey** rpm packages. This is especially important during the
+bootstrapping phase and can be accomplished using an :ref:`server-info`
+file that looks like the following:
+
+.. code-block:: xml
+
+ <FileInfo>
+ <Info owner='root' group='root' perms='0644' important='true'/>
+ </FileInfo>
+
Now, running the client shows only unmanaged Service entries. Woohoo!
Manage services
diff --git a/doc/conf.py b/doc/conf.py
index 77ab6fd94..4b13579d7 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -40,8 +40,13 @@ source_suffix = '.txt'
master_doc = 'index'
# General information about the project.
-project = u'Bcfg2'
-copyright = u'2009-%s, Narayan Desai' % time.strftime('%Y')
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ project = 'Bcfg2'
+ copyright = '2009-%s, Narayan Desai' % time.strftime('%Y')
+else:
+ project = u'Bcfg2'
+ copyright = u'2009-%s, Narayan Desai' % time.strftime('%Y')
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -174,10 +179,17 @@ latex_font_size = '11pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
-latex_documents = [
- ('contents', 'bcfg2.tex', u'Bcfg2 Documentation',
- u'Narayan Desai et al.', 'manual', True),
-]
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ latex_documents = [
+ ('contents', 'bcfg2.tex', 'Bcfg2 Documentation',
+ 'Narayan Desai et al.', 'manual', True),
+ ]
+else:
+ latex_documents = [
+ ('contents', 'bcfg2.tex', u'Bcfg2 Documentation',
+ u'Narayan Desai et al.', 'manual', True),
+ ]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt
index 88a0febc5..5fe1500f4 100644
--- a/doc/help/troubleshooting.txt
+++ b/doc/help/troubleshooting.txt
@@ -81,101 +81,138 @@ Type `help` in bcfg2-info for more information.
Error Messages
==============
-This page describes error messages produced by Bcfg2 and steps that can
+The tables below describe error messages produced by Bcfg2 and steps that can
be taken to remedy them.
+Client Errors
+-------------
+
+------------------------------+----------+---------------------+--------------+
| Error | Location | Meaning | Repair |
+==============================+==========+=====================+==============+
-| Incomplete information for | Client | The described entry | [1]_ |
+| Incomplete information for | Client | The described entry | [c1]_ |
| entry <EntryTag>:<EntryName> | | is not fully | |
| cannot verify | | specified by the | |
| | | server, so no | |
| | | verification can be | |
| | | performed. | |
+------------------------------+----------+---------------------+--------------+
-| Incomplete information for | Client | The described entry | [1]_ |
+| Incomplete information for | Client | The described entry | [c1]_ |
| entry <EntryTag>:<EntryName> | | is not fully | |
| cannot install | | specified by the | |
| | | server, so no | |
| | | verification can be | |
| | | performed. | |
+------------------------------+----------+---------------------+--------------+
-| The following entries are | Client | The client cannot | [2]_ |
+| The following entries are | Client | The client cannot | [c2]_ |
| not handled by any tool: | | figure out how to | |
| <EntryTag>:<EntryName> | | handle this entry. | |
+------------------------------+----------+---------------------+--------------+
-| No ca is specified. Cannot | Client | The client is | [3]_ |
+| No ca is specified. Cannot | Client | The client is | [c3]_ |
| authenticate the server with | | unable to verify | |
| SSL. | | the server | |
+------------------------------+----------+---------------------+--------------+
-| Failed to bind entry: | Server | The server was | [4]_ |
+| GID normalization failed for | Client | The client is | [c4]_ |
+| FILENAME. Does group GROUP | | unable to convert | |
+| exist? | | the group GROUP to | |
+| | | a usable GID | |
++------------------------------+----------+---------------------+--------------+
+| UID normalization failed for | Client | The client is | [c5]_ |
+| FILENAME. Does owner OWNER | | unable to convert | |
+| exist? | | the owner OWNER to | |
+| | | a usable UID | |
++------------------------------+----------+---------------------+--------------+
+
+
+.. [c1] This entry is not being bound. Ensure that a version of this
+ entry applies to this client.
+.. [c2] It is possible that the type attribute for this generator entry
+ is undefined. You may need to add the appropriate type attribute
+ in the literal entry specification.
+.. [c3] Copy the Bcfg2 server's CA certificate to the client and specify it
+ using the **ca** option in the [communication] section of
+ ``bcfg2.conf``
+.. [c4] If the group doesn't exist, you need to specify the correct one
+ in an :ref:`info.xml <server-info>` file or set the default group
+ appropriately.
+.. [c5] If the owner doesn't exist, you need to specify the correct one
+ in an :ref:`info.xml <server-info>` file or set the default owner
+ appropriately.
+
+Server Errors
+-------------
+
++------------------------------+----------+---------------------+--------------+
+| Error | Location | Meaning | Repair |
++==============================+==========+=====================+==============+
+| Failed to bind entry: | Server | The server was | [s1]_ |
| <EntryTag> <EntryName> | | unable to find a | |
| | | suitable version of | |
| | | entry for client. | |
+------------------------------+----------+---------------------+--------------+
-| Failed to bind to socket | Server | The server was | [5]_ |
+| Failed to bind to socket | Server | The server was | [s2]_ |
| | | unable to bind to | |
| | | the tcp server | |
| | | socket. | |
+------------------------------+----------+---------------------+--------------+
-| Failed to load | Server | The server was | [6]_ |
+| Failed to load | Server | The server was | [s3]_ |
| ssl key <path> | | unable to read and | |
| | | process the ssl key.| |
+------------------------------+----------+---------------------+--------------+
-| Failed to read file <path> | Server | The server failed | [7]_ |
+| Failed to read file <path> | Server | The server failed | [s4]_ |
| | | to read the | |
| | | specified file | |
+------------------------------+----------+---------------------+--------------+
-| Failed to parse file <path> | Server | The server failed | [8]_ |
+| Failed to parse file <path> | Server | The server failed | [s5]_ |
| | | to parse the | |
| | | specified XML file | |
+------------------------------+----------+---------------------+--------------+
-| Client metadata resolution | Server | The server cannot | [9]_ |
+| Client metadata resolution | Server | The server cannot | [s6]_ |
| error for <IP> | | resolve the client | |
| | | hostname or the | |
| | | client is | |
| | | associated with a | |
| | | non-profile group. | |
+------------------------------+----------+---------------------+--------------+
-| Failed to decode <filename> | Server | The encoding being | [10]_ |
+| Failed to decode <filename> | Server | The encoding being | [s7]_ |
| Please verify you are using | | used is unable to | |
| the proper encoding | | decode the | |
| | | character present | |
| | | in this file. | |
+------------------------------+----------+---------------------+--------------+
-| Got unknown entries | Server | The Packages plugin | [11]_ |
+| Got unknown entries | Server | The Packages plugin | [s8]_ |
| [list of unknown entries] | | has no knowledge of | |
| | | the listed entries | |
+------------------------------+----------+---------------------+--------------+
-| Failed to import lxml | Server | The server cannot | [12]_ |
+| Failed to import lxml | Server | The server cannot | [s9]_ |
| dependency. Shutting | | import lxml | |
| down server. | | | |
+------------------------------+----------+---------------------+--------------+
+| You need to specify base64 | Server | The server cannot | [s10]_ |
+| encoding for <path> | | send the file as | |
+| | | ascii text | |
++------------------------------+----------+---------------------+--------------+
+| ERROR: Error reading file | Server | The server cannot | [s11]_ |
+| '/path/to/schema': failed to | | find the schema | |
+| load external entity | | file | |
+| "/path/to/schema" | | | |
++------------------------------+----------+---------------------+--------------+
-.. [1] This entry is not being bound. Ensure that a version of this
- entry applies to this client.
-.. [2] It is possible that the type attribute for this generator entry
- is undefined. You may need to add the appropriate type attribute
- in the literal entry specification.
-.. [3] Copy the Bcfg2 server's CA certificate to the client and specify it
- using the **ca** option in the [communication] section of
- ``bcfg2.conf``
-.. [4] This entry is not being bound. Ensure that a version of this
- entry applies to this client.
-.. [5] Ensure that another instance of the daemon (or any other process)
- is not listening on the same port.
-.. [6] Ensure that the key is readable by the user running the daemon
- and that it is well-formed.
-.. [7] Ensure that this file still exists; a frequent cause is the
- deletion of a temp file.
-.. [8] Ensure that the file is properly formed XML.
-.. [9] Fix hostname resolution for the client or ensure that the profile
- group is properly setup.
-.. [10] Ensure the correct encoding is specified in the [components]
+.. [s1] This entry is not being bound. Ensure that a version of this
+ entry applies to this client.
+.. [s2] Ensure that another instance of the daemon (or any other process)
+ is not listening on the same port.
+.. [s3] Ensure that the key is readable by the user running the daemon
+ and that it is well-formed.
+.. [s4] Ensure that this file still exists; a frequent cause is the
+ deletion of a temp file.
+.. [s5] Ensure that the file is properly formed XML.
+.. [s6] Fix hostname resolution for the client or ensure that the profile
+ group is properly setup.
+.. [s7] Ensure the correct encoding is specified in the [components]
section of ``bcfg2.conf``.
-.. [11] For packages listed other than **gpg-pubkey**, this error means
+.. [s8] For packages listed other than **gpg-pubkey**, this error means
that the Packages plugin is unable to find the package in any of
the sources listed in ``Packages/config.xml``. The issue often
arises when the client is not in one of the groups necessary for
@@ -183,8 +220,11 @@ be taken to remedy them.
ignore the message as the Packages plugin has no knowledge of
these packages (however, note that this package is most often
specified as a BoundPackage entry).
-.. [12] Ensure that you have installed all the necessary
+.. [s9] Ensure that you have installed all the necessary
:ref:`installation-prerequisites`.
+.. [s10] You likely need to specify a base64 encoding using an
+ :ref:`server-info` file for this entry.
+.. [s11] Verify that you have the proper prefix set in bcfg2.conf.
FAQs
====
diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt
index 6e0ea8050..4c75cce32 100644
--- a/doc/reports/dynamic.txt
+++ b/doc/reports/dynamic.txt
@@ -188,29 +188,38 @@ displays::
Usage: python bcfg2-reports [option] ...
Options and arguments (and corresponding environment variables):
- -a : shows all hosts, including expired hosts
- -b NAME : single-host mode - shows bad entries from the
- current interaction of NAME
- -c : shows only clean hosts
- -d : shows only dirty hosts
- -e NAME : single-host mode - shows extra entries from the
- current interaction of NAME
- -h : shows help and usage info about bcfg2-reports
- -s NAME : single-host mode - shows bad and extra entries from
- the current interaction of NAME
- -x NAME : toggles expired/unexpired state of NAME
- --badentry=KIND,NAME : shows only hosts whose current interaction has bad
- entries in of KIND kind and NAME name; if a single
- argument ARG1 is given, then KIND,NAME pairs will be
- read from a file of name ARG1
- --extraentry=KIND,NAME : shows only hosts whose current interaction has extra
- entries in of KIND kind and NAME name; if a single
- argument ARG1 is given, then KIND,NAME pairs will be
- read from a file of name ARG1
- --fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,...
- (name,time,state)
- --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state)
- --stale : shows hosts which haven't run in the last 24 hours
+ -a : shows all hosts, including expired hosts
+ -b NAME : single-host mode - shows bad entries from the
+ current interaction of NAME
+ -c : shows only clean hosts
+ -d : shows only dirty hosts
+ -e NAME : single-host mode - shows extra entries from the
+ current interaction of NAME
+ -h : shows help and usage info about bcfg2-reports
+ -m NAME : single-host mode - shows modified entries from the
+ current interaction of NAME
+ -s NAME : single-host mode - shows bad, modified, and extra
+ entries from the current interaction of NAME
+ -t NAME : single-host mode - shows total number of managed and
+ good entries from the current interaction of NAME
+ -x NAME : toggles expired/unexpired state of NAME
+ --badentry=KIND,NAME : shows only hosts whose current interaction has bad
+ entries in of KIND kind and NAME name; if a single
+ argument ARG1 is given, then KIND,NAME pairs will be
+ read from a file of name ARG1
+ --modifiedentry=KIND,NAME : shows only hosts whose current interaction has
+ modified entries in of KIND kind and NAME name; if a
+ single argument ARG1 is given, then KIND,NAME pairs
+ will be read from a file of name ARG1
+ --extraentry=KIND,NAME : shows only hosts whose current interaction has extra
+ entries in of KIND kind and NAME name; if a single
+ argument ARG1 is given, then KIND,NAME pairs will be
+ read from a file of name ARG1
+ --fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,...
+ (name,time,state,total,good,bad)
+ --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,...
+ (name,time,state,total,good,bad)
+ --stale : shows hosts which haven't run in the last 24 hours
Screenshots
===========
diff --git a/doc/server/configurationentries.txt b/doc/server/configurationentries.txt
index 5602da189..0c12ce9c9 100644
--- a/doc/server/configurationentries.txt
+++ b/doc/server/configurationentries.txt
@@ -34,6 +34,22 @@ Non-POSIX entries
| Service | System Services | name, type, status, target |
+-------------+---------------------+-----------------------------+
+.. note::
+
+ PostInstall entries are deprecated in favor of Action entries. In
+ fact, a PostInstall entry is simply a specific type of Action.
+ Basically, the following are equivalent:
+
+ .. code-block:: xml
+
+ <PostInstall name='foo'/>
+
+ and
+
+ .. code-block:: xml
+
+ <Action timing='post' when='modified' name='n' command='foo' status='ignore'/>
+
POSIX entries
=============
@@ -48,7 +64,7 @@ will only contain a *name* attribute. The type will be added by the
plugin that handles the entry in the case of `Cfg`_, `TGenshi`_, or
`TCheetah`_. If the entry is handled by the `Rules`_ plugin (i.e. it is
a device, directory, hardlink, symlink, etc), then you will specify both
-the *type* and any other necessary attributes in `Rules`_.
+the *type* and any other necessary attributes in `Rules`_.
Running ``bcfg2-lint`` will check your configuration specification for
the presence of any mandatory attributes that are necessary for the
@@ -91,8 +107,8 @@ Path type specified.
| | | that should not | |
| | | exist | |
+-------------+----------------------+-----------------+--------------------------+
-| permissions | Replaces Permissions | Permissions of | name, owner, |
-| | entries | POSIX entities | group, perms |
+| permissions | Replaces Permissions | Permissions of | name, owner, group, |
+| | entries | POSIX entities | perms, recursive |
| | | | |
+-------------+----------------------+-----------------+--------------------------+
| vcs | New | Create version | vcstype (git), |
@@ -103,6 +119,19 @@ Path type specified.
Keep in mind that permissions for files served up by Cfg/TGenshi/TCheetah
are still handled via the traditional :ref:`server-info` mechanisms.
+Additional information
+----------------------
+
+This section describes some additional behavior relating to POSIX entry
+attributes.
+
+Recursive permissions
+^^^^^^^^^^^^^^^^^^^^^
+
+As per the request in ticket 871, Path type='permissions' entries allow you to
+set a recursive attribute which allows the owner/group to be set recursively
+for a directory.
+
.. _boundentries:
Bound Entries
diff --git a/doc/server/info.txt b/doc/server/info.txt
index 231a83a52..f1154e665 100644
--- a/doc/server/info.txt
+++ b/doc/server/info.txt
@@ -41,6 +41,8 @@ possible fields in an info file are:
| paranoid: | yes | no | Backup file before replacement? | no |
+------------+-------------------+----------------------------------+---------+
| perms: | Numeric file mode | Sets the permissions of the file | 0644 |
+| | | 'inherit' | (or inherits from the files on | |
+| | | disk if set to inherit) | |
+------------+-------------------+----------------------------------+---------+
A sample info file for CGI script on a web server might look like::
diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt
index 1cbc4cf65..3329f48bd 100644
--- a/doc/server/plugins/connectors/properties.txt
+++ b/doc/server/plugins/connectors/properties.txt
@@ -38,7 +38,7 @@ Usage
Specific property files can be referred to in
templates as ``metadata.Properties[<filename>]``. The
-data attribute is an LXML element object. (Documented
+``xdata`` attribute is an LXML element object. (Documented
`here <http://codespeak.net/lxml/tutorial.html#the-element-class>`_)
Currently, only one access method is defined for this data, ``Match``.
@@ -56,10 +56,21 @@ more details on how Group and Client tags are parsed.) For instance::
As we formulate more common use cases, we will add them to the
!PropertyFile class as methods. This will simplify templates.
+You can also access the XML data that comprises a property file
+directly in one of several ways:
+
+* ``metadata.Properties['prop-file'].xdata`` is an lxml.etree._Element
+ object representing the top-level element in the file.
+* ``metadata.Properties['prop-file'].data`` is the raw contents of the
+ property file as a string.
+* ``metadata.Properties['prop-file'].entries`` is a list of
+ lxml.etree._Element objects representing the direct children of the
+ top-level element. (I.e., everything directly under the
+ ``<Properties>`` tag.)
+
Accessing Properties contents from TGenshi
==========================================
Access contents of ``Properties/auth.xml``::
-
- ${metadata.Properties['auth.xml'].data.find('file').find('bcfg2.key').text}
+ ${metadata.Properties['auth.xml'].xdata.find('file').find('bcfg2.key').text}
diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt
index 612b14bec..1039ff556 100644
--- a/doc/server/plugins/generators/cfg.txt
+++ b/doc/server/plugins/generators/cfg.txt
@@ -132,16 +132,16 @@ server and we have the following configuration files::
motd.G01_web-server
motd.G01_mail-server.cat
motd.G02_file-server.cat
- motd.H_foo.example.cat
+ motd.H_foo.example.com.cat
-If our machine isn't *foo.example.com* then here's what would happen:
+If our machine **isn't** *foo.example.com* then here's what would happen:
Bcfg2 would choose ``motd.G01_web-server`` as the base file. It is
the most specific base file for this host. Bcfg2 would apply the
``motd.G01_mail-server.cat`` delta to the ``motd.G01_web-server``
base file. It is the least specific delta. Bcfg2 would then apply the
``motd.G02_file-server.cat`` delta to the result of the delta before
-it. If our machine is foo.example.com then here's what would happen:
+it. If our machine **is** *foo.example.com* then here's what would happen:
Bcfg2 would choose ``motd.G01_web-server`` as the base file. It
is the most specific base file for this host. Bcfg2 would apply the
diff --git a/doc/server/plugins/generators/decisions.txt b/doc/server/plugins/generators/decisions.txt
index ba01f7fc2..d75a9fede 100644
--- a/doc/server/plugins/generators/decisions.txt
+++ b/doc/server/plugins/generators/decisions.txt
@@ -22,6 +22,12 @@ should be corrected during the current run of the installation tool. The
Decisions plugin is the only stock plugin that generates entries for
client's whitelists or blacklists.
+.. note::
+
+ If the client is not explicitly configured to run in whitelist or
+ blacklist mode, the list of entries is not downloaded and decisions
+ is not used. See `Decision Mode`_ below.
+
The Decisions plugin uses a directory in the Bcfg2 repository called
Decisions. Files in the Decisions subdirectory are named similarly to
files managed by Cfg, probes, TCheetah, and TGenshi (so you can use host-
@@ -37,12 +43,6 @@ the following is an example.
<Decision type='Path' name='/etc/apt/apt.conf'/>
</Decisions>
-.. note:: To add syntax highlighting in vim, you can add a modeline such as this:
-
- <!--
- vim: ft=xml
- -->
-
This example, included as a whitelist due to its name, enables all services,
and the path entry named ``/etc/apt/apt.conf``. All these entries must
already be present in your repository, the Decisions plugin just references
@@ -55,11 +55,27 @@ When a client asks for its whitelist or blacklist, all of the files
pertaining to that client of the correct type are aggregated into a single
list. This list is sent to the client.
-.. note:: This list is only generated when a client is explicitly run with
- the appropriate option (``-l (whitelist|blacklist)``); client
- behavior is not controlled unless this option is used. If you do
- not use Decisions, all your entries will be installed normally.
-.. note:: Also, using this plugin does not present additional prompts or
+.. note:: Using this plugin does not present additional prompts or
safety nets to the administrator running the client, you have to
control these via their respective options (``-I`` or ``-n``, for
example).
+
+To add syntax highlighting to Decisions files in vim and emacs, you
+can add comments such as this::
+
+ <Decisions><!--*- mode: xml; -*-->
+ <!-- vim: set ft=xml : -->
+
+
+=============
+Decision Mode
+=============
+
+The whitelist or blacklist is only generated when a client is run in
+whitelist or blacklist mode. This can either be set at the command
+line with the appropriate option (``-l (whitelist|blacklist)``), or in
+``bcfg2.conf`` by setting ``decision`` in the ``client`` section to
+``whitelist`` or ``blacklist``).
+
+Client behavior is not controlled unless the decision mode is set. If
+you do not use Decisions, all your entries will be installed normally.
diff --git a/doc/server/plugins/generators/tgenshi/index.txt b/doc/server/plugins/generators/tgenshi/index.txt
index 0fc541b52..d7e0b3bf2 100644
--- a/doc/server/plugins/generators/tgenshi/index.txt
+++ b/doc/server/plugins/generators/tgenshi/index.txt
@@ -54,7 +54,10 @@ Inside of templates
available when used in conjunction with the
:ref:`server-plugins-connectors-properties` plugin)
* **name** is the path name specified in bcfg
-* **path** is the path to the TGenshi template
+* **path** is the path to the TGenshi template. It starts with a
+ leading slash, and is relative to the Bcfg2 specification root.
+ E.g., ``/Cfg/etc/foo.conf/foo.conf.genshi`` or
+ ``/TGenshi/etc/foo.conf/template.newtxt.H_foo.example.com``
See the genshi `documentation
<http://genshi.edgewall.org/wiki/Documentation>`_ for examples of
@@ -91,6 +94,56 @@ Produces:
This flexibility provides the ability to build much more compact and
succinct definitions of configuration contents than Cfg can.
+Troubleshooting
+===============
+
+When developing a template, you can see what the template would
+generate on a client with :ref:`bcfg2-info <server-bcfg2-info>`::
+
+ bcfg2-info buildfile <path> <hostname>
+
+E.g.::
+
+ bcfg2-info buildfile /etc/foo.conf foo.example.com
+
+Sometimes, it's useful to be able to do more in-depth troubleshooting
+by running the template manually. (This is also necessary if you want
+to generate a template that depends on an :ref:`altsrc
+<server-plugins-structures-altsrc>` tag.) To do this, run ``bcfg2-info
+debug``, and, once in the Python interpreter, run::
+
+ metadata = self.build_metadata("<hostname>")
+ path = "<relative path to template (see note below)>"
+ bcfg2root = "<path to bcfg2 specification root>"
+
+``path`` should be set to the path to the template file with a leading
+slash, relative to the Bcfg2 specification root. See `Inside of
+Templates`_ for examples.
+
+``bcfg2root`` should be set to the absolute path to the Bcfg2
+specification. (This is ``/var/lib/bcfg2`` by default.)
+
+Then, run::
+
+ import os
+ name = os.path.dirname(path[path.find('/', 1):])
+ from genshi.template import TemplateLoader, NewTextTemplate
+ template = TemplateLoader().load(bcfg2root + path, cls=NewTextTemplate)
+ print template.generate(metadata=metadata, path=path, name=name).render()
+
+This gives you more fine-grained control over how your template is
+rendered.
+
+You can also use this approach to render templates that depend on
+:ref:`altsrc <server-plugins-structures-altsrc>` tags by setting
+``path`` to the path to the template, and setting ``name`` to the path
+to the file to be generated, e.g.::
+
+ metadata = self.build_metadata("foo.example.com")
+ path = "/Cfg/etc/sysconfig/network-scripts/ifcfg-template/ifcfg-template.genshi"
+ bcfg2root = "/var/lib/bcfg2"
+ name = "/etc/sysconfig/network-scripts/ifcfg-bond0"
+
File permissions
================
@@ -101,10 +154,10 @@ Permissions entry and a Path entry to handle the same file.
Error handling
================
-Situations may arrise where a templated file cannot be generated due to
+Situations may arise where a templated file cannot be generated due to
missing or incomplete information. A TemplateError can be raised to
-force a bind failure and prevent sending an incomplete file to the client.
-For example, this template::
+force a bind failure and prevent sending an incomplete file to the
+client. For example, this template::
{% python
from genshi.template import TemplateError
diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt
index 8ef3d5af3..ca1cf459a 100644
--- a/doc/server/plugins/index.txt
+++ b/doc/server/plugins/index.txt
@@ -65,8 +65,8 @@ Literal Configuration (Generators)
:maxdepth: 1
:glob:
- generators/tgenshi/index
generators/*
+ generators/tgenshi/index
Each of these plugins has a corresponding subdirectory with the same
name in the Bcfg2 repository.
diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt
index b9f698a0f..5b1f9e259 100644
--- a/doc/server/plugins/probes/index.txt
+++ b/doc/server/plugins/probes/index.txt
@@ -85,10 +85,12 @@ file looks like::
#end if
Any Probe script you run will store its output in
-``$self.metadata.Probes["scriptname"]``, so we get to our `scratchlocal`
-script's output as seen above. Note that we had to wrap the output in an
-`int()` call; the script output is treated as a string, so it needs to
-be converted before it can be tested numerically.
+``$self.metadata.Probes["scriptname"]``, so we get to our
+`scratchlocal` script's output as seen above. (See `Handling Probe
+Output`_, below, for more information on how this is done.) Note that
+we had to wrap the output in an `int()` call; the script output is
+treated as a string, so it needs to be converted before it can be
+tested numerically.
With all of these pieces in place, the following series of events will
happen when the client is run:
@@ -110,8 +112,37 @@ is to add the ``/etc/auto.master`` to a Bundle:
<Path name='/etc/auto.master'/>
-Host and Group Specific probes
-==============================
+Handling Probe Output
+=====================
+
+Bcfg2 stores output from probes in the ``Probes`` property of a
+client's metadata object. To access this data in TGenshi, for
+instance, you could do::
+
+ ${metadata.Probes['script-name']}
+
+This is not the full output of the probe; any lines that start with
+"group:" have been stripped from the output. The data is a
+string-like object that has some interesting and salient features:
+
+* If the data is a valid XML document, then
+ ``metadata.Probes['script-name'].xdata`` will be an
+ ``lxml.etree._Element`` object representing the XML data.
+* If the data is a valid JSON document, and either the Python ``json``
+ or ``simplejson`` module is installed, then
+ ``metadata.Probes['script-name'].json`` will be a data structure
+ representing the JSON data.
+* If the data is a valid YAML document, and either the Python ``yaml``
+ or ``syck`` module is installed, then
+ ``metadata.Probes['script-name'].yaml`` will be a data structure
+ representing the YAML data.
+
+If these conditions are not met, then the named properties will be
+``None``. In all other fashions, the probe data objects should act
+like strings.
+
+Host- and Group-Specific probes
+===============================
Bcfg2 has the ability to alter probes based on client hostname and group
membership. These files work similarly to files in Cfg.
diff --git a/doc/server/plugins/structures/base.txt b/doc/server/plugins/structures/base.txt
index 66d8062e1..03eae0573 100644
--- a/doc/server/plugins/structures/base.txt
+++ b/doc/server/plugins/structures/base.txt
@@ -6,6 +6,13 @@
Base
====
+.. deprecated:: 1.2.0
+
+.. warning::
+
+ The Base plugin no longer receives new features/functionality.
+ Please use :ref:`server-plugins-structures-bundler-index` instead.
+
The Base plugin is a structure plugin that provides the ability to add
lists of unrelated entries into client configuration entry inventories.
diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt
index 9fd897385..5ca53b010 100644
--- a/doc/server/plugins/structures/bundler/index.txt
+++ b/doc/server/plugins/structures/bundler/index.txt
@@ -146,6 +146,16 @@ The `Genshi templating system`_ is used internally.
Use
---
+.. warning::
+
+ Group tags are not used inside of Genshi templates. You can get the
+ same logic (and more) using Genshi conditionals.
+
+ .. code-block:: xml
+
+ <py:if test="groupname in metadata.groups">
+ </py:if>
+
Bcfg uses the Genshi API for templates, and performs a XML format
stream rendering of the template into an lxml entry, which is included
in the client configuration. :ref:`Client metadata <client-metadata>`
@@ -156,6 +166,23 @@ format is XML.
A Genshi template looks much like a Bundler file, except the Bundle tag
has an additional `xmlns:py` attribute. See the examples.
+Troubleshooting
+---------------
+
+There is no :ref:`bcfg2-info <server-bcfg2-info>` command like
+``buildfile`` for Bundler templates, so if you want to generate a
+Bundler template for a given client, you have to do so manually by
+first invoking ``bcfg2-info debug``, then run::
+
+ metadata = self.build_metadata("<hostname>")
+ path = "<full path to template file>"
+ from genshi.template import TemplateLoader, MarkupTemplate
+ template = TemplateLoader().load(path, cls=MarkupTemplate)
+ print template.generate(metadata=metadata).render('xml')
+
+``path`` needs to be the full path to the template file on the
+filesystem, not just within the Bcfg2 repo.
+
Altsrc
======
diff --git a/doc/server/plugins/generators/deps.txt b/doc/server/plugins/structures/deps.txt
index 7c5861d06..7c5861d06 100644
--- a/doc/server/plugins/generators/deps.txt
+++ b/doc/server/plugins/structures/deps.txt
diff --git a/examples/bcfg2-lint.conf b/examples/bcfg2-lint.conf
index abf969496..9c0d2c72a 100644
--- a/examples/bcfg2-lint.conf
+++ b/examples/bcfg2-lint.conf
@@ -1,5 +1,5 @@
[lint]
-plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate
+plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate,MergeFiles
[errors]
no-infoxml=error
@@ -23,3 +23,6 @@ probe_comments = Maintainer,Purpose,Groups,Other Output
[Validate]
schema=/usr/share/bcfg2/schema
+
+[MergeFiles]
+threshold=85
diff --git a/gentoo/bcfg2-1.1.1.ebuild b/gentoo/bcfg2-1.1.2.ebuild
index 1992ddaa4..4da67d865 100644
--- a/gentoo/bcfg2-1.1.1.ebuild
+++ b/gentoo/bcfg2-1.1.2.ebuild
@@ -33,7 +33,7 @@ PYTHON_MODNAME="Bcfg2"
distutils_src_install_post_hook() {
if ! use server; then
- rm -f "${T}/images/${PYTHON_ABI}${EPREFIX}/usr/sbin/bcfg2-"*
+ rm -f "$(distutils_get_intermediate_installation_image)${EPREFIX}/usr/sbin/bcfg2-"*
fi
}
diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8
index b1fa9244b..25fa30f9e 100644
--- a/man/bcfg2-lint.8
+++ b/man/bcfg2-lint.8
@@ -71,7 +71,7 @@ Require property files to have matching schema files
.SH "PLUGINS"
See
-.BR bcfg-lint.conf(5)
+.BR bcfg2-lint.conf(5)
for more information on the configuration of the plugins listed below.
.TP
@@ -120,6 +120,11 @@ exists for all Cfg files, and that paranoid mode be enabled for all
files.
.TP
+.BR MergeFiles
+Suggest that similar probes and config files be merged into single
+probes or TGenshi templates.
+
+.TP
.BR Pkgmgr
Check for duplicate packages specified in Pkgmgr.
diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5
index 0ae7a27ac..10a812874 100644
--- a/man/bcfg2-lint.conf.5
+++ b/man/bcfg2-lint.conf.5
@@ -43,7 +43,7 @@ section.
A comma-delimited list of plugins to run. By default, all plugins are
run. This can be overridden by listing plugins on the command line.
See
-.B bcfg2-lint(1)
+.B bcfg2-lint(8)
for a list of the available plugins.
.SH ERROR HANDLING
@@ -56,10 +56,10 @@ section. Each option should be the name of an error and one of
, or
.I "silent"
, which tells
-.B bcfg2-lint(1)
+.B bcfg2-lint(8)
how to handle the warning. Error names and their defaults can be
displayed by running
-.B bcfg2-lint(1)
+.B bcfg2-lint(8)
with the
.B --list-errors
option.
@@ -152,6 +152,14 @@ A comma-delimited list of attributes to require on
tags. Default is "owner,group,perms".
.TP
+.BR MergeFiles
+
+\(bu
+.B threshold
+The threshold at which MergeFiles will suggest merging config files
+and probes. Default is 75% similar.
+
+.TP
.BR Validate
\(bu
@@ -162,5 +170,5 @@ The full path to the XML Schema files. Default is
command-line option
.SH SEE ALSO
-.BR bcfg2-lint(1)
+.BR bcfg2-lint(8)
diff --git a/man/bcfg2-reports.8 b/man/bcfg2-reports.8
index bc4c9344b..51399e1c9 100644
--- a/man/bcfg2-reports.8
+++ b/man/bcfg2-reports.8
@@ -39,10 +39,15 @@ of NAME. NAME is the name of the entry.
.RS
Shows help and usage info about bcfg2-reports.
.RE
+.B "\-m NAME"
+.RS
+Single-host mode \- shows modified entries from the current interaction
+of NAME. NAME is the name of the entry.
+.RE
.B "\-s NAME"
.RS
-Single host mode \- shows bad and extra entries from the current
-interaction of NAME. NAME is the name of the entry.
+Single host mode \- shows bad, modified, and extra entries from the
+current interaction of NAME. NAME is the name of the entry.
.RE
.B "\-x NAME"
.RS
@@ -64,11 +69,19 @@ pairs will be read from a file of name ARG1. KIND is the type of entry
.RE
.B "\-\-fields=ARG1,ARG2,..."
.RS
-Only displays the fields ARG1,ARG2,... (name, time, state)
+Only displays the fields ARG1,ARG2,... (name, time, state, total, good,
+bad)
+.RE
+.B "\-\-modifiedentry=KIND,NAME"
+.RS
+Shows only hosts whose current interaction has modified entries in of
+KIND kind and NAME name; if a single argument ARG1 is given, then
+KIND,NAME pairs will be read from a file of name ARG1. KIND is the type
+of entry (Package, Path, Service, etc). NAME is the name of the entry.
.RE
.B "\-\-sort=ARG1,ARG2,..."
.RS
-Sorts output on ARG1,ARG2,... (name, time, state)
+Sorts output on ARG1,ARG2,... (name, time, state, total, good, bad)
.RE
.B "\-\-stale"
.RS
diff --git a/man/bcfg2.1 b/man/bcfg2.1
index 938d41dfe..0ace97e8a 100644
--- a/man/bcfg2.1
+++ b/man/bcfg2.1
@@ -3,7 +3,7 @@
bcfg2 \- reconfigure machine based on settings in Bcfg2
.SH SYNOPSIS
.B bcfg2
-.I [\-d] [\-v] [\-p] [\-c cache file] [\-e] [\-f config file] [\-I] [\-q] [\-b bundle] [\-r removal mode]
+.I [\-d] [\-v] [\-p] [\-c cache file] [\-e] [\-f config file] [\-I] [\-q] [\-z] [\-b bundle] [\-r removal mode] [\-\-ca\-cert=file] [\-\-ssl\-cns=list] [\-\-ssl\-cert=file] [\-\-ssl\-key=file]
.SH DESCRIPTION
.TP
.BR bcfg2
@@ -62,6 +62,10 @@ Run bcfg2 against one or multiple bundles in the configuration.
Cache a copy of the configuration in cachefile.
.TP
+.BR "\-\-ca\-cert=<ca cert>"
+Specifiy the path to the SSL CA certificate.
+
+.TP
.BR "\-d"
Run bcfg2 in debug mode.
@@ -85,10 +89,12 @@ debian toolset; it calls apt\-get update and clean and
dpkg \-\-configure \-\-pending.
.TP
-.BR "\-l <whitelist|blacklist>"
-Run the client in the server decision list mode. This approach is needed
-when particular changes are deemed "high risk". It gives the ability to
-centrally specify these changes, but only install them on clients when
+.BR "\-l <whitelist|blacklist|none>"
+Run the client in the server decision list mode (unless "none" is
+specified, which can be done in order to override the decision list mode
+specified in bcfg2.conf). This approach is needed when particular
+changes are deemed "high risk". It gives the ability to centrally
+specify these changes, but only install them on clients when
administrator supervision is available. Because collaborative
configuration is one of the remaining hard issues in configuration
management, these issues typically crop up in environments with several
@@ -116,7 +122,7 @@ should only be used in safe conditions.
.TP
.BR "\-Q"
Run bcfg2 in "bundle quick" mode, where only entries in a bundle are
-or installed. This runs much faster than -q, but doesn't provide
+verified or installed. This runs much faster than -q, but doesn't provide
statistics to the server at all. In order for this option to work, the
-b option must also be provided. This option is incompatible with -r.
@@ -135,6 +141,14 @@ to stop all services started. disabled suppresses all attempts to
modify services.
.TP
+.BR "\-\-ssl\-cert=<ssl cert>"
+Specifiy the path to the SSL certificate.
+
+.TP
+.BR "\-\-ssl\-cns=<CommonName1:CommonName2 ...>"
+List of acceptable SSL server Common Names.
+
+.TP
.BR "\-\-ssl\-key=<ssl key>"
Specifiy the path to the SSL key.
@@ -147,8 +161,17 @@ Attempt to authenticate as 'user'.
Use 'password' for client communication.
.TP
+.BR "\-t <timeout>"
+Set the timeout (in seconds) for client communication. Default is 90
+seconds.
+
+.TP
.BR "\-v"
Run bcfg2 in verbose mode.
+
+.TP
+.BR "\-z"
+Only configure independent entries, ignore bundles.
.RE
.SH "SEE ALSO"
.BR bcfg2-server(8),
diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5
index f2e47b7ac..7cd04a0b7 100644
--- a/man/bcfg2.conf.5
+++ b/man/bcfg2.conf.5
@@ -38,6 +38,12 @@ The file monitor used to watch for changes in the repository.
Values of 'gamin', 'fam', or 'pseudo' are valid.
.TP
+.B listen_all
+This setting tells the server to listen on all available interfaces. The
+default is to only listen on those interfaces specified by the bcfg2
+setting in the components section of bcfg2.conf.
+
+.TP
.B plugins
A comma-delimited list of enabled server plugins. Currently available
plugins are:
@@ -285,6 +291,10 @@ These options only affect client functionality, specified in the
[client] section.
.TP
+.B decision
+Specify the server decision list mode (whitelist or blacklist).
+
+.TP
.B drivers
Specify tool driver set to use. This option can be used to explicitly
specify the client tool drivers you want to use when the client is run.
@@ -369,6 +379,11 @@ Communication protocol to use. Defaults to xmlrpc/ssl.
A client-only option. Number of times to retry network communication.
.TP
+.B serverCommonNames
+A client-only option. A colon-separated list of Common Names the client
+will accept in the SSL certificate presented by the server.
+
+.TP
.B user
A client-only option. The UUID of the client.
@@ -400,9 +415,7 @@ eg: bcfg2 = https://10.3.1.6:6789
.TP
.B encoding
-Text encoding of configuration files. Defaults to the system default
-encoding.
-
+Text encoding of configuration files. Defaults to UTF-8.
.SH LOGGING OPTIONS
Specified in the [logging] section. These options control the server
diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec
index 1e6b71ea5..d23478c19 100644
--- a/misc/bcfg2.spec
+++ b/misc/bcfg2.spec
@@ -13,7 +13,7 @@
%define lxmldep %(rpm -q %{alt_lxml} 2>&1 > /dev/null && echo %{alt_lxml} || echo %{dfl_lxml})
Name: bcfg2
-Version: 1.2.0pre2
+Version: 1.2.0pre3
Release: %{release}
Summary: Configuration management system
diff --git a/redhat/RELEASE b/redhat/RELEASE
index a0694f914..08f0fd716 100644
--- a/redhat/RELEASE
+++ b/redhat/RELEASE
@@ -1 +1 @@
-0.0pre2
+0.0pre3
diff --git a/schemas/base.xsd b/schemas/base.xsd
index 8eff0c69f..e8d677737 100644
--- a/schemas/base.xsd
+++ b/schemas/base.xsd
@@ -10,26 +10,30 @@
<xsd:include schemaLocation="atom.xsd"/>
<xsd:include schemaLocation="pathentry.xsd"/>
+ <xsd:include schemaLocation="rules.xsd"/>
- <xsd:complexType name='ContainerType'>
- <xsd:choice minOccurs='0' maxOccurs='unbounded'>
+ <xsd:group name='BaseEntries'>
+ <xsd:choice>
+ <xsd:element name='Group' type='ContainerType'/>
+ <xsd:element name='Client' type='ContainerType'/>
<xsd:element name='Package' type='StructureEntry'/>
<xsd:element name='Path' type='PathEntry'/>
<xsd:element name='Service' type='StructureEntry'/>
- <xsd:element name='Group' type='ContainerType'/>
+ <xsd:element name='BoundPackage' type='PackageType'/>
+ <xsd:element name='BoundPath' type='BoundPathEntry'/>
+ <xsd:element name='BoundService' type='ServiceType'/>
</xsd:choice>
+ </xsd:group>
+
+ <xsd:complexType name='ContainerType'>
+ <xsd:group ref='BaseEntries' minOccurs='0' maxOccurs='unbounded'/>
<xsd:attribute name='name' type='xsd:string'/>
<xsd:attribute name='negate' type='xsd:string'/>
</xsd:complexType>
<xsd:element name='Base'>
<xsd:complexType>
- <xsd:choice minOccurs='0' maxOccurs='unbounded'>
- <xsd:element name='Package' type='StructureEntry'/>
- <xsd:element name='Path' type='PathEntry'/>
- <xsd:element name='Service' type='StructureEntry'/>
- <xsd:element name='Group' type='ContainerType'/>
- </xsd:choice>
+ <xsd:group ref='BaseEntries' minOccurs='0' maxOccurs='unbounded'/>
</xsd:complexType>
</xsd:element>
diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd
index b226e1078..c0a7e08ac 100644
--- a/schemas/bundle.xsd
+++ b/schemas/bundle.xsd
@@ -55,6 +55,15 @@
</xsd:documentation>
</xsd:annotation>
</xsd:element>
+ <xsd:element name='PostInstall' type='StructureEntry'>
+ <xsd:annotation>
+ <xsd:documentation>
+ PostInstall entries are deprecated in favor of Action
+ entries. Actions can do everything PostInstall entries can
+ do and more.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
<xsd:element name='BoundPackage' type='PackageType'>
<xsd:annotation>
<xsd:documentation>
diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd
index 112047836..e10dc51dd 100644
--- a/schemas/fileprobes.xsd
+++ b/schemas/fileprobes.xsd
@@ -18,7 +18,7 @@
<xsd:complexType name="FileProbeType">
<xsd:attribute type="xsd:string" name="name" use="required"/>
- <xsd:attribute type="xsd:string" name="base64"/>
+ <xsd:attribute type="xsd:string" name="encoding"/>
<xsd:attribute type="xsd:string" name="update"/>
</xsd:complexType>
diff --git a/schemas/genshi.xsd b/schemas/genshi.xsd
index 853b69c0d..d87c4a6e3 100644
--- a/schemas/genshi.xsd
+++ b/schemas/genshi.xsd
@@ -2,7 +2,8 @@
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:py="http://genshi.edgewall.org/"
xml:lang="en"
- targetNamespace="http://genshi.edgewall.org/">
+ targetNamespace="http://genshi.edgewall.org/"
+ elementFormDefault="qualified">
<xs:annotation>
<xs:documentation>
Genshi schema
diff --git a/schemas/info.xsd b/schemas/info.xsd
index 96ccbe56c..2ff1d937e 100644
--- a/schemas/info.xsd
+++ b/schemas/info.xsd
@@ -11,7 +11,7 @@
<xsd:complexType name='InfoType'>
<xsd:attribute name='encoding' type='xsd:string'/>
<xsd:attribute name='group' type='xsd:string'/>
- <xsd:attribute name='important' type='xsd:boolean'/>
+ <xsd:attribute name='important' type='xsd:string'/>
<xsd:attribute name='owner' type='xsd:string'/>
<xsd:attribute name='perms' type='xsd:string'/>
<xsd:attribute name='paranoid' type='xsd:boolean'/>
diff --git a/schemas/pathentry.xsd b/schemas/pathentry.xsd
index 0c27f9112..24be22612 100644
--- a/schemas/pathentry.xsd
+++ b/schemas/pathentry.xsd
@@ -24,6 +24,7 @@
<xsd:attribute type='xsd:string' name='owner' use='optional'/>
<xsd:attribute type='xsd:string' name='perms' use='optional'/>
<xsd:attribute type='xsd:string' name='prune' use='optional'/>
+ <xsd:attribute type='xsd:string' name='recursive' use='optional'/>
<xsd:attribute type='xsd:string' name='to' use='optional'/>
<xsd:attribute type='xsd:string' name='type' use='optional'/>
<xsd:attributeGroup ref="py:genshiAttrs"/>
diff --git a/schemas/rules.xsd b/schemas/rules.xsd
index bc8a4af80..0a408c35c 100644
--- a/schemas/rules.xsd
+++ b/schemas/rules.xsd
@@ -24,6 +24,10 @@
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
+ <xsd:complexType name='PostInstallType'>
+ <xsd:attribute type='xsd:string' name='name' use='required'/>
+ </xsd:complexType>
+
<xsd:complexType name='PathType'>
<xsd:attribute type='PathTypeEnum' name='type' use='required'/>
<xsd:attribute type='xsd:string' name='name' use='required'/>
@@ -34,6 +38,7 @@
<xsd:attribute type='xsd:string' name='perms'/>
<xsd:attribute type='xsd:string' name='owner'/>
<xsd:attribute type='xsd:string' name='group'/>
+ <xsd:attribute type='xsd:string' name='recursive'/>
<xsd:attribute type='xsd:string' name='prune'/>
<xsd:attribute type='xsd:string' name='to'/>
<xsd:attributeGroup ref="py:genshiAttrs"/>
@@ -68,6 +73,7 @@
<xsd:element name='Package' type='PackageType'/>
<xsd:element name='Path' type='PathType'/>
<xsd:element name='Action' type='ActionType'/>
+ <xsd:element name='PostInstall' type='PostInstallType'/>
<xsd:element name='Group' type='RContainerType'/>
<xsd:element name='Client' type='RContainerType'/>
<xsd:element ref="py:def"/>
diff --git a/setup.py b/setup.py
index 18590cc34..76f92ef0e 100644
--- a/setup.py
+++ b/setup.py
@@ -121,7 +121,7 @@ if sys.hexversion < 0x03000000 and os.path.exists(py3lib):
setup(cmdclass=cmdclass,
name="Bcfg2",
- version="1.2.0pre2",
+ version="1.2.0pre3",
description="Bcfg2 Server",
author="Narayan Desai",
author_email="desai@mcs.anl.gov",
diff --git a/solaris/Makefile b/solaris/Makefile
index 65ab6309e..6be7ed2fb 100644
--- a/solaris/Makefile
+++ b/solaris/Makefile
@@ -1,7 +1,7 @@
#!/usr/sfw/bin/gmake
PYTHON="/opt/csw/bin/python"
-VERS=1.2.0pre2-1
+VERS=1.2.0pre3-1
PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]")
default: clean package
diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2
index cd8215741..293e64632 100644
--- a/solaris/pkginfo.bcfg2
+++ b/solaris/pkginfo.bcfg2
@@ -1,7 +1,7 @@
PKG="SCbcfg2"
NAME="bcfg2"
ARCH="sparc"
-VERSION="1.2.0pre2"
+VERSION="1.2.0pre3"
CATEGORY="application"
VENDOR="Argonne National Labratory"
EMAIL="bcfg-dev@mcs.anl.gov"
diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server
index 8bc069d08..49ea4e294 100644
--- a/solaris/pkginfo.bcfg2-server
+++ b/solaris/pkginfo.bcfg2-server
@@ -1,7 +1,7 @@
PKG="SCbcfg2-server"
NAME="bcfg2-server"
ARCH="sparc"
-VERSION="1.2.0pre2"
+VERSION="1.2.0pre3"
CATEGORY="application"
VENDOR="Argonne National Labratory"
EMAIL="bcfg-dev@mcs.anl.gov"
diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py
index 60d158eb1..d8e308ee0 100644
--- a/src/lib/Client/Frame.py
+++ b/src/lib/Client/Frame.py
@@ -5,6 +5,7 @@ installs entries, and generates statistics.
__revision__ = '$Revision$'
import logging
+import sys
import time
import Bcfg2.Client.Tools
@@ -29,7 +30,7 @@ def promptFilter(prompt, entries):
try:
# py3k compatibility
try:
- ans = raw_input(iprompt)
+ ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace'))
except NameError:
ans = input(iprompt)
if ans in ['y', 'Y']:
@@ -122,22 +123,7 @@ class Frame:
self.logger.info("Loaded tool drivers:")
self.logger.info([tool.name for tool in self.tools])
- if not self.dryrun and not self.setup['bundle']:
- for cfile in [cfl for cfl in config.findall(".//Path") \
- if cfl.get('name') in self.__important__ and \
- cfl.get('type') == 'file']:
- tl = [t for t in self.tools if t.handlesEntry(cfile) \
- and t.canVerify(cfile)]
- if tl:
- if not tl[0].VerifyPath(cfile, []):
- if self.setup['interactive'] and not \
- promptFilter("Install %s: %s? (y/N):", [cfile]):
- continue
- try:
- self.states[cfile] = tl[0].InstallPath(cfile)
- except:
- self.logger.error("Unexpected tool failure",
- exc_info=1)
+
# find entries not handled by any tools
problems = [entry for struct in config for \
entry in struct if entry not in self.handled]
@@ -176,6 +162,57 @@ class Frame:
return self.__dict__[name]
raise AttributeError(name)
+ def InstallImportant(self):
+ """Install important entries
+
+ We also process the decision mode stuff here because we want to prevent
+ non-whitelisted/blacklisted 'important' entries from being installed
+ prior to determining the decision mode on the client.
+ """
+ # Need to process decision stuff early so that dryrun mode works with it
+ self.whitelist = [entry for entry in self.states \
+ if not self.states[entry]]
+ if self.setup['decision'] == 'whitelist':
+ dwl = self.setup['decision_list']
+ w_to_rem = [e for e in self.whitelist \
+ if not matches_white_list(e, dwl)]
+ if w_to_rem:
+ self.logger.info("In whitelist mode: suppressing installation of:")
+ self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem])
+ self.whitelist = [x for x in self.whitelist \
+ if x not in w_to_rem]
+
+ elif self.setup['decision'] == 'blacklist':
+ b_to_rem = [e for e in self.whitelist \
+ if not passes_black_list(e, self.setup['decision_list'])]
+ if b_to_rem:
+ self.logger.info("In blacklist mode: suppressing installation of:")
+ self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem])
+ self.whitelist = [x for x in self.whitelist if x not in b_to_rem]
+
+ # take care of important entries first
+ if not self.dryrun and not self.setup['bundle']:
+ for cfile in [cfl for cfl in self.config.findall(".//Path") \
+ if cfl.get('name') in self.__important__ and \
+ cfl.get('type') == 'file']:
+ if cfile not in self.whitelist:
+ continue
+ tl = [t for t in self.tools if t.handlesEntry(cfile) \
+ and t.canVerify(cfile)]
+ if tl:
+ if self.setup['interactive'] and not \
+ promptFilter("Install %s: %s? (y/N):", [cfile]):
+ self.whitelist.remove(cfile)
+ continue
+ try:
+ self.states[cfile] = tl[0].InstallPath(cfile)
+ except:
+ self.logger.error("Unexpected tool failure",
+ exc_info=1)
+ cfile.set('qtext', '')
+ if tl[0].VerifyPath(cfile, []):
+ self.whitelist.remove(cfile)
+
def Inventory(self):
"""
Verify all entries,
@@ -209,26 +246,6 @@ class Frame:
candidates = [entry for entry in self.states \
if not self.states[entry]]
- self.whitelist = [entry for entry in self.states \
- if not self.states[entry]]
- # Need to process decision stuff early so that dryrun mode works with it
- if self.setup['decision'] == 'whitelist':
- dwl = self.setup['decision_list']
- w_to_rem = [e for e in self.whitelist \
- if not matches_white_list(e, dwl)]
- if w_to_rem:
- self.logger.info("In whitelist mode: suppressing installation of:")
- self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem])
- self.whitelist = [x for x in self.whitelist \
- if x not in w_to_rem]
-
- elif self.setup['decision'] == 'blacklist':
- b_to_rem = [e for e in self.whitelist \
- if not passes_black_list(e, self.setup['decision_list'])]
- if b_to_rem:
- self.logger.info("In blacklist mode: suppressing installation of:")
- self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem])
- self.whitelist = [x for x in self.whitelist if x not in b_to_rem]
if self.dryrun:
if self.whitelist:
@@ -388,6 +405,7 @@ class Frame:
self.Inventory()
self.times['inventory'] = time.time()
self.CondDisplayState('initial')
+ self.InstallImportant()
self.Decide()
self.Install()
self.times['install'] = time.time()
diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py
index a838f5e27..2b8cc3304 100644
--- a/src/lib/Client/Tools/APT.py
+++ b/src/lib/Client/Tools/APT.py
@@ -9,6 +9,8 @@ warnings.filterwarnings("ignore", "Accessed deprecated property Package.installe
warnings.filterwarnings("ignore", "Accessed deprecated property Package.candidateVersion, please see the Version class for alternatives.", DeprecationWarning)
warnings.filterwarnings("ignore", "Deprecated, please use 'is_installed' instead", DeprecationWarning)
warnings.filterwarnings("ignore", "Attribute 'IsUpgradable' of the 'apt_pkg.DepCache' object is deprecated, use 'is_upgradable' instead.", DeprecationWarning)
+warnings.filterwarnings("ignore", "Attribute 'VersionList' of the 'apt_pkg.Package' object is deprecated, use 'version_list' instead.", DeprecationWarning)
+warnings.filterwarnings("ignore", "Attribute 'VerStr' of the 'apt_pkg.Version' object is deprecated, use 'ver_str' instead.", DeprecationWarning)
import apt.cache
import os
@@ -71,7 +73,8 @@ class APT(Bcfg2.Client.Tools.Tool):
self.cmd.run("%s clean" % APTGET)
try:
self.pkg_cache = apt.cache.Cache()
- except SystemError, e:
+ except SystemError:
+ e = sys.exc_info()[1]
self.logger.info("Failed to initialize APT cache: %s" % e)
raise Bcfg2.Client.Tools.toolInstantiationError
self.pkg_cache.update()
diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py
index af3d1a473..faec2e251 100644
--- a/src/lib/Client/Tools/POSIX.py
+++ b/src/lib/Client/Tools/POSIX.py
@@ -14,7 +14,12 @@ import os
import pwd
import shutil
import stat
+import sys
import time
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ unicode = str
+
import Bcfg2.Client.Tools
import Bcfg2.Options
from Bcfg2.Client import XML
@@ -55,7 +60,8 @@ def normGid(entry):
except:
return int(grp.getgrnam(entry.get('group'))[2])
except (OSError, KeyError):
- log.error('GID normalization failed for %s' % (entry.get('name')))
+ log.error('GID normalization failed for %s. Does group %s exist?'
+ % (entry.get('name'), entry.get('group')))
return False
@@ -70,7 +76,23 @@ def normUid(entry):
except:
return int(pwd.getpwnam(entry.get('owner'))[2])
except (OSError, KeyError):
- log.error('UID normalization failed for %s' % (entry.get('name')))
+ log.error('UID normalization failed for %s. Does owner %s exist?'
+ % (entry.get('name'), entry.get('owner')))
+ return False
+
+
+def isString(strng, encoding):
+ """
+ Returns true if the string contains no ASCII control characters
+ and can be decoded from the specified encoding.
+ """
+ for char in strng:
+ if ord(char) < 9 or ord(char) > 13 and ord(char) < 32:
+ return False
+ try:
+ strng.decode(encoding)
+ return True
+ except:
return False
@@ -127,7 +149,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
try:
content = open(entry.get('name')).read()
entry.set('current_bfile', binascii.b2a_base64(content))
- except IOError, error:
+ except IOError:
+ error = sys.exc_info()[1]
self.logger.error("Failed to read %s: %s" % (error.filename,
error.strerror))
@@ -439,12 +462,14 @@ class POSIX(Bcfg2.Client.Tools.Tool):
if type(tempdata) == unicode:
try:
tempdata = tempdata.encode(self.setup['encoding'])
- except UnicodeEncodeError, e:
+ except UnicodeEncodeError:
+ e = sys.exc_info()[1]
self.logger.error("Error encoding file %s:\n %s" % \
(entry.get('name'), e))
try:
content = open(entry.get('name')).read()
- except IOError, error:
+ except IOError:
+ error = sys.exc_info()[1]
if error.strerror == "No such file or directory":
# print diff for files that don't exist (yet)
content = ''
@@ -456,12 +481,7 @@ class POSIX(Bcfg2.Client.Tools.Tool):
# md5sum so it would be faster for big binary files
contentStatus = content == tempdata
if not contentStatus:
- try:
- content.decode('ascii')
- isstring = True
- except:
- isstring = False
- if tbin or not isstring:
+ if tbin or not isString(content, self.setup['encoding']):
entry.set('current_bfile', binascii.b2a_base64(content))
nqtext = entry.get('qtext', '')
nqtext += '\nBinary file, no printable diff'
@@ -491,15 +511,15 @@ class POSIX(Bcfg2.Client.Tools.Tool):
difflib.unified_diff(content.split('\n'), \
tempdata.split('\n'))])
try:
- eudiff = udiff.encode('ascii')
+ dudiff = udiff.decode(self.setup['encoding'])
except:
- eudiff = "Binary file: no diff printed"
+ dudiff = "Binary file: no diff printed"
nqtext = entry.get('qtext', '')
if nqtext:
nqtext += '\n'
- nqtext += eudiff
+ nqtext += dudiff
else:
entry.set('current_bfile', binascii.b2a_base64(content))
nqtext = entry.get('qtext', '')
@@ -547,8 +567,14 @@ class POSIX(Bcfg2.Client.Tools.Tool):
(entry.get('current_exists', 'true') == 'false'):
bkupnam = entry.get('name').replace('/', '_')
# current list of backups for this file
- bkuplist = [f for f in os.listdir(self.ppath) if
- f.startswith(bkupnam)]
+ try:
+ bkuplist = [f for f in os.listdir(self.ppath) if
+ f.startswith(bkupnam)]
+ except OSError:
+ e = sys.exc_info()[1]
+ self.logger.error("Failed to create backup list in %s: %s" %
+ (self.ppath, e.strerror))
+ return False
bkuplist.sort()
while len(bkuplist) >= int(self.max_copies):
# remove the oldest backup available
@@ -567,7 +593,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
datetime.isoformat(datetime.now())))
self.logger.info("Backup of %s saved to %s" %
(entry.get('name'), self.ppath))
- except IOError, e:
+ except IOError:
+ e = sys.exc_info()[1]
self.logger.error("Failed to create backup file for %s" % \
(entry.get('name')))
self.logger.error(e)
@@ -603,7 +630,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
% (entry.get('name')))
return False
return True
- except (OSError, IOError), err:
+ except (OSError, IOError):
+ err = sys.exc_info()[1]
if err.errno == errno.EACCES:
self.logger.info("Failed to open %s for writing" % (entry.get('name')))
else:
@@ -683,7 +711,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
return False
try:
shutil.rmtree(ename)
- except OSError, e:
+ except OSError:
+ e = sys.exc_info()[1]
self.logger.error('Failed to remove %s: %s' % (ename,
e.strerror))
else:
@@ -691,20 +720,63 @@ class POSIX(Bcfg2.Client.Tools.Tool):
try:
os.rmdir(ename)
return True
- except OSError, e:
+ except OSError:
+ e = sys.exc_info()[1]
self.logger.error('Failed to remove %s: %s' % (ename,
e.strerror))
return False
try:
os.remove(ename)
return True
- except OSError, e:
+ except OSError:
+ e = sys.exc_info()[1]
self.logger.error('Failed to remove %s: %s' % (ename,
e.strerror))
return False
def Verifypermissions(self, entry, _):
"""Verify Path type='permissions' entry"""
+ if entry.get('perms') == None or \
+ entry.get('owner') == None or \
+ entry.get('group') == None:
+ self.logger.error('Entry %s not completely specified. '
+ 'Try running bcfg2-lint.' % (entry.get('name')))
+ return False
+ if entry.get('recursive') in ['True', 'true']:
+ # verify ownership information recursively
+ owner = normUid(entry)
+ group = normGid(entry)
+
+ for root, dirs, files in os.walk(entry.get('name')):
+ for p in dirs + files:
+ path = os.path.join(root, p)
+ pstat = os.stat(path)
+ if owner != pstat.st_uid:
+ # owner mismatch for path
+ entry.set('current_owner', str(pstat.st_uid))
+ self.logger.debug("%s %s ownership wrong" % \
+ (entry.tag, path))
+ nqtext = entry.get('qtext', '') + '\n'
+ nqtext += ("Owner for path %s is incorrect. "
+ "Current owner is %s but should be %s\n" % \
+ (path, pstat.st_uid, entry.get('owner')))
+ nqtext += ("\nInstall %s %s: (y/N): " %
+ (entry.tag, entry.get('name')))
+ entry.set('qtext', nqtext)
+ return False
+ if group != pstat.st_gid:
+ # group mismatch for path
+ entry.set('current_group', str(pstat.st_gid))
+ self.logger.debug("%s %s group wrong" % \
+ (entry.tag, path))
+ nqtext = entry.get('qtext', '') + '\n'
+ nqtext += ("Group for path %s is incorrect. "
+ "Current group is %s but should be %s\n" % \
+ (path, pstat.st_gid, entry.get('group')))
+ nqtext += ("\nInstall %s %s: (y/N): " %
+ (entry.tag, entry.get('name')))
+ entry.set('qtext', nqtext)
+ return False
return self.Verifydirectory(entry, _)
def Installpermissions(self, entry):
@@ -715,9 +787,23 @@ class POSIX(Bcfg2.Client.Tools.Tool):
self.logger.error('Entry %s not completely specified. '
'Try running bcfg2-lint.' % (entry.get('name')))
return False
+ plist = [entry.get('name')]
+ if entry.get('recursive') in ['True', 'true']:
+ # verify ownership information recursively
+ owner = normUid(entry)
+ group = normGid(entry)
+
+ for root, dirs, files in os.walk(entry.get('name')):
+ for p in dirs + files:
+ path = os.path.join(root, p)
+ pstat = os.stat(path)
+ if owner != pstat.st_uid or group != pstat.st_gid:
+ # owner mismatch for path
+ plist.append(path)
try:
- os.chown(entry.get('name'), normUid(entry), normGid(entry))
- os.chmod(entry.get('name'), calcPerms(S_IFDIR, entry.get('perms')))
+ for p in plist:
+ os.chown(p, normUid(entry), normGid(entry))
+ os.chmod(p, calcPerms(S_IFDIR, entry.get('perms')))
return True
except (OSError, KeyError):
self.logger.error('Permission fixup failed for %s' % \
diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py
index 082897934..c8c05061c 100644
--- a/src/lib/Client/Tools/Pacman.py
+++ b/src/lib/Client/Tools/Pacman.py
@@ -78,5 +78,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool):
try:
self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline))
self.cmd.run("%s -S %s" % (self.pkgtool, pkgline))
- except Exception, e:
+ except Exception:
+ e = sys.exc_info()[1]
self.logger.error("Error occurred during installation: %s" % e)
diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py
index a1e14b3a6..5376118c2 100644
--- a/src/lib/Client/Tools/RPMng.py
+++ b/src/lib/Client/Tools/RPMng.py
@@ -2,11 +2,12 @@
__revision__ = '$Revision$'
-import ConfigParser
import os.path
import rpm
import rpmtools
import Bcfg2.Client.Tools
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
# Fix for python2.3
try:
diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py
index 04d9f5c07..66768fb34 100644
--- a/src/lib/Client/Tools/YUM24.py
+++ b/src/lib/Client/Tools/YUM24.py
@@ -1,13 +1,14 @@
"""This provides bcfg2 support for yum."""
__revision__ = '$Revision: $'
-import ConfigParser
import copy
import os.path
import sys
import yum
import Bcfg2.Client.XML
import Bcfg2.Client.Tools.RPMng
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
# Fix for python2.3
try:
@@ -76,8 +77,6 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng):
__new_gpg_ireq__ = {'Package': ['name'],
'Instance': ['version', 'release']}
- conflicts = ['YUMng', 'RPMng']
-
def __init__(self, logger, setup, config):
Bcfg2.Client.Tools.RPMng.RPMng.__init__(self, logger, setup, config)
self.__important__ = self.__important__ + \
@@ -153,7 +152,8 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng):
try:
pkgDict = dict([(i.name, i) for i in \
self.yb.returnPackagesByDep(entry.get('name'))])
- except yum.Errors.YumBaseError, e:
+ except yum.Errors.YumBaseError:
+ e = sys.exc_info()[1]
self.logger.error('Yum Error Depsolving for %s: %s' % \
(entry.get('name'), str(e)))
pkgDict = {}
diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py
index c9e7aa15e..24605ca44 100644
--- a/src/lib/Client/Tools/YUMng.py
+++ b/src/lib/Client/Tools/YUMng.py
@@ -1,9 +1,9 @@
"""This provides bcfg2 support for yum."""
__revision__ = '$Revision$'
-import ConfigParser
import copy
import os.path
+import sys
import yum
import yum.packages
import yum.rpmtrans
@@ -13,6 +13,8 @@ import yum.misc
import rpmUtils.arch
import Bcfg2.Client.XML
import Bcfg2.Client.Tools
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
# Fix for python2.3
try:
@@ -141,7 +143,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
'Path': ['type']}
__ireq__ = {'Package': ['name']}
- conflicts = ['RPMng']
+ conflicts = ['YUM24', 'RPMng']
def __init__(self, logger, setup, config):
self.yb = yum.YumBase()
@@ -167,10 +169,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
self.yb.doConfigSetup()
self.yb.doTsSetup()
self.yb.doRpmDBSetup()
- except yum.Errors.RepoError, e:
+ except yum.Errors.RepoError:
+ e = sys.exc_info()[1]
self.logger.error("YUMng Repository error: %s" % e)
raise Bcfg2.Client.Tools.toolInstantiationError
- except yum.Errors.YumBaseError, e:
+ except Exception:
+ e = sys.exc_info()[1]
self.logger.error("YUMng error: %s" % e)
raise Bcfg2.Client.Tools.toolInstantiationError
@@ -447,8 +451,13 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
verify_flags = inst.get('verify_flags', self.verifyFlags)
verify_flags = verify_flags.lower().replace(' ', ',').split(',')
- if len(POs) == 0:
- # Package not installed
+ if 'arch' in nevra:
+ # If arch is specified use it to select the package
+ _POs = [ p for p in POs if p.arch == nevra['arch'] ]
+ else:
+ _POs = POs
+ if len(_POs) == 0:
+ # Package (name, arch) not installed
self.logger.debug(" %s is not installed" % nevraString(nevra))
stat['installed'] = False
package_fail = True
@@ -485,6 +494,9 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
qtext_versions.append("U(%s)" % str(POs[0]))
continue
+ if self.setup.get('quick', False):
+ # Passed -q on the command line
+ continue
if not (pkg_verify and \
inst.get('pkg_verify', 'true').lower() == 'true'):
continue
@@ -502,7 +514,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
try:
vResult = self._verifyHelper(_POs[0])
- except Exception, e:
+ except Exception:
+ e = sys.exc_info()[1]
# Unknown Yum exception
self.logger.warning(" Verify Exception: %s" % str(e))
package_fail = True
@@ -668,38 +681,58 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
return True
def _runYumTransaction(self):
+ def cleanup():
+ self.yb.closeRpmDB()
+ self.RefreshPackages()
+
rDisplay = RPMDisplay(self.logger)
yDisplay = YumDisplay(self.logger)
# Run the Yum Transaction
- rescode, restring = self.yb.buildTransaction()
+ try:
+ rescode, restring = self.yb.buildTransaction()
+ except yum.Errors.YumBaseError:
+ e = sys.exc_info()[1]
+ self.logger.error("Yum transaction error: %s" % str(e))
+ cleanup()
+ return
+
self.logger.debug("Initial Yum buildTransaction() run said:")
self.logger.debug(" resultcode: %s, msgs: %s" \
% (rescode, restring))
if rescode != 1:
# Transaction built successfully, run it
- self.yb.processTransaction(callback=yDisplay,
- rpmDisplay=rDisplay)
- self.logger.info("Single Pass for Install Succeeded")
+ try:
+ self.yb.processTransaction(callback=yDisplay,
+ rpmDisplay=rDisplay)
+ self.logger.info("Single Pass for Install Succeeded")
+ except yum.Errors.YumBaseError:
+ e = sys.exc_info()[1]
+ self.logger.error("Yum transaction error: %s" % str(e))
+ cleanup()
+ return
else:
# The yum command failed. No packages installed.
# Try installing instances individually.
self.logger.error("Single Pass Install of Packages Failed")
skipBroken = self.yb.conf.skip_broken
self.yb.conf.skip_broken = True
- rescode, restring = self.yb.buildTransaction()
- if rescode != 1:
- self.yb.processTransaction(callback=yDisplay,
- rpmDisplay=rDisplay)
- self.logger.debug(
- "Second pass install did not install all packages")
- else:
- self.logger.error("Second pass yum install failed.")
- self.logger.debug(" %s" % restring)
+ try:
+ rescode, restring = self.yb.buildTransaction()
+ if rescode != 1:
+ self.yb.processTransaction(callback=yDisplay,
+ rpmDisplay=rDisplay)
+ self.logger.debug(
+ "Second pass install did not install all packages")
+ else:
+ self.logger.error("Second pass yum install failed.")
+ self.logger.debug(" %s" % restring)
+ except yum.Errors.YumBaseError, e:
+ self.logger.error("Yum transaction error: %s" % str(e))
+
self.yb.conf.skip_broken = skipBroken
- self.yb.closeRpmDB()
- self.RefreshPackages()
+ cleanup()
def Install(self, packages, states):
"""
@@ -801,7 +834,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
pkg_arg = self.instance_status[inst].get('pkg').get('name')
try:
self.yb.install(**build_yname(pkg_arg, inst))
- except yum.Errors.YumBaseError, yume:
+ except yum.Errors.YumBaseError:
+ yume = sys.exc_info()[1]
self.logger.error("Error installing some packages: %s" % yume)
if len(upgrade_pkgs) > 0:
@@ -811,7 +845,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
pkg_arg = self.instance_status[inst].get('pkg').get('name')
try:
self.yb.update(**build_yname(pkg_arg, inst))
- except yum.Errors.YumBaseError, yume:
+ except yum.Errors.YumBaseError:
+ yume = sys.exc_info()[1]
self.logger.error("Error upgrading some packages: %s" % yume)
if len(reinstall_pkgs) > 0:
@@ -820,7 +855,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool):
pkg_arg = self.instance_status[inst].get('pkg').get('name')
try:
self.yb.reinstall(**build_yname(pkg_arg, inst))
- except yum.Errors.YumBaseError, yume:
+ except yum.Errors.YumBaseError:
+ yume = sys.exc_info()[1]
self.logger.error("Error upgrading some packages: %s" \
% yume)
diff --git a/src/lib/Component.py b/src/lib/Component.py
index 88dce906e..b73098d09 100644
--- a/src/lib/Component.py
+++ b/src/lib/Component.py
@@ -23,8 +23,8 @@ logger = logging.getLogger()
class NoExposedMethod (Exception):
"""There is no method exposed with the given name."""
-def run_component(component_cls, location, daemon, pidfile_name, to_file,
- cfile, argv=None, register=True,
+def run_component(component_cls, listen_all, location, daemon, pidfile_name,
+ to_file, cfile, argv=None, register=True,
state_name=False, cls_kwargs={}, extra_getopt='', time_out=10,
protocol='xmlrpc/ssl', certfile=None, keyfile=None, ca=None):
@@ -64,8 +64,13 @@ def run_component(component_cls, location, daemon, pidfile_name, to_file,
port = tuple(up[1].split(':'))
port = (port[0], int(port[1]))
try:
- server = XMLRPCServer(port, keyfile=keyfile, certfile=certfile,
- register=register, timeout=time_out, ca=ca,
+ server = XMLRPCServer(listen_all,
+ port,
+ keyfile=keyfile,
+ certfile=certfile,
+ register=register,
+ timeout=time_out,
+ ca=ca,
protocol=protocol)
except:
logger.error("Server startup failed")
diff --git a/src/lib/Options.py b/src/lib/Options.py
index d5304e696..619b16787 100644
--- a/src/lib/Options.py
+++ b/src/lib/Options.py
@@ -173,6 +173,18 @@ def colon_split(c_string):
return c_string.split(':')
return []
+def get_bool(s):
+ # these values copied from ConfigParser.RawConfigParser.getboolean
+ # with the addition of True and False
+ truelist = ["1", "yes", "True", "true", "on"]
+ falselist = ["0", "no", "False", "false", "off"]
+ if s in truelist:
+ return True
+ elif s in falselist:
+ return False
+ else:
+ raise ValueError
+
# General options
CFILE = Option('Specify configuration file', DEFAULT_CONFIG_LOCATION, cmd='-C',
odesc='<conffile>')
@@ -191,8 +203,10 @@ SENDMAIL_PATH = Option('Path to sendmail', cf=('reports', 'sendmailpath'),
default='/usr/lib/sendmail')
INTERACTIVE = Option('Prompt the user for each change', default=False,
cmd='-I', )
-ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(),
- cmd='-E', odesc='<encoding>',
+ENCODING = Option('Encoding of cfg files',
+ default='UTF-8',
+ cmd='-E',
+ odesc='<encoding>',
cf=('components', 'encoding'))
PARANOID_PATH = Option('Specify path for paranoid file backups',
default='/var/cache/bcfg2', cf=('paranoid', 'path'),
@@ -249,6 +263,13 @@ SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split,
cf=('server', 'connectors'), default=['Probes'], )
SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'),
default='default', odesc='File monitoring driver')
+SERVER_LISTEN_ALL = Option('Listen on all interfaces',
+ cf=('server', 'listen_all'),
+ cmd='--listen-all',
+ default=False,
+ long_arg=True,
+ cook=get_bool,
+ odesc='True|False')
SERVER_LOCATION = Option('Server Location', cf=('components', 'bcfg2'),
default='https://localhost:6789', cmd='-S',
odesc='https://server:port')
@@ -302,12 +323,13 @@ CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[],
cmd='-b', odesc='<bundle:bundle>', cook=colon_split)
CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False,
cmd='-Q')
-CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False,
+CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False,
cmd='-z')
CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False,
cmd='-k', )
-CLIENT_DLIST = Option('Run client in server decision list mode', default=False,
- cmd='-l', odesc='<whitelist|blacklist>')
+CLIENT_DLIST = Option('Run client in server decision list mode', default='none',
+ cf=('client', 'decision'),
+ cmd='-l', odesc='<whitelist|blacklist|none>')
CLIENT_FILE = Option('Configure from a file rather than querying the server',
default=False, cmd='-f', odesc='<specification path>')
CLIENT_QUICK = Option('Disable some checksum verification', default=False,
@@ -316,6 +338,9 @@ CLIENT_USER = Option('The user to provide for authentication', default='root',
cmd='-u', cf=('communication', 'user'), odesc='<user>')
CLIENT_SERVICE_MODE = Option('Set client service mode', default='default',
cmd='-s', odesc='<default|disabled|build>')
+CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90,
+ cmd='-t', cf=('communication', 'timeout'),
+ odesc='<timeout>')
# APT client tool options
CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path',
diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py
index 8a1ad683e..4cb0bbe80 100644
--- a/src/lib/Proxy.py
+++ b/src/lib/Proxy.py
@@ -47,6 +47,9 @@ __all__ = ["ComponentProxy",
class CertificateError(Exception):
def __init__(self, commonName):
self.commonName = commonName
+ def __str__(self):
+ return ("Got unallowed commonName %s from server"
+ % self.commonName)
class RetryMethod(xmlrpclib._Method):
@@ -181,7 +184,7 @@ class SSLHTTPConnection(httplib.HTTPConnection):
other_side_required = ssl.CERT_NONE
self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.")
if self.cert and not self.key:
- self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.")
+ self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.")
self.cert = None
if self.key and not self.cert:
self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.")
@@ -226,7 +229,7 @@ class SSLHTTPConnection(httplib.HTTPConnection):
# authentication to the server
ctx.load_cert(self.cert, self.key)
elif self.cert:
- self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.")
+ self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.")
elif self.key:
self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.")
@@ -332,5 +335,5 @@ def ComponentProxy(url, user=None, password=None,
else:
newurl = url
ssl_trans = XMLRPCTransport(key, cert, ca,
- allowedServerCNs, timeout=timeout)
+ allowedServerCNs, timeout=float(timeout))
return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans)
diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py
index a89beabbb..21bf48d3e 100644
--- a/src/lib/SSLServer.py
+++ b/src/lib/SSLServer.py
@@ -46,7 +46,12 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher):
if '.' not in method:
params = (address, ) + params
response = self.instance._dispatch(method, params, self.funcs)
- response = (response, )
+ # py3k compatibility
+ if isinstance(response, bool) or isinstance(response, str) \
+ or isinstance(response, list):
+ response = (response, )
+ else:
+ response = (response.decode('utf-8'), )
raw_response = xmlrpclib.dumps(response, methodresponse=1,
allow_none=self.allow_none,
encoding=self.encoding)
@@ -79,9 +84,9 @@ class SSLServer (SocketServer.TCPServer, object):
allow_reuse_address = True
logger = logging.getLogger("Cobalt.Server.TCPServer")
- def __init__(self, server_address, RequestHandlerClass, keyfile=None,
- certfile=None, reqCert=False, ca=None, timeout=None,
- protocol='xmlrpc/ssl'):
+ def __init__(self, listen_all, server_address, RequestHandlerClass,
+ keyfile=None, certfile=None, reqCert=False, ca=None,
+ timeout=None, protocol='xmlrpc/ssl'):
"""Initialize the SSL-TCP server.
@@ -97,9 +102,12 @@ class SSLServer (SocketServer.TCPServer, object):
"""
- all_iface_address = ('', server_address[1])
+ if listen_all:
+ listen_address = ('', server_address[1])
+ else:
+ listen_address = (server_address[0], server_address[1])
try:
- SocketServer.TCPServer.__init__(self, all_iface_address,
+ SocketServer.TCPServer.__init__(self, listen_address,
RequestHandlerClass)
except socket.error:
self.logger.error("Failed to bind to socket")
@@ -188,9 +196,18 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
self.logger.error("No authentication data presented")
return False
auth_type, auth_content = header.split()
- auth_content = base64.standard_b64decode(auth_content)
try:
- username, password = auth_content.split(":")
+ # py3k compatibility
+ auth_content = base64.standard_b64decode(auth_content)
+ except TypeError:
+ auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii')))
+ try:
+ # py3k compatibility
+ try:
+ username, password = auth_content.split(":")
+ except TypeError:
+ username, pw = auth_content.split(bytes(":", encoding='utf-8'))
+ password = pw.decode('utf-8')
except ValueError:
username = auth_content
password = ""
@@ -231,11 +248,13 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
print("got select timeout")
raise
chunk_size = min(size_remaining, max_chunk_size)
- L.append(self.rfile.read(chunk_size))
+ L.append(self.rfile.read(chunk_size).decode('utf-8'))
size_remaining -= len(L[-1])
data = ''.join(L)
response = self.server._marshaled_dispatch(self.client_address,
data)
+ if sys.hexversion >= 0x03000000:
+ response = response.encode('utf-8')
except:
try:
self.send_response(500)
@@ -310,7 +329,7 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer,
"""
- def __init__(self, server_address, RequestHandlerClass=None,
+ def __init__(self, listen_all, server_address, RequestHandlerClass=None,
keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl',
timeout=10,
logRequests=False,
@@ -339,8 +358,14 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer,
"""A subclassed request handler to prevent class-attribute conflicts."""
SSLServer.__init__(self,
- server_address, RequestHandlerClass, ca=ca,
- timeout=timeout, keyfile=keyfile, certfile=certfile, protocol=protocol)
+ listen_all,
+ server_address,
+ RequestHandlerClass,
+ ca=ca,
+ timeout=timeout,
+ keyfile=keyfile,
+ certfile=certfile,
+ protocol=protocol)
self.logRequests = logRequests
self.serve = False
self.register = register
diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py
index 3af25b15a..81fc4a1b1 100644
--- a/src/lib/Server/Admin/Client.py
+++ b/src/lib/Server/Admin/Client.py
@@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore):
for i in args[2:]:
attr, val = i.split('=', 1)
if attr not in ['profile', 'uuid', 'password',
- 'location', 'secure', 'address']:
+ 'location', 'secure', 'address',
+ 'auth']:
print("Attribute %s unknown" % attr)
raise SystemExit(1)
attr_d[attr] = val
@@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore):
for i in args[2:]:
attr, val = i.split('=', 1)
if attr not in ['profile', 'uuid', 'password',
- 'location', 'secure', 'address']:
+ 'location', 'secure', 'address',
+ 'auth']:
print("Attribute %s unknown" % attr)
raise SystemExit(1)
attr_d[attr] = val
diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py
index fff8bcd1c..eab030cf8 100644
--- a/src/lib/Server/Admin/Init.py
+++ b/src/lib/Server/Admin/Init.py
@@ -103,8 +103,7 @@ plugin_list = ['Account',
'TGenshi']
# Default list of plugins to use
-default_plugins = ['Base',
- 'Bundler',
+default_plugins = ['Bundler',
'Cfg',
'Metadata',
'Pkgmgr',
diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py
index af1c83072..d03b37d57 100644
--- a/src/lib/Server/Admin/Perf.py
+++ b/src/lib/Server/Admin/Perf.py
@@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode):
'password': Bcfg2.Options.SERVER_PASSWORD,
'server': Bcfg2.Options.SERVER_LOCATION,
'user': Bcfg2.Options.CLIENT_USER,
+ 'timeout': Bcfg2.Options.CLIENT_TIMEOUT,
}
setup = Bcfg2.Options.OptionParser(optinfo)
setup.parse(sys.argv[2:])
@@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode):
setup['password'],
key=setup['key'],
cert=setup['certificate'],
- ca=setup['ca'])
+ ca=setup['ca'],
+ timeout=setup['timeout'])
data = proxy.get_statistics()
for key, value in list(data.items()):
data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]])
diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
index 942477a49..c9f3d3f58 100644
--- a/src/lib/Server/Admin/Reports.py
+++ b/src/lib/Server/Admin/Reports.py
@@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode):
except (IOError, XMLSyntaxError):
self.errExit("StatReports: Failed to parse %s" % (stats_file))
+ try:
+ encoding = self.cfp.get('components', 'encoding')
+ except:
+ encoding = 'UTF-8'
+
if not clientspath:
try:
clientspath = "%s/Metadata/clients.xml" % \
@@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
try:
load_stats(clientsdata,
statsdata,
+ encoding,
verb,
self.log,
quick=quick,
diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py
index fd5794f88..2cb085346 100644
--- a/src/lib/Server/Admin/Xcmd.py
+++ b/src/lib/Server/Admin/Xcmd.py
@@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode):
'password': Bcfg2.Options.SERVER_PASSWORD,
'key': Bcfg2.Options.SERVER_KEY,
'certificate': Bcfg2.Options.CLIENT_CERT,
- 'ca': Bcfg2.Options.CLIENT_CA
+ 'ca': Bcfg2.Options.CLIENT_CA,
+ 'timeout': Bcfg2.Options.CLIENT_TIMEOUT,
}
setup = Bcfg2.Options.OptionParser(optinfo)
setup.parse(sys.argv[2:])
@@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode):
key=setup['key'],
cert=setup['certificate'],
ca=setup['ca'],
- timeout=180)
+ timeout=setup['timeout'])
if len(setup['args']) == 0:
print("Usage: xcmd <xmlrpc method> <optional arguments>")
return
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 8915492a3..b34d7108c 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -113,7 +113,8 @@ class MetadataCore(Mode):
def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None):
Mode.__init__(self, configfile)
options = {'plugins': Bcfg2.Options.SERVER_PLUGINS,
- 'configfile': Bcfg2.Options.CFILE}
+ 'configfile': Bcfg2.Options.CFILE,
+ 'encoding': Bcfg2.Options.ENCODING}
setup = Bcfg2.Options.OptionParser(options)
setup.hm = usage
setup.parse(sys.argv[1:])
@@ -126,7 +127,7 @@ class MetadataCore(Mode):
try:
self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(),
setup['plugins'],
- 'foo', 'UTF-8')
+ 'foo', setup['encoding'])
except Bcfg2.Server.Core.CoreInitError:
msg = sys.exc_info()[1]
self.errExit("Core load failed because %s" % msg)
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index 8f9d3e746..5adfa5381 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -384,7 +384,7 @@ class Core(Component):
# clear dynamic groups
self.metadata.cgroups[meta.hostname] = []
try:
- xpdata = lxml.etree.XML(probedata)
+ xpdata = lxml.etree.XML(probedata.encode('utf-8'))
except:
self.logger.error("Failed to parse probe data from client %s" % \
(address[0]))
@@ -433,7 +433,7 @@ class Core(Component):
@exposed
def RecvStats(self, address, stats):
"""Act on statistics upload."""
- sdata = lxml.etree.XML(stats)
+ sdata = lxml.etree.XML(stats.encode('utf-8'))
client = self.metadata.resolve_client(address)
self.process_statistics(client, sdata)
return "<ok/>"
diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py
index aa822409c..bf774f695 100644
--- a/src/lib/Server/Hostbase/backends.py
+++ b/src/lib/Server/Hostbase/backends.py
@@ -57,12 +57,14 @@ class NISBackend(object):
return user
- except NISAUTHError, e:
+ except NISAUTHError:
+ e = sys.exc_info()[1]
return None
def get_user(self, user_id):
try:
return User.objects.get(pk=user_id)
- except User.DoesNotExist, e:
+ except User.DoesNotExist:
+ e = sys.exc_info()[1]
return None
diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py
index 21b462c86..f3db26f67 100644
--- a/src/lib/Server/Hostbase/ldapauth.py
+++ b/src/lib/Server/Hostbase/ldapauth.py
@@ -69,7 +69,8 @@ class ldapauth(object):
None)
result_type, result_data = conn.result(result_id, 0)
return ('success', 'User profile found', result_data,)
- except ldap.LDAPError, e:
+ except ldap.LDAPError:
+ e = sys.exc_info()[1]
#connection failed
return ('error', 'LDAP connect failed', e,)
@@ -86,7 +87,8 @@ class ldapauth(object):
None)
result_type, result_data = conn.result(result_id, 0)
return ('success', 'User profile found', result_data,)
- except ldap.LDAPError, e:
+ except ldap.LDAPError:
+ e = sys.exc_info()[1]
#connection failed
return ('error', 'LDAP connect failed', e,)
@@ -108,7 +110,8 @@ class ldapauth(object):
raw_obj = result_data[0][1]
distinguishedName = raw_obj['distinguishedName']
return ('success', distinguishedName[0],)
- except ldap.LDAPError, e:
+ except ldap.LDAPError:
+ e = sys.exc_info()[1]
#connection failed
return ('error', 'LDAP connect failed', e,)
@@ -134,7 +137,8 @@ class ldapauth(object):
self.is_superuser = False
return
- except KeyError, e:
+ except KeyError:
+ e = sys.exc_info()[1]
raise LDAPAUTHError("Portions of the LDAP User profile not present")
def member_of(self):
diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py
index c44c7bf16..4e641f13c 100644
--- a/src/lib/Server/Hostbase/settings.py
+++ b/src/lib/Server/Hostbase/settings.py
@@ -1,9 +1,10 @@
-from ConfigParser import ConfigParser, NoSectionError, NoOptionError
import os.path
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
-c = ConfigParser()
+c = ConfigParser.ConfigParser()
#This needs to be configurable one day somehow
c.read(['./bcfg2.conf'])
diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py
index 8e86cc564..09443d4c0 100644
--- a/src/lib/Server/Lint/Comments.py
+++ b/src/lib/Server/Lint/Comments.py
@@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
props = self.core.plugins['Properties']
for propfile, pdata in props.store.entries.items():
if os.path.splitext(propfile)[1] == ".xml":
- self.check_xml(pdata.name, pdata.data, 'properties')
+ self.check_xml(pdata.name, pdata.xdata, 'properties')
def check_metadata(self):
""" check metadata files for required headers """
diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py
new file mode 100644
index 000000000..1e177acff
--- /dev/null
+++ b/src/lib/Server/Lint/MergeFiles.py
@@ -0,0 +1,71 @@
+import os
+from copy import deepcopy
+from difflib import SequenceMatcher
+import Bcfg2.Options
+import Bcfg2.Server.Lint
+
+class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
+ """ find Probes or Cfg files with multiple similar files that
+ might be merged into one """
+
+ @Bcfg2.Server.Lint.returnErrors
+ def Run(self):
+ if 'Cfg' in self.core.plugins:
+ self.check_cfg()
+ if 'Probes' in self.core.plugins:
+ self.check_probes()
+
+ def check_cfg(self):
+ for filename, entryset in self.core.plugins['Cfg'].entries.items():
+ for mset in self.get_similar(entryset.entries):
+ self.LintError("merge-cfg",
+ "The following files are similar: %s. "
+ "Consider merging them into a single Genshi "
+ "template." %
+ ", ".join([os.path.join(filename, p)
+ for p in mset]))
+
+ def check_probes(self):
+ probes = self.core.plugins['Probes'].probes.entries
+ for mset in self.get_similar(probes):
+ self.LintError("merge-cfg",
+ "The following probes are similar: %s. "
+ "Consider merging them into a single probe." %
+ ", ".join([p for p in mset]))
+
+ def get_similar(self, entries):
+ if "threshold" in self.config:
+ # accept threshold either as a percent (e.g., "threshold=75") or
+ # as a ratio (e.g., "threshold=.75")
+ threshold = float(self.config['threshold'])
+ if threshold > 1:
+ threshold /= 100
+ else:
+ threshold = 0.75
+ rv = []
+ elist = entries.items()
+ while elist:
+ result = self._find_similar(elist.pop(0), deepcopy(elist),
+ threshold)
+ if len(result) > 1:
+ elist = [(fname, fdata)
+ for fname, fdata in elist
+ if fname not in result]
+ rv.append(result)
+ return rv
+
+ def _find_similar(self, ftuple, others, threshold):
+ fname, fdata = ftuple
+ rv = [fname]
+ while others:
+ cname, cdata = others.pop(0)
+ sm = SequenceMatcher(None, fdata.data, cdata.data)
+ # perform progressively more expensive comparisons
+ if (sm.real_quick_ratio() > threshold and
+ sm.quick_ratio() > threshold and
+ sm.ratio() > threshold):
+ rv.extend(self._find_similar((cname, cdata), deepcopy(others),
+ threshold))
+ return rv
+
+
diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py
index 834608378..8a8406e73 100644
--- a/src/lib/Server/Lint/Validate.py
+++ b/src/lib/Server/Lint/Validate.py
@@ -1,10 +1,12 @@
+import fnmatch
import glob
import lxml.etree
import os
-import fnmatch
+from subprocess import Popen, PIPE, STDOUT
+import sys
+
import Bcfg2.Options
import Bcfg2.Server.Lint
-from subprocess import Popen, PIPE, STDOUT
class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
""" Ensure that the repo validates """
@@ -14,7 +16,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
self.filesets = {"metadata:groups":"%s/metadata.xsd",
"metadata:clients":"%s/clients.xsd",
"info":"%s/info.xsd",
- "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd",
+ "%s/Bundler/*.xml":"%s/bundle.xsd",
+ "%s/Bundler/*.genshi":"%s/bundle.xsd",
"%s/Pkgmgr/*.xml":"%s/pkglist.xsd",
"%s/Base/*.xml":"%s/base.xsd",
"%s/Rules/*.xml":"%s/rules.xsd",
@@ -33,23 +36,27 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
@Bcfg2.Server.Lint.returnErrors
def Run(self):
- self.schemadir = self.config['schema']
+ schemadir = self.config['schema']
- for schemaname, path in self.filesets.items():
+ for path, schemaname in self.filesets.items():
try:
filelist = self.filelists[path]
except KeyError:
filelist = []
-
+
if filelist:
# avoid loading schemas for empty file lists
try:
schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname %
schemadir))
+ except IOError:
+ e = sys.exc_info()[1]
+ self.LintError("input-output-error", e.message)
+ continue
except:
self.LintError("schema-failed-to-parse",
- "Failed to process schema %s",
- schemaname % schemadir)
+ "Failed to process schema %s" %
+ (schemaname % schemadir))
continue
for filename in filelist:
self.validate(filename, schemaname % schemadir,
diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py
index 3b89d1f9e..63cb2463b 100644
--- a/src/lib/Server/Lint/__init__.py
+++ b/src/lib/Server/Lint/__init__.py
@@ -4,6 +4,7 @@ __all__ = ['Bundles',
'Comments',
'Duplicates',
'InfoXML',
+ 'MergeFiles',
'Pkgmgr',
'RequiredAttrs',
'Validate']
@@ -11,6 +12,7 @@ __all__ = ['Bundles',
import logging
import os.path
from copy import copy
+import textwrap
import lxml.etree
import Bcfg2.Logger
@@ -84,7 +86,10 @@ class ErrorHandler (object):
"properties-schema-not-found":"warning",
"xml-failed-to-parse":"error",
"xml-failed-to-read":"error",
- "xml-failed-to-verify":"error",}
+ "xml-failed-to-verify":"error",
+ "merge-cfg":"warning",
+ "merge-probes":"warning",
+ "input-output-error": "error"}
def __init__(self, config=None):
self.errors = 0
@@ -92,6 +97,9 @@ class ErrorHandler (object):
self.logger = logging.getLogger('bcfg2-lint')
+ self._wrapper = textwrap.TextWrapper(initial_indent = " ",
+ subsequent_indent = " ")
+
self._handlers = {}
if config is not None:
for err, action in config.items():
@@ -116,26 +124,42 @@ class ErrorHandler (object):
self._handlers[err](msg)
self.logger.debug(" (%s)" % err)
else:
- self.logger.info("Unknown error %s" % err)
+ # assume that it's an error, but complain
+ self.error(msg)
+ self.logger.warning("Unknown error %s" % err)
def error(self, msg):
""" log an error condition """
self.errors += 1
- lines = msg.splitlines()
- self.logger.error("ERROR: %s" % lines.pop())
- [self.logger.error(" %s" % l) for l in lines]
+ self._log(msg, self.logger.error, prefix="ERROR: ")
def warn(self, msg):
""" log a warning condition """
self.warnings += 1
- lines = msg.splitlines()
- self.logger.warning("WARNING: %s" % lines.pop())
- [self.logger.warning(" %s" % l) for l in lines]
+ self._log(msg, self.logger.warning, prefix="WARNING: ")
def debug(self, msg):
""" log a silent/debug condition """
- lines = msg.splitlines()
- [self.logger.debug("%s" % l) for l in lines]
+ self._log(msg, self.logger.debug)
+
+ def _log(self, msg, logfunc, prefix=""):
+ # a message may itself consist of multiple lines. wrap() will
+ # elide them all into a single paragraph, which we don't want.
+ # so we split the message into its paragraphs and wrap each
+ # paragraph individually. this means, unfortunately, that we
+ # lose textwrap's built-in initial indent functionality,
+ # because we want to only treat the very first line of the
+ # first paragraph specially. so we do some silliness.
+ rawlines = msg.splitlines()
+ firstline = True
+ for rawline in rawlines:
+ lines = self._wrapper.wrap(rawline)
+ for line in lines:
+ if firstline:
+ logfunc("%s%s" % (prefix, line.lstrip()))
+ firstline = False
+ else:
+ logfunc(line)
class ServerlessPlugin (Plugin):
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index cd2b63656..30c4f9686 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked):
def Index(self):
"""Build local data structures."""
try:
- xdata = XML(self.data)
+ self.xdata = XML(self.data)
except XMLSyntaxError:
logger.error("Failed to parse %s" % (self.name))
return
- self.label = xdata.attrib[self.__identifier__]
- self.entries = xdata.getchildren()
+ self.entries = self.xdata.getchildren()
+ if self.__identifier__ is not None:
+ self.label = self.xdata.attrib[self.__identifier__]
def __iter__(self):
return iter(self.entries)
@@ -455,53 +456,51 @@ class SingleXMLFileBacked(XMLFileBacked):
class StructFile(XMLFileBacked):
"""This file contains a set of structure file formatting logic."""
+ __identifier__ = None
+
def __init__(self, name):
XMLFileBacked.__init__(self, name)
- self.fragments = {}
-
- def Index(self):
- """Build internal data structures."""
- try:
- xdata = lxml.etree.XML(self.data)
- except lxml.etree.XMLSyntaxError:
- logger.error("Failed to parse file %s" % self.name)
- return
- self.fragments = {}
- work = {lambda x: True: xdata.getchildren()}
- while work:
- (predicate, worklist) = work.popitem()
- self.fragments[predicate] = \
- [item for item in worklist
- if (item.tag != 'Group' and
- item.tag != 'Client' and
- not isinstance(item,
- lxml.etree._Comment))]
- for item in worklist:
- cmd = None
- if item.tag == 'Group':
- if item.get('negate', 'false').lower() == 'true':
- cmd = "lambda x:'%s' not in x.groups and predicate(x)"
- else:
- cmd = "lambda x:'%s' in x.groups and predicate(x)"
- elif item.tag == 'Client':
- if item.get('negate', 'false').lower() == 'true':
- cmd = "lambda x:x.hostname != '%s' and predicate(x)"
- else:
- cmd = "lambda x:x.hostname == '%s' and predicate(x)"
- # else, ignore item
- if cmd is not None:
- newpred = eval(cmd % item.get('name'),
- {'predicate':predicate})
- work[newpred] = item.getchildren()
-
+ self.matches = {}
+
+ def _match(self, item, metadata):
+ """ recursive helper for Match() """
+ if isinstance(item, lxml.etree._Comment):
+ return []
+ elif item.tag == 'Group':
+ rv = []
+ if ((item.get('negate', 'false').lower() == 'true' and
+ item.get('name') not in metadata.groups) or
+ (item.get('negate', 'false').lower() == 'false' and
+ item.get('name') in metadata.groups)):
+ for child in item.iterchildren():
+ rv.extend(self._match(child, metadata))
+ return rv
+ elif item.tag == 'Client':
+ rv = []
+ if ((item.get('negate', 'false').lower() == 'true' and
+ item.get('name') != metadata.hostname) or
+ (item.get('negate', 'false').lower() == 'false' and
+ item.get('name') == metadata.hostname)):
+ for child in item.iterchildren():
+ rv.extend(self._match(child, metadata))
+ return rv
+ else:
+ rv = copy.deepcopy(item)
+ for child in rv.iterchildren():
+ rv.remove(child)
+ for child in item.iterchildren():
+ rv.extend(self._match(child, metadata))
+ return [rv]
+
def Match(self, metadata):
"""Return matching fragments of independent."""
- matching = [frag for (pred, frag) in list(self.fragments.items())
- if pred(metadata)]
- if matching:
- return reduce(lambda x, y: x + y, matching)
- logger.error("File %s got null match" % (self.name))
- return []
+ if metadata.hostname not in self.matches:
+ rv = []
+ for child in self.entries:
+ rv.extend(self._match(child, metadata))
+ logger.debug("File %s got %d match(es)" % (self.name, len(rv)))
+ self.matches[metadata.hostname] = rv
+ return self.matches[metadata.hostname]
class INode:
@@ -644,9 +643,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked):
def BindEntry(self, entry, metadata):
"""Check package lists of package entries."""
- [src.Cache(metadata) for src in list(self.entries.values())]
name = entry.get('name')
- if not src.cache:
+ if False in [src.Cache(metadata) for src in
+ list(self.entries.values())]:
self.logger.error("Called before data loaded")
raise PluginExecutionError
matching = [src for src in list(self.entries.values())
@@ -693,6 +692,9 @@ class Specificity:
self.prio = prio
self.delta = delta
+ def __lt__(self, other):
+ return self.__cmp__(other) < 0
+
def matches(self, metadata):
return self.all or \
self.hostname == metadata.hostname or \
diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py
index 5e7d89727..5de57a87c 100644
--- a/src/lib/Server/Plugins/Base.py
+++ b/src/lib/Server/Plugins/Base.py
@@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin,
__version__ = '$Id$'
__author__ = 'bcfg-dev@mcs.anl.gov'
__child__ = Bcfg2.Server.Plugin.StructFile
+ deprecated = True
"""Base creates independent clauses based on client metadata."""
def __init__(self, core, datastore):
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 998bacc19..23ba0a745 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -4,8 +4,11 @@ __revision__ = '$Revision$'
import binascii
import logging
import lxml
+import operator
import os
+import os.path
import re
+import stat
import sys
import tempfile
@@ -22,9 +25,10 @@ except:
logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+# py3k compatibility
def u_str(string, encoding):
if sys.hexversion >= 0x03000000:
- return str(string, encoding)
+ return string.encode(encoding)
else:
return unicode(string, encoding)
@@ -94,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
entry_type, encoding)
self.specific = CfgMatcher(path.split('/')[-1])
+ path = path
def sort_by_specific(self, one, other):
return cmp(one.specific, other.specific)
@@ -104,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
"""
matching = [ent for ent in list(self.entries.values()) if \
ent.specific.matches(metadata)]
- matching.sort(self.sort_by_specific)
+ matching.sort(key=operator.attrgetter('specific'))
non_delta = [matching.index(m) for m in matching
if not m.specific.delta]
if not non_delta:
@@ -118,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
self.bind_info_to_entry(entry, metadata)
used = self.get_pertinent_entries(metadata)
basefile = used.pop(0)
+ if entry.get('perms').lower() == 'inherit':
+ # use on-disk permissions
+ fname = "%s/%s" % (self.path, entry.get('name'))
+ entry.set('perms',
+ str(oct(stat.S_IMODE(os.stat(fname).st_mode))))
if entry.tag == 'Path':
entry.set('type', 'file')
if basefile.name.endswith(".genshi"):
@@ -134,9 +144,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
metadata=metadata,
path=basefile.name).filter(removecomment)
try:
- data = stream.render('text', strip_whitespace=False)
+ data = stream.render('text', encoding=self.encoding,
+ strip_whitespace=False)
except TypeError:
- data = stream.render('text')
+ data = stream.render('text', encoding=self.encoding)
if data == '':
entry.set('empty', 'true')
except Exception:
@@ -160,6 +171,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
logger.error("Please verify you are using the proper encoding.")
raise Bcfg2.Server.Plugin.PluginExecutionError
+ except ValueError:
+ e = sys.exc_info()[1]
+ logger.error("Error in specification for %s" % entry.get('name'))
+ logger.error("%s" % e)
+ logger.error("You need to specify base64 encoding for %s." %
+ entry.get('name'))
+ raise Bcfg2.Server.Plugin.PluginExecutionError
if entry.text in ['', None]:
entry.set('empty', 'true')
@@ -185,10 +203,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
def write_update(self, specific, new_entry, log):
if 'text' in new_entry:
name = self.build_filename(specific)
- if name.endswith(".genshi"):
+ if os.path.exists("%s.genshi" % name):
logger.error("Cfg: Unable to pull data for genshi types")
raise Bcfg2.Server.Plugin.PluginExecutionError
- open(name, 'w').write(new_entry['text'])
+ try:
+ etext = new_entry['text'].encode(self.encoding)
+ except:
+ logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ open(name, 'w').write(etext)
if log:
logger.info("Wrote file %s" % name)
badattr = [attr for attr in ['owner', 'group', 'perms']
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 5ef1920e1..103fb7353 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
try:
Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
container,
+ self.core.encoding,
0,
logger,
True,
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index ca6e43851..6570f2912 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
(address[0]))
return False
# populate the session cache
- if user != 'root':
+ if user.decode('utf-8') != 'root':
self.session_cache[address] = (time.time(), client)
return True
diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py
index ea2e79ccc..ec0f294dd 100644
--- a/src/lib/Server/Plugins/Probes.py
+++ b/src/lib/Server/Plugins/Probes.py
@@ -1,12 +1,100 @@
import lxml.etree
+import operator
import re
+try:
+ import json
+ has_json = True
+except ImportError:
+ try:
+ import simplejson as json
+ has_json = True
+ except ImportError:
+ has_json = False
+
+try:
+ import syck
+ has_syck = True
+except ImportError:
+ has_syck = False
+ try:
+ import yaml
+ has_yaml = True
+ except ImportError:
+ has_yaml = False
+
import Bcfg2.Server.Plugin
specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)")
probe_matcher = re.compile("(.*/)?(?P<basename>\S+)")
+class ProbeData (object):
+ """ a ProbeData object emulates a str object, but also has .xdata
+ and .json properties to provide convenient ways to use ProbeData
+ objects as XML or JSON data """
+ def __init__(self, data):
+ self.data = data
+ self._xdata = None
+ self._json = None
+ self._yaml = None
+
+ def __str__(self):
+ return str(self.data)
+
+ def __repr__(self):
+ return repr(self.data)
+
+ def __getattr__(self, name):
+ """ make ProbeData act like a str object """
+ return getattr(self.data, name)
+
+ def __complex__(self):
+ return complex(self.data)
+
+ def __int__(self):
+ return int(self.data)
+
+ def __long__(self):
+ return long(self.data)
+
+ def __float__(self):
+ return float(self.data)
+
+ @property
+ def xdata(self):
+ if self._xdata is None:
+ try:
+ self._xdata = lxml.etree.XML(self.data)
+ except lxml.etree.XMLSyntaxError:
+ pass
+ return self._xdata
+
+ @property
+ def json(self):
+ if self._json is None and has_json:
+ try:
+ self._json = json.loads(self.data)
+ except ValueError:
+ pass
+ return self._json
+
+ @property
+ def yaml(self):
+ if self._yaml is None:
+ if has_yaml:
+ try:
+ self._yaml = yaml.load(self.data)
+ except yaml.YAMLError:
+ pass
+ elif has_syck:
+ try:
+ self._yaml = syck.load(self.data)
+ except syck.error:
+ pass
+ return self._yaml
+
+
class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
@@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
ret = []
build = dict()
candidates = self.get_matching(metadata)
- candidates.sort(lambda x, y: cmp(x.specific, y.specific))
+ candidates.sort(key=operator.attrgetter('specific'))
for entry in candidates:
rem = specific_probe_matcher.match(entry.name)
if not rem:
@@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
cx = lxml.etree.SubElement(top, 'Client', name=client)
for probe in sorted(probed):
lxml.etree.SubElement(cx, 'Probe', name=probe,
- value=self.probedata[client][probe])
+ value=str(self.probedata[client][probe]))
for group in sorted(self.cgroups[client]):
lxml.etree.SubElement(cx, "Group", name=group)
data = lxml.etree.tostring(top, encoding='UTF-8',
@@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
except IOError:
self.logger.error("Failed to write probed.xml")
- datafile.write(data)
+ datafile.write(data.decode('utf-8'))
def load_data(self):
try:
@@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
self.cgroups[client.get('name')] = []
for pdata in client:
if (pdata.tag == 'Probe'):
- self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value')
+ self.probedata[client.get('name')][pdata.get('name')] = \
+ ProbeData(pdata.get('value'))
elif (pdata.tag == 'Group'):
self.cgroups[client.get('name')].append(pdata.get('name'))
@@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
self.logger.error("Got null response to probe %s from %s" % \
(data.get('name'), client.hostname))
try:
- self.probedata[client.hostname].update({data.get('name'): ''})
+ self.probedata[client.hostname].update({data.get('name'):
+ ProbeData('')})
except KeyError:
- self.probedata[client.hostname] = {data.get('name'): ''}
+ self.probedata[client.hostname] = \
+ {data.get('name'): ProbeData('')}
return
dlines = data.text.split('\n')
self.logger.debug("%s:probe:%s:%s" % (client.hostname,
@@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
if newgroup not in self.cgroups[client.hostname]:
self.cgroups[client.hostname].append(newgroup)
dlines.remove(line)
- dtext = "\n".join(dlines)
+ dobj = ProbeData("\n".join(dlines))
try:
- self.probedata[client.hostname].update({data.get('name'): dtext})
+ self.probedata[client.hostname].update({data.get('name'): dobj})
except KeyError:
- self.probedata[client.hostname] = {data.get('name'): dtext}
+ self.probedata[client.hostname] = {data.get('name'): dobj}
def get_additional_groups(self, meta):
return self.cgroups.get(meta.hostname, list())
diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py
index dea797a10..54c5def57 100644
--- a/src/lib/Server/Plugins/Properties.py
+++ b/src/lib/Server/Plugins/Properties.py
@@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin
class PropertyFile(Bcfg2.Server.Plugin.StructFile):
"""Class for properties files."""
- def Index(self):
- """Build internal data structures."""
- if type(self.data) is not lxml.etree._Element:
- try:
- self.data = lxml.etree.XML(self.data)
- except lxml.etree.XMLSyntaxError:
- Bcfg2.Server.Plugin.logger.error("Failed to parse %s" %
- self.name)
-
- self.fragments = {}
- work = {lambda x: True: self.data.getchildren()}
- while work:
- (predicate, worklist) = work.popitem()
- self.fragments[predicate] = \
- [item for item in worklist
- if (item.tag != 'Group' and
- item.tag != 'Client' and
- not isinstance(item,
- lxml.etree._Comment))]
- for item in worklist:
- cmd = None
- if item.tag == 'Group':
- if item.get('negate', 'false').lower() == 'true':
- cmd = "lambda x:'%s' not in x.groups and predicate(x)"
- else:
- cmd = "lambda x:'%s' in x.groups and predicate(x)"
- elif item.tag == 'Client':
- if item.get('negate', 'false').lower() == 'true':
- cmd = "lambda x:x.hostname != '%s' and predicate(x)"
- else:
- cmd = "lambda x:x.hostname == '%s' and predicate(x)"
- # else, ignore item
- if cmd is not None:
- newpred = eval(cmd % item.get('name'),
- {'predicate':predicate})
- work[newpred] = item.getchildren()
-
-
+ pass
class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
__child__ = PropertyFile
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index cf0998aaa..4a33c0cb0 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
__author__ = 'bcfg-dev@mcs.anl.gov'
pubkeys = ["ssh_host_dsa_key.pub.H_%s",
- "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"]
+ "ssh_host_rsa_key.pub.H_%s",
+ "ssh_host_key.pub.H_%s"]
hostkeys = ["ssh_host_dsa_key.H_%s",
- "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"]
- keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key',
- 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub',
- 'ssh_host_key.pub']
+ "ssh_host_rsa_key.H_%s",
+ "ssh_host_key.H_%s"]
+ keypatterns = ["ssh_host_dsa_key",
+ "ssh_host_rsa_key",
+ "ssh_host_key",
+ "ssh_host_dsa_key.pub",
+ "ssh_host_rsa_key.pub",
+ "ssh_host_key.pub"]
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
@@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
def get_skn(self):
"""Build memory cache of the ssh known hosts file."""
if not self.__skn:
- self.__skn = "\n".join([str(value.data) for key, value in \
+ self.__skn = "\n".join([value.data.decode() for key, value in \
list(self.entries.items()) if \
key.endswith('.static')])
names = dict()
@@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname)
continue
self.__skn += "%s %s" % (','.join(names[hostname]),
- self.entries[pubkey].data)
+ self.entries[pubkey].data.decode())
return self.__skn
def set_skn(self, value):
@@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
hostkeys.sort()
for hostkey in hostkeys:
entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
- self.entries[hostkey].data)
+ self.entries[hostkey].data.decode())
permdata = {'owner': 'root',
'group': 'root',
'type': 'file',
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
index 8b6bad574..aeb3b9f74 100644
--- a/src/lib/Server/Plugins/Snapshots.py
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -28,6 +28,7 @@ datafields = {
}
+# py3k compatibility
def u_str(string):
if sys.hexversion >= 0x03000000:
return string
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index b6a3c2599..68774cec6 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -38,7 +38,7 @@ import platform
from Bcfg2.Bcfg2Py3k import ConfigParser
-def build_reason_kwargs(r_ent):
+def build_reason_kwargs(r_ent, encoding, logger):
binary_file = False
if r_ent.get('current_bfile', False):
binary_file = True
@@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent):
rc_diff = r_ent.get('current_diff')
else:
rc_diff = ''
+ if not binary_file:
+ try:
+ rc_diff = rc_diff.decode(encoding)
+ except:
+ logger.error("Reason isn't %s encoded, cannot decode it" % encoding)
+ rc_diff = ''
return dict(owner=r_ent.get('owner', default=""),
current_owner=r_ent.get('current_owner', default=""),
group=r_ent.get('group', default=""),
@@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent):
is_binary=binary_file)
-def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
+def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''):
clients = {}
[clients.__setitem__(c.name, c) \
for c in Client.objects.all()]
@@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
for (xpath, type) in pattern:
for x in statistics.findall(xpath):
counter_fields[type] = counter_fields[type] + 1
- kargs = build_reason_kwargs(x)
+ kargs = build_reason_kwargs(x, encoding, logger)
try:
rr = None
@@ -270,6 +276,11 @@ if __name__ == '__main__':
print("StatReports: Failed to parse %s" % (statpath))
raise SystemExit(1)
+ try:
+ encoding = cf.get('components', 'encoding')
+ except:
+ encoding = 'UTF-8'
+
if not clientpath:
try:
clientspath = "%s/Metadata/clients.xml" % \
@@ -288,6 +299,7 @@ if __name__ == '__main__':
update_database()
load_stats(clientsdata,
statsdata,
+ encoding,
verb,
logger,
quick=q,
diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html
index 6ef4c9aff..ec9a17468 100644
--- a/src/lib/Server/Reports/reports/templates/base.html
+++ b/src/lib/Server/Reports/reports/templates/base.html
@@ -87,7 +87,7 @@
<div style='clear:both'></div>
</div><!-- document -->
<div id="footer">
- <span>Bcfg2 Version 1.2.0pre2</span>
+ <span>Bcfg2 Version 1.2.0pre3</span>
</div>
<div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div>
diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
index 291528e2e..2e30125f9 100644
--- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
+++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
@@ -15,6 +15,7 @@ try:
except:
colorize = False
+# py3k compatibility
def u_str(string):
if sys.hexversion >= 0x03000000:
return string
diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py
index 2aa35f1ec..f30c38a05 100644
--- a/src/lib/Server/Snapshots/model.py
+++ b/src/lib/Server/Snapshots/model.py
@@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref
from sqlalchemy.ext.declarative import declarative_base
+# py3k compatibility
def u_str(string):
if sys.hexversion >= 0x03000000:
return string
diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2
index 7f7d8f5c6..5ddfd8791 100755
--- a/src/sbin/bcfg2
+++ b/src/sbin/bcfg2
@@ -7,6 +7,7 @@ import fcntl
import logging
import os
import signal
+import socket
import stat
import sys
import tempfile
@@ -75,6 +76,7 @@ class Client:
'certificate': Bcfg2.Options.CLIENT_CERT,
'ca': Bcfg2.Options.CLIENT_CA,
'serverCN': Bcfg2.Options.CLIENT_SCNS,
+ 'timeout': Bcfg2.Options.CLIENT_TIMEOUT,
}
self.setup = Bcfg2.Options.OptionParser(optinfo)
@@ -178,10 +180,11 @@ class Client:
proxy = Bcfg2.Proxy.ComponentProxy(self.setup['server'],
self.setup['user'],
self.setup['password'],
- key = self.setup['key'],
- cert = self.setup['certificate'],
- ca = self.setup['ca'],
- allowedServerCNs = self.setup['serverCN'])
+ key=self.setup['key'],
+ cert=self.setup['certificate'],
+ ca=self.setup['ca'],
+ allowedServerCNs=self.setup['serverCN'],
+ timeout=self.setup['timeout'])
if self.setup['profile']:
try:
@@ -197,6 +200,13 @@ class Client:
self.logger.error("Failed to download probes from bcfg2")
self.logger.error(flt.faultString)
raise SystemExit(1)
+ except (Bcfg2.Proxy.CertificateError,
+ socket.gaierror,
+ socket.error):
+ e = sys.exc_info()[1]
+ self.logger.error("Failed to download probes from bcfg2: %s"
+ % e)
+ raise SystemExit(1)
times['probe_download'] = time.time()
@@ -243,7 +253,7 @@ class Client:
raise SystemExit(1)
try:
- rawconfig = proxy.GetConfig()
+ rawconfig = proxy.GetConfig().encode('UTF-8')
except xmlrpclib.Fault:
self.logger.error("Failed to download configuration from Bcfg2")
raise SystemExit(2)
@@ -252,7 +262,7 @@ class Client:
if self.setup['cache']:
try:
- open(self.setup['cache'], 'w').write(rawconfig.encode(self.setup['encoding']))
+ open(self.setup['cache'], 'w').write(rawconfig)
os.chmod(self.setup['cache'], 33152)
except IOError:
self.logger.warning("Failed to write config cache file %s" %
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index 161fee441..07953ae69 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -375,21 +375,21 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core):
except:
print("Client %s not defined" % client)
continue
- print("Hostname:\t", client_meta.hostname)
- print("Profile:\t", client_meta.profile)
- print("Groups:\t\t", list(client_meta.groups)[0])
+ print("Hostname:\t%s" % client_meta.hostname)
+ print("Profile:\t%s" % client_meta.profile)
+ print("Groups:\t\t%s" % list(client_meta.groups)[0])
for grp in list(client_meta.groups)[1:]:
- print('\t\t%s' % grp)
+ print("\t\t%s" % grp)
if client_meta.bundles:
- print("Bundles:\t", list(client_meta.bundles)[0])
+ print("Bundles:\t%s" % list(client_meta.bundles)[0])
for bnd in list(client_meta.bundles)[1:]:
- print('\t\t%s' % bnd)
+ print("\t\t%s" % bnd)
if client_meta.connectors:
print("Connector data")
print("=" * 80)
for conn in client_meta.connectors:
if getattr(client_meta, conn):
- print("%s:\t" % (conn), getattr(client_meta, conn))
+ print("%s:\t%s" % (conn, getattr(client_meta, conn)))
print("=" * 80)
def do_mappings(self, args):
@@ -487,7 +487,10 @@ if __name__ == '__main__':
})
setup = Bcfg2.Options.OptionParser(optinfo)
setup.parse(sys.argv[1:])
- if setup['profile'] and have_profile:
+ if setup['args'] and setup['args'][0] == 'help':
+ print(USAGE)
+ sys.exit(0)
+ elif setup['profile'] and have_profile:
prof = profile.Profile()
loop = prof.runcall(infoCore, setup['repo'], setup['plugins'],
setup['password'], setup['encoding'],
diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint
index 6bc34433e..464e839e5 100755
--- a/src/sbin/bcfg2-lint
+++ b/src/sbin/bcfg2-lint
@@ -50,10 +50,13 @@ def run_plugin(plugin, plugin_name, setup=None, errorhandler=None,
errorhandler = get_errorhandler(config)
if config is not None and config.has_section(plugin_name):
- args.append(dict(config.items(plugin_name), **setup))
+ arg = setup
+ for key, val in config.items(plugin_name):
+ arg[key] = val
+ args.append(arg)
else:
args.append(setup)
-
+
# older versions of python do not support mixing *-magic and
# non-*-magic (e.g., "plugin(*args, files=files)", so we do this
# all with *-magic
@@ -181,8 +184,20 @@ if __name__ == '__main__':
config=config, setup=setup)
if serverplugins:
- run_server_plugins(serverplugins, errorhandler=errorhandler,
- config=config, setup=setup)
+ if errorhandler.errors:
+ # it would be swell if we could try to start the server
+ # even if there were errors with the serverless plugins,
+ # but since XML parsing errors occur in the FAM thread
+ # (not in the core server thread), there's no way we can
+ # start the server and try to catch exceptions --
+ # bcfg2-lint isn't in the same stack as the exceptions.
+ # so we're forced to assume that a serverless plugin error
+ # will prevent the server from starting
+ print("Serverless plugins encountered errors, skipping server "
+ "plugins")
+ else:
+ run_server_plugins(serverplugins, errorhandler=errorhandler,
+ config=config, setup=setup)
if errorhandler.errors or errorhandler.warnings or setup['verbose']:
print("%d errors" % errorhandler.errors)
diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports
index 20288fc5e..9a4c6e60d 100755
--- a/src/sbin/bcfg2-reports
+++ b/src/sbin/bcfg2-reports
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
"""Query reporting system for client status."""
__revision__ = '$Revision$'
@@ -47,6 +47,23 @@ def statecompare(client1, client2):
else:
return 0
+def totalcompare(client1, client2):
+ """Compares two clients by their total entry counts."""
+ return cmp(client2.current_interaction.totalcount, \
+ client1.current_interaction.totalcount)
+
+def goodcompare(client1, client2):
+ """Compares two clients by their good entry counts."""
+ return cmp(client2.current_interaction.goodcount, \
+ client1.current_interaction.goodcount)
+
+def badcompare(client1, client2):
+ """Compares two clients by their bad entry counts."""
+ return cmp(client2.current_interaction.totalcount - \
+ client2.current_interaction.goodcount, \
+ client1.current_interaction.totalcount - \
+ client1.current_interaction.goodcount)
+
def crit_compare(criterion, client1, client2):
"""Compares two clients by the criteria provided in criterion."""
for crit in criterion:
@@ -57,6 +74,12 @@ def crit_compare(criterion, client1, client2):
comp = statecompare(client1, client2)
elif crit == 'time':
comp = timecompare(client1, client2)
+ elif crit == 'total':
+ comp = totalcompare(client1, client2)
+ elif crit == 'good':
+ comp = goodcompare(client1, client2)
+ elif crit == 'bad':
+ comp = badcompare(client1, client2)
if comp != 0:
return comp
@@ -83,6 +106,13 @@ def print_fields(fields, cli, max_name, entrydict):
fdata.append("clean")
else:
fdata.append("dirty")
+ elif field == 'total':
+ fdata.append("%5d" % cli.current_interaction.totalcount)
+ elif field == 'good':
+ fdata.append("%5d" % cli.current_interaction.goodcount)
+ elif field == 'bad':
+ fdata.append("%5d" % cli.current_interaction.totalcount \
+ - cli.current_interaction.goodcount)
else:
try:
fdata.append(getattr(cli, field))
@@ -104,6 +134,7 @@ def print_entry(item, max_name):
fields = ""
sort = ""
badentry = ""
+modifiedentry = ""
extraentry = ""
expire = ""
singlehost = ""
@@ -114,8 +145,8 @@ result = list()
entrydict = dict()
args = sys.argv[1:]
-opts, pargs = getopt(args, 'ab:cde:hs:x:',
- ['stale', 'sort=', 'fields=', 'badentry=', 'extraentry='])
+opts, pargs = getopt(args, 'ab:cde:hm:s:t:x:',
+ ['stale', 'sort=', 'fields=', 'badentry=', 'modifiedentry=', 'extraentry='])
for option in opts:
if len(option) > 0:
@@ -125,11 +156,17 @@ for option in opts:
sort = option[1]
if option[0] == '--badentry':
badentry = option[1]
+ if option[0] == '--modifiedentry':
+ modifiedentry = option[1]
if option[0] == '--extraentry':
extraentry = option[1]
if option[0] == '-x':
expire = option[1]
- if option[0] == '-s' or option[0] == '-b' or option[0] == '-e':
+ if option[0] == '-s' or \
+ option[0] == '-t' or \
+ option[0] == '-b' or \
+ option[0] == '-m' or \
+ option[0] == '-e':
singlehost = option[1]
if expire != "":
@@ -147,33 +184,45 @@ elif '-h' in args:
print("""Usage: bcfg2-reports [option] ...
Options and arguments (and corresponding environment variables):
--a : shows all hosts, including expired hosts
--b NAME : single-host mode - shows bad entries from the
- current interaction of NAME
--c : shows only clean hosts
--d : shows only dirty hosts
--e NAME : single-host mode - shows extra entries from the
- current interaction of NAME
--h : shows help and usage info about bcfg2-reports
--s NAME : single-host mode - shows bad and extra entries from
- the current interaction of NAME
--x NAME : toggles expired/unexpired state of NAME
---badentry=KIND,NAME : shows only hosts whose current interaction has bad
- entries in of KIND kind and NAME name; if a single
- argument ARG1 is given, then KIND,NAME pairs will be
- read from a file of name ARG1
---extraentry=KIND,NAME : shows only hosts whose current interaction has extra
- entries in of KIND kind and NAME name; if a single
- argument ARG1 is given, then KIND,NAME pairs will be
- read from a file of name ARG1
---fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,...
- (name,time,state)
---sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state)
---stale : shows hosts which haven't run in the last 24 hours
+-a : shows all hosts, including expired hosts
+-b NAME : single-host mode - shows bad entries from the
+ current interaction of NAME
+-c : shows only clean hosts
+-d : shows only dirty hosts
+-e NAME : single-host mode - shows extra entries from the
+ current interaction of NAME
+-h : shows help and usage info about bcfg2-reports
+-m NAME : single-host mode - shows modified entries from the
+ current interaction of NAME
+-s NAME : single-host mode - shows bad, modified, and extra
+ entries from the current interaction of NAME
+-t NAME : single-host mode - shows total number of managed and
+ good entries from the current interaction of NAME
+-x NAME : toggles expired/unexpired state of NAME
+--badentry=KIND,NAME : shows only hosts whose current interaction has bad
+ entries in of KIND kind and NAME name; if a single
+ argument ARG1 is given, then KIND,NAME pairs will be
+ read from a file of name ARG1
+--modifiedentry=KIND,NAME : shows only hosts whose current interaction has
+ modified entries in of KIND kind and NAME name; if a
+ single argument ARG1 is given, then KIND,NAME pairs
+ will be read from a file of name ARG1
+--extraentry=KIND,NAME : shows only hosts whose current interaction has extra
+ entries in of KIND kind and NAME name; if a single
+ argument ARG1 is given, then KIND,NAME pairs will be
+ read from a file of name ARG1
+--fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,...
+ (name,time,state)
+--sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state)
+--stale : shows hosts which haven't run in the last 24 hours
""")
elif singlehost != "":
for c_inst in c_list:
if singlehost == c_inst.name:
+ if '-t' in args:
+ managed = c_inst.current_interaction.totalcount
+ good = c_inst.current_interaction.goodcount
+ print("Total managed entries: %d (good: %d)" % (managed, good))
baditems = c_inst.current_interaction.bad()
if len(baditems) > 0 and ('-b' in args or '-s' in args):
print("Bad Entries:")
@@ -183,6 +232,15 @@ elif singlehost != "":
max_name = len(item.entry.name)
for item in baditems:
print_entry(item, max_name)
+ modifieditems = c_inst.current_interaction.modified()
+ if len(modifieditems) > 0 and ('-m' in args or '-s' in args):
+ print "Modified Entries:"
+ max_name = -1
+ for item in modifieditems:
+ if len(item.entry.name) > max_name:
+ max_name = len(item.entry.name)
+ for item in modifieditems:
+ print_entry(item, max_name)
extraitems = c_inst.current_interaction.extra()
if len(extraitems) > 0 and ('-e' in args or '-s' in args):
print("Extra Entries:")
@@ -206,6 +264,9 @@ else:
if badentry != "":
badentry = badentry.split(',')
+ if modifiedentry != "":
+ modifiedentry = modifiedentry.split(',')
+
if extraentry != "":
extraentry = extraentry.split(',')
@@ -233,7 +294,7 @@ else:
for c_inst in c_list:
baditems = c_inst.current_interaction.bad()
for item in baditems:
- if item.name == badentry[1] and item.kind == badentry[0]:
+ if item.entry.name == badentry[1] and item.entry.kind == badentry[0]:
result.append(c_inst)
if c_inst in entrydict:
entrydict.get(c_inst).append(badentry[1])
@@ -244,7 +305,29 @@ else:
for c_inst in c_list:
baditems = c_inst.current_interaction.bad()
for item in baditems:
- if item.name == badentry[1] and item.kind == badentry[0]:
+ if item.entry.name == badentry[1] and item.entry.kind == badentry[0]:
+ result.append(c_inst)
+ break
+ elif modifiedentry != "":
+ if len(modifiedentry) == 1:
+ fileread = fileinput.input(modifiedentry[0])
+ for line in fileread:
+ modifiedentry = line.strip().split(',')
+ for c_inst in c_list:
+ modifieditems = c_inst.current_interaction.modified()
+ for item in modifieditems:
+ if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]:
+ result.append(c_inst)
+ if c_inst in entrydict:
+ entrydict.get(c_inst).append(modifiedentry[1])
+ else:
+ entrydict[c_inst] = [modifiedentry[1]]
+ break
+ else:
+ for c_inst in c_list:
+ modifieditems = c_inst.current_interaction.modified()
+ for item in modifieditems:
+ if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]:
result.append(c_inst)
break
elif extraentry != "":
@@ -255,7 +338,7 @@ else:
for c_inst in c_list:
extraitems = c_inst.current_interaction.extra()
for item in extraitems:
- if item.name == extraentry[1] and item.kind == extraentry[0]:
+ if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]:
result.append(c_inst)
if c_inst in entrydict:
entrydict.get(c_inst).append(extraentry[1])
@@ -266,7 +349,7 @@ else:
for c_inst in c_list:
extraitems = c_inst.current_interaction.extra()
for item in extraitems:
- if item.name == extraentry[1] and item.kind == extraentry[0]:
+ if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]:
result.append(c_inst)
break
diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server
index f4bd5e5b7..546d5a249 100755
--- a/src/sbin/bcfg2-server
+++ b/src/sbin/bcfg2-server
@@ -35,6 +35,7 @@ if __name__ == '__main__':
OPTINFO.update({'key' : Bcfg2.Options.SERVER_KEY,
'cert' : Bcfg2.Options.SERVER_CERT,
'ca' : Bcfg2.Options.SERVER_CA,
+ 'listen_all' : Bcfg2.Options.SERVER_LISTEN_ALL,
'location' : Bcfg2.Options.SERVER_LOCATION,
'passwd' : Bcfg2.Options.SERVER_PASSWORD,
'static' : Bcfg2.Options.SERVER_STATIC,
@@ -51,6 +52,7 @@ if __name__ == '__main__':
print("Could not read %s" % setup['configfile'])
sys.exit(1)
Bcfg2.Component.run_component(Bcfg2.Server.Core.Core,
+ listen_all=setup['listen_all'],
location=setup['location'],
daemon = setup['daemon'],
pidfile_name = setup['daemon'],
diff --git a/tools/export.py b/tools/export.py
index d637c166c..2885625d5 100755
--- a/tools/export.py
+++ b/tools/export.py
@@ -8,8 +8,11 @@ import fileinput
from subprocess import Popen, PIPE
import sys
-# Compatibility import
-from Bcfg2.Bcfg2Py3k import formatdate
+# py3k compatibility
+try:
+ from email.Utils import formatdate
+except ImportError:
+ from email.utils import formatdate
pkgname = 'bcfg2'
ftphost = 'terra.mcs.anl.gov'
@@ -25,6 +28,17 @@ tarname = '/tmp/%s-%s.tar.gz' % (pkgname, version)
def run(command):
return Popen(command, shell=True, stdout=PIPE).communicate()
+def find_and_replace(f, iftest, rline, startswith=False):
+ for line in fileinput.input(f, inplace=1):
+ if startswith:
+ if line.startswith(iftest):
+ line = line.replace(line, rline)
+ sys.stdout.write(line)
+ else:
+ if iftest in line and line != "Version: %{version}\n":
+ line = line.replace(line, rline)
+ sys.stdout.write(line)
+
# update the version
majorver = version[:5]
minorver = version[5:]
@@ -36,7 +50,7 @@ except NameError:
name = input("Your name: ")
email = input("Your email: ")
newchangelog = \
-"""bcfg2 (%s-0.0%s) unstable; urgency=low
+"""bcfg2 (%s%s-0.0) unstable; urgency=low
* New upstream release
@@ -49,16 +63,6 @@ with open('debian/changelog', 'r+') as f:
f.seek(0)
f.write(newchangelog + old)
f.close()
-# set new version in setup.py
-for line in fileinput.input('setup.py', inplace=1):
- if 'version' in line:
- line = line.replace(line, ' version="%s",\n' % version)
- sys.stdout.write(line)
-# replace version in misc/bcfg2.spec
-for line in fileinput.input('misc/bcfg2.spec', inplace=1):
- if 'Version:' in line and line != "Version: %{version}\n":
- line = line.replace(line, 'Version: %s\n' % version)
- sys.stdout.write(line)
# Update redhat directory versions
with open('redhat/VERSION', 'w') as f:
f.write("%s\n" % majorver)
@@ -67,24 +71,25 @@ with open('redhat/RELEASE', 'w') as f:
f.write("0.0%s\n" % minorver)
f.close()
# update solaris version
-for line in fileinput.input('solaris/Makefile', inplace=1):
- if line.startswith('VERS='):
- line = line.replace(line, 'VERS=%s-1\n' % version)
- sys.stdout.write(line)
-for line in fileinput.input('solaris/pkginfo.bcfg2', inplace=1):
- if line.startswith('VERSION='):
- line = line.replace(line, 'VERSION="%s"\n' % version)
- sys.stdout.write(line)
-for line in fileinput.input('solaris/pkginfo.bcfg2-server', inplace=1):
- if line.startswith('VERSION='):
- line = line.replace(line, 'VERSION="%s"\n' % version)
- sys.stdout.write(line)
+find_and_replace('solaris/Makefile', 'VERS=',
+ 'VERS=%s-1\n' % version, startswith=True)
+find_and_replace('solaris/pkginfo.bcfg2', 'VERSION=',
+ 'VERSION="%s"\n' % version, startswith=True)
+find_and_replace('solaris/pkginfo.bcfg2-server', 'VERSION=',
+ 'VERSION="%s"\n' % version, startswith=True)
+# set new version in setup.py
+find_and_replace('setup.py', 'version=', ' version="%s",\n' % version)
+# replace version in misc/bcfg2.spec
+find_and_replace('misc/bcfg2.spec', 'Version:',
+ 'Version: %s\n' % version)
# update the version in reports
-for line in fileinput.input('src/lib/Server/Reports/reports/templates/base.html',
- inplace=1):
- if 'Bcfg2 Version' in line:
- line = line.replace(line, ' <span>Bcfg2 Version %s</span>\n' % version)
- sys.stdout.write(line)
+find_and_replace('src/lib/Server/Reports/reports/templates/base.html',
+ 'Bcfg2 Version', ' <span>Bcfg2 Version %s</span>\n' % version)
+# update the version in the docs
+find_and_replace('doc/conf.py', 'version =',
+ 'version = \'%s\'\n' % majorver[0:3], startswith=True)
+find_and_replace('doc/conf.py', 'release =',
+ 'release = \'%s\'\n' % (majorver), startswith=True)
# tag the release
#FIXME: do this using python-dulwich