diff options
97 files changed, 2553 insertions, 737 deletions
diff --git a/.travis.yml b/.travis.yml index 8786dcc77..f5aade735 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,21 +11,22 @@ matrix: exclude: - python: "3.2" env: WITH_OPTIONAL_DEPS=yes + - python: "3.3" + env: WITH_OPTIONAL_DEPS=yes before_install: - testsuite/before_install.sh install: - testsuite/install.sh - - pip install -e . + - pip install --use-mirrors -e . script: - nosetests testsuite branches: except: - - maint + - maint-1.2 - 1.1.0-stable - - py3k notifications: email: chris.a.st.pierre@gmail.com - irc: + irc: channels: - "irc.freenode.org#bcfg2" use_notice: true diff --git a/debian/bcfg2-report-collector.init b/debian/bcfg2-report-collector.init index 2d182385a..df7b751cb 100755 --- a/debian/bcfg2-report-collector.init +++ b/debian/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -85,22 +86,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index 8de16b9b5..04774c063 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -41,6 +41,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Management Server: " @@ -91,22 +92,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2-server.install b/debian/bcfg2-server.install index eab9cd117..3f1a73b06 100644 --- a/debian/bcfg2-server.install +++ b/debian/bcfg2-server.install @@ -1,6 +1,7 @@ debian/bcfg2-server.default usr/share/bcfg2 debian/tmp/usr/bin/bcfg2-* usr/sbin debian/tmp/usr/lib/python*/*-packages/Bcfg2/Server/* +debian/tmp/usr/lib/python*/*-packages/Bcfg2/Reporting/* debian/tmp/usr/share/bcfg2/schemas/* debian/tmp/usr/share/bcfg2/xsl-transforms/* debian/tmp/usr/share/man/man8/* diff --git a/debian/bcfg2.init b/debian/bcfg2.init index 4f83adbf6..b2e47b346 100755 --- a/debian/bcfg2.init +++ b/debian/bcfg2.init @@ -47,6 +47,7 @@ fi # Internal variables BINARY=$(basename $BCFG2) +RETVAL=0 # Include lsb functions . /lib/lsb/init-functions @@ -70,17 +71,19 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/changelog b/debian/changelog index 3a38bf02a..298e695c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +bcfg2 (1.3.1-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Thu, 21 Mar 2013 09:32:16 -0500 + +bcfg2 (1.3.0-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Fri, 15 Mar 2013 08:45:18 -0500 + bcfg2 (1.3.0rc2-0.0) unstable; urgency=low * New upstream release diff --git a/doc/conf.py b/doc/conf.py index a1bceb1b1..d3d30687b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -66,7 +66,7 @@ else: # The short X.Y version. version = '1.3' # The full version, including alpha/beta/rc tags. -release = '1.3.0' +release = '1.3.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/exts/xmlschema.py b/doc/exts/xmlschema.py index 727b4bbd0..24cbf2e2d 100644 --- a/doc/exts/xmlschema.py +++ b/doc/exts/xmlschema.py @@ -76,6 +76,11 @@ from sphinx.util.nodes import make_refnode, split_explicit_title, \ from sphinx.util.compat import Directive from sphinx.domains import ObjType, Domain +try: + from new import classobj +except ImportError: + classobj = type + XS = "http://www.w3.org/2001/XMLSchema" XS_NS = "{%s}" % XS NSMAP = dict(xs=XS) @@ -653,7 +658,7 @@ def append_node(parent, cls_or_node, *contents): def build_node(cls_or_node, *contents): - if isinstance(cls_or_node, type): + if isinstance(cls_or_node, (type, classobj)): rv = cls_or_node() else: rv = cls_or_node diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index c3abab9e7..5ae668c89 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -56,8 +56,8 @@ the debug level individually on a given plugin, e.g.:: Finally, the File Activity Monitor has its own analogue to these two methods, for setting the debug level of the FAM: - bcfg2-admin xcmd toggle_fam_debug - bcfg2-admin xcmd set_fam_debug false + bcfg2-admin xcmd Inotify.toggle_debug + bcfg2-admin xcmd Inotify.set_debug false Check if all repository XML files conform to schemas ==================================================== diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index eaa2a0393..81ac12632 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -59,3 +59,17 @@ Bcfg2 Server | included in python versions | | | | 2.6 and later | | | +-------------------------------+----------+--------------------------------+ + +Bcfg2 Reporting +--------------- + +A webserver capabable of running wsgi applications is required for web reporting, +such as Apache + mod_wsgi or nginx. + ++-------------------------------+----------+--------------------------------+ +| Software | Version | Requires | ++===============================+==========+================================+ +| django | 1.2.0+ | | ++-------------------------------+----------+--------------------------------+ +| south | 0.7.0+ | | ++-------------------------------+----------+--------------------------------+ diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index ffc202f7f..f5516cbbd 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -719,11 +719,21 @@ control the database connection of the server. port Port for database connections. Not used for sqlite3. +Reporting options +----------------- + + config + Specifies the location of the reporting configuration (default + is /etc/bcfg2-web.conf. + time_zone - Specify a time zone other than that used on the system. (Note + Specifies a time zone other than that used on the system. (Note that this will cause the Bcfg2 server to log messages in this time zone as well). + web_debug + Turn on Django debugging. + See Also -------- diff --git a/doc/man/bcfg2.txt b/doc/man/bcfg2.txt index 6a77a4a3c..6df4f9b4f 100644 --- a/doc/man/bcfg2.txt +++ b/doc/man/bcfg2.txt @@ -88,10 +88,13 @@ Options the constraints of correctness, and thus should only be used in safe conditions. -r mode Cause bcfg2 to remove extra configuration elements - it detects. Mode is one of "all", "Services", or - "Packages". "all" removes all entries. Likewise, - "Services" and "Packages" remove only the extra + it detects. Mode is one of "all", "Services", + "Packages", or "Users". "all" removes all extra entries. + "Services", "Packages", and "Users" remove only the extra configuration elements of the respective type. + ("Services" actually just disables extra services, + since they can't be removed, and "Users" removes + extra POSIXUser and POSIXUser entries.) -s servicemode Set bcfg2 interaction level for services. Default behavior is to modify all services affected by reconfiguration. "build" mode attempts to stop all diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 19c947a71..b3028e9e1 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -56,7 +56,7 @@ Install Be sure to include the specified fields included in the example ``bcfg2.conf`` file. These can be specified in either ``/etc/bcfg2.conf``, if it is readable by the webserver user, or ``/etc/bcfg2-web.conf``. Any -database supported by `Django <http://www.djangoproject.com>`_ can be used. +database supported by `Django <http://www.djangoproject.com>`_ can be used. As of version 1.3, `South <http://south.aeracode.org>`_ is used to control schema changes. If your database is not supported by South, any updates will need to be applied manually. Sqlite is configured by default. @@ -78,11 +78,11 @@ databases. After configuring your database be sure to run `bcfg2-admin reports init` to create the schema. -To enable statistics collection in the bcfg2-server, add +To enable statistics collection in the bcfg2-server, add :ref:`server-plugins-statistics-reporting` to the **plugins** line in your ``bcfg2.conf`` and restart the bcfg2-server. A report collecting -daemon should be run to import the collected statistics into the backend. -Please see the section :ref:`Report Collector <report_collector>` for more +daemon should be run to import the collected statistics into the backend. +Please see the section :ref:`Report Collector <report_collector>` for more information. Detailed installation instructions can be found :ref:`here @@ -155,7 +155,7 @@ http://localhost/bcfg2 and see the new reports. Upgrading ============ -1. Convert database config +1. Convert database config Run `tools/upgrade/1.3/migrate_configs.py` @@ -166,32 +166,34 @@ Upgrading 2. Replace the DBStats plugin with the Reporting plugin. 3. Migrate historic data. - Run `tools/upgrade/1.3/migrate_dbstats.py` + Run ``tools/upgrade/1.3/migrate_dbstats.py`` The reporting schema is now managed using `South <http://south.aeracode.org>`_ instead of a set of custom scripts. This creates the new schema and imports all of the historic data to the new format. -.. Note: + .. note:: - After the database is upgraded all of the old tables are left intact. To - remove them any table starting with reports_ can be dropped. + After the database is upgraded all of the old tables are left + intact. To remove them any table starting with reports_ can + be dropped. 4. `(Optional)` Run the :ref:`Report Collector <report_collector>` - Add "transport = LocalFilesystem" under "[reporting]" in `bcfg2.conf`. - Restart the bcfg2-server and start the bcfg2-report-collector. + Add "transport = LocalFilesystem" under "[reporting]" in + ``bcfg2.conf``. Restart the bcfg2-server and start the + bcfg2-report-collector. Configuring =========== -Most of the configuration is handled through the ``/etc/bcfg2.conf`` or alternatively -``/etc/bcfg2-web.conf``. +Most of the configuration is handled through the ``/etc/bcfg2.conf`` +or alternatively ``/etc/bcfg2-web.conf``. An example using the defaults is listed below:: [database] engine = sqlite3 - name = '/var/lib/bcfg2/etc/bcfg2.sqlite' + name = /var/lib/bcfg2/etc/bcfg2.sqlite user = password = host = @@ -260,7 +262,7 @@ reporting Statistics Transports --------------------- -A transport is required to pass the data collected from the bcfg2-server +A transport is required to pass the data collected from the bcfg2-server to the bcfg2-report-collector. At the time of this writing two transports are available: @@ -271,7 +273,7 @@ are available: Future transports will allow multiple servers to pass data to a single or multiple bcfg2-report-collector processes. New installations default to and should use the -LocalFilesystem transport. Upgrades will use DirectStore by default in the 1.3 +LocalFilesystem transport. Upgrades will use DirectStore by default in the 1.3 release. .. Note:: @@ -293,7 +295,7 @@ An example configuration with the default values:: redis_port = 6379 redis_db = 0 -bcfg2-admin commands operate slightly differently in this mode. Instead of querying the +bcfg2-admin commands operate slightly differently in this mode. Instead of querying the database directly, rpc commands are issued to the report collectors. This only affects the minestruct and pull commands. diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index b11d1ebc5..95bf4238a 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -217,7 +217,8 @@ something like this: <Source type="apt" recommended="true" ...> .. warning:: You must regenerate the Packages cache when adding or - removing the recommended attribute. + removing the recommended attribute (``bcfg2-admin xcmd + Packages.Refresh``). .. [#f1] Bcfg2 will by default add **Essential** packages to the client specification. You can disable this behavior by @@ -321,9 +322,9 @@ will report information like:: Packages: Updating http://mirror.centos.org/centos/5/extras/x86_64/repodata/filelists.xml.gz Packages: Updating http://mirror.centos.org/centos/5/extras/x86_64/repodata/primary.xml.gz -Once line per file download needed. ``Packages/sources.xml`` will -be reloaded at this time, so any source specification changes (new -or modified sources in this file) will be reflected by the server at +One line per file download needed. ``Packages/sources.xml`` will be +reloaded at this time, so any source specification changes (new or +modified sources in this file) will be reflected by the server at this point. This process is much, much faster if you use the :ref:`native yum @@ -426,7 +427,6 @@ Benefits to this include: * Much lower memory usage by the ``bcfg2-server`` process. * Much faster ``Packages.Refresh`` behavior. * More accurate dependency resolution. -* Support for package groups. Drawbacks include: @@ -475,9 +475,9 @@ generally be overridden: Package Groups -------------- -Yum package groups are supported by the native Yum libraries. To -include a package group, use the -:xml:attribute:`PackageStructure:group` attribute of the +Yum package groups are supported by both the native Yum libraries and +Bcfg2's internal dependency resolver. To include a package group, use +the :xml:attribute:`PackageStructure:group` attribute of the :xml:element:`Package` tag. You can use either the short group ID or the long group name: diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index f690aac8b..9fe87c878 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -204,74 +204,9 @@ A special client metadata class is available to :ref:`server-plugins-generators-cfg-genshi` and :ref:`server-plugins-generators-cfg-cheetah`. -+------------+------------------------------------------------+---------------+ -| Attribute | Description | Value | -+============+================================================+===============+ -| hostname | Client hostname | String | -+------------+------------------------------------------------+---------------+ -| profile | Client profile | String | -+------------+------------------------------------------------+---------------+ -| aliases | Client aliases | List | -+------------+------------------------------------------------+---------------+ -| addresses | Adresses this client is known by | List | -+------------+------------------------------------------------+---------------+ -| groups | Groups this client is a member of | List | -+------------+------------------------------------------------+---------------+ -| categories | Categories of this clients groups | List | -+------------+------------------------------------------------+---------------+ -| uuid | uuid identifier for this client | String | -+------------+------------------------------------------------+---------------+ -| password | bcfg password for this client | String | -+------------+------------------------------------------------+---------------+ -| connectors | connector plugins known to this client | List | -+------------+------------------------------------------------+---------------+ -| query | `MetadataQuery`_ object | MetadataQuery | -+------------+------------------------------------------------+---------------+ - - -+-----------------------------+------------------------------------------------+-------------------+ -| Method | Description | Value | -+=============================+================================================+===================+ -| inGroup(group) | True if this client is a memnber of 'group' | Boolean | -+-----------------------------+------------------------------------------------+-------------------+ -| group_in_category(category) | Returns the group in 'category' if the client | String | -| | is a member of 'category', otherwise '' | | -+-----------------------------+------------------------------------------------+-------------------+ +.. autoclass:: Bcfg2.Server.Plugins.Metadata.ClientMetadata MetadataQuery ------------- -This class provides query methods for the metadata of all clients -known to the Bcfg2 server. Note that ``*by_groups()`` and -``*by_profiles()`` behave differently; for a client to be included in -the return value of a ``by_groups()`` method, it must be a member of -*all* groups listed in the argument; for a client to be included in -the return value of a ``by_profiles()`` method, it must have any group -listed as its profile group. - -+------------------------------+------------------------------------------------+-------------------+ -| Method | Description | Value | -+==============================+================================================+===================+ -| by_name(client) | Get ClientMetadata object for 'client' | ClientMetadata | -+------------------------------+------------------------------------------------+-------------------+ -| by_groups(groups) | Get ClientMetadata object for clients in all | List of | -| | listed groups | ClientMetadata | -+------------------------------+------------------------------------------------+-------------------+ -| by_profiles(client) | Get ClientMetadata objects for clients whose | List of | -| | profile matches any listed profile group | ClientMetadata | -+------------------------------+------------------------------------------------+-------------------+ -| names_by_groups(groups) | Get the names of all clients in all listed | List of strings | -| | groups | | -+------------------------------+------------------------------------------------+-------------------+ -| names_by_profiles(profiles) | Get the names of clients whose profile matches | List of strings | -| | any listed profile group | | -+------------------------------+------------------------------------------------+-------------------+ -| all_clients() | All known client hostnames | List of strings | -+------------------------------+------------------------------------------------+-------------------+ -| all_groups() | All known group names | List of strings | -+------------------------------+------------------------------------------------+-------------------+ -| all_groups_in_category(cat) | The names of all groups in category 'cat' | List of strings | -+------------------------------+------------------------------------------------+-------------------+ -| all() | Get ClientMetadata for all clients | List of | -| | | ClientMetadata | -+------------------------------+------------------------------------------------+-------------------+ +.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataQuery diff --git a/doc/server/plugins/version/git.txt b/doc/server/plugins/version/git.txt index 3f7ab9d9b..64ff422ca 100644 --- a/doc/server/plugins/version/git.txt +++ b/doc/server/plugins/version/git.txt @@ -13,12 +13,6 @@ reporting purposes. Once the plugin is enabled, every time a client checks in, it will include the current repository revision in the reports/statistics. -As with the other Version plugins, the Git plugin enables you to get -revision information out of your repository for reporting -purposes. Once the plugin is enabled, every time a client checks in, -it will include the current repository revision in the -reports/statistics. - Additionally, if the ``GitPython`` library is installed, the Git plugin exposes an additional XML-RPC method call, ``Git.Update``. With no arguments, ``Git.Update`` updates the working copy to the diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 279ee7e0b..06cbeec0b 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-ADMIN" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-ADMIN" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-admin \- Perform repository administration tasks . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -249,5 +249,4 @@ Add a shape/color key. .sp \fIbcfg2\-info(8)\fP, \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-build-reports.8 b/man/bcfg2-build-reports.8 index 8d3d58ebb..3b4582820 100644 --- a/man/bcfg2-build-reports.8 +++ b/man/bcfg2-build-reports.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-BUILD-REPORTS" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-BUILD-REPORTS" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-build-reports \- Generate state reports for Bcfg2 clients . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -58,5 +58,4 @@ default is \fBrepo/etc/statistics.xml\fP. .sp \fIbcfg2(1)\fP, \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-crypt.8 b/man/bcfg2-crypt.8 index 7c11eb66c..3cdef3f84 100644 --- a/man/bcfg2-crypt.8 +++ b/man/bcfg2-crypt.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-CRYPT" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-CRYPT" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-crypt \- Bcfg2 encryption and decryption utility . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -150,5 +150,4 @@ produced and the file being encrypted or decrypted is skipped. .sp \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-info.8 b/man/bcfg2-info.8 index ee650b5da..a52194372 100644 --- a/man/bcfg2-info.8 +++ b/man/bcfg2-info.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-INFO" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-INFO" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-info \- Creates a local version of the Bcfg2 server core for state observation . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -136,5 +136,4 @@ Print version of this tool. .sp \fIbcfg2(1)\fP, \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8 index c81c305f1..22632f5dd 100644 --- a/man/bcfg2-lint.8 +++ b/man/bcfg2-lint.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-LINT" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-LINT" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-lint \- Check Bcfg2 specification for validity, common mistakes, and style . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -143,5 +143,4 @@ than by a mix of Cfg and TGenshi or TCheetah. \fIbcfg2(1)\fP, \fIbcfg2\-server(8)\fP, \fIbcfg2\-lint.conf(5)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5 index d6d299616..a2b34e601 100644 --- a/man/bcfg2-lint.conf.5 +++ b/man/bcfg2-lint.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2-LINT.CONF" "5" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-LINT.CONF" "5" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-lint.conf \- Configuration parameters for bcfg2-lint . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH DESCRIPTION .sp @@ -158,5 +158,4 @@ The full path to the XML Schema files. Default is .sp \fIbcfg2\-lint(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-reports.8 b/man/bcfg2-reports.8 index 7b1622a5e..6d0db36fa 100644 --- a/man/bcfg2-reports.8 +++ b/man/bcfg2-reports.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-REPORTS" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-REPORTS" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-reports \- Query reporting system for client status . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -142,5 +142,4 @@ specified file instead of the command line. .sp \fIbcfg2(1)\fP, \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index 8df8ebfd5..27f6a7b01 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-SERVER" "8" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2-SERVER" "8" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-server \- Server for client configuration specifications . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -76,5 +76,4 @@ Specify the path to the SSL key. .sp \fIbcfg2(1)\fP, \fIbcfg2\-lint(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2.1 b/man/bcfg2.1 index 1e0f526c8..5b9449fda 100644 --- a/man/bcfg2.1 +++ b/man/bcfg2.1 @@ -1,4 +1,4 @@ -.TH "BCFG2" "1" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2" "1" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2 \- Bcfg2 client tool . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH SYNOPSIS .sp @@ -164,10 +164,13 @@ be used in safe conditions. .TP .BI \-r \ mode Cause bcfg2 to remove extra configuration elements -it detects. Mode is one of "all", "Services", or -"Packages". "all" removes all entries. Likewise, -"Services" and "Packages" remove only the extra +it detects. Mode is one of "all", "Services", +"Packages", or "Users". "all" removes all extra entries. +"Services", "Packages", and "Users" remove only the extra configuration elements of the respective type. +("Services" actually just disables extra services, +since they can\(aqt be removed, and "Users" removes +extra POSIXUser and POSIXUser entries.) .TP .BI \-s \ servicemode Set bcfg2 interaction level for services. Default @@ -206,5 +209,4 @@ Only configure independent entries, ignore bundles. .sp \fIbcfg2\-server(8)\fP, \fIbcfg2\-info(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 0a982b8f0..b0db91a5b 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "January 14, 2013" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "March 18, 2013" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructeredText. +.\" Man page generated from reStructuredText. . .SH DESCRIPTION .sp @@ -62,6 +62,8 @@ the \fIbcfg2\-admin init\fP command. The file monitor used to watch for changes in the repository. The default is the best available monitor. The following values are valid: +.INDENT 7.0 +.INDENT 3.5 .sp .nf .ft C @@ -71,10 +73,14 @@ fam pseudo .ft P .fi +.UNINDENT +.UNINDENT .TP .B ignore_files A comma\-separated list of globs that should be ignored by the file monitor. Default values are: +.INDENT 7.0 +.INDENT 3.5 .sp .nf .ft C @@ -90,6 +96,8 @@ SCCS \&.gitignore .ft P .fi +.UNINDENT +.UNINDENT .TP .B listen_all This setting tells the server to listen on all available interfaces. @@ -99,6 +107,8 @@ bcfg2 setting in the components section of \fBbcfg2.conf\fP. .B plugins A comma\-delimited list of enabled server plugins. Currently available plugins are: +.INDENT 7.0 +.INDENT 3.5 .sp .nf .ft C @@ -145,6 +155,8 @@ TGenshi Trigger .ft P .fi +.UNINDENT +.UNINDENT .sp Descriptions of each plugin can be found in their respective sections below. @@ -156,6 +168,8 @@ default location (e.g. \fB/usr/local\fP). .B backend Specifies which server core backend to use. Current available options are: +.INDENT 7.0 +.INDENT 3.5 .sp .nf .ft C @@ -164,6 +178,8 @@ builtin best .ft P .fi +.UNINDENT +.UNINDENT .sp The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP. More details on the backends can be found in the official @@ -725,6 +741,8 @@ control the database connection of the server. .B engine The database engine used by the statistics module. One of the following: +.INDENT 7.0 +.INDENT 3.5 .sp .nf .ft C @@ -734,6 +752,8 @@ sqlite3 ado_mssql .ft P .fi +.UNINDENT +.UNINDENT .TP .B name The name of the database to use for statistics data. If @@ -751,11 +771,25 @@ Host for database connections. Not used for sqlite3. .TP .B port Port for database connections. Not used for sqlite3. +.UNINDENT +.UNINDENT +.UNINDENT +.SH REPORTING OPTIONS +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.B config +Specifies the location of the reporting configuration (default +is /etc/bcfg2\-web.conf. .TP .B time_zone -Specify a time zone other than that used on the system. (Note +Specifies a time zone other than that used on the system. (Note that this will cause the Bcfg2 server to log messages in this time zone as well). +.TP +.B web_debug +Turn on Django debugging. .UNINDENT .UNINDENT .UNINDENT @@ -763,5 +797,4 @@ time zone as well). .sp \fIbcfg2(1)\fP, \fIbcfg2\-server(8)\fP .\" Generated by docutils manpage writer. -.\" . diff --git a/misc/apache/bcfg2.conf b/misc/apache/bcfg2.conf index 6cd5addf5..b8409a513 100644 --- a/misc/apache/bcfg2.conf +++ b/misc/apache/bcfg2.conf @@ -4,7 +4,7 @@ # WSGIScriptAlias /bcfg2 "/usr/share/bcfg2/reports.wsgi" - WSGISocketPrefix /var/run/httpd/wsgi + WSGISocketPrefix /var/run/apache2/wsgi WSGIDaemonProcess Bcfg2.Server.Reports processes=1 threads=10 WSGIProcessGroup Bcfg2.Server.Reports diff --git a/misc/bcfg2-selinux.spec b/misc/bcfg2-selinux.spec index e5a0eed16..4c05f4959 100644 --- a/misc/bcfg2-selinux.spec +++ b/misc/bcfg2-selinux.spec @@ -8,8 +8,8 @@ %global selinux_variants %([ -z "%{selinux_types}" ] && echo mls strict targeted || echo %{selinux_types}) Name: bcfg2-selinux -Version: 1.3.0 -Release: 0.0rc2 +Version: 1.3.1 +Release: 1 Summary: Bcfg2 Client and Server SELinux policy %if 0%{?suse_version} @@ -24,8 +24,8 @@ Conflicts: selinux-policy = 3.11.1 License: BSD URL: http://bcfg2.org -Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}rc2.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}rc2-%{release}-root-%(%{__id_u} -n) +Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch BuildRequires: checkpolicy, selinux-policy-devel, hardlink @@ -65,7 +65,7 @@ deployment strategies. This package includes the Bcfg2 server and client SELinux policy. %prep -%setup -q -n %{name}-%{version}rc2 +%setup -q -n %{name}-%{version} %build cd redhat/selinux @@ -120,6 +120,12 @@ if [ $1 -eq 0 ] ; then fi %changelog +* Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 +- New upstream release + +* Fri Mar 15 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0 +- New upstream release + * Tue Jan 29 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0rc2 - New upstream release diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 4aaf15e1c..13bc6474e 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -5,8 +5,8 @@ %{!?_initrddir: %global _initrddir %{_sysconfdir}/rc.d/init.d} Name: bcfg2 -Version: 1.3.0 -Release: 0.0rc2 +Version: 1.3.1 +Release: 1 Summary: Configuration management system %if 0%{?suse_version} @@ -17,8 +17,8 @@ Group: Applications/System %endif License: BSD URL: http://bcfg2.org -Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}rc2.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}rc2-%{release}-root-%(%{__id_u} -n) +Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch BuildRequires: python-devel @@ -87,7 +87,7 @@ deployment strategies. This package includes the Bcfg2 client software. %package server -Version: 1.3.0 +Version: 1.3.1 Summary: Bcfg2 Server %if 0%{?suse_version} Group: System/Management @@ -140,7 +140,7 @@ deployment strategies. This package includes the Bcfg2 server software. %package server-cherrypy -Version: 1.3.0 +Version: 1.3.1 Summary: Bcfg2 Server - CherryPy backend %if 0%{?suse_version} Group: System/Management @@ -220,7 +220,7 @@ deployment strategies. This package includes the Bcfg2 documentation. %package web -Version: 1.3.0 +Version: 1.3.1 Summary: Bcfg2 Web Reporting Interface %if 0%{?suse_version} Group: System/Management @@ -267,7 +267,7 @@ deployment strategies. This package includes the Bcfg2 reports web frontend. %prep -%setup -q -n %{name}-%{version}rc2 +%setup -q -n %{name}-%{version} %build %{__python}%{pythonversion} setup.py build @@ -311,6 +311,7 @@ cp -r tools/* %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} cp -r build/sphinx/html/* %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version} %{__install} -d %{buildroot}%{apache_conf}/conf.d +sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf %{__install} -m 644 misc/apache/bcfg2.conf %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf %{__mkdir_p} %{buildroot}%{_localstatedir}/cache/%{name} @@ -453,6 +454,12 @@ fi %endif %changelog +* Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 +- New upstream release + +* Fri Mar 15 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0 +- New upstream release + * Tue Jan 29 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0rc2 - New upstream release diff --git a/osx/Makefile b/osx/Makefile index f7c4b8ffa..9c5d30254 100644 --- a/osx/Makefile +++ b/osx/Makefile @@ -28,9 +28,9 @@ SITELIBDIR = /Library/Python/${PYVERSION}/site-packages # an Info.plist file for packagemaker to look at for package creation # and substitute the version strings. Major/Minor versions can only be # integers (e.g. "1" and "00" for bcfg2 version 1.0.0. -BCFGVER = 1.3.0rc2 +BCFGVER = 1.3.1 MAJOR = 1 -MINOR = 30 +MINOR = 31 default: clean client diff --git a/osx/macports/Portfile b/osx/macports/Portfile index f53974670..45cf3dd2b 100644 --- a/osx/macports/Portfile +++ b/osx/macports/Portfile @@ -5,7 +5,7 @@ PortSystem 1.0 PortGroup python26 1.0 name bcfg2 -version 1.3.0rc2 +version 1.3.1 categories sysutils python maintainers gmail.com:sol.jerome license BSD diff --git a/redhat/RELEASE b/redhat/RELEASE index c39908ef3..ba66466c2 100644 --- a/redhat/RELEASE +++ b/redhat/RELEASE @@ -1 +1 @@ -0.0rc2 +0.0 diff --git a/redhat/VERSION b/redhat/VERSION index f0bb29e76..3a3cd8cc8 100644 --- a/redhat/VERSION +++ b/redhat/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 diff --git a/redhat/bcfg2.spec.in b/redhat/bcfg2.spec.in index 871f3e86a..5d0d54d08 100644 --- a/redhat/bcfg2.spec.in +++ b/redhat/bcfg2.spec.in @@ -256,6 +256,12 @@ fi %doc %{_defaultdocdir}/bcfg2-doc-%{version} %changelog +* Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 +- New upstream release + +* Fri Mar 15 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0 +- New upstream release + * Tue Jan 29 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0rc2 - New upstream release diff --git a/redhat/scripts/bcfg2-report-collector.init b/redhat/scripts/bcfg2-report-collector.init index a8e23f080..43e875a6b 100755 --- a/redhat/scripts/bcfg2-report-collector.init +++ b/redhat/scripts/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -79,22 +80,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/redhat/scripts/bcfg2-server.init b/redhat/scripts/bcfg2-server.init index ffac6ac3d..c4412d1c3 100755 --- a/redhat/scripts/bcfg2-server.init +++ b/redhat/scripts/bcfg2-server.init @@ -59,9 +59,11 @@ stop () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status $prog @@ -71,6 +73,7 @@ case "$1" in stop sleep 5 start + RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|status|restart|reload|force-reload}" diff --git a/redhat/scripts/bcfg2.init b/redhat/scripts/bcfg2.init index 5cfdf47bc..9c26434ff 100755 --- a/redhat/scripts/bcfg2.init +++ b/redhat/scripts/bcfg2.init @@ -54,12 +54,14 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" @@ -23,7 +23,7 @@ if need_m2crypto: inst_reqs.append('M2Crypto') setup(name="Bcfg2", - version="1.3.0rc2", + version="1.3.1", description="Bcfg2 Server", author="Narayan Desai", author_email="desai@mcs.anl.gov", diff --git a/solaris/Makefile b/solaris/Makefile index d81dd0292..fd2c254bb 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -1,7 +1,7 @@ #!/usr/sfw/bin/gmake PYTHON="/usr/local/bin/python" -VERS=1.3.0rc2-1 +VERS=1.3.1-1 PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2 index b95b24f75..2bf3abaf5 100644 --- a/solaris/pkginfo.bcfg2 +++ b/solaris/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.0rc2" +VERSION="1.3.1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server index 25b0bfa4b..4425220c2 100644 --- a/solaris/pkginfo.bcfg2-server +++ b/solaris/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.0rc2" +VERSION="1.3.1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index e2521c0f8..66c1ce430 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -105,6 +105,7 @@ class Client(object): self._probe_failure(name, "Return value %s" % rv) self.logger.info("Probe %s has result:" % name) self.logger.info(rv.stdout) + ret.text = rv.stdout finally: os.unlink(scriptname) except SystemExit: diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 82512130c..6ef686c10 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -481,7 +481,8 @@ class Frame(object): len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) if phase == 'final' and self.setup['extra']: - for entry in self.extra: + for entry in sorted(self.extra, key=lambda e: e.tag + ":" + + e.get('name')): etype = entry.get('type') if etype: self.logger.info("%s:%s:%s" % (entry.tag, etype, diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index 1276c3ce9..f8817bb27 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -1,6 +1,6 @@ -import logging import re import socket +import logging # The ssl module is provided by either Python 2.6 or a separate ssl # package that works on older versions of Python (see @@ -20,7 +20,7 @@ import sys import time # Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse +from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) @@ -51,13 +51,16 @@ class ProxyError(Exception): msg = str(err) Exception.__init__(self, msg) + class CertificateError(Exception): def __init__(self, commonName): self.commonName = commonName + def __str__(self): return ("Got unallowed commonName %s from server" % self.commonName) + _orig_Method = xmlrpclib._Method class RetryMethod(xmlrpclib._Method): @@ -350,7 +353,8 @@ def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, if user and password: method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, user, password, path) + newurl = "%s://%s:%s@%s" % (method, quote_plus(user, ''), + quote_plus(password, ''), path) else: newurl = url ssl_trans = XMLRPCTransport(key, cert, ca, diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cc2f657d0..f449557aa 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -228,8 +228,13 @@ class APT(Bcfg2.Client.Tools.Tool): continue if pkg.get('version') in ['auto', 'any']: if self._newapi: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidate.version)) + try: + ipkgs.append("%s=%s" % (pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) + except AttributeError: + self.logger.error("Failed to find %s in apt package cache" % + pkg.get('name')) + continue else: ipkgs.append("%s=%s" % (pkg.get('name'), self.pkg_cache[pkg.get('name')].candidateVersion)) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index b867fa3d8..f46875743 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -687,7 +687,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if path is None: path = entry.get("name") cur = path - while cur != '/': + while cur and cur != '/': if not os.path.exists(cur): created.append(cur) cur = os.path.dirname(cur) diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index aa1254b46..32afa8cbf 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -36,7 +36,8 @@ class Portage(Bcfg2.Client.Tools.PkgTool): return self.logger.info('Getting list of installed packages') self.installed = {} - for pkg in self.cmd.run("equery -q list '*'").stdout.splitlines(): + for pkg in self.cmd.run(["equery", "-q", + "list", "*"]).stdout.splitlines(): if self._pkg_pattern.match(pkg): name = self._pkg_pattern.match(pkg).group(1) version = self._pkg_pattern.match(pkg).group(2) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 2e58f2564..552b27842 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -22,8 +22,8 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return True # check if service is enabled - cmd = '/sbin/rc-update show default | grep %s' - is_enabled = self.cmd.run(cmd % entry.get('name')).success + result = self.cmd.run(["/sbin/rc-update", "show", "default"]) + is_enabled = entry.get("name") in result.stdout # check if init script exists try: @@ -34,8 +34,8 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return False # check if service is enabled - cmd = '/etc/init.d/%s status | grep started' - is_running = self.cmd.run(cmd % entry.attrib['name']).success + result = self.cmd.run(self.get_svc_command(entry, "status")) + is_running = "started" in result.stdout if entry.get('status') == 'on' and not (is_enabled and is_running): entry.set('current_status', 'off') @@ -70,9 +70,9 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra rc-update services.""" - cmd = '/bin/rc-status -s' allsrv = [line.split()[0] - for line in self.cmd.run(cmd).stdout.splitlines() + for line in self.cmd.run(['/bin/rc-status', + '-s']).stdout.splitlines() if 'started' in line] self.logger.debug('Found active services:') self.logger.debug(allsrv) diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index fa3d22e1d..1ab867215 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -120,8 +120,9 @@ class VCS(Bcfg2.Client.Tools.Tool): def Installsvn(self, entry): """Checkout contents from a svn repository""" # pylint: disable=E1101 + client = pysvn.Client() try: - client = pysvn.Client.update(entry.get('name'), recurse=True) + client.update(entry.get('name'), recurse=True) except pysvn.ClientError: self.logger.error("Failed to update repository", exc_info=1) return False diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 648d30d15..57ca06e77 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -125,7 +125,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): ('Package', 'rpm'), ('Path', 'ignore')] - __req__ = {'Package': ['name'], + __req__ = {'Package': ['type'], 'Path': ['type']} conflicts = ['RPM'] @@ -290,6 +290,17 @@ class YUM(Bcfg2.Client.Tools.PkgTool): return self.yumbase.rpmdb.returnGPGPubkeyPackages() return self.yumbase.rpmdb.searchNevra(name='gpg-pubkey') + def missing_attrs(self, entry): + """ Implementing from superclass to check for existence of either + name or group attribute for Package entry in the case of a YUM + group. """ + missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) + + if entry.get('name', None) == None and \ + entry.get('group', None) == None: + missing += ['name', 'group'] + return missing + def _verifyHelper(self, pkg_obj): """ _verifyHelper primarly deals with a yum bug where the pkg_obj.verify() method does not properly take into count multilib @@ -412,8 +423,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if entry.get('version', False) == 'auto': self._fixAutoVersion(entry) - self.logger.debug("Verifying package instances for %s" % - entry.get('name')) + if entry.get('group'): + self.logger.debug("Verifying packages for group %s" % + entry.get('group')) + else: + self.logger.debug("Verifying package instances for %s" % + entry.get('name')) self.verify_cache = dict() # Used for checking multilib packages self.modlists[entry] = modlist @@ -426,14 +441,58 @@ class YUM(Bcfg2.Client.Tools.PkgTool): entry.get('pkg_checks', 'true').lower() == 'true' pkg_verify = self.pkg_verify and \ entry.get('pkg_verify', 'true').lower() == 'true' + yum_group = False if entry.get('name') == 'gpg-pubkey': all_pkg_objs = self._getGPGKeysAsPackages() pkg_verify = False # No files here to verify + elif entry.get('group'): + entry.set('name', 'group:%s' % entry.get('group')) + yum_group = True + all_pkg_objs = [] + instances = [] + if self.yumbase.comps.has_group(entry.get('group')): + group = self.yumbase.comps.return_group(entry.get('group')) + group_packages = [p + for p, d in group.mandatory_packages.items() + if d] + group_type = entry.get('choose', 'default') + if group_type in ['default', 'optional', 'all']: + group_packages += [p + for p, d in + group.default_packages.items() + if d] + if group_type in ['optional', 'all']: + group_packages += [p + for p, d in + group.optional_packages.items() + if d] + if len(group_packages) == 0: + self.logger.error("No packages found for group %s" % + entry.get("group")) + for pkg in group_packages: + # create package instances for each package in yum group + instance = Bcfg2.Client.XML.SubElement(entry, 'Package') + instance.attrib['name'] = pkg + instance.attrib['type'] = 'yum' + try: + newest = \ + self.yumbase.pkgSack.returnNewestByName(pkg)[0] + instance.attrib['version'] = newest['version'] + instance.attrib['epoch'] = newest['epoch'] + instance.attrib['release'] = newest['release'] + except: # pylint: disable=W0702 + self.logger.info("Error finding newest package " + "for %s" % + pkg) + instance.attrib['version'] = 'any' + instances.append(instance) + else: + self.logger.error("Group not found: %s" % entry.get("group")) else: all_pkg_objs = \ self.yumbase.rpmdb.searchNevra(name=entry.get('name')) - if len(all_pkg_objs) == 0: + if len(all_pkg_objs) == 0 and yum_group != True: # Some sort of virtual capability? Try to resolve it all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name')) if len(all_pkg_objs) > 0: @@ -444,7 +503,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.info(" %s" % pkg) for inst in instances: - nevra = build_yname(entry.get('name'), inst) + if yum_group: + # the entry is not the name of the package + nevra = build_yname(inst.get('name'), inst) + all_pkg_objs = \ + self.yumbase.rpmdb.searchNevra(name=inst.get('name')) + else: + nevra = build_yname(entry.get('name'), inst) if nevra in pkg_cache: continue # Ignore duplicate instances else: @@ -458,7 +523,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): stat['version_fail'] = False stat['verify'] = {} stat['verify_fail'] = False - stat['pkg'] = entry + if yum_group: + stat['pkg'] = inst + else: + stat['pkg'] = entry stat['modlist'] = modlist if inst.get('verify_flags'): # this splits on either space or comma @@ -627,7 +695,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: install_only = False - if virt_pkg or (install_only and not self.setup['kevlar']): + if virt_pkg or \ + (install_only and not self.setup['kevlar']) or \ + yum_group: # virtual capability supplied, we are probably dealing # with multiple packages of different names. This check # doesn't make a lot of since in this case. diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 39a664df1..473d2bf09 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -130,7 +130,6 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) - def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -516,8 +515,8 @@ class SvcTool(Tool): :param service: The service entry to modify :type service: lxml.etree._Element - :returns: tuple - The return value from - :class:`Bcfg2.Client.Tools.Executor.run` + :returns: Bcfg2.Utils.ExecutorResult - The return value from + :class:`Bcfg2.Utils.Executor.run` """ self.logger.debug('Starting service %s' % service.get('name')) return self.cmd.run(self.get_svc_command(service, 'start')) @@ -527,8 +526,8 @@ class SvcTool(Tool): :param service: The service entry to modify :type service: lxml.etree._Element - :returns: tuple - The return value from - :class:`Bcfg2.Client.Tools.Executor.run` + :returns: Bcfg2.Utils.ExecutorResult - The return value from + :class:`Bcfg2.Utils.Executor.run` """ self.logger.debug('Stopping service %s' % service.get('name')) return self.cmd.run(self.get_svc_command(service, 'stop')) @@ -538,8 +537,8 @@ class SvcTool(Tool): :param service: The service entry to modify :type service: lxml.etree._Element - :returns: tuple - The return value from - :class:`Bcfg2.Client.Tools.Executor.run` + :returns: Bcfg2.Utils.ExecutorResult - The return value from + :class:`Bcfg2.Utils.Executor.run` """ self.logger.debug('Restarting service %s' % service.get('name')) restart_target = service.get('target', 'restart') @@ -553,7 +552,7 @@ class SvcTool(Tool): :returns: bool - True if the status command returned 0, False otherwise """ - return self.cmd.run(self.get_svc_command(service, 'status')) + return bool(self.cmd.run(self.get_svc_command(service, 'status'))) def Remove(self, services): if self.setup['servicemode'] != 'disabled': diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index beb534791..44c76303c 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -19,12 +19,13 @@ except ImportError: # urllib imports try: + from urllib import quote_plus from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ urlopen, HTTPError, URLError except ImportError: - from urllib.parse import urljoin, urlparse + from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen from urllib.error import HTTPError, URLError @@ -51,7 +52,8 @@ except ImportError: # xmlrpc imports try: - import xmlrpclib, SimpleXMLRPCServer + import xmlrpclib + import SimpleXMLRPCServer except ImportError: import xmlrpc.client as xmlrpclib import xmlrpc.server as SimpleXMLRPCServer @@ -73,6 +75,7 @@ try: except NameError: unicode = str + def u_str(string, encoding=None): """ print to file compatibility """ if sys.hexversion >= 0x03000000: diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index c2eac1e60..5bbc9ff96 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -105,7 +105,11 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): (self.encodePriority(self.facility, newrec.levelname.lower()), self.format(newrec)) try: - self.socket.send(msg.encode('ascii')) + try: + encoded = msg.encode('utf-8') + except UnicodeDecodeError: + encoded = msg + self.socket.send(encoded) except socket.error: for i in range(10): # pylint: disable=W0612 try: @@ -139,6 +143,10 @@ def add_console_handler(level=logging.DEBUG): console.setLevel(level) # tell the handler to use this format console.setFormatter(TermiosFormatter()) + try: + console.set_name("console") + except AttributeError: + console.name = "console" logging.root.addHandler(console) @@ -153,6 +161,10 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): syslog = FragmentingSysLogHandler(procname, ('localhost', 514), syslog_facility) + try: + syslog.set_name("syslog") + except AttributeError: + syslog.name = "syslog" syslog.setLevel(level) syslog.setFormatter( logging.Formatter('%(name)s[%(process)d]: %(message)s')) @@ -166,6 +178,10 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): def add_file_handler(to_file, level=logging.DEBUG): """Add a logging handler that logs to to_file.""" filelog = logging.FileHandler(to_file) + try: + filelog.set_name("file") + except AttributeError: + filelog.name = "file" filelog.setLevel(level) filelog.setFormatter( logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 74c488b45..e5aeccf4d 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -528,6 +528,11 @@ SERVER_FAM_IGNORE = \ 'SCCS', '.svn', '4913', '.gitignore'], cf=('server', 'ignore_files'), cook=list_split) +SERVER_FAM_BLOCK = \ + Option('FAM blocks on startup until all events are processed', + default=False, + cook=get_bool, + cf=('server', 'fam_blocking')) SERVER_LISTEN_ALL = \ Option('Listen on all interfaces', default=False, @@ -630,17 +635,19 @@ WEB_CFILE = \ default="/etc/bcfg2-web.conf", cmd='-W', odesc='<conffile>', - cf=('statistics', 'config'),) + cf=('reporting', 'config'), + deprecated_cf=('statistics', 'web_prefix'),) DJANGO_TIME_ZONE = \ Option('Django timezone', default=None, - cf=('statistics', 'time_zone'),) + cf=('reporting', 'time_zone'), + deprecated_cf=('statistics', 'web_prefix'),) DJANGO_DEBUG = \ Option('Django debug', default=None, - cf=('statistics', 'web_debug'), + cf=('reporting', 'web_debug'), + deprecated_cf=('statistics', 'web_prefix'), cook=get_bool,) -# Django options DJANGO_WEB_PREFIX = \ Option('Web prefix', default=None, @@ -824,7 +831,7 @@ CLIENT_COMMAND_TIMEOUT = \ # bcfg2-test and bcfg2-lint options TEST_NOSEOPTS = \ - Option('Options to pass to nosetests', + Option('Options to pass to nosetests. Only honored with --children 0', default=[], cmd='--nose-options', odesc='<opts>', @@ -839,6 +846,21 @@ TEST_IGNORE = \ cf=('bcfg2_test', 'ignore_entries'), cook=list_split, long_arg=True) +TEST_CHILDREN = \ + Option('Spawn this number of children for bcfg2-test (python 2.6+)', + default=0, + cmd='--children', + odesc='<children>', + cf=('bcfg2_test', 'children'), + cook=int, + long_arg=True) +TEST_XUNIT = \ + Option('Output an XUnit result file with --children', + default=None, + cmd='--xunit', + odesc='<xunit file>', + cf=('bcfg2_test', 'xunit'), + long_arg=True) LINT_CONFIG = \ Option('Specify bcfg2-lint configuration file', default='/etc/bcfg2-lint.conf', @@ -1019,7 +1041,7 @@ CRYPT_STDOUT = \ cmd='--stdout', long_arg=True) CRYPT_PASSPHRASE = \ - Option('Encryption passphrase (name or passphrase)', + Option('Encryption passphrase name', default=None, cmd='-p', odesc='<passphrase>') @@ -1066,6 +1088,7 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, password=SERVER_PASSWORD, filemonitor=SERVER_FILEMONITOR, ignore=SERVER_FAM_IGNORE, + fam_blocking=SERVER_FAM_BLOCK, location=SERVER_LOCATION, key=SERVER_KEY, cert=SERVER_CERT, @@ -1172,6 +1195,16 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, REPORTING_COMMON_OPTIONS = dict(reporting_file_limit=REPORTING_FILE_LIMIT, reporting_transport=REPORTING_TRANSPORT) +TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, + test_ignore=TEST_IGNORE, + children=TEST_CHILDREN, + xunit=TEST_XUNIT, + validate=CFG_VALIDATION) + +INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, + max_copies=PARANOID_MAX_COPIES) +INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) +INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) class OptionParser(OptionSet): """ OptionParser bootstraps option parsing, getting the value of diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index bca4a9c1e..3b2c0ccfa 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -16,6 +16,7 @@ from Bcfg2.Reporting.Storage.base import StorageBase, StorageError from Bcfg2.Server.Plugin.exceptions import PluginExecutionError from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.db.models import FieldDoesNotExist from django.core.cache import cache from django.db import transaction @@ -30,6 +31,230 @@ class DjangoORM(StorageBase): super(DjangoORM, self).__init__(setup) self.size_limit = setup.get('reporting_file_limit') + def _import_default(self, entry, state, entrytype=None, defaults=None, + mapping=None, boolean=None, xforms=None): + """ Default entry importer. Maps the entry (in state + ``state``) to an appropriate *Entry object; by default, this + is determined by the entry tag, e.g., from an Action entry an + ActionEntry object is created. This can be overridden with + ``entrytype``, which should be the class to instantiate for + this entry. + + ``defaults`` is an optional mapping of <attribute + name>:<value> that will be used to set the default values for + various attributes. + + ``mapping`` is a mapping of <field name>:<attribute name> that + can be used to map fields that are named differently on the + XML entry and in the database model. + + ``boolean`` is a list of attribute names that should be + treated as booleans. + + ``xforms`` is a dict of <attribute name>:<function>, where the + given function will be applied to the value of the named + attribute before trying to store it in the database. + """ + if entrytype is None: + entrytype = globals()["%sEntry" % entry.tag] + if defaults is None: + defaults = dict() + if mapping is None: + mapping = dict() + if boolean is None: + boolean = [] + if xforms is None: + xforms = dict() + mapping['exists'] = 'current_exists' + defaults['current_exists'] = 'true' + boolean.append("current_exists") + + def boolean_xform(val): + try: + return val.lower() == "true" + except AttributeError: + return False + + for attr in boolean + ["current_exists"]: + xforms[attr] = boolean_xform + act_dict = dict(state=state) + for fieldname in entrytype._meta.get_all_field_names(): + if fieldname in ['id', 'hash_key', 'state']: + continue + try: + field = entrytype._meta.get_field(fieldname) + except FieldDoesNotExist: + continue + attrname = mapping.get(fieldname, fieldname) + val = entry.get(fieldname, defaults.get(attrname)) + act_dict[fieldname] = xforms.get(attrname, lambda v: v)(val) + self.logger.debug("Adding %s:%s" % (entry.tag, entry.get("name"))) + return entrytype.entry_get_or_create(act_dict) + + def _import_Action(self, entry, state): + return self._import_default(entry, state, + defaults=dict(status='check', rc=-1), + mapping=dict(output="rc")) + + def _import_Package(self, entry, state): + name = entry.get('name') + exists = entry.get('current_exists', default="true").lower() == "true" + act_dict = dict(name=name, state=state, exists=exists, + target_version=entry.get('version', default=''), + current_version=entry.get('current_version', + default='')) + + # extra entries are a bit different. They can have Instance + # objects + if not act_dict['target_version']: + for instance in entry.findall("Instance"): + # FIXME - this probably only works for rpms + release = instance.get('release', '') + arch = instance.get('arch', '') + act_dict['current_version'] = instance.get('version') + if release: + act_dict['current_version'] += "-" + release + if arch: + act_dict['current_version'] += "." + arch + self.logger.debug("Adding package %s %s" % + (name, act_dict['current_version'])) + return PackageEntry.entry_get_or_create(act_dict) + else: + self.logger.debug("Adding package %s %s" % + (name, act_dict['target_version'])) + + # not implemented yet + act_dict['verification_details'] = \ + entry.get('verification_details', '') + return PackageEntry.entry_get_or_create(act_dict) + + def _import_Path(self, entry, state): + name = entry.get('name') + exists = entry.get('current_exists', default="true").lower() == "true" + path_type = entry.get("type").lower() + act_dict = dict(name=name, state=state, exists=exists, + path_type=path_type) + + target_dict = dict( + owner=entry.get('owner', default="root"), + group=entry.get('group', default="root"), + mode=entry.get('mode', default=entry.get('perms', + default="")) + ) + fperm, created = FilePerms.objects.get_or_create(**target_dict) + act_dict['target_perms'] = fperm + + current_dict = dict( + owner=entry.get('current_owner', default=""), + group=entry.get('current_group', default=""), + mode=entry.get('current_mode', + default=entry.get('current_perms', default="")) + ) + fperm, created = FilePerms.objects.get_or_create(**current_dict) + act_dict['current_perms'] = fperm + + if path_type in ('symlink', 'hardlink'): + act_dict['target_path'] = entry.get('to', default="") + act_dict['current_path'] = entry.get('current_to', default="") + self.logger.debug("Adding link %s" % name) + return LinkEntry.entry_get_or_create(act_dict) + elif path_type == 'device': + # TODO devices + self.logger.warn("device path types are not supported yet") + return + + # TODO - vcs output + act_dict['detail_type'] = PathEntry.DETAIL_UNUSED + if path_type == 'directory' and entry.get('prune', 'false') == 'true': + unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + if unpruned_elist: + act_dict['detail_type'] = PathEntry.DETAIL_PRUNED + act_dict['details'] = "\n".join(unpruned_elist) + elif entry.get('sensitive', 'false').lower() == 'true': + act_dict['detail_type'] = PathEntry.DETAIL_SENSITIVE + else: + cdata = None + if entry.get('current_bfile', None): + act_dict['detail_type'] = PathEntry.DETAIL_BINARY + cdata = entry.get('current_bfile') + elif entry.get('current_bdiff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = b64decode(entry.get('current_bdiff')) + elif entry.get('current_diff', None): + act_dict['detail_type'] = PathEntry.DETAIL_DIFF + cdata = entry.get('current_bdiff') + if cdata: + if len(cdata) > self.size_limit: + act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT + act_dict['details'] = md5(cdata).hexdigest() + else: + act_dict['details'] = cdata + self.logger.debug("Adding path %s" % name) + return PathEntry.entry_get_or_create(act_dict) + # TODO - secontext + # TODO - acls + + def _import_Service(self, entry, state): + return self._import_default(entry, state, + defaults=dict(status='', + current_status=''), + mapping=dict(status='target_status')) + + def _import_SEBoolean(self, entry, state): + return self._import_default( + entry, state, + xforms=dict(value=lambda v: v.lower() == "on")) + + def _import_SEFcontext(self, entry, state): + return self._import_default(entry, state, + defaults=dict(filetype='all')) + + def _import_SEInterface(self, entry, state): + return self._import_default(entry, state) + + def _import_SEPort(self, entry, state): + return self._import_default(entry, state) + + def _import_SENode(self, entry, state): + return self._import_default(entry, state) + + def _import_SELogin(self, entry, state): + return self._import_default(entry, state) + + def _import_SEUser(self, entry, state): + return self._import_default(entry, state) + + def _import_SEPermissive(self, entry, state): + return self._import_default(entry, state) + + def _import_SEModule(self, entry, state): + return self._import_default(entry, state, + defaults=dict(disabled='false'), + boolean=['disabled', 'current_disabled']) + + def _import_POSIXUser(self, entry, state): + defaults = dict(group=entry.get("name"), + gecos=entry.get("name"), + shell='/bin/bash', + uid=entry.get("current_uid")) + if entry.get('name') == 'root': + defaults['home'] = '/root' + else: + defaults['home'] = '/home/%s' % entry.get('name') + + # TODO: supplementary group membership + return self._import_default(entry, state, defaults=defaults) + + def _import_POSIXGroup(self, entry, state): + return self._import_default( + entry, state, + defaults=dict(gid=entry.get("current_gid"))) + + def _import_unknown(self, entry, _): + self.logger.error("Unknown type %s not handled by reporting yet" % + entry.tag) + return None + @transaction.commit_on_success def _import_interaction(self, interaction): """Real import function""" @@ -46,13 +271,15 @@ class DjangoORM(StorageBase): cache.set(hostname, client) timestamp = datetime(*strptime(stats.get('time'))[0:6]) - if len(Interaction.objects.filter(client=client, timestamp=timestamp)) > 0: + if len(Interaction.objects.filter(client=client, + timestamp=timestamp)) > 0: self.logger.warn("Interaction for %s at %s already exists" % (hostname, timestamp)) return if 'profile' in metadata: - profile, created = Group.objects.get_or_create(name=metadata['profile']) + profile, created = \ + Group.objects.get_or_create(name=metadata['profile']) else: profile = None inter = Interaction(client=client, @@ -65,10 +292,10 @@ class DjangoORM(StorageBase): server=server, profile=profile) inter.save() - self.logger.debug("Interaction for %s at %s with INSERTED in to db" % + self.logger.debug("Interaction for %s at %s with INSERTED in to db" % (client.id, timestamp)) - #FIXME - this should be more efficient + # FIXME - this should be more efficient for group_name in metadata['groups']: group = cache.get("GROUP_" + group_name) if not group: @@ -76,12 +303,13 @@ class DjangoORM(StorageBase): if created: self.logger.debug("Added group %s" % group) cache.set("GROUP_" + group_name, group) - + inter.groups.add(group) - for bundle_name in metadata['bundles']: + for bundle_name in metadata.get('bundles', []): bundle = cache.get("BUNDLE_" + bundle_name) if not bundle: - bundle, created = Bundle.objects.get_or_create(name=bundle_name) + bundle, created = \ + Bundle.objects.get_or_create(name=bundle_name) if created: self.logger.debug("Added bundle %s" % bundle) cache.set("BUNDLE_" + bundle_name, bundle) @@ -94,130 +322,26 @@ class DjangoORM(StorageBase): pattern = [('Bad/*', TYPE_BAD), ('Extra/*', TYPE_EXTRA), ('Modified/*', TYPE_MODIFIED)] - updates = dict(failures=[], paths=[], packages=[], actions=[], services=[]) + updates = dict([(etype, []) for etype in Interaction.entry_types]) for (xpath, state) in pattern: for entry in stats.findall(xpath): counter_fields[state] = counter_fields[state] + 1 - entry_type = entry.tag - name = entry.get('name') - exists = entry.get('current_exists', default="true").lower() == "true" - # handle server failures differently failure = entry.get('failure', '') if failure: - act_dict = dict(name=name, entry_type=entry_type, - message=failure) + act_dict = dict(name=entry.get("name"), + entry_type=entry.tag, + message=failure) newact = FailureEntry.entry_get_or_create(act_dict) updates['failures'].append(newact) continue - act_dict = dict(name=name, state=state, exists=exists) - - if entry_type == 'Action': - act_dict['status'] = entry.get('status', default="check") - act_dict['output'] = entry.get('rc', default=-1) - self.logger.debug("Adding action %s" % name) - updates['actions'].append(ActionEntry.entry_get_or_create(act_dict)) - elif entry_type == 'Package': - act_dict['target_version'] = entry.get('version', default='') - act_dict['current_version'] = entry.get('current_version', default='') - - # extra entries are a bit different. They can have Instance objects - if not act_dict['target_version']: - for instance in entry.findall("Instance"): - #TODO - this probably only works for rpms - release = instance.get('release', '') - arch = instance.get('arch', '') - act_dict['current_version'] = instance.get('version') - if release: - act_dict['current_version'] += "-" + release - if arch: - act_dict['current_version'] += "." + arch - self.logger.debug("Adding package %s %s" % (name, act_dict['current_version'])) - updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) - else: - - self.logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) - - # not implemented yet - act_dict['verification_details'] = entry.get('verification_details', '') - updates['packages'].append(PackageEntry.entry_get_or_create(act_dict)) - - elif entry_type == 'Path': - path_type = entry.get("type").lower() - act_dict['path_type'] = path_type - - target_dict = dict( - owner=entry.get('owner', default="root"), - group=entry.get('group', default="root"), - mode=entry.get('mode', default=entry.get('perms', default="")) - ) - fperm, created = FilePerms.objects.get_or_create(**target_dict) - act_dict['target_perms'] = fperm - - current_dict = dict( - owner=entry.get('current_owner', default=""), - group=entry.get('current_group', default=""), - mode=entry.get('current_mode', - default=entry.get('current_perms', default="")) - ) - fperm, created = FilePerms.objects.get_or_create(**current_dict) - act_dict['current_perms'] = fperm - - if path_type in ('symlink', 'hardlink'): - act_dict['target_path'] = entry.get('to', default="") - act_dict['current_path'] = entry.get('current_to', default="") - self.logger.debug("Adding link %s" % name) - updates['paths'].append(LinkEntry.entry_get_or_create(act_dict)) - continue - elif path_type == 'device': - #TODO devices - self.logger.warn("device path types are not supported yet") - continue - - # TODO - vcs output - act_dict['detail_type'] = PathEntry.DETAIL_UNUSED - if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] - if unpruned_elist: - act_dict['detail_type'] = PathEntry.DETAIL_PRUNED - act_dict['details'] = "\n".join(unpruned_elist) - elif entry.get('sensitive', 'false').lower() == 'true': - act_dict['detail_type'] = PathEntry.DETAIL_SENSITIVE - else: - cdata = None - if entry.get('current_bfile', None): - act_dict['detail_type'] = PathEntry.DETAIL_BINARY - cdata = entry.get('current_bfile') - elif entry.get('current_bdiff', None): - act_dict['detail_type'] = PathEntry.DETAIL_DIFF - cdata = b64decode(entry.get('current_bdiff')) - elif entry.get('current_diff', None): - act_dict['detail_type'] = PathEntry.DETAIL_DIFF - cdata = entry.get('current_bdiff') - if cdata: - if len(cdata) > self.size_limit: - act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT - act_dict['details'] = md5(cdata).hexdigest() - else: - act_dict['details'] = cdata - self.logger.debug("Adding path %s" % name) - updates['paths'].append(PathEntry.entry_get_or_create(act_dict)) - - - #TODO - secontext - #TODO - acls - - elif entry_type == 'Service': - act_dict['target_status'] = entry.get('status', default='') - act_dict['current_status'] = entry.get('current_status', default='') - self.logger.debug("Adding service %s" % name) - updates['services'].append(ServiceEntry.entry_get_or_create(act_dict)) - elif entry_type == 'SELinux': - self.logger.info("SELinux not implemented yet") - else: - self.logger.error("Unknown type %s not handled by reporting yet" % entry_type) + updatetype = entry.tag.lower() + "s" + update = getattr(self, "_import_%s" % entry.tag, + self._import_unknown)(entry, state) + if update is not None: + updates[updatetype].append(update) inter.bad_count = counter_fields[TYPE_BAD] inter.modified_count = counter_fields[TYPE_MODIFIED] @@ -227,15 +351,16 @@ class DjangoORM(StorageBase): # batch this for sqlite i = 0 while(i < len(updates[entry_type])): - getattr(inter, entry_type).add(*updates[entry_type][i:i+100]) + getattr(inter, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 # performance metrics for times in stats.findall('OpStamps'): for metric, value in list(times.items()): - Performance(interaction=inter, metric=metric, value=value).save() + Performance(interaction=inter, + metric=metric, + value=value).save() - def import_interaction(self, interaction): """Import the data into the backend""" @@ -245,7 +370,6 @@ class DjangoORM(StorageBase): self.logger.error("Failed to import interaction: %s" % traceback.format_exc().splitlines()[-1]) - def validate(self): """Validate backend storage. Should be called once when loaded""" diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index 30ea39263..0a0f032e5 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -36,7 +36,8 @@ class LocalFilesystem(TransportBase): def set_debug(self, debug): rv = TransportBase.set_debug(self, debug) - self.fmon.set_debug(debug) + if self.fmon is not None: + self.fmon.set_debug(debug) return rv def start_monitor(self, collector): @@ -48,7 +49,8 @@ class LocalFilesystem(TransportBase): self.logger.error("File monitor driver %s not available; " "forcing to default" % setup['filemonitor']) fmon = Bcfg2.Server.FileMonitor.available['default'] - + if self.debug_flag: + self.fmon.set_debug(self.debug_flag) try: self.fmon = fmon(debug=self.debug_flag) self.logger.info("Using the %s file monitor" % diff --git a/src/lib/Bcfg2/Reporting/migrations/0005_add_selinux_entry_support.py b/src/lib/Bcfg2/Reporting/migrations/0005_add_selinux_entry_support.py new file mode 100644 index 000000000..d5f5d801a --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0005_add_selinux_entry_support.py @@ -0,0 +1,485 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'SELoginEntry' + db.create_table('Reporting_seloginentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('selinuxuser', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_selinuxuser', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + )) + db.send_create_signal('Reporting', ['SELoginEntry']) + + # Adding model 'SEUserEntry' + db.create_table('Reporting_seuserentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('roles', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_roles', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + ('prefix', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_prefix', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + )) + db.send_create_signal('Reporting', ['SEUserEntry']) + + # Adding model 'SEBooleanEntry' + db.create_table('Reporting_sebooleanentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('value', self.gf('django.db.models.fields.BooleanField')(default=True)), + )) + db.send_create_signal('Reporting', ['SEBooleanEntry']) + + # Adding model 'SENodeEntry' + db.create_table('Reporting_senodeentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + ('proto', self.gf('django.db.models.fields.CharField')(max_length=4)), + )) + db.send_create_signal('Reporting', ['SENodeEntry']) + + # Adding model 'SEFcontextEntry' + db.create_table('Reporting_sefcontextentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + ('filetype', self.gf('django.db.models.fields.CharField')(max_length=16)), + )) + db.send_create_signal('Reporting', ['SEFcontextEntry']) + + # Adding model 'SEInterfaceEntry' + db.create_table('Reporting_seinterfaceentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + )) + db.send_create_signal('Reporting', ['SEInterfaceEntry']) + + # Adding model 'SEPermissiveEntry' + db.create_table('Reporting_sepermissiveentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + )) + db.send_create_signal('Reporting', ['SEPermissiveEntry']) + + # Adding model 'SEModuleEntry' + db.create_table('Reporting_semoduleentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('disabled', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('current_disabled', self.gf('django.db.models.fields.BooleanField')(default=False)), + )) + db.send_create_signal('Reporting', ['SEModuleEntry']) + + # Adding model 'SEPortEntry' + db.create_table('Reporting_seportentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('current_selinuxtype', self.gf('django.db.models.fields.CharField')(max_length=128, null=True)), + )) + db.send_create_signal('Reporting', ['SEPortEntry']) + + # Adding M2M table for field sebooleans on 'Interaction' + db.create_table('Reporting_interaction_sebooleans', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('sebooleanentry', models.ForeignKey(orm['Reporting.sebooleanentry'], null=False)) + )) + db.create_unique('Reporting_interaction_sebooleans', ['interaction_id', 'sebooleanentry_id']) + + # Adding M2M table for field seports on 'Interaction' + db.create_table('Reporting_interaction_seports', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('seportentry', models.ForeignKey(orm['Reporting.seportentry'], null=False)) + )) + db.create_unique('Reporting_interaction_seports', ['interaction_id', 'seportentry_id']) + + # Adding M2M table for field sefcontexts on 'Interaction' + db.create_table('Reporting_interaction_sefcontexts', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('sefcontextentry', models.ForeignKey(orm['Reporting.sefcontextentry'], null=False)) + )) + db.create_unique('Reporting_interaction_sefcontexts', ['interaction_id', 'sefcontextentry_id']) + + # Adding M2M table for field senodes on 'Interaction' + db.create_table('Reporting_interaction_senodes', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('senodeentry', models.ForeignKey(orm['Reporting.senodeentry'], null=False)) + )) + db.create_unique('Reporting_interaction_senodes', ['interaction_id', 'senodeentry_id']) + + # Adding M2M table for field selogins on 'Interaction' + db.create_table('Reporting_interaction_selogins', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('seloginentry', models.ForeignKey(orm['Reporting.seloginentry'], null=False)) + )) + db.create_unique('Reporting_interaction_selogins', ['interaction_id', 'seloginentry_id']) + + # Adding M2M table for field seusers on 'Interaction' + db.create_table('Reporting_interaction_seusers', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('seuserentry', models.ForeignKey(orm['Reporting.seuserentry'], null=False)) + )) + db.create_unique('Reporting_interaction_seusers', ['interaction_id', 'seuserentry_id']) + + # Adding M2M table for field seinterfaces on 'Interaction' + db.create_table('Reporting_interaction_seinterfaces', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('seinterfaceentry', models.ForeignKey(orm['Reporting.seinterfaceentry'], null=False)) + )) + db.create_unique('Reporting_interaction_seinterfaces', ['interaction_id', 'seinterfaceentry_id']) + + # Adding M2M table for field sepermissives on 'Interaction' + db.create_table('Reporting_interaction_sepermissives', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('sepermissiveentry', models.ForeignKey(orm['Reporting.sepermissiveentry'], null=False)) + )) + db.create_unique('Reporting_interaction_sepermissives', ['interaction_id', 'sepermissiveentry_id']) + + # Adding M2M table for field semodules on 'Interaction' + db.create_table('Reporting_interaction_semodules', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('semoduleentry', models.ForeignKey(orm['Reporting.semoduleentry'], null=False)) + )) + db.create_unique('Reporting_interaction_semodules', ['interaction_id', 'semoduleentry_id']) + + + def backwards(self, orm): + # Deleting model 'SELoginEntry' + db.delete_table('Reporting_seloginentry') + + # Deleting model 'SEUserEntry' + db.delete_table('Reporting_seuserentry') + + # Deleting model 'SEBooleanEntry' + db.delete_table('Reporting_sebooleanentry') + + # Deleting model 'SENodeEntry' + db.delete_table('Reporting_senodeentry') + + # Deleting model 'SEFcontextEntry' + db.delete_table('Reporting_sefcontextentry') + + # Deleting model 'SEInterfaceEntry' + db.delete_table('Reporting_seinterfaceentry') + + # Deleting model 'SEPermissiveEntry' + db.delete_table('Reporting_sepermissiveentry') + + # Deleting model 'SEModuleEntry' + db.delete_table('Reporting_semoduleentry') + + # Deleting model 'SEPortEntry' + db.delete_table('Reporting_seportentry') + + # Removing M2M table for field sebooleans on 'Interaction' + db.delete_table('Reporting_interaction_sebooleans') + + # Removing M2M table for field seports on 'Interaction' + db.delete_table('Reporting_interaction_seports') + + # Removing M2M table for field sefcontexts on 'Interaction' + db.delete_table('Reporting_interaction_sefcontexts') + + # Removing M2M table for field senodes on 'Interaction' + db.delete_table('Reporting_interaction_senodes') + + # Removing M2M table for field selogins on 'Interaction' + db.delete_table('Reporting_interaction_selogins') + + # Removing M2M table for field seusers on 'Interaction' + db.delete_table('Reporting_interaction_seusers') + + # Removing M2M table for field seinterfaces on 'Interaction' + db.delete_table('Reporting_interaction_seinterfaces') + + # Removing M2M table for field sepermissives on 'Interaction' + db.delete_table('Reporting_interaction_sepermissives') + + # Removing M2M table for field semodules on 'Interaction' + db.delete_table('Reporting_interaction_semodules') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'mode'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': "orm['Reporting.Group']"}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'sebooleans': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEBooleanEntry']", 'symmetrical': 'False'}), + 'sefcontexts': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEFcontextEntry']", 'symmetrical': 'False'}), + 'seinterfaces': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEInterfaceEntry']", 'symmetrical': 'False'}), + 'selogins': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SELoginEntry']", 'symmetrical': 'False'}), + 'semodules': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEModuleEntry']", 'symmetrical': 'False'}), + 'senodes': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SENodeEntry']", 'symmetrical': 'False'}), + 'sepermissives': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPermissiveEntry']", 'symmetrical': 'False'}), + 'seports': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPortEntry']", 'symmetrical': 'False'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'seusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEUserEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.sebooleanentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEBooleanEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'value': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'Reporting.sefcontextentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEFcontextEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'filetype': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seinterfaceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEInterfaceEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seloginentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SELoginEntry'}, + 'current_selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.semoduleentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEModuleEntry'}, + 'current_disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.senodeentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SENodeEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'proto': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.sepermissiveentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPermissiveEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seportentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPortEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + }, + 'Reporting.seuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEUserEntry'}, + 'current_prefix': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'current_roles': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'prefix': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['Reporting']
\ No newline at end of file diff --git a/src/lib/Bcfg2/Reporting/migrations/0006_add_user_group_entry_support.py b/src/lib/Bcfg2/Reporting/migrations/0006_add_user_group_entry_support.py new file mode 100644 index 000000000..d86e663d5 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0006_add_user_group_entry_support.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'POSIXGroupEntry' + db.create_table('Reporting_posixgroupentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('gid', self.gf('django.db.models.fields.IntegerField')(null=True)), + ('current_gid', self.gf('django.db.models.fields.IntegerField')(null=True)), + )) + db.send_create_signal('Reporting', ['POSIXGroupEntry']) + + # Adding model 'POSIXUserEntry' + db.create_table('Reporting_posixuserentry', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)), + ('hash_key', self.gf('django.db.models.fields.BigIntegerField')(db_index=True)), + ('state', self.gf('django.db.models.fields.IntegerField')()), + ('exists', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('uid', self.gf('django.db.models.fields.IntegerField')(null=True)), + ('current_uid', self.gf('django.db.models.fields.IntegerField')(null=True)), + ('group', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('current_group', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)), + ('gecos', self.gf('django.db.models.fields.CharField')(max_length=1024)), + ('current_gecos', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)), + ('home', self.gf('django.db.models.fields.CharField')(max_length=1024)), + ('current_home', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)), + ('shell', self.gf('django.db.models.fields.CharField')(default='/bin/bash', max_length=1024)), + ('current_shell', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)), + )) + db.send_create_signal('Reporting', ['POSIXUserEntry']) + + # Adding M2M table for field posixusers on 'Interaction' + db.create_table('Reporting_interaction_posixusers', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('posixuserentry', models.ForeignKey(orm['Reporting.posixuserentry'], null=False)) + )) + db.create_unique('Reporting_interaction_posixusers', ['interaction_id', 'posixuserentry_id']) + + # Adding M2M table for field posixgroups on 'Interaction' + db.create_table('Reporting_interaction_posixgroups', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('interaction', models.ForeignKey(orm['Reporting.interaction'], null=False)), + ('posixgroupentry', models.ForeignKey(orm['Reporting.posixgroupentry'], null=False)) + )) + db.create_unique('Reporting_interaction_posixgroups', ['interaction_id', 'posixgroupentry_id']) + + + def backwards(self, orm): + # Deleting model 'POSIXGroupEntry' + db.delete_table('Reporting_posixgroupentry') + + # Deleting model 'POSIXUserEntry' + db.delete_table('Reporting_posixuserentry') + + # Removing M2M table for field posixusers on 'Interaction' + db.delete_table('Reporting_interaction_posixusers') + + # Removing M2M table for field posixgroups on 'Interaction' + db.delete_table('Reporting_interaction_posixgroups') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'mode'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'posixgroups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXGroupEntry']", 'symmetrical': 'False'}), + 'posixusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXUserEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': "orm['Reporting.Group']"}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'sebooleans': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEBooleanEntry']", 'symmetrical': 'False'}), + 'sefcontexts': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEFcontextEntry']", 'symmetrical': 'False'}), + 'seinterfaces': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEInterfaceEntry']", 'symmetrical': 'False'}), + 'selogins': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SELoginEntry']", 'symmetrical': 'False'}), + 'semodules': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEModuleEntry']", 'symmetrical': 'False'}), + 'senodes': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SENodeEntry']", 'symmetrical': 'False'}), + 'sepermissives': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPermissiveEntry']", 'symmetrical': 'False'}), + 'seports': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPortEntry']", 'symmetrical': 'False'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'seusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEUserEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.posixgroupentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXGroupEntry'}, + 'current_gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.posixuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXUserEntry'}, + 'current_gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_group': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'current_home': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_shell': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'group': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'home': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'shell': ('django.db.models.fields.CharField', [], {'default': "'/bin/bash'", 'max_length': '1024'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + 'Reporting.sebooleanentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEBooleanEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'value': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'Reporting.sefcontextentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEFcontextEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'filetype': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seinterfaceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEInterfaceEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seloginentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SELoginEntry'}, + 'current_selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.semoduleentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEModuleEntry'}, + 'current_disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.senodeentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SENodeEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'proto': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.sepermissiveentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPermissiveEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seportentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPortEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + }, + 'Reporting.seuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEUserEntry'}, + 'current_prefix': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'current_roles': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'prefix': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['Reporting']
\ No newline at end of file diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index ab2dc8418..4be509f53 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -11,21 +11,9 @@ except ImproperlyConfigured: from django.core.cache import cache from datetime import datetime, timedelta +from Bcfg2.Compat import cPickle + -try: - import cPickle as pickle -except: - import pickle - -KIND_CHOICES = ( - #These are the kinds of config elements - ('Package', 'Package'), - ('Path', 'directory'), - ('Path', 'file'), - ('Path', 'permissions'), - ('Path', 'symlink'), - ('Service', 'Service'), -) TYPE_GOOD = 0 TYPE_BAD = 1 TYPE_MODIFIED = 2 @@ -57,8 +45,8 @@ def hash_entry(entry_dict): for key in sorted(entry_dict.keys()): if key in ('id', 'hash_key') or key.startswith('_'): continue - dataset.append( (key, entry_dict[key]) ) - return hash(pickle.dumps(dataset)) + dataset.append((key, entry_dict[key])) + return hash(cPickle.dumps(dataset)) class Client(models.Model): @@ -121,7 +109,8 @@ class InteractionManager(models.Manager): class Interaction(models.Model): - """Models each reconfiguration operation interaction between client and server.""" + """ Models each reconfiguration operation interaction between + client and server. """ client = models.ForeignKey(Client, related_name="interactions") timestamp = models.DateTimeField(db_index=True) # Timestamp for this record state = models.CharField(max_length=32) # good/bad/modified/etc @@ -137,8 +126,24 @@ class Interaction(models.Model): packages = models.ManyToManyField("PackageEntry") paths = models.ManyToManyField("PathEntry") services = models.ManyToManyField("ServiceEntry") + sebooleans = models.ManyToManyField("SEBooleanEntry") + seports = models.ManyToManyField("SEPortEntry") + sefcontexts = models.ManyToManyField("SEFcontextEntry") + senodes = models.ManyToManyField("SENodeEntry") + selogins = models.ManyToManyField("SELoginEntry") + seusers = models.ManyToManyField("SEUserEntry") + seinterfaces = models.ManyToManyField("SEInterfaceEntry") + sepermissives = models.ManyToManyField("SEPermissiveEntry") + semodules = models.ManyToManyField("SEModuleEntry") + posixusers = models.ManyToManyField("POSIXUserEntry") + posixgroups = models.ManyToManyField("POSIXGroupEntry") failures = models.ManyToManyField("FailureEntry") + entry_types = ('actions', 'packages', 'paths', 'services', 'sebooleans', + 'seports', 'sefcontexts', 'senodes', 'selogins', 'seusers', + 'seinterfaces', 'sepermissives', 'semodules', 'posixusers', + 'posixgroups') + # Formerly InteractionMetadata profile = models.ForeignKey("Group", related_name="+", null=True) groups = models.ManyToManyField("Group") @@ -157,7 +162,8 @@ class Interaction(models.Model): def percentbad(self): if not self.total_count == 0: - return ((self.total_count - self.good_count) / (float(self.total_count))) * 100 + return ((self.total_count - self.good_count) / + (float(self.total_count))) * 100 else: return 0 @@ -189,7 +195,8 @@ class Interaction(models.Model): self.client.save() # save again post update def delete(self): - '''Override the default delete. Allows us to remove Performance items''' + '''Override the default delete. Allows us to remove + Performance items ''' pitems = list(self.performance_items.all()) super(Interaction, self).delete() for perf in pitems: @@ -201,19 +208,19 @@ class Interaction(models.Model): def bad(self): rv = [] - for entry in ('actions', 'packages', 'paths', 'services'): + for entry in self.entry_types: rv.extend(getattr(self, entry).filter(state=TYPE_BAD)) return rv def modified(self): rv = [] - for entry in ('actions', 'packages', 'paths', 'services'): + for entry in self.entry_types: rv.extend(getattr(self, entry).filter(state=TYPE_MODIFIED)) return rv def extra(self): rv = [] - for entry in ('actions', 'packages', 'paths', 'services'): + for entry in self.entry_types: rv.extend(getattr(self, entry).filter(state=TYPE_EXTRA)) return rv @@ -325,7 +332,6 @@ class BaseEntry(models.Model): self.hash_key = hash_entry(self.__dict__) super(BaseEntry, self).save(*args, **kwargs) - def class_name(self): return self.__class__.__name__ @@ -333,7 +339,6 @@ class BaseEntry(models.Model): """todo""" return [] - @classmethod def entry_from_name(cls, name): try: @@ -344,28 +349,26 @@ class BaseEntry(models.Model): except KeyError: raise ValueError("Invalid type %s" % name) - @classmethod def entry_from_type(cls, etype): - for entry_cls in (ActionEntry, PackageEntry, PathEntry, ServiceEntry): + for entry_cls in ENTRY_CLASSES: if etype == entry_cls.ENTRY_TYPE: return entry_cls else: raise ValueError("Invalid type %s" % etype) - @classmethod def entry_get_or_create(cls, act_dict): """Helper to quickly lookup an object""" cls_name = cls().__class__.__name__ act_hash = hash_entry(act_dict) - + # TODO - get form cache and validate act_key = "%s_%s" % (cls_name, act_hash) newact = cache.get(act_key) if newact: return newact - + acts = cls.objects.filter(hash_key=act_hash) if len(acts) > 0: for act in acts: @@ -375,20 +378,18 @@ class BaseEntry(models.Model): #match found newact = act break - + # worst case, its new if not newact: newact = cls(**act_dict) newact.save(hash_key=act_hash) - + cache.set(act_key, newact, 60 * 60) return newact - def is_failure(self): return isinstance(self, FailureEntry) - @classmethod def prune_orphans(cls): '''Remove unused entries''' @@ -397,7 +398,7 @@ class BaseEntry(models.Model): for x in cls.objects.filter(interaction__isnull=True).values("id")] i = 0 while i < len(cls_orphans): - cls.objects.filter(id__in=cls_orphans[i:i+100]).delete() + cls.objects.filter(id__in=cls_orphans[i:i + 100]).delete() i += 100 @@ -439,13 +440,161 @@ class FailureEntry(BaseEntry): class ActionEntry(SuccessEntry): - """ The new model for package information """ + """ Action entry """ status = models.CharField(max_length=128, default="check") output = models.IntegerField(default=0) ENTRY_TYPE = r"Action" +class SEBooleanEntry(SuccessEntry): + """ SELinux boolean """ + value = models.BooleanField(default=True) + + ENTRY_TYPE = r"SEBoolean" + + +class SEPortEntry(SuccessEntry): + """ SELinux port """ + selinuxtype = models.CharField(max_length=128) + current_selinuxtype = models.CharField(max_length=128, null=True) + + ENTRY_TYPE = r"SEPort" + + def selinuxtype_problem(self): + """Check for an selinux type problem.""" + if not self.current_selinuxtype: + return True + return self.selinuxtype != self.current_selinuxtype + + def short_list(self): + """Return a list of problems""" + rv = super(SEPortEntry, self).short_list() + if self.selinuxtype_problem(): + rv.append("Wrong SELinux type") + return rv + + +class SEFcontextEntry(SuccessEntry): + """ SELinux file context """ + selinuxtype = models.CharField(max_length=128) + current_selinuxtype = models.CharField(max_length=128, null=True) + filetype = models.CharField(max_length=16) + + ENTRY_TYPE = r"SEFcontext" + + def selinuxtype_problem(self): + """Check for an selinux type problem.""" + if not self.current_selinuxtype: + return True + return self.selinuxtype != self.current_selinuxtype + + def short_list(self): + """Return a list of problems""" + rv = super(SEFcontextEntry, self).short_list() + if self.selinuxtype_problem(): + rv.append("Wrong SELinux type") + return rv + + +class SENodeEntry(SuccessEntry): + """ SELinux node """ + selinuxtype = models.CharField(max_length=128) + current_selinuxtype = models.CharField(max_length=128, null=True) + proto = models.CharField(max_length=4) + + ENTRY_TYPE = r"SENode" + + def selinuxtype_problem(self): + """Check for an selinux type problem.""" + if not self.current_selinuxtype: + return True + return self.selinuxtype != self.current_selinuxtype + + def short_list(self): + """Return a list of problems""" + rv = super(SENodeEntry, self).short_list() + if self.selinuxtype_problem(): + rv.append("Wrong SELinux type") + return rv + + +class SELoginEntry(SuccessEntry): + """ SELinux login """ + selinuxuser = models.CharField(max_length=128) + current_selinuxuser = models.CharField(max_length=128, null=True) + + ENTRY_TYPE = r"SELogin" + + +class SEUserEntry(SuccessEntry): + """ SELinux user """ + roles = models.CharField(max_length=128) + current_roles = models.CharField(max_length=128, null=True) + prefix = models.CharField(max_length=128) + current_prefix = models.CharField(max_length=128, null=True) + + ENTRY_TYPE = r"SEUser" + + +class SEInterfaceEntry(SuccessEntry): + """ SELinux interface """ + selinuxtype = models.CharField(max_length=128) + current_selinuxtype = models.CharField(max_length=128, null=True) + + ENTRY_TYPE = r"SEInterface" + + def selinuxtype_problem(self): + """Check for an selinux type problem.""" + if not self.current_selinuxtype: + return True + return self.selinuxtype != self.current_selinuxtype + + def short_list(self): + """Return a list of problems""" + rv = super(SEInterfaceEntry, self).short_list() + if self.selinuxtype_problem(): + rv.append("Wrong SELinux type") + return rv + + +class SEPermissiveEntry(SuccessEntry): + """ SELinux permissive domain """ + ENTRY_TYPE = r"SEPermissive" + + +class SEModuleEntry(SuccessEntry): + """ SELinux module """ + disabled = models.BooleanField(default=False) + current_disabled = models.BooleanField(default=False) + + ENTRY_TYPE = r"SEModule" + + +class POSIXUserEntry(SuccessEntry): + """ POSIX user """ + uid = models.IntegerField(null=True) + current_uid = models.IntegerField(null=True) + group = models.CharField(max_length=64) + current_group = models.CharField(max_length=64, null=True) + gecos = models.CharField(max_length=1024) + current_gecos = models.CharField(max_length=1024, null=True) + home = models.CharField(max_length=1024) + current_home = models.CharField(max_length=1024, null=True) + shell = models.CharField(max_length=1024, default='/bin/bash') + current_shell = models.CharField(max_length=1024, null=True) + + ENTRY_TYPE = r"POSIXUser" + + +class POSIXGroupEntry(SuccessEntry): + """ POSIX group """ + gid = models.IntegerField(null=True) + current_gid = models.IntegerField(null=True) + + ENTRY_TYPE = r"POSIXGroup" + + class PackageEntry(SuccessEntry): """ The new model for package information """ @@ -455,7 +604,7 @@ class PackageEntry(SuccessEntry): verification_details = models.TextField(default="") ENTRY_TYPE = r"Package" - #TODO - prune + # TODO - prune def version_problem(self): """Check for a version problem.""" @@ -612,3 +761,7 @@ class ServiceEntry(SuccessEntry): return rv +ENTRY_TYPES = (ActionEntry, PackageEntry, PathEntry, ServiceEntry, + SEBooleanEntry, SEPortEntry, SEFcontextEntry, SENodeEntry, + SELoginEntry, SEUserEntry, SEInterfaceEntry, SEPermissiveEntry, + SEModuleEntry) diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 533dcc79e..c73339911 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -88,7 +88,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.0rc2</span> + <span>Bcfg2 Version 1.3.1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 737760252..259414399 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -45,36 +45,47 @@ div.entry_list h3 { {% endif %} {# Really need a better test here #} -{% if item.mdoe_problem or item.status_problem or item.linkentry.link_problem or item.version_problem %} +{% if item.mode_problem or item.status_problem or item.linkentry.link_problem or item.version_problem %} <table class='entry_list'> <tr id='table_list_header'> <td style='text-align: right;'>Problem Type</td><td>Expected</td><td style='border-bottom: 1px solid #98DBCC;'>Found</td></tr> {% if item.mode_problem %} {% if item.current_perms.owner %} - <tr><td style='text-align: right'><b>Owner</b></td><td>{{item.target_perms.owner}}</td> + <tr><td style='text-align: right'><b>Owner</b></td> + <td>{{item.target_perms.owner}}</td> <td>{{item.current_perms.owner}}</td></tr> {% endif %} {% if item.current_perms.group %} - <tr><td style='text-align: right'><b>Group</b></td><td>{{item.target_perms.group}}</td> + <tr><td style='text-align: right'><b>Group</b></td> + <td>{{item.target_perms.group}}</td> <td>{{item.current_perms.group}}</td></tr> {% endif %} {% if item.current_perms.mode%} - <tr><td style='text-align: right'><b>Mode</b></td><td>{{item.target_perms.mode}}</td> + <tr><td style='text-align: right'><b>Permissions</b> + </td><td>{{item.target_perms.mode}}</td> <td>{{item.current_perms.mode}}</td></tr> {% endif %} {% endif %} {% if item.status_problem %} - <tr><td style='text-align: right'><b>Status</b></td><td>{{item.target_status}}</td> - <td>{{item.current_status}}</td></tr> + <tr><td style='text-align: right'><b>Status</b></td> + <td>{{item.target_status}}</td> + <td>{{item.current_status}}</td></tr> {% endif %} {% if item.linkentry.link_problem %} - <tr><td style='text-align: right'><b>{{item.get_path_type_display}}</b></td><td>{{item.linkentry.target_path}}</td> - <td>{{item.linkentry.current_path}}</td></tr> + <tr><td style='text-align: right'><b>{{item.get_path_type_display}}</b></td> + <td>{{item.linkentry.target_path}}</td> + <td>{{item.linkentry.current_path}}</td></tr> {% endif %} {% if item.version_problem %} - <tr><td style='text-align: right'><b>Package Version</b></td><td>{{item.target_version|cut:"("|cut:")"}}</td> + <tr><td style='text-align: right'><b>Package Version</b></td> + <td>{{item.target_version|cut:"("|cut:")"}}</td> <td>{{item.current_version|cut:"("|cut:")"}}</td></tr> {% endif %} + {% if item.selinuxtype_problem %} + <tr><td style='text-align: right'><b>SELinux Type</b></td> + <td>{{item.selinuxtype}}</td> + <td>{{item.current_selinuxtype}}</td></tr> + {% endif %} </table> {% endif %} @@ -92,7 +103,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html index 759415507..bb4f650d1 100644 --- a/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html +++ b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html @@ -16,7 +16,7 @@ <label for="id_group">Group filter:</label> <select id="id_group" name="group" onchange="javascript:url=document.forms['filter_form'].group.value; if(url) { location.href=url }"> {% for group, group_url, selected in groups %} - <option label="{{group}}" value="{{group_url}}" {% if selected %}selected {% endif %}/> + <option value="{{group_url}}" {% if selected %}selected {% endif %}>{{group}}</option> {% endfor %} </select> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 0341a18af..6cba7bf8c 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -161,7 +161,7 @@ def config_item(request, pk, entry_type, interaction=None): ts_end = ts_start + timedelta(days=1) associated_list = item.interaction_set.select_related('client').filter(\ timestamp__gte=ts_start, timestamp__lt=ts_end) - + if item.is_failure(): template = 'config_items/item-failure.html' else: @@ -184,7 +184,7 @@ def config_item_list(request, item_state, timestamp=None, **kwargs): current_clients = [q['id'] for q in _handle_filters(current_clients, **kwargs).values('id')] lists = [] - for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: + for etype in ENTRY_TYPES: ldata = etype.objects.filter(state=state, interaction__in=current_clients)\ .annotate(num_entries=Count('id')).select_related('linkentry', 'target_perms', 'current_perms') if len(ldata) > 0: @@ -218,7 +218,7 @@ def entry_status(request, entry_type, pk, timestamp=None, **kwargs): if it.pk not in seen: items.append((it, it.interaction_set.filter(pk__in=current_clients).order_by('client__name').select_related('client'))) seen.append(it.pk) - + return render_to_response('config_items/entry_status.html', {'entry': item, 'items': items, @@ -254,8 +254,8 @@ def common_problems(request, timestamp=None, threshold=None, group=None): else: current_clients = Interaction.objects.recent_ids(timestamp) lists = [] - for etype in ActionEntry, PackageEntry, PathEntry, ServiceEntry: - ldata = etype.objects.exclude(state=TYPE_GOOD).filter( + for etype in ENTRY_TYPES: + ldata = etype.objects.exclude(state=TYPE_GOOD).filter( interaction__in=current_clients).annotate(num_entries=Count('id')).filter(num_entries__gte=threshold)\ .order_by('-num_entries', 'name') if len(ldata) > 0: @@ -315,7 +315,8 @@ def client_detailed_list(request, timestamp=None, **kwargs): kwargs['orderby'] = "client__name" kwargs['sort'] = "client" - kwargs['interaction_base'] = Interaction.objects.recent(timestamp).select_related() + kwargs['interaction_base'] = \ + Interaction.objects.recent(timestamp).select_related() kwargs['page_limit'] = 0 return render_history_view(request, 'clients/detailed-list.html', **kwargs) @@ -330,16 +331,18 @@ def client_detail(request, hostname=None, pk=None): inter = client.interactions.get(pk=pk) maxdate = inter.timestamp - etypes = { TYPE_BAD: 'bad', TYPE_MODIFIED: 'modified', TYPE_EXTRA: 'extra' } + etypes = {TYPE_BAD: 'bad', + TYPE_MODIFIED: 'modified', + TYPE_EXTRA: 'extra'} edict = dict() for label in etypes.values(): edict[label] = [] - for ekind in ('actions', 'packages', 'paths', 'services'): + for ekind in inter.entry_types: for ent in getattr(inter, ekind).all(): edict[etypes[ent.state]].append(ent) context['entry_types'] = edict - context['interaction']=inter + context['interaction'] = inter return render_history_view(request, 'clients/detail.html', page_limit=5, client=client, maxdate=maxdate, context=context) @@ -356,7 +359,8 @@ def client_manage(request): client.expiration = datetime.now() client.save() message = "Expiration for %s set to %s." % \ - (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) + (client_name, + client.expiration.strftime("%Y-%m-%d %H:%M:%S")) elif client_action == 'unexpire': client.expiration = None client.save() diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 724da124b..884405786 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -12,6 +12,7 @@ from Bcfg2.Utils import Executor import Bcfg2.Server.Admin import Bcfg2.Server.Plugin import Bcfg2.Options +import Bcfg2.Server.Plugins.Metadata from Bcfg2.Compat import input # pylint: disable=W0622 # default config file @@ -171,8 +172,6 @@ class Init(Bcfg2.Server.Admin.Mode): self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - # Parse options setup = Bcfg2.Options.get_option_parser() setup.add_options(dict(configfile=Bcfg2.Options.CFILE, @@ -218,7 +217,7 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the repository path.""" while True: newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) + self.data['repopath']) if newrepo != '': self.data['repopath'] = os.path.abspath(newrepo) if os.path.isdir(self.data['repopath']): @@ -296,7 +295,7 @@ class Init(Bcfg2.Server.Admin.Mode): "created [%s]: " % self.data['keypath']) if keypath: self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created" + certpath = safe_input("Path where Bcfg2 server cert will be created " "[%s]: " % self.data['certpath']) if certpath: self.data['certpath'] = certpath @@ -324,6 +323,16 @@ class Init(Bcfg2.Server.Admin.Mode): def init_repo(self): """Setup a new repo and create the content of the configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + confdata = CONFIG % (self.data['repopath'], ','.join(self.plugins), self.data['sendmail'], @@ -339,13 +348,3 @@ class Init(Bcfg2.Server.Admin.Mode): create_key(self.data['shostname'], self.data['keypath'], self.data['certpath'], self.data['country'], self.data['state'], self.data['location']) - - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 07f9e0588..c69e8b055 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -4,8 +4,9 @@ implementations inherit from. """ import os import sys import time -import select import atexit +import select +import signal import logging import inspect import threading @@ -119,13 +120,36 @@ class BaseCore(object): #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') + #: Log levels for the various logging handlers with debug True + #: and False. Each loglevel dict is a dict of ``logger name + #: => log level``; the logger names are set in + #: :mod:`Bcfg2.Logger`. The logger name ``default`` is + #: special, and will be used for any log handlers whose name + #: does not appear elsewhere in the dict. At a minimum, + #: ``default`` must be provided. + self._loglevels = {True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=level)} + + #: Used to keep track of the current debug state of the core. + self.debug_flag = False + + # enable debugging on the core now. debugging is enabled on + # everything else later + if self.setup['debug']: + self.set_core_debug(None, setup['debug']) + if 'ignore' not in self.setup: self.setup.add_option('ignore', SERVER_FAM_IGNORE) self.setup.reparse() + famargs = dict(filemonitor=self.setup['filemonitor'], debug=self.setup['debug'], ignore=self.setup['ignore']) - if self.setup['filemonitor'] not in Bcfg2.Server.FileMonitor.available: + try: + filemonitor = \ + Bcfg2.Server.FileMonitor.available[setup['filemonitor']] + except KeyError: self.logger.error("File monitor driver %s not available; " "forcing to default" % self.setup['filemonitor']) famargs['filemonitor'] = 'default' @@ -281,6 +305,14 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = self.setup['ca'] + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + self.shutdown() + os._exit(1) # pylint: disable=W0212 + + signal.signal(signal.SIGINT, hdlr) + #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -295,6 +327,10 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + if self.debug_flag: + # enable debugging on everything else. + self.plugins[plugin].set_debug(self.debug_flag) + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -392,6 +428,7 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ + self.logger.debug("Shutting down core...") if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() @@ -427,6 +464,8 @@ class BaseCore(object): hook. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Running %s hooks for %s" % (hook, + metadata.hostname)) start = time.time() try: for plugin in \ @@ -460,6 +499,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating structures for %s" % metadata.hostname) for plugin in \ self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): try: @@ -486,6 +526,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating goals for %s" % metadata.hostname) for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): try: plugin.validate_goals(metadata, data) @@ -506,6 +547,7 @@ class BaseCore(object): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of :class:`lxml.etree._Element` objects """ + self.logger.debug("Getting structures for %s" % metadata.hostname) structures = list(chain(*[struct.BuildStructures(metadata) for struct in self.structures])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] @@ -528,6 +570,7 @@ class BaseCore(object): structures to. Modified in-place. :type config: lxml.etree._Element """ + self.logger.debug("Binding structures for %s" % metadata.hostname) for astruct in structures: try: self.BindStructure(astruct, metadata) @@ -544,6 +587,9 @@ class BaseCore(object): :param metadata: Client metadata to bind structure for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Binding structure %s for %s" % + (structure.get("name", "unknown"), + metadata.hostname)) for entry in structure.getchildren(): if entry.tag.startswith("Bound"): entry.tag = entry.tag[5:] @@ -619,6 +665,7 @@ class BaseCore(object): :type client: string :returns: :class:`lxml.etree._Element` - A complete Bcfg2 configuration document """ + self.logger.debug("Building configuration for %s" % client) start = time.time() config = lxml.etree.Element("Configuration", version='2.0', revision=self.revision) @@ -718,6 +765,12 @@ class BaseCore(object): self.shutdown() raise + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): @@ -746,6 +799,7 @@ class BaseCore(object): :type mode: string :returns: list of Decision tuples ``(<entry tag>, <entry name>)`` """ + self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: @@ -816,6 +870,7 @@ class BaseCore(object): else: imd = self.metadata_cache.get(client_name, None) if not imd: + self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) for conn in self.connectors: grps = conn.get_additional_groups(imd) @@ -837,6 +892,7 @@ class BaseCore(object): :param statistics: The statistics document to process :type statistics: lxml.etree._Element """ + self.logger.debug("Processing statistics for %s" % client_name) meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': @@ -907,10 +963,12 @@ class BaseCore(object): def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - if self.plugins: - for pname, pinst in list(self.plugins.items()): - for mname in pinst.__rmi__: - rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + for pname, pinst in list(self.plugins.items()): + for mname in pinst.__rmi__: + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + famname = self.fam.__class__.__name__ + for mname in self.fam.__rmi__: + rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname) return rmi def _resolve_exposed_method(self, method_name): @@ -964,6 +1022,7 @@ class BaseCore(object): return func.__doc__ @exposed + @track_statistics() def DeclareVersion(self, address, version): """ Declare the client version. @@ -974,7 +1033,9 @@ class BaseCore(object): :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` """ - client = self.resolve_client(address)[0] + client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s is running Bcfg2 client version %s" % (client, + version)) try: self.metadata.set_version(client, version) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -996,6 +1057,7 @@ class BaseCore(object): """ resp = lxml.etree.Element('probes') client, metadata = self.resolve_client(address, cleanup_cache=True) + self.logger.debug("Getting probes for %s" % client) try: for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(metadata): @@ -1017,6 +1079,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client, metadata = self.resolve_client(address) + self.logger.debug("Receiving probe data from %s" % client) if self.metadata_cache_mode == 'cautious': # clear the metadata cache right after building the # metadata object; that way the cache is cleared for any @@ -1063,6 +1126,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -1171,22 +1235,35 @@ class BaseCore(object): :type address: tuple :returns: bool - The new debug state of the FAM """ - for plugin in self.plugins.values(): - plugin.toggle_debug() - return self.toggle_fam_debug(address) + return self.set_debug(address, not self.debug_flag) @exposed - def toggle_fam_debug(self, _): + def toggle_core_debug(self, address): + """ Toggle debug status of the server core + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - The new debug state of the FAM + """ + return self.set_core_debug(address, not self.debug_flag) + + @exposed + def toggle_fam_debug(self, address): """ Toggle debug status of the FAM :returns: bool - The new debug state of the FAM """ - return self.fam.toggle_debug() + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.toggle_debug() @exposed def set_debug(self, address, debug): """ Explicitly set debug status of the FAM and all plugins + :param address: Client (address, hostname) pair + :type address: tuple :param debug: The new debug status. This can either be a boolean, or a string describing the state (e.g., "true" or "false"; case-insensitive) @@ -1197,10 +1274,33 @@ class BaseCore(object): debug = debug.lower() == "true" for plugin in self.plugins.values(): plugin.set_debug(debug) - return self.set_fam_debug(address, debug) + rv = self.set_core_debug(address, debug) + return self.fam.set_debug(debug) and rv + + @exposed + def set_core_debug(self, _, debug): + """ Explicity set debug status of the server core + + :param debug: The new debug status. This can either be a + boolean, or a string describing the state (e.g., + "true" or "false"; case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state of the FAM + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + self.debug_flag = debug + self.logger.info("Core: debug = %s" % debug) + levels = self._loglevels[self.debug_flag] + for handler in logging.root.handlers: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + handler.setLevel(level) + return self.debug_flag @exposed - def set_fam_debug(self, _, debug): + def set_fam_debug(self, address, debug): """ Explicitly set debug status of the FAM :param debug: The new debug status of the FAM. This can @@ -1212,4 +1312,7 @@ class BaseCore(object): """ if debug not in [True, False]: debug = debug.lower() == "true" - return self.fam.set_debug(debug) + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 178a47b1a..cdd52dbb9 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -2,6 +2,7 @@ support. """ import os +import errno import logging import pyinotify from Bcfg2.Compat import reduce # pylint: disable=W0622 @@ -15,6 +16,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): """ File monitor backend with `inotify <http://inotify.aiken.cz/>`_ support. """ + __rmi__ = Pseudo.__rmi__ + ["list_watches", "list_paths"] + #: Inotify is the best FAM backend, so it gets a very high #: priority __priority__ = 99 @@ -182,6 +185,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): try: watchdir = self.watches_by_path[watch_path] except KeyError: + if not os.path.exists(watch_path): + raise OSError(errno.ENOENT, + "No such file or directory: '%s'" % path) watchdir = self.watchmgr.add_watch(watch_path, self.mask, quiet=False)[watch_path] self.watches_by_path[watch_path] = watchdir @@ -211,3 +217,20 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ + + def list_watches(self): + """ XML-RPC that returns a list of current inotify watches for + debugging purposes. """ + return list(self.watches_by_path.keys()) + + def list_paths(self): + """ XML-RPC that returns a list of paths that are handled for + debugging purposes. Because inotify doesn't like watching + files, but prefers to watch directories, this will be + different from + :func:`Bcfg2.Server.FileMonitor.Inotify.Inotify.ListWatches`. For + instance, if a plugin adds a monitor to + ``/var/lib/bcfg2/Plugin/foo.xml``, :func:`ListPaths` will + return ``/var/lib/bcfg2/Plugin/foo.xml``, while + :func:`ListWatches` will return ``/var/lib/bcfg2/Plugin``. """ + return list(self.handles.keys()) diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d77f21b93..522ddb705 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -116,6 +116,9 @@ class FileMonitor(Debuggable): #: should have higher priorities. __priority__ = -1 + #: List of names of methods to be exposed as XML-RPC functions + __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] + def __init__(self, ignore=None, debug=False): """ :param ignore: A list of filename globs describing events that @@ -288,6 +291,8 @@ class FileMonitor(Debuggable): def shutdown(self): """ Handle any tasks required to shut down the monitor. """ + self.debug_log("Shutting down %s file monitor" % + self.__class__.__name__) self.started = False def AddMonitor(self, path, obj, handleID=None): @@ -310,6 +315,15 @@ class FileMonitor(Debuggable): """ raise NotImplementedError + def list_event_handlers(self): + """ XML-RPC that returns + :attr:`Bcfg2.Server.FileMonitor.FileMonitor.handles` for + debugging purposes. """ + rv = dict() + for watch, handler in self.handles.items(): + rv[watch] = getattr(handler, "name", handler.__class__.__name__) + return rv + #: A module-level FAM object that all plugins, etc., can use. This #: should not be used directly, but retrieved via :func:`get_fam`. diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 60525d5a1..497e8fac6 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -37,7 +37,7 @@ def is_octal_mode(val): def is_username(val): """ Return True if val is a string giving either a positive integer uid, or a valid Unix username """ - return re.match(r'^([a-z]\w{0,30}|\d+)$', val) + return re.match(r'^([A-z][-_A-z0-9]{0,30}|\d+)$', val) def is_device_mode(val): diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 25a687874..f7bc08717 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -34,8 +34,8 @@ class Debuggable(object): :returns: bool - The new value of the debug flag """ self.debug_flag = debug - self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, - self.debug_flag), + self.debug_log("%s: debug = %s" % (self.__class__.__name__, + self.debug_flag), flag=True) return debug @@ -122,6 +122,7 @@ class Plugin(Debuggable): """ Perform shutdown tasks for the plugin :returns: None """ + self.debug_log("Shutting down %s plugin" % self.name) self.running = False def __str__(self): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 187c594fd..ded7dd8dc 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -478,6 +478,7 @@ class XMLFileBacked(FileBacked): def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, parser=Bcfg2.Server.XMLParser) + self.extras = [] self._follow_xincludes() if self.extras: try: diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 3ef29775d..11a61ff9c 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -313,6 +313,7 @@ class Threaded(object): """ raise NotImplementedError + class ThreadedStatistics(Statistics, Threaded, threading.Thread): """ ThreadedStatistics plugins process client statistics in a separate thread. """ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 5f10879be..b3781e299 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -77,6 +77,10 @@ class CfgGenshiGenerator(CfgGenerator): __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): + if self.template is None: + raise PluginExecutionError("Failed to load template %s" % + self.name) + fname = entry.get('realname', entry.get('name')) stream = self.template.generate( name=fname, diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e0794f019..e6ef50fa1 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -9,7 +9,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) self.logger.error("DBStats has been replaced with Reporting") - self.logger.error("DBStats: Be sure to migrate your data "\ - "before running the report collector") + self.logger.error("DBStats: Be sure to migrate your data " + "before running the report collector") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b053e65d3..7f8db7b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -138,7 +138,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): self.logger.error('Failed to parse %s' % self.basefile) return self.extras = [] - self.basedata = copy.copy(xdata) + self.basedata = copy.deepcopy(xdata) self._follow_xincludes(xdata=xdata) if self.extras: try: @@ -263,31 +263,63 @@ class ClientMetadata(object): # pylint: disable=R0913 def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): + #: The client hostname (as a string) self.hostname = client + + #: The client profile (as a string) self.profile = profile + + #: The set of all bundles this client gets self.bundles = bundles + + #: A list of all client aliases self.aliases = aliases + + #: A list of all addresses this client is known by self.addresses = addresses + + #: A list of groups this client is a member of self.groups = groups + + #: A dict of categories of this client's groups. Keys are + #: category names, values are corresponding group names. self.categories = categories + + #: The UUID identifier for this client self.uuid = uuid + + #: The Bcfg2 password for this client self.password = password + + #: Connector plugins known to this client self.connectors = [] + + #: The version of the Bcfg2 client this client is running, as + #: a string self.version = version try: + #: The version of the Bcfg2 client this client is running, + #: as a :class:`Bcfg2.version.Bcfg2VersionInfo` object. self.version_info = Bcfg2VersionInfo(version) except (ValueError, AttributeError): self.version_info = None + + #: A :class:`Bcfg2.Server.Plugins.Metadata.MetadataQuery` + #: object for this client. self.query = query # pylint: enable=R0913 def inGroup(self, group): - """Test to see if client is a member of group.""" + """Test to see if client is a member of group. + + :returns: bool """ return group in self.groups def group_in_category(self, category): - """ return the group in the given category that the client is - a member of, or the empty string """ + """ Return the group in the given category that the client is + a member of, or an empty string. + + :returns: string """ for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp @@ -295,17 +327,59 @@ class ClientMetadata(object): class MetadataQuery(object): - """ object supplied to client metadata to allow client metadata - objects to query metadata without being able to modify it """ + """ This class provides query methods for the metadata of all + clients known to the Bcfg2 server, without being able to modify + that data. + + Note that ``*by_groups()`` and ``*by_profiles()`` behave + differently; for a client to be included in the return value of a + ``*by_groups()`` method, it must be a member of *all* groups + listed in the argument; for a client to be included in the return + value of a ``*by_profiles()`` method, it must have *any* group + listed as its profile group. """ def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): - # resolver is set later + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + #: object for the given hostname. + #: + #: :returns: Bcfg2.Server.Plugins.Metadata.ClientMetadata self.by_name = by_name + + #: Get a list of hostnames of clients that are in all given + #: groups. + #: + #: :param groups: The groups to check clients for membership in + #: :type groups: list + #: + #: :returns: list of strings self.names_by_groups = self._warn_string(by_groups) + + #: Get a list of hostnames of clients whose profile matches + #: any given profile group. + #: + #: :param profiles: The profiles to check clients for + #: membership in. + #: :type profiles: list + #: :returns: list of strings self.names_by_profiles = self._warn_string(by_profiles) + + #: Get all known client hostnames. + #: + #: :returns: list of strings self.all_clients = get_clients + + #: Get all known group names. + #: + #: :returns: list of strings self.all_groups = all_groups + + #: Get the names of all groups in the given category. + #: + #: :param category: The category to query for groups that + #: belong to it. + #: :type category: string + #: :returns: list of strings self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): @@ -326,22 +400,41 @@ class MetadataQuery(object): return inner def by_groups(self, groups): - """ get a list of ClientMetadata objects that are in all given - groups """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that are in all given groups. + + :param groups: The groups to check clients for membership in. + :type groups: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): - """ get a list of ClientMetadata objects that are in any of - the given profiles """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that have any of the given groups as their profile. + + :param profiles: The profiles to check clients for membership + in. + :type profiles: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_profiles is decorated return [self.by_name(name) for name in self.names_by_profiles(profiles)] def all(self): - """ get a list of all ClientMetadata objects """ + """ Get a list of all + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects. + + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ return [self.by_name(name) for name in self.all_clients()] @@ -1255,7 +1348,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup('authentication')) == 'bootstrap': + self.core.setup['authentication']) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 0dd42c9cb..490ee6f20 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -15,6 +15,11 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" + if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + # do not care about a client that is _any_ 1.3.0 release + # (including prereleases and RCs) + return + for goal in goals: for entry in goal.getchildren(): if entry.tag == 'Path' and 'mode' in entry.keys(): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 3799b1723..4535fb76d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -313,9 +313,7 @@ class YumCollection(Collection): @property def __package_groups__(self): - """ YumCollections support package groups only if - :attr:`use_yum` is True """ - return self.use_yum + return True @property def helper(self): @@ -663,11 +661,6 @@ class YumCollection(Collection): In this implementation the packages may be strings or tuples. See :ref:`yum-pkg-objects` for more information. """ - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported " - "by Bcfg2's internal Yum dependency generator") - return dict() - if not grouplist: return dict() @@ -679,7 +672,15 @@ class YumCollection(Collection): ptype = "default" gdicts.append(dict(group=group, type=ptype)) - return self.call_helper("get_groups", inputdata=gdicts) + if self.use_yum: + return self.call_helper("get_groups", inputdata=gdicts) + else: + pkgs = dict() + for gdict in gdicts: + pkgs[gdict['group']] = Collection.get_group(self, + gdict['group'], + gdict['type']) + return pkgs def _element_to_pkg(self, el, name): """ Convert a Package or Instance element to a package tuple """ @@ -975,6 +976,7 @@ class YumSource(Source): for x in ['global'] + self.arches]) self.needed_paths = set() self.file_to_arch = dict() + self.yumgroups = dict() __init__.__doc__ = Source.__init__.__doc__ @property @@ -992,7 +994,8 @@ class YumSource(Source): if not self.use_yum: cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) + self.filemap, self.url_map, + self.yumgroups), cache, 2) cache.close() def load_state(self): @@ -1002,7 +1005,7 @@ class YumSource(Source): if not self.use_yum: data = open(self.cachefile) (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) + self.filemap, self.url_map, self.yumgroups) = cPickle.load(data) @property def urls(self): @@ -1057,7 +1060,7 @@ class YumSource(Source): urls = [] for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: + if elt.get('type') in ['filelists', 'primary', 'group']: floc = elt.find(RPO + 'location') fullurl = url + floc.get('href') urls.append(fullurl) @@ -1074,11 +1077,14 @@ class YumSource(Source): # we have to read primary.xml first, and filelists.xml afterwards; primaries = list() filelists = list() + groups = list() for fname in self.files: if fname.endswith('primary.xml.gz'): primaries.append(fname) elif fname.endswith('filelists.xml.gz'): filelists.append(fname) + elif fname.find('comps'): + groups.append(fname) for fname in primaries: farch = self.file_to_arch[fname] @@ -1088,6 +1094,9 @@ class YumSource(Source): farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) + for fname in groups: + fdata = lxml.etree.parse(fname).getroot() + self.parse_group(fdata) # merge data sdata = list(self.packages.values()) @@ -1151,6 +1160,35 @@ class YumSource(Source): self.provides[arch][prov] = list() self.provides[arch][prov].append(pkgname) + @Bcfg2.Server.Plugin.track_statistics() + def parse_group(self, data): + """ parse comps.xml.gz data """ + for group in data.getchildren(): + if not group.tag.endswith('group'): + continue + try: + groupid = group.xpath('id')[0].text + self.yumgroups[groupid] = {'mandatory': list(), + 'default': list(), + 'optional': list(), + 'conditional': list()} + except IndexError: + continue + try: + packagelist = group.xpath('packagelist')[0] + except IndexError: + continue + for pkgreq in packagelist.getchildren(): + pkgtype = pkgreq.get('type', None) + if pkgtype == 'mandatory': + self.yumgroups[groupid]['mandatory'].append(pkgreq.text) + elif pkgtype == 'default': + self.yumgroups[groupid]['default'].append(pkgreq.text) + elif pkgtype == 'optional': + self.yumgroups[groupid]['optional'].append(pkgreq.text) + elif pkgtype == 'conditional': + self.yumgroups[groupid]['conditional'].append(pkgreq.text) + def is_package(self, metadata, package): arch = [a for a in self.arches if a in metadata.groups] if not arch: @@ -1230,3 +1268,26 @@ class YumSource(Source): return self.pulp_id else: return Source.get_repo_name(self, url_map) + + def get_group(self, metadata, group, ptype=None): # pylint: disable=W0613 + """ Get the list of packages of the given type in a package + group. + + :param group: The name of the group to query + :type group: string + :param ptype: The type of packages to get, for backends that + support multiple package types in package groups + (e.g., "recommended," "optional," etc.) + :type ptype: string + :returns: list of strings - package names + """ + try: + yumgroup = self.yumgroups[group] + except KeyError: + return [] + packages = yumgroup['conditional'] + yumgroup['mandatory'] + if ptype in ['default', 'optional', 'all']: + packages += yumgroup['default'] + if ptype in ['optional', 'all']: + packages += yumgroup['optional'] + return packages diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5d3fbae2e..2175cf0aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -5,6 +5,7 @@ determine the completeness of the client configuration. """ import os import sys import glob +import copy import shutil import lxml.etree import Bcfg2.Logger @@ -296,20 +297,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if self.disableResolver: # Config requests no resolver + for struct in structures: + for pkg in struct.xpath('//Package | //BoundPackage'): + if pkg.get("group"): + if pkg.get("type"): + pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) - # base is the set of initial packages -- explicitly - # given in the specification, from expanded package groups, - # and essential to the distribution - base = set() + initial = set() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - base.update(collection.packages_from_entry(pkg)) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -321,6 +324,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pkg, xml_declaration=False).decode('UTF-8')) + # base is the set of initial packages explicitly given in the + # specification, packages from expanded package groups, and + # packages essential to the distribution + base = set(initial) + # remove package groups for el in to_remove: el.getparent().remove(el) @@ -336,7 +344,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = collection.get_new_packages(base, packages) + newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() @@ -514,7 +522,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data()) + return dict(sources=collection.get_additional_data(), + allsources=copy.deepcopy(self.sources)) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index d072f1a33..a6dc2c1ef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -65,10 +65,13 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): (self.name, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise PluginInitError(msg) + if self.debug_flag: + self.transport.set_debug(self.debug_flag) def set_debug(self, debug): rv = Debuggable.set_debug(self, debug) - self.transport.set_debug(debug) + if self.transport is not None: + self.transport.set_debug(debug) return rv def process_statistics(self, client, xdata): diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 0d1246d85..28450aa1a 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -412,12 +412,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, name = instance.name except AttributeError: name = "unknown" - if hasattr(instance, 'plugins'): - for pname, pinst in list(instance.plugins.items()): - for mname in pinst.__rmi__: - xmname = "%s.%s" % (pname, mname) - fn = getattr(pinst, mname) - self.register_function(fn, name=xmname) + if hasattr(instance, '_get_rmi'): + for fname, func in instance._get_rmi().items(): + self.register_function(func, name=fname) self.logger.info("serving %s at %s" % (name, self.url)) def serve_forever(self): diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 29c27257a..dd76f04d3 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -165,23 +165,19 @@ class Executor(object): self.logger = logging.getLogger(self.__class__.__name__) self.timeout = timeout - def _timeout_callback(self, proc): - """ Get a callback (suitable for passing to - :class:`threading.Timer`) that kills the given process. + def _timeout(self, proc): + """ A function suitable for passing to + :class:`threading.Timer` that kills the given process. :param proc: The process to kill upon timeout. :type proc: subprocess.Popen - :returns: function """ - def _timeout(): - """ Callback that kills ``proc`` """ - if proc.poll() == None: - try: - proc.kill() - self.logger.warning("Process exceeeded timeout, killing") - except OSError: - pass - - return _timeout + :returns: None """ + if proc.poll() == None: + try: + proc.kill() + self.logger.warning("Process exceeeded timeout, killing") + except OSError: + pass def run(self, command, inputdata=None, timeout=None, **kwargs): """ Run a command, given as a list, optionally giving it the @@ -213,9 +209,7 @@ class Executor(object): if timeout is None: timeout = self.timeout if timeout is not None: - timer = threading.Timer(float(timeout), - self._timeout_callback(proc), - [proc]) + timer = threading.Timer(float(timeout), self._timeout, [proc]) timer.start() try: if inputdata: diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index d819ee534..87f2a0df0 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -32,7 +32,7 @@ TIME_ZONE = None DEBUG = False TEMPLATE_DEBUG = DEBUG -MEDIA_URL = '/site_media' +MEDIA_URL = '/site_media/' def _default_config(): @@ -86,14 +86,6 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None): HOST=setup['db_host'], PORT=setup['db_port']) - if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - DATABASE_ENGINE = setup['db_engine'] - DATABASE_NAME = DATABASES['default']['NAME'] - DATABASE_USER = DATABASES['default']['USER'] - DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] - DATABASE_HOST = DATABASES['default']['HOST'] - DATABASE_PORT = DATABASES['default']['PORT'] - # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] @@ -106,7 +98,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None): if setup['web_prefix']: MEDIA_URL = setup['web_prefix'].rstrip('/') + MEDIA_URL else: - MEDIA_URL = '/site_media' + MEDIA_URL = '/site_media/' # initialize settings from /etc/bcfg2-web.conf or /etc/bcfg2.conf, or # set up basic defaults. this lets manage.py work in all cases @@ -144,7 +136,7 @@ MEDIA_ROOT = '' # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a # trailing slash. -ADMIN_MEDIA_PREFIX = '/media/' +STATIC_URL = '/media/' #TODO - make this unique # Make this unique, and don't share it with anybody. @@ -159,16 +151,10 @@ else: } } -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.load_template_source', - 'django.template.loaders.app_directories.load_template_source', - ) -else: - TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - ) +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', +) #TODO - review these. auth and sessions aren't really used MIDDLEWARE_CLASSES = ( @@ -194,20 +180,10 @@ TEMPLATE_DIRS = ( '/usr/share/python-support/python-django/django/contrib/admin/templates/', ) -# TODO - sanitize this -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.core.context_processors.auth', - 'django.core.context_processors.debug', - 'django.core.context_processors.i18n', - 'django.core.context_processors.media', - 'django.core.context_processors.request' - ) -else: - TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.contrib.auth.context_processors.auth', - 'django.core.context_processors.debug', - 'django.core.context_processors.i18n', - 'django.core.context_processors.media', - 'django.core.context_processors.request' - ) +TEMPLATE_CONTEXT_PROCESSORS = ( + 'django.contrib.auth.context_processors.auth', + 'django.core.context_processors.debug', + 'django.core.context_processors.i18n', + 'django.core.context_processors.media', + 'django.core.context_processors.request' +) diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 8223d7543..6f3ba3e49 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.0rc2" +__version__ = "1.3.1" class Bcfg2VersionInfo(tuple): diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 3c63cd3f5..3bce7fdab 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -10,6 +10,7 @@ import Bcfg2.Options import Bcfg2.Server.Admin from Bcfg2.Compat import StringIO + def mode_import(modename): """Load Bcfg2.Server.Admin.<mode>.""" modname = modename.capitalize() diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 810406567..f7deba90c 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -258,7 +258,7 @@ class Encryptor(object): (self.pname, pname)) return (passphrase, pname) - def _get_passphrase(self, chunk): # pylint: disable=W0613 + def _get_passphrase(self, chunk): # pylint: disable=W0613 """ get the passphrase for a chunk of a file """ return None diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 287d0a161..ad35bbeeb 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -473,7 +473,7 @@ Bcfg2 client itself.""") ('Password', self.setup['password']), ('Server Metadata Connector', self.setup['mconnect']), ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), + ('Server address', self.setup['location']), ('Path to key', self.setup['key']), ('Path to SSL certificate', self.setup['cert']), ('Path to SSL CA certificate', self.setup['ca']), @@ -640,21 +640,24 @@ Bcfg2 client itself.""") if 'Packages' not in self.plugins: print("Packages plugin not enabled") return + self.plugins['Packages'].toggle_debug() + + indep = lxml.etree.Element("Independent") + structures = [lxml.etree.Element("Bundle", name="packages")] + for arg in arglist[1:]: + lxml.etree.SubElement(structures[0], "Package", name=arg) + hostname = arglist[0] - initial = arglist[1:] metadata = self.build_metadata(hostname) - self.plugins['Packages'].toggle_debug() - collection = self.plugins['Packages'].get_collection(metadata) - packages, unknown = collection.complete(initial) - newpkgs = list(packages.difference(initial)) - print("%d initial packages" % len(initial)) - print(" %s" % "\n ".join(initial)) - print("%d new packages added" % len(newpkgs)) - if newpkgs: - print(" %s" % "\n ".join(newpkgs)) - print("%d unknown packages" % len(unknown)) - if unknown: - print(" %s" % "\n ".join(unknown)) + + # pylint: disable=W0212 + self.plugins['Packages']._build_packages(metadata, indep, structures) + # pylint: enable=W0212 + + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) def do_packagesources(self, args): """ packagesources <hostname> - Show package sources """ @@ -726,17 +729,19 @@ Bcfg2 client itself.""") pass - def build_usage(): """ build usage message """ cmd_blacklist = ["do_loop", "do_EOF"] usage = dict() for attrname in dir(InfoCore): attr = getattr(InfoCore, attrname) - if (hasattr(attr, "__func__") and - attr.__func__.func_name not in cmd_blacklist and - attr.__func__.func_name.startswith("do_") and - attr.__func__.func_doc): + + # shim for python 2.4, __func__ is im_func + funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) + if (funcattr != None and + funcattr.func_name not in cmd_blacklist and + funcattr.func_name.startswith("do_") and + funcattr.func_doc): usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) @@ -748,9 +753,8 @@ def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, interpreter=Bcfg2.Options.INTERPRETER) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) + optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", "Options:", setup.buildHelpMessage(), diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 9f2ff96c2..2c4a918be 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -10,7 +10,7 @@ from Bcfg2.Compat import ConfigParser try: import Bcfg2.settings except ConfigParser.NoSectionError: - print("Your bcfg2.conf is currently missing the statistics section which " + print("Your bcfg2.conf is currently missing the [database] section which " "is necessary for the reporting interface. Please see bcfg2.conf(5) " "for more details.") sys.exit(1) @@ -121,7 +121,7 @@ def main(): help="Show hosts that haven't run in the last 24 " "hours") parser.add_option_group(allhostmodes) - + # entry modes entrymodes = \ OptionGroup(parser, "Entry Modes", @@ -166,7 +166,7 @@ def main(): (mode.get_opt_string(), opt.get_opt_string())) mode = opt mode_family = parser.get_option_group(opt.get_opt_string()) - + # you can specify more than one of --bad, --extra, --modified, --show, so # consider single-host options separately if not mode_family: @@ -174,7 +174,7 @@ def main(): if getattr(options, opt.dest): mode_family = parser.get_option_group(opt.get_opt_string()) break - + if not mode_family: parser.error("You must specify a mode") @@ -243,7 +243,7 @@ def main(): parser.error("%s require either a list of entries on the " "command line or the --file options" % mode_family.title) - + if options.badentry: result = hosts_by_entry_type(clients, "bad", entries) elif options.modifiedentry: @@ -263,7 +263,7 @@ def main(): # todo batch fetch this. sqlite could break for client in clients: - ents = entry_cls.objects.filter(name=entries[0][1], + ents = entry_cls.objects.filter(name=entries[0][1], interaction=client.current_interaction) if len(ents) == 0: continue diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 8e96d65f2..33ee327fc 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -11,6 +11,7 @@ from Bcfg2.Server.Core import CoreInitError LOGGER = logging.getLogger('bcfg2-server') + def main(): optinfo = dict() optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 75079ec3f..510bb898b 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -5,29 +5,87 @@ without failures""" import os import sys -import signal import fnmatch import logging import Bcfg2.Logger import Bcfg2.Server.Core +from math import ceil from nose.core import TestProgram from nose.suite import LazySuite from unittest import TestCase +try: + from multiprocessing import Process, Queue, active_children + HAS_MULTIPROC = True +except ImportError: + HAS_MULTIPROC = False + active_children = lambda: [] # pylint: disable=C0103 + + +class CapturingLogger(object): + """ Fake logger that captures logging output so that errors are + only displayed for clients that fail tests """ + def __init__(self, *args, **kwargs): # pylint: disable=W0613 + self.output = [] + + def error(self, msg): + """ discard error messages """ + self.output.append(msg) + + def warning(self, msg): + """ discard error messages """ + self.output.append(msg) + + def info(self, msg): + """ discard error messages """ + self.output.append(msg) + + def debug(self, msg): + """ discard error messages """ + self.output.append(msg) + + def reset_output(self): + """ Reset the captured output """ + self.output = [] + + +class ClientTestFromQueue(TestCase): + """ A test case that tests a value that has been enqueued by a + child test process. ``client`` is the name of the client that has + been tested; ``result`` is the result from the :class:`ClientTest` + test. ``None`` indicates a successful test; a string value + indicates a failed test; and an exception indicates an error while + running the test. """ + __test__ = False # Do not collect + + def __init__(self, client, result): + TestCase.__init__(self) + self.client = client + self.result = result + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ parse the result from this test """ + if isinstance(self.result, Exception): + raise self.result + assert self.result is None, self.result + class ClientTest(TestCase): - """ - A test case representing the build of all of the configuration for + """ A test case representing the build of all of the configuration for a single host. Checks that none of the build config entities has had a failure when it is building. Optionally ignores some config files that we know will cause errors (because they are private - files we don't have access to, for instance) - """ + files we don't have access to, for instance) """ __test__ = False # Do not collect + divider = "-" * 70 - def __init__(self, bcfg2_core, client, ignore=None): + def __init__(self, core, client, ignore=None): TestCase.__init__(self) - self.bcfg2_core = bcfg2_core + self.core = core + self.core.logger = CapturingLogger() self.client = client if ignore is None: self.ignore = dict() @@ -52,13 +110,34 @@ class ClientTest(TestCase): def runTest(self): """ run this individual test """ - config = self.bcfg2_core.BuildConfiguration(self.client) + config = self.core.BuildConfiguration(self.client) + output = self.core.logger.output[:] + if output: + output.append(self.divider) + self.core.logger.reset_output() + # check for empty client configuration assert len(config.findall("Bundle")) > 0, \ - "%s has no content" % self.client + "\n".join(output + ["%s has no content" % self.client]) + + # check for missing bundles + metadata = self.core.build_metadata(self.client) + sbundles = [el.get('name') for el in config.findall("Bundle")] + missing = [b for b in metadata.bundles if b not in sbundles] + assert len(missing) == 0, \ + "\n".join(output + ["Configuration is missing bundle(s): %s" % + ':'.join(missing)]) + + # check for unknown packages + unknown_pkgs = [el.get("name") + for el in config.xpath('//Package[@type="unknown"]') + if not self.ignore_entry(el.tag, el.get("name"))] + assert len(unknown_pkgs) == 0, \ + "Configuration contains unknown packages: %s" % \ + ", ".join(unknown_pkgs) failures = [] - msg = ["Failures:"] + msg = output + ["Failures:"] for failure in config.xpath('//*[@failure]'): if not self.ignore_entry(failure.tag, failure.get('name')): failures.append(failure) @@ -73,23 +152,46 @@ class ClientTest(TestCase): id = __str__ -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" +def get_core(setup): + """ Get a server core, with events handled """ + core = Bcfg2.Server.Core.BaseCore(setup) + core.fam.handle_events_in_interval(0.1) + return core - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - return hdlr +def get_ignore(setup): + """ Given an options dict, get a dict of entry tags and names to + ignore errors from """ + ignore = dict() + for entry in setup['test_ignore']: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + return ignore -def main(): - optinfo = dict(noseopts=Bcfg2.Options.TEST_NOSEOPTS, - test_ignore=Bcfg2.Options.TEST_IGNORE, - validate=Bcfg2.Options.CFG_VALIDATION) +def run_child(setup, clients, queue): + """ Run tests for the given clients in a child process, returning + results via the given Queue """ + core = get_core(setup) + ignore = get_ignore(setup) + for client in clients: + try: + ClientTest(core, client, ignore).runTest() + queue.put((client, None)) + except AssertionError: + queue.put((client, str(sys.exc_info()[1]))) + except: + queue.put((client, sys.exc_info()[1])) + + core.shutdown() + + +def parse_args(): + """ Parse command line arguments. """ + optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.load_option_parser(optinfo) @@ -109,37 +211,88 @@ def main(): to_syslog=False, to_file=setup['logging'], level=level) + logger = logging.getLogger(sys.argv[0]) if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']: setup['noseopts'].append("-v") - core = Bcfg2.Server.Core.BaseCore() - signal.signal(signal.SIGINT, get_sigint_handler(core)) + if setup['children'] and not HAS_MULTIPROC: + logger.warning("Python multiprocessing library not found, running " + "with no children") + setup['children'] = 0 - ignore = dict() - for entry in setup['test_ignore']: - tag, name = entry.split(":") - try: - ignore[tag].append(name) - except KeyError: - ignore[tag] = [name] + if (setup['children'] and ('--with-xunit' in setup['noseopts'] or + '--xunit-file' in setup['noseopts'])): + logger.warning("Use the --xunit option to bcfg2-test instead of the " + "--with-xunit or --xunit-file options to nosetest") + xunitfile = None + if '--with-xunit' in setup['noseopts']: + setup['noseopts'].remove('--with-xunit') + xunitfile = "nosetests.xml" + if '--xunit-file' in setup['noseopts']: + idx = setup['noseopts'].index('--xunit-file') + try: + setup['noseopts'].pop(idx) # remove --xunit-file + # remove the argument to it + xunitfile = setup['noseopts'].pop(idx) + except IndexError: + pass + if xunitfile and not setup['xunit']: + setup['xunit'] = xunitfile + return setup - core.fam.handle_events_in_interval(0.1) + +def main(): + setup = parse_args() + logger = logging.getLogger(sys.argv[0]) + core = get_core(setup) if setup['args']: clients = setup['args'] else: clients = core.metadata.clients - def run_tests(): - """ Run the test suite """ - for client in clients: - yield ClientTest(core, client, ignore) + ignore = get_ignore(setup) - TestProgram(argv=sys.argv[0:1] + setup['noseopts'], - suite=LazySuite(run_tests)) + if setup['children']: + if setup['children'] > len(clients): + logger.info("Refusing to spawn more children than clients to test," + " setting children=%s" % len(clients)) + setup['children'] = len(clients) + perchild = int(ceil(len(clients) / float(setup['children'] + 1))) + queue = Queue() + for child in range(setup['children']): + start = child * perchild + end = (child + 1) * perchild + child = Process(target=run_child, + args=(setup, clients[start:end], queue)) + child.start() + + def generate_tests(): + """ Read test results for the clients """ + start = setup['children'] * perchild + for client in clients[start:]: + yield ClientTest(core, client, ignore) + + for i in range(start): # pylint: disable=W0612 + yield ClientTestFromQueue(*queue.get()) + else: + def generate_tests(): + """ Run tests for the clients """ + for client in clients: + yield ClientTest(core, client, ignore) + + TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], + suite=LazySuite(generate_tests), exit=False) + + # block until all children have completed -- should be + # immediate since we've already gotten all the results we + # expect + for child in active_children(): + child.join() core.shutdown() os._exit(0) # pylint: disable=W0212 + if __name__ == "__main__": sys.exit(main()) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index ba6f30406..7e5c03fd5 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -129,7 +129,7 @@ class DepSolver(object): err = sys.exc_info()[1] self.logger.warning(err) return [] - + if ptype == "default": return [p for p, d in list(group.default_packages.items()) @@ -254,6 +254,6 @@ def main(): rv[gdata['group']] = list(packages) print(json.dumps(rv)) - + if __name__ == '__main__': sys.exit(main()) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index 063257863..2dc763730 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -380,7 +380,7 @@ class TestPkgTool(TestTool): # test single-pass install success reset() - pt.cmd.run.return_value = (0, '') + pt.cmd.run.return_value = True states = pt.Install(packages) pt._get_package_command.assert_called_with(packages) pt.cmd.run.assert_called_with([p.get("name") for p in packages]) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 27667b3fe..415b316fd 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -25,14 +25,6 @@ srcpath = os.path.abspath(os.path.join(testdir, "..", "src")) # path to pylint rc file rcfile = os.path.join(testdir, "pylintrc.conf") -# test for pylint existence -try: - Popen(['pylint'], stdout=PIPE, stderr=STDOUT).wait() - HAS_PYLINT = True -except OSError: - HAS_PYLINT = False - - # perform checks on the listed files only if the module listed in the # keys can be imported contingent_checks = { @@ -57,7 +49,6 @@ contingent_checks = { # perform only error checking on the listed files error_checks = { - "sbin": ["bcfg2-reports"], "lib/Bcfg2": ["Reporting"], "lib/Bcfg2/Client": ["Proxy.py"], "lib/Bcfg2/Server": ["Reports", "SchemaUpdater", "SSLServer.py"], @@ -128,13 +119,8 @@ def blacklist_filter(filelist, blacklist): return rv -class TestPylint(Bcfg2TestCase): - pylint_cmd = ["pylint", "--rcfile", rcfile, "--init-hook", - "import sys;sys.path.append('%s')" % - os.path.join(srcpath, "lib")] - - # regex to find errors and fatal errors - error_re = re.compile(r':\d+:\s+\[[EF]\d{4}') +class CodeTestCase(Bcfg2TestCase): + __test__ = False # build the blacklists blacklist = expand_path_dict(no_checks) @@ -146,118 +132,185 @@ class TestPylint(Bcfg2TestCase): full_blacklist = expand_path_dict(error_checks) + contingent_blacklist + \ blacklist + command = [None] + + has_command = None + + # extra arguments when running tests on sbin/* + sbin_args = [] + + # extra arguments when running tests on lib/* + lib_args = [] + + # extra arguments for full tests + full_args = [] + + # extra arguments for error tests + error_args = [] + + def has_exec(self): + if self.has_command is None: + try: + Popen(self.command, + stdin=PIPE, stdout=PIPE, stderr=STDOUT).wait() + self.has_command = True + except OSError: + self.has_command = False + return self.has_command + def get_env(self): - env = copy.copy(os.environ) - if 'PYTHONPATH' in os.environ: - env['PYTHONPATH'] = '%s:%s' % (env['PYTHONPATH'], testdir) + if ('PYTHONPATH' not in os.environ or + testdir not in os.environ['PYTHONPATH'].split(":")): + env = copy.copy(os.environ) + env['PYTHONPATH'] = ':'.join([env.get("PYTHONPATH", ""), + testdir]) + return env else: - env['PYTHONPATH'] = testdir - return env + return os.environ + + def _test_full(self, files, extra_args=None): + """ test select files for all problems """ + if not len(files): + return + if extra_args is None: + extra_args = [] + cmd = self.command + self.full_args + extra_args + \ + [os.path.join(srcpath, f) for f in files] + proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, env=self.get_env()) + print(proc.communicate()[0]) + self.assertEqual(proc.wait(), 0) + + def _test_errors(self, files, extra_args=None): + """ test select files for errors """ + if not len(files): + return + if extra_args is None: + extra_args = [] + cmd = self.command + self.error_args + extra_args + \ + [os.path.join(srcpath, f) for f in files] + proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, env=self.get_env()) + print(proc.communicate()[0]) + self.assertEqual(proc.wait(), 0) @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") def test_lib_full(self): - full_list = [] - for root, _, files in os.walk(os.path.join(srcpath, "lib")): - full_list.extend(blacklist_filter([os.path.join(root, f) - for f in files - if f.endswith(".py")], - self.full_blacklist)) - self._pylint_full(full_list) + @skipUnless(self.has_exec(), + "%s not found, skipping" % self.command[0]) + def inner(): + full_list = [] + for root, _, files in os.walk(os.path.join(srcpath, "lib")): + full_list.extend(blacklist_filter([os.path.join(root, f) + for f in files + if f.endswith(".py")], + self.full_blacklist)) + self._test_full(full_list, extra_args=self.lib_args) + + inner() @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") def test_contingent_full(self): - blacklist = set(expand_path_dict(error_checks) + self.blacklist) - for (mods, filedict) in contingent_checks.items(): - if "django" in mods: - # there's some issue with running pylint on modules - # that use django in Travis CI (but not elsewhere), so - # skip these for now - continue - try: - for mod in mods: - __import__(mod) - except ImportError: - continue - self._pylint_full(blacklist_filter(expand_path_dict(filedict), - blacklist)) + @skipUnless(self.has_exec(), + "%s not found, skipping" % self.command[0]) + def inner(): + filelist = [] + blacklist = set(expand_path_dict(error_checks) + self.blacklist) + for (mods, filedict) in contingent_checks.items(): + try: + for mod in mods: + __import__(mod) + except ImportError: + continue + filelist.extend(expand_path_dict(filedict)) + self._test_full(blacklist_filter(filelist, blacklist), + extra_args=self.lib_args) + + inner() @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") - def test_sbin_full(self): - all_sbin = [os.path.join(srcpath, "sbin", f) - for f in glob.glob(os.path.join(srcpath, "sbin", "*"))] - sbin_list = blacklist_filter([f for f in all_sbin - if not os.path.islink(f)], - self.full_blacklist) - self._pylint_full(sbin_list, - extra_args=["--module-rgx", - "[a-z_-][a-z0-9_-]*$"]) - - def _pylint_full(self, paths, extra_args=None): - """ test select files for all pylint errors """ - if extra_args is None: - extra_args = [] - args = self.pylint_cmd + extra_args + \ - [os.path.join(srcpath, p) for p in paths] - pylint = Popen(args, stdout=PIPE, stderr=STDOUT, env=self.get_env()) - print(pylint.communicate()[0]) - self.assertEqual(pylint.wait(), 0) + def test_sbin(self): + @skipUnless(self.has_exec(), + "%s not found, skipping" % self.command[0]) + def inner(): + all_sbin = [os.path.join(srcpath, "sbin", f) + for f in glob.glob(os.path.join(srcpath, "sbin", "*"))] + full_list = blacklist_filter([f for f in all_sbin + if not os.path.islink(f)], + self.full_blacklist) + self._test_full(full_list, extra_args=self.sbin_args) + + errors_list = blacklist_filter([f for f in all_sbin + if not os.path.islink(f)], + self.contingent_blacklist) + self._test_errors(errors_list, extra_args=self.sbin_args) + + inner() @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") - def test_sbin_errors(self): - all_sbin = [os.path.join(srcpath, "sbin", f) - for f in glob.glob(os.path.join(srcpath, "sbin", "*"))] - sbin_list = blacklist_filter([f for f in all_sbin - if not os.path.islink(f)], - self.contingent_blacklist) - self._pylint_errors(sbin_list, - extra_args=["--module-rgx", - "[a-z_-][a-z0-9_-]*$"]) + def test_contingent_errors(self): + @skipUnless(self.has_exec(), + "%s not found, skipping" % self.command[0]) + def inner(): + filelist = [] + whitelist = expand_path_dict(error_checks) + for (mods, filedict) in contingent_checks.items(): + try: + for mod in mods: + __import__(mod) + except ImportError: + continue + filelist.extend(expand_path_dict(filedict)) + flist = blacklist_filter(whitelist_filter(filelist, whitelist), + self.blacklist) + self._test_errors(flist, extra_args=self.lib_args) + + inner() @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") - def test_contingent_errors(self): - whitelist = expand_path_dict(error_checks) - for (mods, filedict) in contingent_checks.items(): + def test_lib_errors(self): + @skipUnless(self.has_exec(), + "%s not found, skipping" % self.command[0]) + def inner(): + filelist = blacklist_filter(expand_path_dict(error_checks), + self.contingent_blacklist) + return self._test_errors(filelist, extra_args=self.lib_args) + + inner() + + +class TestPylint(CodeTestCase): + __test__ = True + command = ["pylint", "--rcfile", rcfile, "--init-hook", + "import sys;sys.path.append('%s')" % + os.path.join(srcpath, "lib")] + + sbin_args = ["--module-rgx", "[a-z_-][a-z0-9_-]*$"] + error_args = ["-f", "parseable", "-d", "R0801,E1103"] + + # regex to find errors and fatal errors + error_re = re.compile(r':\d+:\s+\[[EF]\d{4}') + + def __init__(self, *args, **kwargs): + CodeTestCase.__init__(self, *args, **kwargs) + for mods, filedict in contingent_checks.items(): if "django" in mods: # there's some issue with running pylint on modules # that use django in Travis CI (but not elsewhere), so # skip these for now - continue - try: - for mod in mods: - __import__(mod) - except ImportError: - continue - flist = \ - blacklist_filter(whitelist_filter(expand_path_dict(filedict), - whitelist), - self.blacklist) - self._pylint_errors(flist) - - @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) - @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) - @skipUnless(HAS_PYLINT, "pylint not found, skipping") - def test_lib_errors(self): - filelist = blacklist_filter(expand_path_dict(error_checks), - self.contingent_blacklist) - return self._pylint_errors(filelist) + self.blacklist += expand_path_dict(filedict) - def _pylint_errors(self, paths, extra_args=None): + def _test_errors(self, files, extra_args=None): """ test all files for fatals and errors """ + if not len(files): + return if extra_args is None: extra_args = [] - args = self.pylint_cmd + extra_args + \ - ["-f", "parseable", "-d", "R0801,E1103"] + \ - [os.path.join(srcpath, p) for p in paths] + args = self.command + self.error_args + extra_args + \ + [os.path.join(srcpath, p) for p in files] pylint = Popen(args, stdout=PIPE, stderr=STDOUT, env=self.get_env()) output = pylint.communicate()[0] rv = pylint.wait() @@ -268,3 +321,11 @@ class TestPylint(Bcfg2TestCase): # pylint returns a bitmask, where 1 means fatal errors # were encountered and 2 means errors were encountered. self.assertEqual(rv & 3, 0) + + +class TestPEP8(CodeTestCase): + __test__ = True + command = ["pep8", "--ignore=E125,E501"] + + def _test_errors(self, files, extra_args=None): + pass diff --git a/testsuite/install.sh b/testsuite/install.sh index 2c962e171..c1685f831 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -4,7 +4,7 @@ pip install -r testsuite/requirements.txt --use-mirrors -PYVER=$(python -c 'import sys;print ".".join(str(v) for v in sys.version_info[0:2])') +PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then if [[ $PYVER == "2.5" ]]; then diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index cc7a1a3d8..3cd3786f9 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -4,7 +4,6 @@ import os import sys import time -import signal import logging import operator import Bcfg2.Logger @@ -13,19 +12,6 @@ import Bcfg2.Server.Core LOGGER = None -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - def main(): optinfo = \ dict(client=Bcfg2.Options.Option("Benchmark templates for one client", @@ -53,7 +39,6 @@ def main(): logger = logging.getLogger(sys.argv[0]) core = Bcfg2.Server.Core.BaseCore(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) logger.info("Bcfg2 server core loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") diff --git a/tools/export.py b/tools/export.py index 33c42d238..716c831d9 100755 --- a/tools/export.py +++ b/tools/export.py @@ -164,11 +164,17 @@ E.G. 1.2.0pre1 is a valid version. print(help_message) quit() - rpmchangelog = ["* %s %s <%s> %s-0.0%s\n" % - (datetime.datetime.now().strftime("%a %b %d %Y"), - name, email, - version_release, version_info['build']), - "- New upstream release\n", "\n"] + if version_info['build'] == '': + rpmchangelog = ["* %s %s <%s> %s-1\n" % + (datetime.datetime.now().strftime("%a %b %d %Y"), + name, email, version_release), + "- New upstream release\n", "\n"] + else: + rpmchangelog = ["* %s %s <%s> %s-0.%s.%s\n" % + (datetime.datetime.now().strftime("%a %b %d %Y"), + name, email, version_release, + version_info['build'][-1], version_info['build']), + "- New upstream release\n", "\n"] # write out the new RPM changelog specs = ["misc/bcfg2.spec", "misc/bcfg2-selinux.spec", "redhat/bcfg2.spec.in"] @@ -236,12 +242,22 @@ E.G. 1.2.0pre1 is a valid version. find_and_replace('misc/bcfg2-selinux.spec', 'Version:', 'Version: %s\n' % version_release, dryrun=options.dryrun) - find_and_replace('misc/bcfg2.spec', 'Release: ', - 'Release: 0.0%s\n' % version_info['build'], - dryrun=options.dryrun) - find_and_replace('misc/bcfg2-selinux.spec', 'Release: ', - 'Release: 0.0%s\n' % version_info['build'], - dryrun=options.dryrun) + if version_info['build'] == '': + find_and_replace('misc/bcfg2.spec', 'Release: ', + 'Release: 1\n', + dryrun=options.dryrun) + find_and_replace('misc/bcfg2-selinux.spec', 'Release: ', + 'Release: 1\n', + dryrun=options.dryrun) + else: + find_and_replace('misc/bcfg2.spec', 'Release: ', + 'Release: 0.%s.%s\n' % + (version_info['build'][-1], version_info['build']), + dryrun=options.dryrun) + find_and_replace('misc/bcfg2-selinux.spec', 'Release: ', + 'Release: 0.%s.%s\n' % + (version_info['build'][-1], version_info['build']), + dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', '%setup', '%%setup -q -n %%{name}-%%{version}%s\n' % version_info['build'], diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index 69d9514df..cbd2a6099 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -246,6 +246,9 @@ def _restructure(): failures = [] int_count = legacy_models.Interaction.objects.count() + if int_count == 0: + logger.error("Found no legacy interactions") + return False int_ctr = 0 start_time = 0 for inter in BatchFetch(legacy_models.Interaction.objects.\ diff --git a/tools/upgrade/1.3/migrate_info.py b/tools/upgrade/1.3/migrate_info.py index 5ff4b73b7..e72599daf 100755 --- a/tools/upgrade/1.3/migrate_info.py +++ b/tools/upgrade/1.3/migrate_info.py @@ -4,7 +4,8 @@ import os import sys import lxml.etree import Bcfg2.Options -from Bcfg2.Server.Plugin import info_regex +from Bcfg2.Server.Plugin import INFO_REGEX + def convert(info_file): info_xml = os.path.join(os.path.dirname(info_file), "info.xml") @@ -15,7 +16,7 @@ def convert(info_file): fileinfo = lxml.etree.Element("FileInfo") info = lxml.etree.SubElement(fileinfo, "Info") for line in open(info_file).readlines(): - match = info_regex.match(line) + match = INFO_REGEX.match(line) if match: mgd = match.groupdict() for key, value in list(mgd.items()): @@ -25,6 +26,7 @@ def convert(info_file): open(info_xml, "w").write(lxml.etree.tostring(fileinfo, pretty_print=True)) os.unlink(info_file) + def main(): opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, configfile=Bcfg2.Options.CFILE, diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index 0aa9c574c..e061558d3 100644..100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -24,7 +24,12 @@ def writefile(f, xdata): def convertinfo(ifile): """Do perms -> mode conversion for info.xml files.""" - xdata = lxml.etree.parse(ifile) + try: + xdata = lxml.etree.parse(ifile) + except lxml.etree.XMLSyntaxError: + err = sys.exc_info()[1] + print("Could not parse %s, skipping: %s" % (ifile, err)) + return found = False for i in xdata.findall('//Info'): found = setmodeattr(i) @@ -34,11 +39,14 @@ def convertinfo(ifile): def convertstructure(structfile): """Do perms -> mode conversion for structure files.""" - xdata = lxml.etree.parse(structfile) + try: + xdata = lxml.etree.parse(structfile) + except lxml.etree.XMLSyntaxError: + err = sys.exc_info()[1] + print("Could not parse %s, skipping: %s" % (structfile, err)) + return found = False - for path in xdata.findall('//BoundPath'): - found = setmodeattr(path) - for path in xdata.findall('//Path'): + for path in xdata.xpath('//BoundPath|//Path'): found = setmodeattr(path) if found: writefile(structfile, xdata) @@ -57,7 +65,7 @@ def main(): for root, dirs, files in os.walk(os.path.join(repo, plugin)): for fname in files: convertstructure(os.path.join(root, fname)) - if plugin not in ['Cfg', 'TGenshi', 'TCheetah']: + if plugin not in ['Cfg', 'TGenshi', 'TCheetah', 'SSHbase', 'SSLCA']: continue for root, dirs, files in os.walk(os.path.join(repo, plugin)): for fname in files: |