diff options
61 files changed, 630 insertions, 299 deletions
diff --git a/.travis.yml b/.travis.yml index 54f2215de..8b336e7f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.6" - - "2.7" + - "2.7_with_system_site_packages" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index b1c3aba21..cd2ad858e 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -33,8 +33,8 @@ BCFG2_SERVER_ENABLED=0 test -f "/etc/default/bcfg2-server" && . /etc/default/bcfg2-server if [ "$BCFG2_SERVER_ENABLED" -eq 0 ] ; then - log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" - exit 0 + log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" + exit 0 fi # Exit if $DAEMON doesn't exist and is not executable @@ -63,10 +63,11 @@ stop () { killproc -p $PIDFILE ${BINARY} STATUS=$? if [ "$STATUS" = 0 ]; then - log_success_msg "bcfg2-server" - test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server + [ -e $PIDFILE ] && rm -f $PIDFILE + log_success_msg "bcfg2-server" + test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server else - log_failure_msg "bcfg2-server" + log_failure_msg "bcfg2-server" fi return $STATUS } @@ -75,15 +76,15 @@ status () { # Inspired by redhat /etc/init.d/functions status() call PID=$(pidof -x $BINARY -o %PPID) if [ -n "$PID" ]; then - echo "$BINARY (pid $PID) is running..." - return 0 + echo "$BINARY (pid $PID) is running..." + return 0 fi if [ -f $PIDFILE ]; then - if [ -n "$PID" ]; then - log_failure_msg "$BINARY dead but pid file exists..." - return 1 - fi + if [ -n "$PID" ]; then + log_failure_msg "$BINARY dead but pid file exists..." + return 1 + fi fi log_failure_msg "$BINARY is not running" diff --git a/debian/changelog b/debian/changelog index e30fba546..c41b2ecc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,12 @@ bcfg2 (1.4.0pre1-0.0) unstable; urgency=low -- Sol Jerome <sol.jerome@gmail.com> Mon, 16 Jun 2014 09:36:13 -0500 +bcfg2 (1.3.5-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Fri, 05 Sep 2014 07:54:48 -0500 + bcfg2 (1.3.4-0.0) unstable; urgency=low * New upstream release diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 09ea76230..93eb11925 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -152,7 +152,36 @@ Systemd service support. SYSV ---- -Handles System V Packaging format that is available on Solaris. +Handles `System V Packaging <http://docs.oracle.com/cd/E19683-01/806-7008/index.html>`_ format that is available on Solaris. + +.. note:: + + If the Packages specified in the PackageList are datastream format + packages distributed via HTTP, you must specify a simplefile attribute. + Such packages will be downloaded and installed from a local path. + + Note the use of the uri attribute in the datastream format example. If + the simplefile attribute exists, the + :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` plugin will + automatically construct the url attribute by concatenating the uri and + simplefile attributes (with an intervening slash). + + Datastream format over HTTP: + + .. code-block:: xml + + <PackageList uri='http://install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' simplefile='bcfg-1.3.4-1' /> + </PackageList> + + File system format over NFS or local path: + + .. code-block:: xml + + <PackageList url='/mnt/install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' /> + </PackageList> + Upstart ------- diff --git a/doc/development/compat.txt b/doc/development/compat.txt index f90274ce5..8700c46d3 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -60,6 +60,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects. +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | urlparse | :func:`urlparse.urlparse` | :func:`urllib.parse.urlparse` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ +| urlretrieve | :func:`urllib.urlretrieve` | :func:`urllib.request.urlretrieve` | ++---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPBasicAuthHandler | :class:`urllib2.HTTPBasicAuthHandler` | :class:`urllib.request.HTTPBasicAuthHandler` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPPasswordMgrWithDefaultRealm | :class:`urllib2.HTTPPasswordMgrWithDefaultRealm` | :class:`urllib.request.HTTPPasswordMgrWithDefaultRealm` | diff --git a/doc/installation/distributions.txt b/doc/installation/distributions.txt index 306439485..5dad4d860 100644 --- a/doc/installation/distributions.txt +++ b/doc/installation/distributions.txt @@ -36,9 +36,9 @@ Just use `pacman` to perform the installation :: Debian ====== -Packages of Bcfg2 are available for Debian Lenny, Debian Squeeze, and -Debian Sid. The fastest way to get Bcfg2 onto your Debian system -is to use ``apt-get`` or ``aptitude``. :: +Packages of Bcfg2 are available for all current versions of Debian. +The fastest way to get Bcfg2 onto your Debian system is to use ``apt-get`` +or ``aptitude``. :: sudo aptitude install bcfg2 bcfg2-server diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index a30a3b26b..8119be06b 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -76,17 +76,3 @@ reporting, such as Apache + mod_wsgi or nginx. +-------------------------------+----------+--------------------------------+ | south | 0.7.5+ | | +-------------------------------+----------+--------------------------------+ - -Bcfg2 Reporting ---------------- - -A webserver capabable of running wsgi applications is required for web reporting, -such as Apache + mod_wsgi or nginx. - -+-------------------------------+----------+--------------------------------+ -| Software | Version | Requires | -+===============================+==========+================================+ -| django | 1.2.0+ | | -+-------------------------------+----------+--------------------------------+ -| south | 0.7.0+ | | -+-------------------------------+----------+--------------------------------+ diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 825ab2121..7c265f263 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -662,9 +662,8 @@ control the database connection of the server. Port for database connections. Not used for sqlite3. options - Various options for the database connection. The value is - expected as multiple key=value pairs, separated with commas. - The concrete value depends on the database engine. + Various options for the database connection. The value expected + is the literal value of the django OPTIONS setting. reporting_engine The database engine used by the Reporting plugin. One of the @@ -716,6 +715,10 @@ Reporting options web_debug Turn on Django debugging. + max_children + Maximum number of children for the reporting collector. Use 0 to + disable the limit. (default is 0) + See Also -------- diff --git a/doc/releases/1.3.5.txt b/doc/releases/1.3.5.txt new file mode 100644 index 000000000..893cdbf39 --- /dev/null +++ b/doc/releases/1.3.5.txt @@ -0,0 +1,33 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.5: + +1.3.5 +===== + +We are happy to announce the release of Bcfg2 1.3.5. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Properly close db connections +* Improved error messages +* Fix yum upgrade/downgrade +* Enable bcfg2-yum-helper to depsolve for arches incompatible with + server +* Spec file fixes +* bcfg2-crypet: Default to only (En|De)crypt vars that need it +* Fix email reporting bug +* Fix debsums parsing +* Fix solaris makefile +* SYSV: Implement downloading and installing SYSV packages from HTTP: + http://docs.bcfg2.org/client/tools.html#sysv +* Fix debian bcfg2-server init script + + +Special thanks to the following contributors for this release: John +Morris, Jonathan Billings, Chris Brinker, Tim Laszlo, Matt Kemp, Michael +Fenn, Pavel Labushev, Nathan Olla, Alexander Sulfrian. diff --git a/doc/releases/1.3.6.txt b/doc/releases/1.3.6.txt new file mode 100644 index 000000000..757fbf6f5 --- /dev/null +++ b/doc/releases/1.3.6.txt @@ -0,0 +1,34 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.6: + +1.3.6 +===== + +We are happy to announce the release of Bcfg2 1.3.6. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Fix python 2.4 compatibility +* Fix stale lockfile detection and behavior +* Reporting: fix filter urls +* Fix client protocol option handling +* YUM: Add options to enable and disable Yum plugins +* Packages: add name to sources +* Reporting: better exception handling +* Various interrupt handling fixes +* Fix client decision whitelist/blacklist handling +* Fix database OPTIONS parsing + + This change requires you to set the *options* value of the + ``[database`` section in ``bcfg2.conf`` to the literal value which is + passed through to the django OPTIONS setting. + + https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-OPTIONS + +Special thanks to the following contributors for this release: Michael +Fenn, Matt Kemp, Alexander Sulfrian, Jonathan Billings. diff --git a/doc/releases/index.txt b/doc/releases/index.txt index 479aa19de..271fc23cc 100644 --- a/doc/releases/index.txt +++ b/doc/releases/index.txt @@ -10,4 +10,5 @@ Release Announcements .. toctree:: 1.4.0pre1 + 1.3.5 1.3.4 diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 53bdef24e..38d4c7e3a 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -270,6 +270,9 @@ reporting * web_prefix: Prefix to be added to Django's MEDIA_URL * file_limit: The maximum size of a diff or binary data to store in the database. +* max_children: Maximum number of children for the reporting + collector. Use 0 to disable the limit and spawn a thread + as soon as a working file is available. .. _dynamic_transports: diff --git a/doc/server/database.txt b/doc/server/database.txt index 986914171..67cb065f4 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -69,8 +69,8 @@ of ``/etc/bcfg2.conf``. +--------------------+------------------------------------------------------------+---------------------------------------+ | options | Extra parameters to use when connecting to the database. | None | | | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_engine | The name of the Django database backend to use for the | None | | | reporting database. Takes the same values as ``engine``. | | @@ -85,10 +85,10 @@ of ``/etc/bcfg2.conf``. +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_port | The port to connect to for the reporting database | None | +--------------------+------------------------------------------------------------+---------------------------------------+ -| reporting_options | Extra parameters to use when connecting to the reporting | None | -| | database. Available parameters vary depending on your | | -| | database backend. The parameters are supplied as comma | | -| | separated key=value pairs. | | +| reporting_options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt index b884ca065..5ac2fbc28 100644 --- a/doc/server/plugins/connectors/awstags.txt +++ b/doc/server/plugins/connectors/awstags.txt @@ -7,8 +7,8 @@ ========= The AWSTags plugin is a connector that retrieves tags from instances -in EC2, and can assign optionally assign -group membership pased on patterns in the tags. See `Using Tags +in EC2, and can optionally assign group membership based on patterns +in the tags. See `Using Tags <http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ for details on using tags in EC2. diff --git a/doc/server/plugins/connectors/templatehelper.txt b/doc/server/plugins/connectors/templatehelper.txt index c719f93cb..d113dcab7 100644 --- a/doc/server/plugins/connectors/templatehelper.txt +++ b/doc/server/plugins/connectors/templatehelper.txt @@ -56,7 +56,7 @@ Usage Specific helpers can be referred to in templates as ``metadata.TemplateHelper[<modulename>]``. That accesses -a HelperModule object will has, as attributes, all symbols listed in +a HelperModule object will have, as attributes, all symbols listed in ``__export__``. For example, consider this helper module:: __export__ = ["hello"] diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 8b49e244b..c991f20c9 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -321,15 +321,6 @@ processed for any Genshi, Cheetah, or Jinja2 base file. Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah -Genshi templates take precence over cheetah templates. For example, if -two files exist named:: - - Cfg/etc/fstab/fstab.genshi - Cfg/etc/fstab/fstab.cheetah - -The Cheetah template is ignored. Exploiting this fact is probably a -pretty bad idea in practice. - You can mix Genshi and Cheetah when using different host-specific or group-specific files. For example:: diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index a8366721a..234a6c030 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "April 06, 2014" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "November 04, 2014" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -28,8 +28,6 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructuredText. -. .SH DESCRIPTION .sp bcfg2.conf includes configuration parameters for the Bcfg2 server and @@ -107,7 +105,7 @@ SCCS .B listen_all This setting tells the server to listen on all available interfaces. The default is to only listen on those interfaces specified by the -bcfg2 setting in the components section of \fBbcfg2.conf\fP. +bcfg2 setting in the components section of \fBbcfg2.conf\fP\&. .TP .B plugins A comma\-delimited list of enabled server plugins. Currently @@ -179,24 +177,24 @@ best .UNINDENT .UNINDENT .sp -The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP. +The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP\&. More details on the backends can be found in the official documentation. .TP .B user -The username or UID to run the daemon as. Default is \fI0\fP. +The username or UID to run the daemon as. Default is \fI0\fP\&. .TP .B group -The group name or GID to run the daemon as. Default is \fI0\fP. +The group name or GID to run the daemon as. Default is \fI0\fP\&. .TP .B vcs_root Specifies the path to the root of the VCS working copy that holds -your Bcfg2 specification, if it is different from \fIrepository\fP. +your Bcfg2 specification, if it is different from \fIrepository\fP\&. E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. .TP .B umask -The umask to set for the server. Default is \fI0077\fP. +The umask to set for the server. Default is \fI0077\fP\&. .UNINDENT .SH SERVER PLUGINS .sp @@ -554,6 +552,10 @@ The following options are specified in the \fB[packages]\fP section. .INDENT 3.5 .INDENT 0.0 .TP +.B backends +Comma separated list of backends for the dependency resolution. +Default is "Yum,Apt,Pac,Pkgng". +.TP .B resolver Enable dependency resolution. Default is 1 (true). .TP @@ -569,7 +571,7 @@ The path at which to generate APT configs. No default. .TP .B gpg_keypath The path on the client where RPM GPG keys will be copied before -they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP. +they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP\&. .TP .B version Set the version attribute used when binding Packages. Default is @@ -628,7 +630,7 @@ the configuration file. .TP .B path Custom path for backups created in paranoid mode. The default is -in \fB/var/cache/bcfg2\fP. +in \fB/var/cache/bcfg2\fP\&. .TP .B max_copies Specify a maximum number of copies for the server to keep when @@ -707,9 +709,8 @@ Host for database connections. Not used for sqlite3. Port for database connections. Not used for sqlite3. .TP .B options -Various options for the database connection. The value is -expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .TP .B reporting_engine The database engine used by the Reporting plugin. One of the @@ -750,9 +751,8 @@ Host for reporting database connections. Not used for sqlite3. Port for reporting database connections. Not used for sqlite3. .TP .B reporting_options -Various options for the reporting database connection. The value -is expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .UNINDENT .UNINDENT .UNINDENT @@ -772,6 +772,10 @@ time zone as well). .TP .B web_debug Turn on Django debugging. +.TP +.B max_children +Maximum number of children for the reporting collector. Use 0 to +disable the limit. (default is 0) .UNINDENT .UNINDENT .UNINDENT diff --git a/schemas/packages.xsd b/schemas/packages.xsd index ae7b0430a..fc5a1356c 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -173,7 +173,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:attribute type="xsd:boolean" name="debsrc" default="false"> <xsd:annotation> <xsd:documentation> Include ``deb-src`` lines in the generated APT @@ -217,6 +217,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:string" name="name"> + <xsd:annotation> + <xsd:documentation> + Specifiy an explicit name for the source and do not generate + it automatically. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/solaris/Makefile b/solaris/Makefile index cdf61d8f7..a8a673e3e 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -2,14 +2,14 @@ PYTHON="/usr/local/bin/python" VERS=1.4.0pre1-1 -PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") +export PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package package: -mkdir tmp tmp/bcfg2-server tmp/bcfg2 -mkdir -p build/lib/$(PYVERSION)/site-packages - -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python2.6/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build + -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python$(PYVERSION)/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build #setuptools appears to use a restictive umask -chmod -R o+r build/ -cat build/bin/bcfg2 | sed -e 's!/usr/bin/python!$(PYTHON)!' > build/bin/bcfg2.new && mv build/bin/bcfg2.new build/bin/bcfg2 diff --git a/solaris/gen-prototypes.sh b/solaris/gen-prototypes.sh index 64aff9edb..b0006df16 100644 --- a/solaris/gen-prototypes.sh +++ b/solaris/gen-prototypes.sh @@ -1,6 +1,6 @@ #!/bin/sh cd build -PP="./lib/python/site-packages/" +PP="./lib/python${PYVERSION}/site-packages/" #bcfg2 echo "i pkginfo=./pkginfo.bcfg2" > ../prototype.tmp diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cf4e7c7ea..1003ab842 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -5,6 +5,7 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import os +import sys import apt.cache import Bcfg2.Options import Bcfg2.Client.Tools @@ -12,7 +13,7 @@ import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations - and inherits the rest from Tools.Tool. """ + and inherits the rest from Toolset.Toolset.""" options = Bcfg2.Client.Tools.Tool.options + [ Bcfg2.Options.PathOption( @@ -79,10 +80,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -103,9 +108,10 @@ class APT(Bcfg2.Client.Tools.Tool): for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) @@ -127,11 +133,11 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" % - entry.get('name')) + self.logger.error("Package %s is not fully installed" + % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" % - item) + self.logger.error("Got Unsupported pattern %s from debsums" + % item) files.append(item) files = list(set(files) - set(self.ignores)) # We check if there is file in the checksum to do @@ -142,30 +148,31 @@ class APT(Bcfg2.Client.Tools.Tool): bad = [filename for filename in files if filename not in modlist] if bad: self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.info("Package %s failed validation. Bad files " - "are:" % entry.get('name')) + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? " - "(y/N) " % (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): # nopep8 + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: # nopep8 + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -178,31 +185,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = \ - self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = \ - self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if (not Bcfg2.Options.setup.quick and - entry.get('verify', 'true') == 'true' - and checksums): + if not Bcfg2.Options.setup.quick \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -220,7 +229,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -240,24 +249,26 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): # nopep8 - self.logger.error("APT has no information about package %s" % - (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - cversion = \ - self.pkg_cache[pkg.get('name')].candidate.version - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: self.logger.error("Failed to find %s in apt package " "cache" % pkg.get('name')) continue else: - cversion = self.pkg_cache[pkg.get('name')].candidateVersion - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: avail_vers = [ x.ver_str for x in @@ -266,13 +277,14 @@ class APT(Bcfg2.Client.Tools.Tool): avail_vers = [ x.VerStr for x in self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" % - (pkg.get('name'), pkg.get('version'), - avail_vers)) + self.logger.error("Package %s: desired version %s not in %s" + % (pkg.get('name'), pkg.get('version'), + avail_vers)) bad_pkgs.append(pkg.get('name')) if bad_pkgs: self.logger.error("Cannot find correct versions of packages:") @@ -290,6 +302,6 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified.append(package) return states - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index dedc50d89..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,7 +2,6 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import safe_input -from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -11,23 +10,6 @@ class Action(Bcfg2.Client.Tools.Tool): __handles__ = [('Action', None)] __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if (Bcfg2.Options.setup.decision == 'whitelist' and - not matches_white_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if (Bcfg2.Options.setup.decision == 'blacklist' and - not passes_black_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -76,7 +58,7 @@ class Action(Bcfg2.Client.Tools.Tool): states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states @@ -87,7 +69,7 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if (action.get('timing') in ['post', 'both'] and action.get('when') != 'modified'): - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index a7fcb6709..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 5698f237a..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +from Bcfg2.Compat import urlretrieve + # pylint: disable=C0103 noask = ''' @@ -37,6 +39,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -45,6 +49,41 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool + + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplefile' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. + """ + for pkg in packages: + if pkg.get('simplefile'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) + + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd + + def Install(self, packages): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -80,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (Bcfg2.Options.setup.quick or - entry.attrib.get('verify', 'true') == 'false'): + if Bcfg2.Options.setup.quick or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..3b60c8285 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,15 +13,25 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + def get_svc_name(self, service): + """Append .service to name if name doesn't specify a unit type.""" + svc = service.get('name') + if svc.endswith(('.service', '.socket', '.device', '.mount', + '.automount', '.swap', '.target', '.path', + '.timer', '.snapshot', '.slice', '.scope')): + return svc + else: + return '%s.service' % svc + def get_svc_command(self, service, action): - return "/bin/systemctl %s %s.service" % (action, service.get('name')) + return "/bin/systemctl %s %s" % (action, self.get_svc_name(service)) def VerifyService(self, entry, _): """Verify Service status for entry.""" if entry.get('status') == 'ignore': return True - cmd = "/bin/systemctl status %s.service " % (entry.get('name')) + cmd = "/bin/systemctl status %s" % (self.get_svc_name(entry)) rv = self.cmd.run(cmd) if 'Loaded: error' in rv.stdout: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 85839cf46..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -11,6 +11,7 @@ import yum.callbacks import yum.Errors import yum.misc import rpmUtils.arch +import rpmUtils.miscutils import Bcfg2.Client.XML import Bcfg2.Client.Tools import Bcfg2.Options @@ -146,14 +147,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): Bcfg2.Options.Option( cf=('YUM', 'verify_flags'), default=[], dest="yum_verify_flags", type=Bcfg2.Options.Types.comma_list, - help="YUM verify flags")] + help="YUM verify flags"), Bcfg2.Options.Option( - cf=('YUM', 'disabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", - help="YUM disabled plugins")] + cf=('YUM', 'disabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", + help="YUM disabled plugins"), Bcfg2.Options.Option( - cf=('YUM', 'enabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", + cf=('YUM', 'enabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", help="YUM enabled plugins")] pkgtype = 'yum' @@ -254,12 +255,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): debuglevel = 0 if len(Bcfg2.Options.setup.yum_disabled_plugins) > 0: - rv.preconf.disabled_plugins= - Bcfg2.Options.setup.yum_disabled_plugins + rv.preconf.disabled_plugins = \ + Bcfg2.Options.setup.yum_disabled_plugins if len(Bcfg2.Options.setup.yum_enabled_plugins) > 0: - rv.preconf.enabled_plugins= - Bcfg2.Options.setup.yum_enabled_plugins + rv.preconf.enabled_plugins = \ + Bcfg2.Options.setup.yum_enabled_plugins # pylint: disable=E1121,W0212 try: @@ -660,7 +661,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra.get('release', 'any')) entry.set('current_version', "%s:%s-%s" % current_evr) entry.set('version', "%s:%s-%s" % wanted_evr) - if yum.compareEVR(current_evr, wanted_evr) == 1: + if rpmUtils.miscutils.compareEVR(current_evr, wanted_evr) == 1: entry.set("package_fail_action", "downgrade") else: entry.set("package_fail_action", "update") @@ -976,8 +977,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd294db98..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -129,6 +129,23 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) + return False + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) + return False + return True + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -587,7 +604,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 073aa7694..5f4f15dcc 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -873,15 +873,21 @@ class Client(object): def GenerateStats(self): """Generate XML summary of execution statistics.""" + states = {} + for (item, val) in list(self.states.items()): + if not Bcfg2.Options.setup.only_important or \ + item.get('important', 'false').lower() == 'true': + states[item] = val + feedback = XML.Element("upload-statistics") stats = XML.SubElement(feedback, - 'Statistics', total=str(len(self.states)), + 'Statistics', total=str(len(states)), version='2.0', revision=self.config.get('revision', '-1')) - good_entries = [key for key, val in list(self.states.items()) if val] + good_entries = [key for key, val in list(states.items()) if val] good = len(good_entries) stats.set('good', str(good)) - if any(not val for val in list(self.states.values())): + if any(not val for val in list(states.values())): stats.set('state', 'dirty') else: stats.set('state', 'clean') @@ -890,8 +896,8 @@ class Client(object): for (data, ename) in [(self.modified, 'Modified'), (self.extra, "Extra"), (good_entries, "Good"), - ([entry for entry in self.states - if not self.states[entry]], "Bad")]: + ([entry for entry in states + if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: item.set('qtext', '') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 982a299c0..f5b5d16aa 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -8,6 +8,7 @@ import Bcfg2.Options try: import django + import django.core.management import django.conf HAS_DJANGO = True except ImportError: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 12c9cdaa8..90b9f0ec7 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -5,13 +6,12 @@ import logging import time import threading -# pylint: disable=E0611 from lockfile import LockFailed, LockTimeout +# pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -30,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -38,26 +38,37 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - sys.exc_info()[1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, Bcfg2.Options.Common.reporting_transport, - Bcfg2.Options.Common.daemon] + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] def __init__(self): """Setup the collector. This may be called by the daemon or though @@ -67,6 +78,10 @@ class ReportingCollector(object): self.children = [] self.cleanup_threshold = 25 + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + if Bcfg2.Options.setup.debug: level = logging.DEBUG elif Bcfg2.Options.setup.verbose: @@ -113,25 +128,31 @@ class ReportingCollector(object): if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - sys.exc_info()[1]) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + Bcfg2.Options.setup.daemon) + self.shutdown() + return + else: + self.context.pidfile.release() + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) @@ -141,7 +162,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - store_thread = ReportingStoreThread(interaction, self.storage) + if Bcfg2.Options.setup.children > 0: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 406216861..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -168,7 +168,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 2d96990b1..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -717,9 +717,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 4a93e77e0..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', - '/group/(?P<group>[\w\-\.]+)', - '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): + '/group/(?P<group>[^/]+)', + '/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)', + '/server/(?P<server>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 769addf55..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -85,20 +86,30 @@ class BuiltinCore(NetworkCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, - Bcfg2.Options.setup.daemon)) - return False + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + Bcfg2.Options.setup.daemon)) + return False + else: + self.context.pidfile.release() + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 892f2832a..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -84,7 +84,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv @@ -783,13 +783,13 @@ class Core(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + self.block_for_fam_events() + self._block() except: self.shutdown() raise - self.block_for_fam_events() - self._block() - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d0fd70c5c..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -238,6 +238,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py index 25ad4ef61..c6d7a3a30 100644 --- a/src/lib/Bcfg2/Server/Lint/AWSTags.py +++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py @@ -9,6 +9,7 @@ import Bcfg2.Server.Lint class AWSTags(Bcfg2.Server.Lint.ServerPlugin): """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags <server-plugins-connectors-awstags>` patterns for validity. """ + __serverplugin__ = 'AWSTags' def Run(self): cfg = self.core.plugins['AWSTags'].config diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py index aee15cb5d..576e157ad 100644 --- a/src/lib/Bcfg2/Server/Lint/Bundler.py +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -7,6 +7,7 @@ from Bcfg2.Server.Lint import ServerPlugin class Bundler(ServerPlugin): """ Perform various :ref:`Bundler <server-plugins-structures-bundler>` checks. """ + __serverplugin__ = 'Bundler' def Run(self): self.missing_bundles() diff --git a/src/lib/Bcfg2/Server/Lint/Cfg.py b/src/lib/Bcfg2/Server/Lint/Cfg.py index 7716cd5c7..13b04a6b8 100644 --- a/src/lib/Bcfg2/Server/Lint/Cfg.py +++ b/src/lib/Bcfg2/Server/Lint/Cfg.py @@ -10,6 +10,7 @@ from Bcfg2.Server.Plugins.Cfg import CfgGenerator class Cfg(ServerPlugin): """ warn about Cfg issues """ + __serverplugin__ = 'Cfg' def Run(self): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index fc4506c12..fbe84de87 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -93,13 +93,21 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for info.xml files"), Bcfg2.Options.Option( - cf=("Comments", "probe_keywords"), + cf=("Comments", "probes_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for probes"), Bcfg2.Options.Option( - cf=("Comments", "probe_comments"), + cf=("Comments", "probes_comments"), type=Bcfg2.Options.Types.comma_list, default=[], - help="Required comments for probes")] + help="Required comments for probes"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for metadata files"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for metadata files")] def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) @@ -248,7 +256,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, - entry.infoxml.pnode.data, + entry.infoxml.xdata, "infoxml") continue if rtype: diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index a2581e70b..a2581e70b 100755..100644 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py index d8142cab9..8ddb9e796 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -13,6 +13,7 @@ class GroupPatterns(ServerPlugin): :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for each pattern, and catching exceptions and presenting them as ``bcfg2-lint`` errors.""" + __serverplugin__ = 'GroupPatterns' def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 4b1513a11..950a86f01 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -15,6 +15,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): * Paranoid mode disabled in an ``info.xml`` file; * Required attributes missing from ``info.xml`` """ + __serverplugin__ = 'Cfg' options = Bcfg2.Server.Lint.ServerPlugin.options + [ Bcfg2.Options.Common.default_paranoid, diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py index 333249cc2..333249cc2 100755..100644 --- a/src/lib/Bcfg2/Server/Lint/Jinja2.py +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py diff --git a/src/lib/Bcfg2/Server/Lint/Metadata.py b/src/lib/Bcfg2/Server/Lint/Metadata.py index 248b1610c..e445892d1 100644 --- a/src/lib/Bcfg2/Server/Lint/Metadata.py +++ b/src/lib/Bcfg2/Server/Lint/Metadata.py @@ -15,6 +15,7 @@ class Metadata(ServerPlugin): * Multiple default groups or a default group that isn't a profile group. """ + __serverplugin__ = 'Metadata' def Run(self): self.nested_clients() diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py index 3f0b9477c..eed6d4c19 100644 --- a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -12,6 +12,7 @@ class Pkgmgr(ServerlessPlugin): """ Find duplicate :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` entries with the same priority. """ + __serverplugin__ = 'Pkgmgr' def Run(self): pset = set() diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py index 5a80a5884..a437c1318 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -62,7 +62,7 @@ class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): # finally, check for executable permissions in info.xml for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): - for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + for pinfo in entry.infoxml.xdata.xpath("//FileInfo/Info"): try: mode = int( pinfo.get("mode", diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index a952da724..9d05516f1 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -20,6 +20,7 @@ class TemplateHelper(ServerPlugin): * Bogus symbols listed in ``__export__``, including symbols that don't exist, that are reserved, or that start with underscores. """ + __serverplugin__ = 'TemplateHelper' def __init__(self, *args, **kwargs): ServerPlugin.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 0b3f1e24d..cab5d248d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -18,7 +18,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): options = Bcfg2.Server.Lint.ServerlessPlugin.options + [ Bcfg2.Options.PathOption( "--schema", cf=("Validate", "schema"), - default="/usr/share/bcfg2/schema", + default="/usr/share/bcfg2/schemas", help="The full path to the XML schema files")] def __init__(self, *args, **kwargs): diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 9b3e6ece2..526bdf159 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -14,6 +14,7 @@ import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins from Bcfg2.Compat import walk_packages +from Bcfg2.Options import _debug def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -48,6 +49,11 @@ def get_termsize(): class Plugin(object): """ Base class for all bcfg2-lint plugins """ + #: Name of the matching server plugin or None if there is no + #: matching one. If this is None the lint plugin will only loaded + #: by default if the matching server plugin is enabled, too. + __serverplugin__ = None + options = [Bcfg2.Options.Common.repository] def __init__(self, errorhandler=None, files=None): @@ -291,19 +297,41 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 class LintPluginAction(Bcfg2.Options.ComponentAction): - """ We want to load all lint plugins that pertain to server - plugins. In order to do this, we hijack the __call__() method of - this action and add all of the server plugins on the fly """ - + """ Option parser action to load lint plugins """ bases = ['Bcfg2.Server.Lint'] - def __call__(self, parser, namespace, values, option_string=None): - plugins = getattr(Bcfg2.Options.setup, "plugins", []) - for lint_plugin in walk_packages(path=__path__): - if lint_plugin[1] in plugins: - values.append(lint_plugin[1]) - Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, - option_string) + +class LintPluginOption(Bcfg2.Options.Option): + """ Option class for the lint_plugins """ + + def early_parsing_hook(self, namespace): + """ + We want a usefull default for the enabled lint plugins. + Therfore we use all importable plugins, that either pertain + with enabled server plugins or that has no matching plugin. + """ + + plugins = [p.__name__ for p in namespace.plugins] + for loader, name, _is_pkg in walk_packages(path=__path__): + try: + module = loader.find_module(name).load_module(name) + plugin = getattr(module, name) + if plugin.__serverplugin__ is None or \ + plugin.__serverplugin__ in plugins: + _debug("Automatically adding lint plugin %s" % + plugin.__name__) + self.default.append(plugin.__name__) + except ImportError: + pass + + +class _EarlyOptions(object): + """ We need the server.plugins options in an early parsing hook + for determining the default value for the lint_plugins. So we + create a component that is parsed before the other options. """ + + parse_first = True + options = [Bcfg2.Options.Common.plugins] class CLI(object): @@ -313,7 +341,7 @@ class CLI(object): '--lint-config', default='/etc/bcfg2-lint.conf', action=Bcfg2.Options.ConfigFileAction, help='Specify bcfg2-lint configuration file'), - Bcfg2.Options.Option( + LintPluginOption( "--lint-plugins", cf=('lint', 'plugins'), default=[], type=Bcfg2.Options.Types.comma_list, action=LintPluginAction, help='bcfg2-lint plugin list'), @@ -328,28 +356,11 @@ class CLI(object): def __init__(self): parser = Bcfg2.Options.get_parser( description="Manage a running Bcfg2 server", - components=[self]) + components=[self, _EarlyOptions]) parser.parse() self.logger = logging.getLogger(parser.prog) - # automatically add Lint plugins for loaded server plugins - for plugin in Bcfg2.Options.setup.plugins: - try: - Bcfg2.Options.setup.lint_plugins.append( - getattr( - __import__("Bcfg2.Server.Lint.%s" % plugin.__name__, - fromlist=[plugin.__name__]), - plugin.__name__)) - self.logger.debug("Automatically adding lint plugin %s" % - plugin.__name__) - except ImportError: - # no lint plugin for this server plugin - self.logger.debug("No lint plugin for %s" % plugin.__name__) - except AttributeError: - self.logger.error("Failed to load plugin %s: %s" % - (plugin.__name__, sys.exc_info()[1])) - self.logger.debug("Running lint with plugins: %s" % [p.__name__ for p in Bcfg2.Options.setup.lint_plugins]) @@ -428,9 +439,9 @@ class CLI(object): def run_server_plugins(self): """ run plugins that require a running server to run """ core = Bcfg2.Server.Core.Core() - core.load_plugins() - core.block_for_fam_events(handle_events=True) try: + core.load_plugins() + core.block_for_fam_events(handle_events=True) self.logger.debug("Running server plugins: %s" % [p.__name__ for p in self.serverplugins]) for plugin in self.serverplugins: diff --git a/src/lib/Bcfg2/Server/Plugins/ACL.py b/src/lib/Bcfg2/Server/Plugins/ACL.py index 1c1e54312..37f51a2a1 100644 --- a/src/lib/Bcfg2/Server/Plugins/ACL.py +++ b/src/lib/Bcfg2/Server/Plugins/ACL.py @@ -62,6 +62,7 @@ def ip_matches(ip, entry): class IPACLFile(Bcfg2.Server.Plugin.XMLFileBacked): """ representation of ACL ip.xml, for IP-based ACLs """ + __identifier__ = None actions = dict(Allow=True, Deny=False, Defer=None) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 3d5c68e3f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -102,7 +102,8 @@ class AptSource(Source): bdeps[barch][pkgname] = [] brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 24db2963d..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -199,6 +199,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -274,11 +277,11 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -286,6 +289,7 @@ class Source(Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property @@ -353,7 +357,7 @@ class Source(Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) @@ -388,8 +392,10 @@ class Source(Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -419,6 +425,9 @@ class Source(Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index 047905fc3..654e792b8 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -32,11 +32,11 @@ class TestTool(Bcfg2TestCase): def setUp(self): set_setup_default('command_timeout') set_setup_default('interactive', False) + set_setup_default('decision') def get_obj(self, config=None): if config is None: config = lxml.etree.Element("Configuration") - execs = self.test_obj.__execs__ self.test_obj.__execs__ = [] rv = self.test_obj(config) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py index cfb0c023b..3da323262 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os import sys -from Bcfg2.Compat import b64decode +from Bcfg2.Compat import b64decode, b64encode from mock import Mock, MagicMock, patch # add all parent testsuite directories to sys.path to allow (most) @@ -125,10 +125,28 @@ baz passwd, "also bogus"])) - # test with no good passphrase given nor in config + # test with no good passphrase given nor in config. we use + # something that isn't a valid ciphertext here since a + # ciphertext encrypted with one key may be technically + # decryptable with a different key, although it will decrypt + # to gibberish. nonetheless, it doesn't raise the requisite + # EVPError, so the test fails. self.assertRaises(EVPError, bruteforce_decrypt, - crypted, passphrases=["bogus", "also bogus"]) + b64encode("not an actual ciphertext!"), + passphrases=["bogus", "also bogus"]) + + # test with no good passphrase given nor in config. this + # version of the test uses a valid ciphertext, and looks for + # *either* EVPError or a failed decrypt. + try: + plaintext = bruteforce_decrypt(crypted, + passphrases=["bogus", "also bogus"]) + if plaintext == passwd: + self.fail("Successfully decrypted ciphertext with wrong key") + except EVPError: + # success! + pass # test with good passphrase in config file Bcfg2.Options.setup.passphrases = dict(bogus="bogus", diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 77b170809..79eff7959 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -35,6 +35,7 @@ contingent_checks = { "lib/Bcfg2/Server/Admin": ["Reports.py", "Syncdb.py"], "sbin": ["bcfg2-reports"]}, ("pyinotify",): {"lib/Bcfg2/Server/FileMonitor": ["Inotify.py"]}, + ("apt",): {"lib/Bcfg2/Client/Tools": ["APT.py"]}, ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, @@ -68,7 +69,8 @@ error_checks = { # perform no checks at all on the listed files no_checks = { - "lib/Bcfg2/Client/Tools": ["APT.py", "RPM.py", "rpmtools.py"], + "lib/Bcfg2/Client/Tools": ["RPM.py", "rpmtools.py"], + "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], "lib/Bcfg2/Server/Plugins": ["Base.py"], diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 1a919f869..29fd9886b 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -24,3 +24,7 @@ migrate_probe_groups_to_db.py - Migrate Probe host and group data from XML to DB backend for Metadata and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. + +migrate_sysv_simplename.py + - Migrate any Pkgmgr entries which may have been using the simplename + attribute introduced in 1.3.5 to the simplefile attribute diff --git a/tools/upgrade/1.3/migrate_sysv_simplename.py b/tools/upgrade/1.3/migrate_sysv_simplename.py new file mode 100755 index 000000000..f6599756b --- /dev/null +++ b/tools/upgrade/1.3/migrate_sysv_simplename.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import lxml.etree +import Bcfg2.Options + +def main(): + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + files = [] + for plugin in ['Pkgmgr']: + files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + + for bfile in files: + bdata = lxml.etree.parse(bfile) + changed = False + + if not bdata.xpath("//@type='sysv'"): + print("%s doesn't contain any sysv packages, skipping" % bfile) + continue + + pkglist = bdata.getroot() + if pkglist.tag != "PackageList": + print("%s doesn't look like a PackageList, skipping" % bfile) + continue + + for pkg in bdata.xpath("//Package"): + if "simplename" in pkg.attrib: + pkg.set("simplefile", pkg.get("simplename")) + del pkg.attrib["simplename"] + changed = True + + # if we switched to simplefile, we also need to switch to uri + if changed and "url" in pkglist.attrib: + pkglist.set("uri", pkglist.get("url")) + del pkglist.attrib["url"] + + if changed: + print("Writing %s" % bfile) + try: + open(bfile, "w").write(lxml.etree.tostring(bdata)) + except IOError: + err = sys.exc_info()[1] + print("Could not write %s: %s" % (bfile, err)) + +if __name__ == '__main__': + sys.exit(main()) |