diff options
| author | Alexander Sulfrian <alexander@sulfrian.net> | 2014-11-10 19:55:16 +0100 |
|---|---|---|
| committer | Alexander Sulfrian <alexander@sulfrian.net> | 2014-11-10 19:55:16 +0100 |
| commit | da93fb540c28be3341ec0d37d1fbd90153fb750c (patch) | |
| tree | 9615b3ddcc594ba5fb6a89dbce2516adab641153 | |
| parent | 389ce1a86b704222ddc9458cd49c281e7601b803 (diff) | |
| parent | f6b3d6dc89e4ba5c2ec643931394db45b8c7f176 (diff) | |
| download | bcfg2-da93fb540c28be3341ec0d37d1fbd90153fb750c.tar.gz bcfg2-da93fb540c28be3341ec0d37d1fbd90153fb750c.tar.bz2 bcfg2-da93fb540c28be3341ec0d37d1fbd90153fb750c.zip | |
Merge remote branch 'upstream/maint'
* upstream/maint: (59 commits)
Client/Tools/APT: fix pylint errors, enable check
Client/Tools/APT: fix exception name
SYSV: add a migration tool to change simplename attributes to simplefile
doc: Update SYSV datastream format docs
SYSV: change instances of simplename to simplefile
Client/Tools/APT: handle exceptions gracefully
Server/Plugins/Packages: add debsrc attribute to url_map
schemas/packages.xsd: add default value for debsrc
Options.py: Fix database OPTIONS parsing
doc: Fix typos
doc: Add note about Decisions fix for 1.3.6
testsuite: Fix tools __init__.py unit test
Tools: Fix install of non-whitelisted services
doc: Add 1.3.6 release notes
fix keyboard interrupt during intial event handling
Server/Core: also shutdown on exceptions during fam blocking
Reporting: better exception handling
Packages: add name to additional_data for Sources
Packages: add name to sources
YUM: Add options to enable and disable Yum plugins
...
Conflicts:
debian/changelog
doc/conf.py
doc/releases/index.txt
doc/server/database.txt
doc/server/plugins/generators/cfg.txt
doc/server/plugins/generators/sslca.txt
man/bcfg2.conf.5
misc/bcfg2-selinux.spec
misc/bcfg2.spec
osx/Makefile
osx/macports/Portfile
schemas/packages.xsd
solaris-ips/MANIFEST.bcfg2-server.header
solaris-ips/MANIFEST.bcfg2.header
solaris-ips/Makefile
solaris-ips/pkginfo.bcfg2
solaris-ips/pkginfo.bcfg2-server
solaris/Makefile
solaris/pkginfo.bcfg2
solaris/pkginfo.bcfg2-server
src/lib/Bcfg2/Client/Client.py
src/lib/Bcfg2/Client/Proxy.py
src/lib/Bcfg2/Client/Tools/APT.py
src/lib/Bcfg2/Client/Tools/Action.py
src/lib/Bcfg2/Client/Tools/SYSV.py
src/lib/Bcfg2/Client/Tools/YUM.py
src/lib/Bcfg2/Client/Tools/__init__.py
src/lib/Bcfg2/Options.py
src/lib/Bcfg2/Reporting/Collector.py
src/lib/Bcfg2/Reporting/templates/base.html
src/lib/Bcfg2/Server/Admin/__init__.py
src/lib/Bcfg2/Server/BuiltinCore.py
src/lib/Bcfg2/Server/Core.py
src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
src/lib/Bcfg2/Server/Plugins/Probes.py
src/lib/Bcfg2/settings.py
src/lib/Bcfg2/version.py
testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py
testsuite/Testsrc/test_code_checks.py
39 files changed, 461 insertions, 213 deletions
diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index b1c3aba21..cd2ad858e 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -33,8 +33,8 @@ BCFG2_SERVER_ENABLED=0 test -f "/etc/default/bcfg2-server" && . /etc/default/bcfg2-server if [ "$BCFG2_SERVER_ENABLED" -eq 0 ] ; then - log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" - exit 0 + log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" + exit 0 fi # Exit if $DAEMON doesn't exist and is not executable @@ -63,10 +63,11 @@ stop () { killproc -p $PIDFILE ${BINARY} STATUS=$? if [ "$STATUS" = 0 ]; then - log_success_msg "bcfg2-server" - test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server + [ -e $PIDFILE ] && rm -f $PIDFILE + log_success_msg "bcfg2-server" + test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server else - log_failure_msg "bcfg2-server" + log_failure_msg "bcfg2-server" fi return $STATUS } @@ -75,15 +76,15 @@ status () { # Inspired by redhat /etc/init.d/functions status() call PID=$(pidof -x $BINARY -o %PPID) if [ -n "$PID" ]; then - echo "$BINARY (pid $PID) is running..." - return 0 + echo "$BINARY (pid $PID) is running..." + return 0 fi if [ -f $PIDFILE ]; then - if [ -n "$PID" ]; then - log_failure_msg "$BINARY dead but pid file exists..." - return 1 - fi + if [ -n "$PID" ]; then + log_failure_msg "$BINARY dead but pid file exists..." + return 1 + fi fi log_failure_msg "$BINARY is not running" diff --git a/debian/changelog b/debian/changelog index e30fba546..c41b2ecc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,12 @@ bcfg2 (1.4.0pre1-0.0) unstable; urgency=low -- Sol Jerome <sol.jerome@gmail.com> Mon, 16 Jun 2014 09:36:13 -0500 +bcfg2 (1.3.5-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Fri, 05 Sep 2014 07:54:48 -0500 + bcfg2 (1.3.4-0.0) unstable; urgency=low * New upstream release diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 09ea76230..93eb11925 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -152,7 +152,36 @@ Systemd service support. SYSV ---- -Handles System V Packaging format that is available on Solaris. +Handles `System V Packaging <http://docs.oracle.com/cd/E19683-01/806-7008/index.html>`_ format that is available on Solaris. + +.. note:: + + If the Packages specified in the PackageList are datastream format + packages distributed via HTTP, you must specify a simplefile attribute. + Such packages will be downloaded and installed from a local path. + + Note the use of the uri attribute in the datastream format example. If + the simplefile attribute exists, the + :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` plugin will + automatically construct the url attribute by concatenating the uri and + simplefile attributes (with an intervening slash). + + Datastream format over HTTP: + + .. code-block:: xml + + <PackageList uri='http://install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' simplefile='bcfg-1.3.4-1' /> + </PackageList> + + File system format over NFS or local path: + + .. code-block:: xml + + <PackageList url='/mnt/install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' /> + </PackageList> + Upstart ------- diff --git a/doc/development/compat.txt b/doc/development/compat.txt index f90274ce5..8700c46d3 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -60,6 +60,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects. +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | urlparse | :func:`urlparse.urlparse` | :func:`urllib.parse.urlparse` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ +| urlretrieve | :func:`urllib.urlretrieve` | :func:`urllib.request.urlretrieve` | ++---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPBasicAuthHandler | :class:`urllib2.HTTPBasicAuthHandler` | :class:`urllib.request.HTTPBasicAuthHandler` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPPasswordMgrWithDefaultRealm | :class:`urllib2.HTTPPasswordMgrWithDefaultRealm` | :class:`urllib.request.HTTPPasswordMgrWithDefaultRealm` | diff --git a/doc/installation/distributions.txt b/doc/installation/distributions.txt index 306439485..5dad4d860 100644 --- a/doc/installation/distributions.txt +++ b/doc/installation/distributions.txt @@ -36,9 +36,9 @@ Just use `pacman` to perform the installation :: Debian ====== -Packages of Bcfg2 are available for Debian Lenny, Debian Squeeze, and -Debian Sid. The fastest way to get Bcfg2 onto your Debian system -is to use ``apt-get`` or ``aptitude``. :: +Packages of Bcfg2 are available for all current versions of Debian. +The fastest way to get Bcfg2 onto your Debian system is to use ``apt-get`` +or ``aptitude``. :: sudo aptitude install bcfg2 bcfg2-server diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 40766b88a..7c265f263 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -662,9 +662,8 @@ control the database connection of the server. Port for database connections. Not used for sqlite3. options - Various options for the database connection. The value is - expected as multiple key=value pairs, separated with commas. - The concrete value depends on the database engine. + Various options for the database connection. The value expected + is the literal value of the django OPTIONS setting. reporting_engine The database engine used by the Reporting plugin. One of the diff --git a/doc/releases/1.3.5.txt b/doc/releases/1.3.5.txt new file mode 100644 index 000000000..893cdbf39 --- /dev/null +++ b/doc/releases/1.3.5.txt @@ -0,0 +1,33 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.5: + +1.3.5 +===== + +We are happy to announce the release of Bcfg2 1.3.5. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Properly close db connections +* Improved error messages +* Fix yum upgrade/downgrade +* Enable bcfg2-yum-helper to depsolve for arches incompatible with + server +* Spec file fixes +* bcfg2-crypet: Default to only (En|De)crypt vars that need it +* Fix email reporting bug +* Fix debsums parsing +* Fix solaris makefile +* SYSV: Implement downloading and installing SYSV packages from HTTP: + http://docs.bcfg2.org/client/tools.html#sysv +* Fix debian bcfg2-server init script + + +Special thanks to the following contributors for this release: John +Morris, Jonathan Billings, Chris Brinker, Tim Laszlo, Matt Kemp, Michael +Fenn, Pavel Labushev, Nathan Olla, Alexander Sulfrian. diff --git a/doc/releases/1.3.6.txt b/doc/releases/1.3.6.txt new file mode 100644 index 000000000..757fbf6f5 --- /dev/null +++ b/doc/releases/1.3.6.txt @@ -0,0 +1,34 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.6: + +1.3.6 +===== + +We are happy to announce the release of Bcfg2 1.3.6. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Fix python 2.4 compatibility +* Fix stale lockfile detection and behavior +* Reporting: fix filter urls +* Fix client protocol option handling +* YUM: Add options to enable and disable Yum plugins +* Packages: add name to sources +* Reporting: better exception handling +* Various interrupt handling fixes +* Fix client decision whitelist/blacklist handling +* Fix database OPTIONS parsing + + This change requires you to set the *options* value of the + ``[database`` section in ``bcfg2.conf`` to the literal value which is + passed through to the django OPTIONS setting. + + https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-OPTIONS + +Special thanks to the following contributors for this release: Michael +Fenn, Matt Kemp, Alexander Sulfrian, Jonathan Billings. diff --git a/doc/releases/index.txt b/doc/releases/index.txt index 479aa19de..271fc23cc 100644 --- a/doc/releases/index.txt +++ b/doc/releases/index.txt @@ -10,4 +10,5 @@ Release Announcements .. toctree:: 1.4.0pre1 + 1.3.5 1.3.4 diff --git a/doc/server/database.txt b/doc/server/database.txt index 986914171..67cb065f4 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -69,8 +69,8 @@ of ``/etc/bcfg2.conf``. +--------------------+------------------------------------------------------------+---------------------------------------+ | options | Extra parameters to use when connecting to the database. | None | | | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_engine | The name of the Django database backend to use for the | None | | | reporting database. Takes the same values as ``engine``. | | @@ -85,10 +85,10 @@ of ``/etc/bcfg2.conf``. +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_port | The port to connect to for the reporting database | None | +--------------------+------------------------------------------------------------+---------------------------------------+ -| reporting_options | Extra parameters to use when connecting to the reporting | None | -| | database. Available parameters vary depending on your | | -| | database backend. The parameters are supplied as comma | | -| | separated key=value pairs. | | +| reporting_options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt index b884ca065..5ac2fbc28 100644 --- a/doc/server/plugins/connectors/awstags.txt +++ b/doc/server/plugins/connectors/awstags.txt @@ -7,8 +7,8 @@ ========= The AWSTags plugin is a connector that retrieves tags from instances -in EC2, and can assign optionally assign -group membership pased on patterns in the tags. See `Using Tags +in EC2, and can optionally assign group membership based on patterns +in the tags. See `Using Tags <http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ for details on using tags in EC2. diff --git a/doc/server/plugins/connectors/templatehelper.txt b/doc/server/plugins/connectors/templatehelper.txt index c719f93cb..d113dcab7 100644 --- a/doc/server/plugins/connectors/templatehelper.txt +++ b/doc/server/plugins/connectors/templatehelper.txt @@ -56,7 +56,7 @@ Usage Specific helpers can be referred to in templates as ``metadata.TemplateHelper[<modulename>]``. That accesses -a HelperModule object will has, as attributes, all symbols listed in +a HelperModule object will have, as attributes, all symbols listed in ``__export__``. For example, consider this helper module:: __export__ = ["hello"] diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 8b49e244b..c991f20c9 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -321,15 +321,6 @@ processed for any Genshi, Cheetah, or Jinja2 base file. Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah -Genshi templates take precence over cheetah templates. For example, if -two files exist named:: - - Cfg/etc/fstab/fstab.genshi - Cfg/etc/fstab/fstab.cheetah - -The Cheetah template is ignored. Exploiting this fact is probably a -pretty bad idea in practice. - You can mix Genshi and Cheetah when using different host-specific or group-specific files. For example:: diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index db72add64..234a6c030 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "April 06, 2014" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "November 04, 2014" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -28,8 +28,6 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructuredText. -. .SH DESCRIPTION .sp bcfg2.conf includes configuration parameters for the Bcfg2 server and @@ -107,7 +105,7 @@ SCCS .B listen_all This setting tells the server to listen on all available interfaces. The default is to only listen on those interfaces specified by the -bcfg2 setting in the components section of \fBbcfg2.conf\fP. +bcfg2 setting in the components section of \fBbcfg2.conf\fP\&. .TP .B plugins A comma\-delimited list of enabled server plugins. Currently @@ -179,24 +177,24 @@ best .UNINDENT .UNINDENT .sp -The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP. +The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP\&. More details on the backends can be found in the official documentation. .TP .B user -The username or UID to run the daemon as. Default is \fI0\fP. +The username or UID to run the daemon as. Default is \fI0\fP\&. .TP .B group -The group name or GID to run the daemon as. Default is \fI0\fP. +The group name or GID to run the daemon as. Default is \fI0\fP\&. .TP .B vcs_root Specifies the path to the root of the VCS working copy that holds -your Bcfg2 specification, if it is different from \fIrepository\fP. +your Bcfg2 specification, if it is different from \fIrepository\fP\&. E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. .TP .B umask -The umask to set for the server. Default is \fI0077\fP. +The umask to set for the server. Default is \fI0077\fP\&. .UNINDENT .SH SERVER PLUGINS .sp @@ -573,7 +571,7 @@ The path at which to generate APT configs. No default. .TP .B gpg_keypath The path on the client where RPM GPG keys will be copied before -they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP. +they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP\&. .TP .B version Set the version attribute used when binding Packages. Default is @@ -632,7 +630,7 @@ the configuration file. .TP .B path Custom path for backups created in paranoid mode. The default is -in \fB/var/cache/bcfg2\fP. +in \fB/var/cache/bcfg2\fP\&. .TP .B max_copies Specify a maximum number of copies for the server to keep when @@ -711,9 +709,8 @@ Host for database connections. Not used for sqlite3. Port for database connections. Not used for sqlite3. .TP .B options -Various options for the database connection. The value is -expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .TP .B reporting_engine The database engine used by the Reporting plugin. One of the @@ -754,9 +751,8 @@ Host for reporting database connections. Not used for sqlite3. Port for reporting database connections. Not used for sqlite3. .TP .B reporting_options -Various options for the reporting database connection. The value -is expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .UNINDENT .UNINDENT .UNINDENT diff --git a/schemas/packages.xsd b/schemas/packages.xsd index ae7b0430a..fc5a1356c 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -173,7 +173,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:attribute type="xsd:boolean" name="debsrc" default="false"> <xsd:annotation> <xsd:documentation> Include ``deb-src`` lines in the generated APT @@ -217,6 +217,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:string" name="name"> + <xsd:annotation> + <xsd:documentation> + Specifiy an explicit name for the source and do not generate + it automatically. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/solaris/Makefile b/solaris/Makefile index cdf61d8f7..a8a673e3e 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -2,14 +2,14 @@ PYTHON="/usr/local/bin/python" VERS=1.4.0pre1-1 -PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") +export PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package package: -mkdir tmp tmp/bcfg2-server tmp/bcfg2 -mkdir -p build/lib/$(PYVERSION)/site-packages - -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python2.6/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build + -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python$(PYVERSION)/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build #setuptools appears to use a restictive umask -chmod -R o+r build/ -cat build/bin/bcfg2 | sed -e 's!/usr/bin/python!$(PYTHON)!' > build/bin/bcfg2.new && mv build/bin/bcfg2.new build/bin/bcfg2 diff --git a/solaris/gen-prototypes.sh b/solaris/gen-prototypes.sh index 64aff9edb..b0006df16 100644 --- a/solaris/gen-prototypes.sh +++ b/solaris/gen-prototypes.sh @@ -1,6 +1,6 @@ #!/bin/sh cd build -PP="./lib/python/site-packages/" +PP="./lib/python${PYVERSION}/site-packages/" #bcfg2 echo "i pkginfo=./pkginfo.bcfg2" > ../prototype.tmp diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cf4e7c7ea..1003ab842 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -5,6 +5,7 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import os +import sys import apt.cache import Bcfg2.Options import Bcfg2.Client.Tools @@ -12,7 +13,7 @@ import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations - and inherits the rest from Tools.Tool. """ + and inherits the rest from Toolset.Toolset.""" options = Bcfg2.Client.Tools.Tool.options + [ Bcfg2.Options.PathOption( @@ -79,10 +80,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -103,9 +108,10 @@ class APT(Bcfg2.Client.Tools.Tool): for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) @@ -127,11 +133,11 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" % - entry.get('name')) + self.logger.error("Package %s is not fully installed" + % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" % - item) + self.logger.error("Got Unsupported pattern %s from debsums" + % item) files.append(item) files = list(set(files) - set(self.ignores)) # We check if there is file in the checksum to do @@ -142,30 +148,31 @@ class APT(Bcfg2.Client.Tools.Tool): bad = [filename for filename in files if filename not in modlist] if bad: self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.info("Package %s failed validation. Bad files " - "are:" % entry.get('name')) + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? " - "(y/N) " % (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): # nopep8 + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: # nopep8 + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -178,31 +185,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = \ - self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = \ - self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if (not Bcfg2.Options.setup.quick and - entry.get('verify', 'true') == 'true' - and checksums): + if not Bcfg2.Options.setup.quick \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -220,7 +229,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -240,24 +249,26 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): # nopep8 - self.logger.error("APT has no information about package %s" % - (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - cversion = \ - self.pkg_cache[pkg.get('name')].candidate.version - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: self.logger.error("Failed to find %s in apt package " "cache" % pkg.get('name')) continue else: - cversion = self.pkg_cache[pkg.get('name')].candidateVersion - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: avail_vers = [ x.ver_str for x in @@ -266,13 +277,14 @@ class APT(Bcfg2.Client.Tools.Tool): avail_vers = [ x.VerStr for x in self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" % - (pkg.get('name'), pkg.get('version'), - avail_vers)) + self.logger.error("Package %s: desired version %s not in %s" + % (pkg.get('name'), pkg.get('version'), + avail_vers)) bad_pkgs.append(pkg.get('name')) if bad_pkgs: self.logger.error("Cannot find correct versions of packages:") @@ -290,6 +302,6 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified.append(package) return states - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index dedc50d89..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,7 +2,6 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import safe_input -from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -11,23 +10,6 @@ class Action(Bcfg2.Client.Tools.Tool): __handles__ = [('Action', None)] __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if (Bcfg2.Options.setup.decision == 'whitelist' and - not matches_white_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if (Bcfg2.Options.setup.decision == 'blacklist' and - not passes_black_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -76,7 +58,7 @@ class Action(Bcfg2.Client.Tools.Tool): states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states @@ -87,7 +69,7 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if (action.get('timing') in ['post', 'both'] and action.get('when') != 'modified'): - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index a7fcb6709..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 5698f237a..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +from Bcfg2.Compat import urlretrieve + # pylint: disable=C0103 noask = ''' @@ -37,6 +39,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -45,6 +49,41 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool + + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplefile' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. + """ + for pkg in packages: + if pkg.get('simplefile'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) + + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd + + def Install(self, packages): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -80,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (Bcfg2.Options.setup.quick or - entry.attrib.get('verify', 'true') == 'false'): + if Bcfg2.Options.setup.quick or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index b1caf3332..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -977,8 +977,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd294db98..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -129,6 +129,23 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) + return False + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) + return False + return True + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -587,7 +604,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 88e8f68eb..90b9f0ec7 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -5,13 +6,12 @@ import logging import time import threading -# pylint: disable=E0611 from lockfile import LockFailed, LockTimeout +# pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -128,25 +128,31 @@ class ReportingCollector(object): if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - sys.exc_info()[1]) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + Bcfg2.Options.setup.daemon) + self.shutdown() + return + else: + self.context.pidfile.release() + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 406216861..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -168,7 +168,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 2d96990b1..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -717,9 +717,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 4a93e77e0..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', - '/group/(?P<group>[\w\-\.]+)', - '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): + '/group/(?P<group>[^/]+)', + '/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)', + '/server/(?P<server>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 769addf55..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -85,20 +86,30 @@ class BuiltinCore(NetworkCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, - Bcfg2.Options.setup.daemon)) - return False + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + Bcfg2.Options.setup.daemon)) + return False + else: + self.context.pidfile.release() + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 892f2832a..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -84,7 +84,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv @@ -783,13 +783,13 @@ class Core(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + self.block_for_fam_events() + self._block() except: self.shutdown() raise - self.block_for_fam_events() - self._block() - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d0fd70c5c..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -238,6 +238,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 3d5c68e3f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -102,7 +102,8 @@ class AptSource(Source): bdeps[barch][pkgname] = [] brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 24db2963d..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -199,6 +199,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -274,11 +277,11 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -286,6 +289,7 @@ class Source(Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property @@ -353,7 +357,7 @@ class Source(Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) @@ -388,8 +392,10 @@ class Source(Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -419,6 +425,9 @@ class Source(Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index 0e9e3a141..740b90251 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -32,11 +32,11 @@ class TestTool(Bcfg2TestCase): def setUp(self): set_setup_default('command_timeout') set_setup_default('interactive', False) + set_setup_default('decision') def get_obj(self, config=None): if config is None: config = lxml.etree.Element("Configuration") - execs = self.test_obj.__execs__ self.test_obj.__execs__ = [] rv = self.test_obj(config) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 77b170809..79eff7959 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -35,6 +35,7 @@ contingent_checks = { "lib/Bcfg2/Server/Admin": ["Reports.py", "Syncdb.py"], "sbin": ["bcfg2-reports"]}, ("pyinotify",): {"lib/Bcfg2/Server/FileMonitor": ["Inotify.py"]}, + ("apt",): {"lib/Bcfg2/Client/Tools": ["APT.py"]}, ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, @@ -68,7 +69,8 @@ error_checks = { # perform no checks at all on the listed files no_checks = { - "lib/Bcfg2/Client/Tools": ["APT.py", "RPM.py", "rpmtools.py"], + "lib/Bcfg2/Client/Tools": ["RPM.py", "rpmtools.py"], + "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], "lib/Bcfg2/Server/Plugins": ["Base.py"], diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 1a919f869..29fd9886b 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -24,3 +24,7 @@ migrate_probe_groups_to_db.py - Migrate Probe host and group data from XML to DB backend for Metadata and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. + +migrate_sysv_simplename.py + - Migrate any Pkgmgr entries which may have been using the simplename + attribute introduced in 1.3.5 to the simplefile attribute diff --git a/tools/upgrade/1.3/migrate_sysv_simplename.py b/tools/upgrade/1.3/migrate_sysv_simplename.py new file mode 100755 index 000000000..f6599756b --- /dev/null +++ b/tools/upgrade/1.3/migrate_sysv_simplename.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import lxml.etree +import Bcfg2.Options + +def main(): + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + files = [] + for plugin in ['Pkgmgr']: + files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + + for bfile in files: + bdata = lxml.etree.parse(bfile) + changed = False + + if not bdata.xpath("//@type='sysv'"): + print("%s doesn't contain any sysv packages, skipping" % bfile) + continue + + pkglist = bdata.getroot() + if pkglist.tag != "PackageList": + print("%s doesn't look like a PackageList, skipping" % bfile) + continue + + for pkg in bdata.xpath("//Package"): + if "simplename" in pkg.attrib: + pkg.set("simplefile", pkg.get("simplename")) + del pkg.attrib["simplename"] + changed = True + + # if we switched to simplefile, we also need to switch to uri + if changed and "url" in pkglist.attrib: + pkglist.set("uri", pkglist.get("url")) + del pkglist.attrib["url"] + + if changed: + print("Writing %s" % bfile) + try: + open(bfile, "w").write(lxml.etree.tostring(bdata)) + except IOError: + err = sys.exc_info()[1] + print("Could not write %s: %s" % (bfile, err)) + +if __name__ == '__main__': + sys.exit(main()) |