diff options
44 files changed, 604 insertions, 279 deletions
diff --git a/doc/appendix/guides/authentication.txt b/doc/appendix/guides/authentication.txt index b8ec82590..1af061170 100644 --- a/doc/appendix/guides/authentication.txt +++ b/doc/appendix/guides/authentication.txt @@ -146,7 +146,7 @@ Allowed values are: +-------------------+------------------------------------------+ ``cert+password`` is the default. This can be changed by setting the -``authentication`` parameter in the ``[communcation]`` section of +``authentication`` parameter in the ``[communication]`` section of ``bcfg2.conf``. For instance, to set ``bootstrap`` mode as the global default, you would add the following to ``bcfg2.conf``:: diff --git a/doc/client/tools.txt b/doc/client/tools.txt index ce8732454..170b30b2e 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -158,14 +158,22 @@ Handles `System V Packaging <http://docs.oracle.com/cd/E19683-01/806-7008/index. .. note:: - If the Packages specified in the PackageList are datastream format packages distributed via HTTP, you must specify a simplename attribute. Such packages will be downloaded and installed from a local path. + If the Packages specified in the PackageList are datastream format + packages distributed via HTTP, you must specify a simplefile attribute. + Such packages will be downloaded and installed from a local path. - datastream format over HTTP: + Note the use of the uri attribute in the datastream format example. If + the simplefile attribute exists, the + :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` plugin will + automatically construct the url attribute by concatenating the uri and + simplefile attributes (with an intervening slash). + + Datastream format over HTTP: .. code-block:: xml - <PackageList url='http://install/packages' type='sysv' priority='0'> - <Package name='SCbcfg2' version='1.3.4' simplename='bcfg-1.3.4-1' /> + <PackageList uri='http://install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' simplefile='bcfg-1.3.4-1' /> </PackageList> File system format over NFS or local path: diff --git a/doc/development/compat.txt b/doc/development/compat.txt index 8700c46d3..132bf67c0 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -113,7 +113,7 @@ with Python 2.4 (and occasionally 2.5). Be sure to read the notes below, since some of these implementations may be feature-incomplete. +----------------+--------------------------------+--------------------------------------------+ -| Name | Python 2.4 | Python 2.4+ | +| Name | Python 2.4 | Python 2.5+ | +================+================================+============================================+ | formatdate | :func:`email.Utils.formatdate` | :func:`email.utils.formatdate` | +----------------+--------------------------------+--------------------------------------------+ @@ -129,6 +129,8 @@ below, since some of these implementations may be feature-incomplete. +----------------+--------------------------------+--------------------------------------------+ | MutableMapping | :class:`UserDict.DictMixin` | :class:`collections.MutableMapping` (2.6+) | +----------------+--------------------------------+--------------------------------------------+ +| literal_eval | :func:`eval` | :func:`ast.literal_eval`(2.6+) | ++----------------+--------------------------------+--------------------------------------------+ walk_packages ~~~~~~~~~~~~~ @@ -171,6 +173,14 @@ mind. :class:`collections.MutableMapping` is available in Python 2.6+, and will be used if available. +literal_eval +~~~~~~~~~~~~ + +:func:`ast.literal_eval` is a safe version of :func:`eval` that will only +allow delaration of literal strings, ints, list, dicts, etc. This was +introduced in Python 2.6, and as such Python 2.4 uses the plain-old +:func:`eval`. + Other Symbols ------------- diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 6faf48a1a..f55540968 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -736,9 +736,8 @@ control the database connection of the server. Port for database connections. Not used for sqlite3. options - Various options for the database connection. The value is - expected as multiple key=value pairs, separated with commas. - The concrete value depends on the database engine. + Various options for the database connection. The value expected + is the literal value of the django OPTIONS setting. Reporting options ----------------- diff --git a/doc/releases/1.3.6.txt b/doc/releases/1.3.6.txt index e27b102bf..f41320f1a 100644 --- a/doc/releases/1.3.6.txt +++ b/doc/releases/1.3.6.txt @@ -22,6 +22,15 @@ This is primarily a bugfix release. * Reporting: better exception handling * Various interrupt handling fixes * Fix client decision whitelist/blacklist handling +* Fix database OPTIONS parsing + + This change requires you to set the *options* value of the + ``[database`` section in ``bcfg2.conf`` to the literal value which is + passed through to the django OPTIONS setting. + + https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-OPTIONS + +* Authentication: Reject passwd auth, if authentication is set to "cert" Special thanks to the following contributors for this release: Michael Fenn, Matt Kemp, Alexander Sulfrian, Jonathan Billings. diff --git a/doc/server/database.txt b/doc/server/database.txt index 3c8970f68..15c66754f 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -51,8 +51,8 @@ of ``/etc/bcfg2.conf``. +-------------+------------------------------------------------------------+-------------------------------+ | options | Extra parameters to use when connecting to the database. | None | | | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +-------------+------------------------------------------------------------+-------------------------------+ diff --git a/doc/server/plugins/generators/nagiosgen.txt b/doc/server/plugins/generators/nagiosgen.txt index 0ae922fa3..2cbc9fb5e 100644 --- a/doc/server/plugins/generators/nagiosgen.txt +++ b/doc/server/plugins/generators/nagiosgen.txt @@ -29,7 +29,6 @@ Create default host, and group specs in: check_period 24x7 contact_groups admins event_handler_enabled 1 - failure_prediction_enabled 1 flap_detection_enabled 1 initial_state o max_check_attempts 10 diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 5e64caae9..c503d6421 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,6 @@ -.TH "BCFG2.CONF" "5" "July 19, 2013" "1.3" "Bcfg2" +.\" Man page generated from reStructuredText. +. +.TH "BCFG2.CONF" "5" "November 04, 2014" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -28,8 +30,6 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructuredText. -. .SH DESCRIPTION .sp bcfg2.conf includes configuration parameters for the Bcfg2 server and @@ -77,7 +77,6 @@ pseudo .UNINDENT .TP .B fam_blocking -. Whether the server should block at startup until the file monitor backend has processed all events. This can cause a slower startup, but ensure that all files are recognized before the first client @@ -109,7 +108,7 @@ SCCS .B listen_all This setting tells the server to listen on all available interfaces. The default is to only listen on those interfaces specified by the -bcfg2 setting in the components section of \fBbcfg2.conf\fP. +bcfg2 setting in the components section of \fBbcfg2.conf\fP\&. .TP .B plugins A comma\-delimited list of enabled server plugins. Currently @@ -188,24 +187,24 @@ best .UNINDENT .UNINDENT .sp -The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP. +The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP\&. More details on the backends can be found in the official documentation. .TP .B user -The username or UID to run the daemon as. Default is \fI0\fP. +The username or UID to run the daemon as. Default is \fI0\fP\&. .TP .B group -The group name or GID to run the daemon as. Default is \fI0\fP. +The group name or GID to run the daemon as. Default is \fI0\fP\&. .TP .B vcs_root Specifies the path to the root of the VCS working copy that holds -your Bcfg2 specification, if it is different from \fIrepository\fP. +your Bcfg2 specification, if it is different from \fIrepository\fP\&. E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. .TP .B umask -The umask to set for the server. Default is \fI0077\fP. +The umask to set for the server. Default is \fI0077\fP\&. .UNINDENT .SH SERVER PLUGINS .sp @@ -625,7 +624,7 @@ The path at which to generate APT configs. No default. .TP .B gpg_keypath The path on the client where RPM GPG keys will be copied before -they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP. +they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP\&. .TP .B version Set the version attribute used when binding Packages. Default is @@ -684,7 +683,7 @@ the configuration file. .TP .B path Custom path for backups created in paranoid mode. The default is -in \fB/var/cache/bcfg2\fP. +in \fB/var/cache/bcfg2\fP\&. .TP .B max_copies Specify a maximum number of copies for the server to keep when @@ -765,7 +764,7 @@ ado_mssql .B name The name of the database to use for statistics data. If \(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to -the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/brpt.sqlite\fP. +the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/brpt.sqlite\fP\&. .TP .B user User for database connections. Not used for sqlite3. @@ -780,9 +779,8 @@ Host for database connections. Not used for sqlite3. Port for database connections. Not used for sqlite3. .TP .B options -Various options for the database connection. The value is -expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .UNINDENT .UNINDENT .UNINDENT diff --git a/schemas/clients.xsd b/schemas/clients.xsd index 20a77b0dd..b6d04b3bf 100644 --- a/schemas/clients.xsd +++ b/schemas/clients.xsd @@ -60,7 +60,7 @@ <xsd:attribute type='xsd:string' name='profile' use='required'> <xsd:annotation> <xsd:documentation> - Profile group naem to associate this client with. + Profile group name to associate this client with. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/packages.xsd b/schemas/packages.xsd index 8ed07baa9..46ae3c663 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -167,7 +167,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:attribute type="xsd:boolean" name="debsrc" default="false"> <xsd:annotation> <xsd:documentation> Include ``deb-src`` lines in the generated APT diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 0a8fe387f..1d9b0dd70 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -1,16 +1,16 @@ """This is the Bcfg2 support for apt-get.""" -# suppress apt API warnings -import warnings -warnings.filterwarnings("ignore", "apt API not stable yet", - FutureWarning) -import apt.cache import os +import sys + +import apt.cache + import Bcfg2.Client.Tools + class APT(Bcfg2.Client.Tools.Tool): - """The Debian toolset implements package and service operations and inherits - the rest from Toolset.Toolset. + """The Debian toolset implements package and service operations and + inherits the rest from Toolset.Toolset. """ name = 'APT' @@ -41,21 +41,24 @@ class APT(Bcfg2.Client.Tools.Tool): if not self.setup['debug']: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and entry.get('type') == 'ignore'] - self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % self.var_path, - "%s/cache/debconf/templates.dat" % self.var_path, - '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % self.etc_path, - '%s/dpkg/dpkg.cfg' % self.etc_path] + \ - [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('name').startswith('%s/apt/sources.list' % self.etc_path)] - self.nonexistent = [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] + self.__important__ = self.__important__ + [ + "%s/cache/debconf/config.dat" % self.var_path, + "%s/cache/debconf/templates.dat" % self.var_path, + '/etc/passwd', '/etc/group', + '%s/apt/apt.conf' % self.etc_path, + '%s/dpkg/dpkg.cfg' % self.etc_path] + \ + [entry.get('name') for struct in config for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith( + '%s/apt/sources.list' % self.etc_path))] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('type') == 'nonexistent')] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} if self.setup['kevlar'] and not self.setup['dryrun']: @@ -65,10 +68,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -84,16 +91,17 @@ class APT(Bcfg2.Client.Tools.Tool): else: extras = [(p.name, p.installedVersion) for p in self.pkg_cache if p.isInstalled and p.name not in packages] - return [Bcfg2.Client.XML.Element('Package', name=name, \ - type='deb', version=version) \ - for (name, version) in extras] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='deb', version=version) + for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: - self.logger.info("Package %s has no md5sums. Cannot verify" % \ + self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) entry.set('qtext', "Reinstall Package %s-%s to setup md5sums? (y/N) " % @@ -113,10 +121,10 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" \ + self.logger.error("Package %s is not fully installed" % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" \ + self.logger.error("Got Unsupported pattern %s from debsums" % item) files.append(item) files = list(set(files) - set(self.ignores)) @@ -127,31 +135,32 @@ class APT(Bcfg2.Client.Tools.Tool): modlist = [os.path.realpath(filename) for filename in modlist] bad = [filename for filename in files if filename not in modlist] if bad: - self.logger.debug("It is suggested that you either manage these " - "files, revert the changes, or ignore false " - "failures:") - self.logger.info("Package %s failed validation. Bad files are:" % \ - entry.get('name')) + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? (y/N) " % \ - (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -164,28 +173,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) - entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % \ + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if not self.setup['quick'] and entry.get('verify', 'true') == 'true' \ - and checksums: + if not self.setup['quick'] \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -203,7 +217,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -223,33 +237,40 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): - self.logger.error("APT has no information about package %s" % (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidate.version)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: - self.logger.error("Failed to find %s in apt package cache" % - pkg.get('name')) + self.logger.error("Failed to find %s in apt package " + "cache" % pkg.get('name')) continue else: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidateVersion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: - avail_vers = [x.ver_str for x in \ - self.pkg_cache[pkg.get('name')]._pkg.version_list] + avail_vers = [ + x.ver_str for x in + self.pkg_cache[pkg.get('name')]._pkg.version_list] else: - avail_vers = [x.VerStr for x in \ - self.pkg_cache[pkg.get('name')]._pkg.VersionList] + avail_vers = [ + x.VerStr for x in + self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" \ + self.logger.error("Package %s: desired version %s not in %s" % (pkg.get('name'), pkg.get('version'), avail_vers)) bad_pkgs.append(pkg.get('name')) @@ -267,6 +288,6 @@ class APT(Bcfg2.Client.Tools.Tool): if states[package]: self.modified.append(package) - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 8ff26d8f3..917d3ccdb 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py @@ -16,7 +16,7 @@ class FreeBSDInit(Bcfg2.Client.Tools.SvcTool): __req__ = {'Service': ['name', 'status']} def __init__(self, logger, cfg, setup): - Bcfg2.Client.Tools.Tool.__init__(self, logger, cfg, setup) + Bcfg2.Client.Tools.SvcTool.__init__(self, logger, cfg, setup) if os.uname()[0] != 'FreeBSD': raise Bcfg2.Client.Tools.ToolInstantiationError diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 40d90eec9..116b24687 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -42,10 +42,9 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): return False if entry.attrib['name'] in self.installed: - if (self.installed[entry.attrib['name']] == entry.attrib['version'] - or entry.attrib['version'] == 'any'): - # if (not self.setup['quick'] and - # entry.get('verify', 'true') == 'true'): + if (entry.attrib['version'] == 'any' or + self.installed[entry.attrib['name']] == + entry.attrib['version']): # FIXME: We should be able to check this once # http://trac.macports.org/ticket/15709 is implemented return True diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 3d1358ce0..1786fa83a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -6,6 +6,7 @@ import pwd import grp import stat import copy +import errno import shutil import Bcfg2.Client.Tools import Bcfg2.Client.XML @@ -272,7 +273,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): rv &= self._apply_acl(defacl, path, posix1e.ACL_TYPE_DEFAULT) return rv - def _set_secontext(self, entry, path=None): + def _set_secontext(self, entry, path=None): # pylint: disable=R0911 """ set the SELinux context of the file on disk according to the config""" if not HAS_SELINUX: @@ -284,25 +285,28 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if not context: # no context listed return True - - if context == '__default__': - try: + secontext = selinux.lgetfilecon(path)[1].split(":")[2] + if secontext in self.setup["posix_secontext_ignore"]: + return True + try: + if context == '__default__': selinux.restorecon(path) - rv = True - except OSError: - err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to restore SELinux context " - "for %s: %s" % (path, err)) - rv = False - else: - try: - rv = selinux.lsetfilecon(path, context) == 0 - except OSError: - err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to restore SELinux context " - "for %s: %s" % (path, err)) - rv = False - return rv + return True + else: + return selinux.lsetfilecon(path, context) == 0 + except OSError: + err = sys.exc_info()[1] + if err.errno == errno.EOPNOTSUPP: + # Operation not supported + if context != '__default__': + self.logger.debug("POSIX: Failed to set SELinux context " + "for %s: %s" % (path, err)) + return False + return True + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to set or restore SELinux " + "context for %s: %s" % (path, err)) + return False def _norm_gid(self, gid): """ This takes a group name or gid and returns the diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index bbae7abcc..b8451d9d6 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -146,8 +146,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] - and self._in_managed_range('POSIXGroup', g[2])] + if entry.get("name") in g[3] and + self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index a29b49efa..27c3d3785 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -52,18 +52,17 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.origpkgtool = self.pkgtool def pkgmogrify(self, packages): - """ Take a list of pkg objects, check for a 'simplename' attribute. + """ Take a list of pkg objects, check for a 'simplefile' attribute. If present, insert a _sysv_pkg_path attribute to the package and download the datastream format SYSV package to a temporary file. """ for pkg in packages: - if pkg.get('simplename'): + if pkg.get('simplefile'): tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) - self.logger.info("Downloading %s%s to %s" % (pkg.get('url'), - pkg.get('simplename'), tmpfile.name)) - urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) pkg.set('_sysv_pkg_path', tmpfile.name) def _get_package_command(self, packages): diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index a584fec86..d1fcb0f4a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -590,8 +590,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): "an RPM release") continue pkg_objs = [p for p in all_pkg_objs - if (p.version == nevra['version'] - and p.release == nevra['release'])] + if (p.version == nevra['version'] and + p.release == nevra['release'])] else: pkg_objs = self.yumbase.rpmdb.searchNevra(**short_yname(nevra)) if len(pkg_objs) == 0: diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 0bec71e20..b5bfdb5de 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -471,9 +471,9 @@ class PkgTool(Tool): # set all package states to true and flush workqueues pkgnames = [pkg.get('name') for pkg in packages] for entry in list(states.keys()): - if (entry.tag == 'Package' - and entry.get('type') == self.pkgtype - and entry.get('name') in pkgnames): + if (entry.tag == 'Package' and + entry.get('type') == self.pkgtype and + entry.get('name') in pkgnames): self.logger.debug('Setting state to true for pkg %s' % entry.get('name')) states[entry] = True @@ -575,7 +575,7 @@ class SvcTool(Tool): return self.cmd.run(self.get_svc_command(service, 'stop')) def restart_service(self, service): - """ Restart a service. + """Restart a service. :param service: The service entry to modify :type service: lxml.etree._Element @@ -608,8 +608,8 @@ class SvcTool(Tool): return for entry in bundle: - if (not self.handlesEntry(entry) - or not self._install_allowed(entry)): + if (not self.handlesEntry(entry) or + not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index b8a75a0c5..1c2420ccf 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -286,3 +286,9 @@ except NameError: def cmp(a, b): """ Py3k implementation of cmp() """ return (a > b) - (a < b) + +# ast was introduced in python 2.6 +try: + from ast import literal_eval +except ImportError: + literal_eval = eval diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 9752ab758..4565ec9a3 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -2,15 +2,17 @@ import copy import getopt +import grp import inspect import os +import pwd import re import shlex import sys -import grp -import pwd + import Bcfg2.Client.Tools from Bcfg2.Compat import ConfigParser +from Bcfg2.Compat import literal_eval from Bcfg2.version import __version__ @@ -329,25 +331,9 @@ def colon_split(c_string): def dict_split(c_string): - """ split an option string on commas, optionally surrounded by - whitespace and split the resulting items again on equals signs, - returning a dict """ - result = dict() - if c_string: - items = re.split(r'\s*,\s*', c_string) - for item in items: - if r'=' in item: - key, value = item.split(r'=', 1) - try: - result[key] = get_bool(value) - except ValueError: - try: - result[key] = get_int(value) - except ValueError: - result[key] = value - else: - result[item] = True - return result + """ literally evaluate the option in order to allow for arbitrarily nested + dictionaries """ + return literal_eval(c_string) def get_bool(val): @@ -1129,6 +1115,18 @@ CLIENT_POSIX_GID_BLACKLIST = \ default=[], cf=('POSIXUsers', 'gid_blacklist'), cook=list_split) +CLIENT_POSIX_SECONTEXT_IGNORE = \ + Option("secontext types to ignore labeling errors", + default=['anon_inodefs_t', 'bdev_t', 'binfmt_misc_fs_t', + 'capifs_t', 'configfs_t', 'cpusetfs_t', 'ecryptfs_t', + 'eventpollfs_t', 'futexfs_t', 'hugetlbfs_t', 'ibmasmfs_t', + 'inotifyfs_t', 'mvfs_t', 'nfsd_fs_t', 'oprofilefs_t', + 'ramfs_t', 'romfs_t', 'rpc_pipefs_t', 'spufs_t', + 'squash_t', 'vmblock_t', 'vxfs_t', 'xenfs_t', 'autofs_t', + 'cifs_t', 'dosfs_t', 'fusefs_t', 'iso9660_t', + 'removable_t', 'nfs_t'], + cf=('POSIX', 'secontext_ignore'), + cook=list_split) # Logging options LOGGING_FILE_PATH = \ @@ -1295,7 +1293,8 @@ DRIVER_OPTIONS = \ posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, - posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST) + posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST, + posix_secontext_ignore=CLIENT_POSIX_SECONTEXT_IGNORE) CLIENT_COMMON_OPTIONS = \ dict(extra=CLIENT_EXTRA_DISPLAY, @@ -1362,7 +1361,9 @@ TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, validate=CFG_VALIDATION) INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, - max_copies=PARANOID_MAX_COPIES) + max_copies=PARANOID_MAX_COPIES, + daemon_uid=SERVER_DAEMON_USER, + daemon_gid=SERVER_DAEMON_GROUP) INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 8e2fe1cb1..8e8e8b1fa 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -105,7 +105,7 @@ class ReportingCollector(object): self.storage.__class__.__name__) self.storage.validate() except: - self.logger.error("Storage backed %s failed to validate: %s" % + self.logger.error("Storage backend %s failed to validate: %s" % (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 71fa66086..29f08e787 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -628,7 +628,7 @@ class POSIXGroupEntry(SuccessEntry): class PackageEntry(SuccessEntry): """ The new model for package information """ - # if this is an extra entry trget_version will be empty + # if this is an extra entry target_version will be empty target_version = models.CharField(max_length=1024, default='') current_version = models.CharField(max_length=1024) verification_details = models.TextField(default="") diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index d097fd08f..c1581679c 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -103,17 +103,21 @@ class Core(BaseCore): return cherrypy.serving.response.body def _daemonize(self): - """ Drop privileges with - :class:`cherrypy.process.plugins.DropPrivileges`, daemonize - with :class:`cherrypy.process.plugins.Daemonizer`, and write a + """ Drop privileges, daemonize + with :class:`cherrypy.process.plugins.Daemonizer` and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ + self._drop_privileges() + Daemonizer(cherrypy.engine).subscribe() + PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() + return True + + def _drop_privileges(self): + """ Drop privileges with + :class:`cherrypy.process.plugins.DropPrivileges` """ DropPrivileges(cherrypy.engine, uid=self.setup['daemon_uid'], gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)).subscribe() - Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() - return True def _run(self): """ Start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6dfe4df1f..0369da8f2 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -11,6 +11,7 @@ import threading import time import inspect import lxml.etree +import daemon import Bcfg2.settings import Bcfg2.Server import Bcfg2.Logger @@ -112,6 +113,7 @@ class BaseCore(object): :type setup: Bcfg2.Options.OptionParser .. automethod:: _daemonize + .. automethod:: _drop_privileges .. automethod:: _run .. automethod:: _block .. ----- @@ -803,7 +805,8 @@ class BaseCore(object): self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): - """ Run the server core. This calls :func:`_daemonize`, + """ Run the server core. This calls :func:`_daemonize` + (or :func:`_drop_privileges` if not in daemon mode), :func:`_run`, starts the :attr:`fam_thread`, and calls :func:`_block`, but note that it is the responsibility of the server core implementation to call :func:`shutdown` under @@ -830,6 +833,8 @@ class BaseCore(object): # dropped os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] else: + if os.getuid() == 0: + self._drop_privileges() os.umask(int(self.setup['umask'], 8)) if not self._run(): @@ -861,6 +866,16 @@ class BaseCore(object): overridden by a core implementation. """ raise NotImplementedError + def _drop_privileges(self): + """ This is called if not daemonized and running as root to + drop the privileges to the configured daemon_uid and daemon_gid. + """ + daemon.daemon.change_process_owner( + self.setup['daemon_uid'], + self.setup['daemon_gid']) + self.logger.debug("Dropped privileges to %s:%s." % + (os.getuid(), os.getgid())) + def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 7a5d901fd..083e50fe6 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -336,8 +336,11 @@ class FileMonitor(Debuggable): available = dict() # pylint: disable=C0103 # TODO: loading the monitor drivers should be automatic -from Bcfg2.Server.FileMonitor.Pseudo import Pseudo -available['pseudo'] = Pseudo +try: + from Bcfg2.Server.FileMonitor.Pseudo import Pseudo + available['pseudo'] = Pseudo +except ImportError: + pass try: from Bcfg2.Server.FileMonitor.Fam import Fam diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 3efcc890d..1e33ec398 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,12 +1,16 @@ -""" Ensure that all XML files in the Bcfg2 repository validate -according to their respective schemas. """ +"""Validate XML files. +Ensure that all XML files in the Bcfg2 repository validate according +to their respective schemas. +""" + +import glob import os import sys -import glob -import fnmatch + import lxml.etree from subprocess import Popen, PIPE, STDOUT + import Bcfg2.Server.Lint @@ -204,17 +208,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): values are lists of the full paths to all files in the Bcfg2 repository (or given with ``bcfg2-lint --stdin``) that match the glob.""" - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, - os.path.join('*', p)) - else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) - for path in self.filesets.keys(): if '/**/' in path: if self.files is not None: - self.filelists[path] = listfiles(path) + self.filelists[path] = self.list_matching_files(path) else: # self.files is None fpath, fname = path.split('/**/') self.filelists[path] = [] @@ -225,9 +222,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == fname]) else: - self.filelists[path] = listfiles(path) + self.filelists[path] = self.list_matching_files(path) - self.filelists['props'] = listfiles("Properties/*.xml") + self.filelists['props'] = self.list_matching_files("Properties/*.xml") def _load_schema(self, filename): """ Load an XML schema document, returning the Schema object diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index 1f55962eb..bdbe6a271 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -1,11 +1,13 @@ -"""Ensure that all JSON files in the Bcfg2 repository are +"""Validate JSON files. + +Ensure that all JSON files in the Bcfg2 repository are valid. Currently, the only plugins that uses JSON are Ohai and -Properties.""" +Properties. +""" import os import sys -import glob -import fnmatch + import Bcfg2.Server.Lint try: @@ -48,18 +50,11 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): def get_files(self): """Return a list of all JSON files to validate, based on :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, - os.path.join('*', p)) - else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) - rv = [] for path in self.globs: if '/**/' in path: if self.files is not None: - rv.extend(listfiles(path)) + rv.extend(self.list_matching_files(path)) else: # self.files is None fpath, fname = path.split('/**/') for root, _, files in \ @@ -68,5 +63,5 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): rv.extend([os.path.join(root, f) for f in files if f == fname]) else: - rv.extend(listfiles(path)) + rv.extend(self.list_matching_files(path)) return rv diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 28644263f..ae2b81a61 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,14 +1,19 @@ """ Base classes for Lint plugins and error handling """ import os +import fnmatch +import glob import sys import logging from copy import copy import textwrap +import time + import lxml.etree import fcntl import termios import struct + from Bcfg2.Compat import walk_packages plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 @@ -139,6 +144,13 @@ class Plugin(object): xml_declaration=False).decode("UTF-8").strip() return " line %s: %s" % (element.sourceline, xml) + def list_matching_files(self, path): + """list all files matching the path in self.files or the bcfg2 repo.""" + if self.files is not None: + return fnmatch.filter(self.files, os.path.join('*', path)) + else: + return glob.glob(os.path.join(self.config['repo'], path)) + class ErrorHandler(object): """ A class to handle errors for bcfg2-lint plugins """ diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 2cb3adae3..4986aac60 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -210,6 +210,9 @@ class ChildCore(BaseCore): def _daemonize(self): return True + def _drop_privileges(self): + pass + def _dispatch(self, address, data): """ Method dispatcher used for commands received from the RPC queue. """ diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index ed1282ba0..05bb92223 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -11,11 +11,6 @@ documentation it's not necessary to use the submodules. E.g., you can from Bcfg2.Server.Plugin.base import Plugin """ - -import os -import sys -sys.path.append(os.path.dirname(__file__)) - # pylint: disable=W0401 from Bcfg2.Server.Plugin.base import * from Bcfg2.Server.Plugin.interfaces import * diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 09685d972..eadd918b7 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -149,15 +149,13 @@ class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): def check(self, entry, groups, ptype="NamePattern"): """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - for el in entry.findall(ptype): pat = el.text try: - pmap(pat) + if ptype == "NamePattern": + PatternMap(pat, None, groups) + else: + PatternMap(None, pat, groups) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("pattern-fails-to-initialize", diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 1e5544c6b..f805772a7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1391,8 +1391,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # look at cert.cN client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) - auth_type = self.auth.get(client, - self.core.setup['authentication']) elif user == 'root': id_method = 'address' try: @@ -1414,6 +1412,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # we have the client name self.debug_log("Authenticating client %s" % client) + # validate id_method + auth_type = self.auth.get(client, self.core.setup['authentication']) + if auth_type == 'cert' and id_method != 'cert': + self.logger.error("Client %s does not provide a cert, but only " + "cert auth is allowed" % client) + return False + # next we validate the address if (id_method != 'uuid' and not self.validate_client_address(client, address)): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 538215c85..7aa688f12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -145,22 +145,22 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: A list of the text of all 'Component' attributes of this #: source from XML - self.components = [item.text for item in xsource.findall('Component')] + self.components = [] #: A list of the arches supported by this source - self.arches = [item.text for item in xsource.findall('Arch')] + self.arches = [] #: A list of the the names of packages that are blacklisted #: from this source - self.blacklist = [item.text for item in xsource.findall('Blacklist')] + self.blacklist = [] #: A list of the the names of packages that are whitelisted in #: this source - self.whitelist = [item.text for item in xsource.findall('Whitelist')] + self.whitelist = [] #: Whether or not to include deb-src lines in the generated APT #: configuration - self.debsrc = xsource.get('debsrc', 'false') == 'true' + self.debsrc = False #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is @@ -172,51 +172,38 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: configuration generated for the client (if that is #: supported by the backend) self.client_options = dict() - opts = xsource.findall("Options") - for el in opts: - repoopts = dict([(k, v) - for k, v in el.attrib.items() - if k != "clientonly" and k != "serveronly"]) - if el.get("clientonly", "false").lower() == "false": - self.server_options.update(repoopts) - if el.get("serveronly", "false").lower() == "false": - self.client_options.update(repoopts) #: A list of URLs to GPG keys that apply to this source - self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] + self.gpgkeys = [] #: Whether or not to include essential packages from this source - self.essential = xsource.get('essential', 'true').lower() == 'true' + self.essential = True #: Whether or not to include recommended packages from this source - self.recommended = xsource.get('recommended', - 'false').lower() == 'true' + self.recommended = False #: The "rawurl" attribute from :attr:`xsource`, if applicable. #: A trailing slash is automatically appended to this if there #: wasn't one already present. - self.rawurl = xsource.get('rawurl', '') - if self.rawurl and not self.rawurl.endswith("/"): - self.rawurl += "/" + self.rawurl = None #: The "url" attribute from :attr:`xsource`, if applicable. A #: trailing slash is automatically appended to this if there #: wasn't one already present. - self.url = xsource.get('url', '') - if self.url and not self.url.endswith("/"): - self.url += "/" + self.url = None #: The "version" attribute from :attr:`xsource` - self.version = xsource.get('version', '') + self.version = None #: The "name" attribute from :attr:`xsource` - self.name = xsource.get('name', None) + self.name = None #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` #: object. self.conditions = [] + #: Formerly, :ref:`server-plugins-generators-packages` only #: supported applying package sources to groups; that is, they #: could not be assigned by more complicated logic like @@ -224,22 +211,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: attribute attempts to provide for some limited backwards #: compat with older code that relies on this. self.groups = [] - for el in xsource.iterancestors(): - if el.tag == "Group": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") not in m.groups) - else: - self.groups.append(el.get("name")) - self.conditions.append(lambda m, el=el: - el.get("name") in m.groups) - elif el.tag == "Client": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") != m.hostname) - else: - self.conditions.append(lambda m, el=el: - el.get("name") == m.hostname) #: A set of all package names in this source. This will not #: necessarily be populated, particularly by backends that @@ -259,6 +230,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + self._init_attributes(xsource) + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -283,11 +256,11 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -298,6 +271,69 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) + def _init_attributes(self, xsource): + """ + This functions evaluates the Source tag and parses all + attributes. Override this function in a sub class to + parse specific attributes. Do not use ``__init__`` because + ``Source.__init__`` may call other functions that already + need this specific fields. This functions is called before + any other function. + + :param xsource: The XML tag that describes this source + :type source: lxml.etree._Element + """ + + self.components = [item.text for item in xsource.findall('Component')] + self.arches = [item.text for item in xsource.findall('Arch')] + self.blacklist = [item.text for item in xsource.findall('Blacklist')] + self.whitelist = [item.text for item in xsource.findall('Whitelist')] + self.debsrc = xsource.get('debsrc', 'false') == 'true' + + opts = xsource.findall("Options") + for el in opts: + repoopts = dict([(k, v) + for k, v in el.attrib.items() + if k != "clientonly" and k != "serveronly"]) + if el.get("clientonly", "false").lower() == "false": + self.server_options.update(repoopts) + if el.get("serveronly", "false").lower() == "false": + self.client_options.update(repoopts) + + self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] + + self.essential = xsource.get('essential', 'true').lower() == 'true' + self.recommended = xsource.get('recommended', + 'false').lower() == 'true' + + self.rawurl = xsource.get('rawurl', '') + if self.rawurl and not self.rawurl.endswith("/"): + self.rawurl += "/" + + self.url = xsource.get('url', '') + if self.url and not self.url.endswith("/"): + self.url += "/" + + self.version = xsource.get('version', '') + self.name = xsource.get('name', None) + + for el in xsource.iterancestors(): + if el.tag == "Group": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") not in m.groups) + else: + self.groups.append(el.get("name")) + self.conditions.append(lambda m, el=el: + el.get("name") in m.groups) + elif el.tag == "Client": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") != m.hostname) + else: + self.conditions.append(lambda m, el=el: + el.get("name") == m.hostname) + @property def cachekey(self): """ A unique key for this source that will be used to generate diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 0e4f93246..b877e7541 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1029,8 +1029,20 @@ class YumSource(Source): ptype = 'yum' def __init__(self, basepath, xsource, setup): - Source.__init__(self, basepath, xsource, setup) + self.filemap = dict() + self.file_to_arch = dict() + self.needed_paths = set() + self.packages = dict() + self.yumgroups = dict() self.pulp_id = None + self.repo = None + + Source.__init__(self, basepath, xsource, setup) + __init__.__doc__ = Source.__init__.__doc__ + + def _init_attributes(self, xsource): + Source._init_attributes(self, xsource) + if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") @@ -1063,15 +1075,11 @@ class YumSource(Source): self.repo['relative_path']) self.arches = [self.repo['arch']] - self.packages = dict() self.deps = dict([('global', dict())]) self.provides = dict([('global', dict())]) self.filemap = dict([(x, dict()) for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - self.yumgroups = dict() - __init__.__doc__ = Source.__init__.__doc__ + _init_attributes.__doc__ = Source._init_attributes.__doc__ @property def use_yum(self): @@ -1161,6 +1169,94 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls + # pylint: disable=R0911,R0912 + # disabling the pylint errors above because we are interesting in + # replicating the flow of the RPM code. + def _compare_rpm_versions(self, str1, str2): + """ Compare RPM versions. + + This is an attempt to reimplement RPM's rpmvercmp method in python. + + :param str1: package 1 version string + :param str2: package 2 version string + :return: 1 - str1 is newer than str2 + 0 - str1 and str2 are the same version + -1 - str2 is newer than str1""" + if str1 == str2: + return 0 + + front_strip_re = re.compile('^[^A-Za-z0-9~]+') + risdigit = re.compile('(^[0-9]+)') + risalpha = re.compile('(^[A-Za-z])') + lzeroes = re.compile('^0+') + + while len(str1) > 0 or len(str2) > 0: + str1 = front_strip_re.sub('', str1) + str2 = front_strip_re.sub('', str2) + + if len(str1) == 0 or len(str2) == 0: + break + + # handle the tilde separator + if str1[0] == '~' and str2[0] == '~': + str1 = str1[1:] + str2 = str2[1:] + elif str1[0] == '~': + return -1 + elif str2[0] == '~': + return 1 + + # grab continuous segments from each string + isnum = False + if risdigit.match(str1): + segment1 = risdigit.split(str1)[1] + str1 = risdigit.split(str1)[2] + if risdigit.match(str2): + segment2 = risdigit.split(str2)[1] + str2 = risdigit.split(str2)[2] + else: + segment2 = '' + isnum = True + else: + segment1 = risalpha.split(str1)[1] + str1 = risalpha.split(str1)[2] + if risalpha.match(str2): + segment2 = risalpha.split(str2)[1] + str2 = risalpha.split(str2)[2] + else: + segment2 = '' + + # numeric segments are always newer than alpha segments + if len(segment2) == 0: + if isnum: + return 1 + return -1 + + if isnum: + # discard leading zeroes + segment1 = lzeroes.sub('', segment1) + segment2 = lzeroes.sub('', segment2) + # higher number has more digits + if len(segment1) > len(segment2): + return 1 + elif len(segment2) > len(segment1): + return -1 + # do a simple string comparison + if segment1 > segment2: + return 1 + elif segment2 > segment1: + return -1 + + # if one of the strings is empty, the version of the longer + # string is higher + if len(str1) > len(str2): + return 1 + elif len(str2) > len(str1): + return -1 + else: + return 0 + # pylint: enable=R0911,R0912 + @Bcfg2.Server.Plugin.track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally @@ -1229,13 +1325,33 @@ class YumSource(Source): if arch not in self.packages: self.packages[arch] = set() if arch not in self.deps: - self.deps[arch] = dict() + self.deps[arch] = {} if arch not in self.provides: - self.provides[arch] = dict() + self.provides[arch] = {} + versionmap = {} for pkg in data.getchildren(): if not pkg.tag.endswith('package'): continue pkgname = pkg.find(XP + 'name').text + vtag = pkg.find(XP + 'version') + epoch = vtag.get('epoch') + version = vtag.get('ver') + release = vtag.get('rel') + if pkgname in self.packages[arch]: + # skip if version older than a previous version + if (self._compare_rpm_versions( + epoch, versionmap[pkgname]['epoch']) < 0): + continue + elif (self._compare_rpm_versions( + version, versionmap[pkgname]['version']) < 0): + continue + elif (self._compare_rpm_versions( + release, versionmap[pkgname]['release']) < 0): + continue + versionmap[pkgname] = {} + versionmap[pkgname]['epoch'] = epoch + versionmap[pkgname]['version'] = version + versionmap[pkgname]['release'] = release self.packages[arch].add(pkgname) pdata = pkg.find(XP + 'format') diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 2deea5f07..0152fcf70 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -93,6 +93,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -127,6 +128,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -444,3 +446,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("Failed to pull %s. This file does not " "currently exist on the client" % entry.get('name')) + + def get_additional_data(self, metadata): + data = dict() + for key in self.keypatterns: + if key.endswith(".pub"): + try: + keyfile = "/etc/ssh/" + key + entry = self.entries[keyfile].best_matching(metadata) + data[key] = entry.data + except Bcfg2.Server.Plugin.PluginExecutionError: + pass + return data diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index f111ffc60..a33b2f448 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -150,15 +150,13 @@ class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): if passphrase: cmd.extend(["-passin", "pass:%s" % passphrase]) - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a + def _scrub_pass(arg): + """ helper to scrub the passphrase from the + argument list for debugging. """ + if arg.startswith("pass:"): + return "pass:******" + else: + return arg self.debug_log("SSLCA: Generating new certificate: %s" % " ".join(_scrub_pass(a) for a in cmd)) @@ -362,10 +360,13 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile + def es_cls(self, *args): + """Fake entry set 'class' that sets this as the parent.""" + # python 2.5 doesn't support mixing *magic and keyword arguments + return SSLCAEntrySet(*args, **dict(parent=self)) + def get_ca(self, name): """ get a dict describing a CA from the config file """ return dict(self.core.setup.cfp.items("sslca_%s" % name)) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index a6c3149bc..2c97a9b91 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -726,6 +726,8 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: + if os.getuid() == 0: + self._drop_privileges() self.load_plugins() self.block_for_fam_events(handle_events=True) if args: diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 7c38a65d8..1d89210c9 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -20,7 +20,10 @@ try: HAS_MULTIPROC = True except ImportError: HAS_MULTIPROC = False - active_children = lambda: [] # pylint: disable=C0103 + + def active_children(): + """active_children() when multiprocessing lib is missing.""" + return [] class CapturingLogger(object): diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index c368d40ce..d9f985104 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -35,6 +35,7 @@ contingent_checks = { "lib/Bcfg2/Server/Admin": ["Reports.py", "Syncdb.py"], "sbin": ["bcfg2-reports"]}, ("pyinotify",): {"lib/Bcfg2/Server/FileMonitor": ["Inotify.py"]}, + ("apt",): {"lib/Bcfg2/Client/Tools": ["APT.py"]}, ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, @@ -66,7 +67,7 @@ error_checks = { # perform no checks at all on the listed files no_checks = { - "lib/Bcfg2/Client/Tools": ["APT.py", "RPM.py", "rpmtools.py"], + "lib/Bcfg2/Client/Tools": ["RPM.py", "rpmtools.py"], "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], diff --git a/testsuite/ext/ssl_protocols.py b/testsuite/ext/ssl_protocols.py new file mode 100644 index 000000000..66068d2a9 --- /dev/null +++ b/testsuite/ext/ssl_protocols.py @@ -0,0 +1,17 @@ +try: + from logilab.astng import MANAGER, scoped_nodes, node_classes + PYLINT=0 +except ImportError: + from astroid import MANAGER, scoped_nodes, node_classes + PYLINT=1 + +def ssl_transform(module): + if module.name == 'ssl': + for proto in ('SSLv23', 'TLSv1'): + module.locals['PROTOCOL_%s' % proto] = [node_classes.Const()] + +def register(linter): + if PYLINT == 0: + MANAGER.register_transformer(ssl_transform) + else: + MANAGER.register_transform(scoped_nodes.Module, ssl_transform) diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 94904877b..b3340cb4f 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -19,7 +19,7 @@ persistent=no # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins=ext.exception_messages +load-plugins=ext.exception_messages,ext.ssl_protocols [MESSAGES CONTROL] diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 898249389..0dd460b87 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -2,6 +2,6 @@ lxml nose mock sphinx -pylint<1.0 +pylint<0.29 pep8 -python-daemon +python-daemon<2.0.0 diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 1a919f869..29fd9886b 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -24,3 +24,7 @@ migrate_probe_groups_to_db.py - Migrate Probe host and group data from XML to DB backend for Metadata and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. + +migrate_sysv_simplename.py + - Migrate any Pkgmgr entries which may have been using the simplename + attribute introduced in 1.3.5 to the simplefile attribute diff --git a/tools/upgrade/1.3/migrate_sysv_simplename.py b/tools/upgrade/1.3/migrate_sysv_simplename.py new file mode 100755 index 000000000..f6599756b --- /dev/null +++ b/tools/upgrade/1.3/migrate_sysv_simplename.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import lxml.etree +import Bcfg2.Options + +def main(): + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + files = [] + for plugin in ['Pkgmgr']: + files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + + for bfile in files: + bdata = lxml.etree.parse(bfile) + changed = False + + if not bdata.xpath("//@type='sysv'"): + print("%s doesn't contain any sysv packages, skipping" % bfile) + continue + + pkglist = bdata.getroot() + if pkglist.tag != "PackageList": + print("%s doesn't look like a PackageList, skipping" % bfile) + continue + + for pkg in bdata.xpath("//Package"): + if "simplename" in pkg.attrib: + pkg.set("simplefile", pkg.get("simplename")) + del pkg.attrib["simplename"] + changed = True + + # if we switched to simplefile, we also need to switch to uri + if changed and "url" in pkglist.attrib: + pkglist.set("uri", pkglist.get("url")) + del pkglist.attrib["url"] + + if changed: + print("Writing %s" % bfile) + try: + open(bfile, "w").write(lxml.etree.tostring(bdata)) + except IOError: + err = sys.exc_info()[1] + print("Could not write %s: %s" % (bfile, err)) + +if __name__ == '__main__': + sys.exit(main()) |