diff options
Diffstat (limited to 'src')
22 files changed, 481 insertions, 373 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 3254da9e9..1b26450a6 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,6 +1,7 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import copy import time import fnmatch import logging @@ -522,7 +523,7 @@ class Frame(object): container = Bcfg2.Client.XML.SubElement(stats, ename) for item in data: item.set('qtext', '') - container.append(item) + container.append(copy.deepcopy(item)) item.text = None timeinfo = Bcfg2.Client.XML.Element("OpStamps") diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index ac874c94c..4833f3f68 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -89,7 +89,7 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot - bootcmd = '/sbin/chkconfig %s %s --level 0123456' % \ + bootcmd = '/sbin/chkconfig %s %s' % \ (entry.get('name'), bootstatus) elif bootstatus == 'off': # make sure service is disabled on boot @@ -116,8 +116,8 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra chkconfig Services.""" allsrv = [line.split()[0] - for line in self.cmd.run("/sbin/chkconfig", - "--list").stdout.splitlines() + for line in + self.cmd.run("/sbin/chkconfig --list").stdout.splitlines() if ":on" in line] self.logger.debug('Found active services:') self.logger.debug(allsrv) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 16fe0acb5..3778569a6 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -706,16 +706,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): (path, err)) rv = False - # we need to make sure that we give +x to everyone who needs - # it. E.g., if the file that's been distributed is 0600, we - # can't make the parent directories 0600 also; that'd be - # pretty useless. They need to be 0700. + # set auto-created directories to mode 755, if you need + # something else, you should specify it in your config tmpentry = copy.deepcopy(entry) - newmode = int(entry.get('mode'), 8) - for i in range(0, 3): - if newmode & (6 * pow(8, i)): - newmode |= 1 * pow(8, i) - tmpentry.set('mode', oct_mode(newmode)) + tmpentry.set('mode', '0755') for acl in tmpentry.findall('ACL'): acl.set('perms', oct_mode(self._norm_acl_perms(acl.get('perms')) | diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8226392f9..bb684899d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -249,7 +249,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('gid'): cmd.extend(['-g', entry.get('gid')]) elif entry.tag == 'POSIXUser': - cmd.append('-m') if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 25603186e..6d1cb9d40 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -21,6 +21,9 @@ def prompt(msg): try: ans = input(msg) return ans in ['y', 'Y'] + except UnicodeEncodeError: + ans = input(msg.encode('utf-8')) + return ans in ['y', 'Y'] except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 243c4ed2a..dba3e96ef 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -319,6 +319,28 @@ def colon_split(c_string): return [] +def dict_split(c_string): + """ split an option string on commas, optionally surrounded by + whitespace and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if c_string: + items = re.split(r'\s*,\s*', c_string) + for item in items: + if r'=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = get_bool(value) + except ValueError: + try: + result[key] = get_int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + def get_bool(val): """ given a string value of a boolean configuration option, return an actual bool (True or False) """ @@ -652,6 +674,12 @@ DB_PORT = \ cf=('database', 'port'), deprecated_cf=('statistics', 'database_port')) +DB_OPTIONS = \ + Option('Database options', + default=dict(), + cf=('database', 'options'), + cook=dict_split) + # Django options WEB_CFILE = \ Option('Web interface configuration file', @@ -1285,6 +1313,7 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, db_password=DB_PASSWORD, db_host=DB_HOST, db_port=DB_PORT, + db_options=DB_OPTIONS, time_zone=DJANGO_TIME_ZONE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7f1fcba3b..0b2b7dd36 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.1</span> + <span>Bcfg2 Version 1.3.2</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index ecd68e1e4..0cd4bea3e 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -728,6 +728,26 @@ class BaseCore(object): self.setup.reparse() self.metadata_cache.expire() + def block_for_fam_events(self, handle_events=False): + """ Block until all fam events have been handleed, optionally + handling events as well. (Setting ``handle_events=True`` is + useful for local server cores that don't spawn an event + handling thread.)""" + slept = 0 + log_interval = 3 + if handle_events: + self.fam.handle_events_in_interval(1) + slept += 1 + if self.setup['fam_blocking']: + time.sleep(1) + slept += 1 + while self.fam.pending() != 0: + time.sleep(1) + slept += 1 + if slept % log_interval == 0: + self.logger.debug("Sleeping to handle FAM events...") + self.logger.debug("Slept %s seconds while handling FAM events" % slept) + def run(self): """ Run the server core. This calls :func:`_daemonize`, :func:`_run`, starts the :attr:`fam_thread`, and calls @@ -777,13 +797,9 @@ class BaseCore(object): self.shutdown() raise - if self.setup['fam_blocking']: - time.sleep(1) - while self.fam.pending() != 0: - time.sleep(1) - if self.debug_flag: self.set_debug(None, self.debug_flag) + self.block_for_fam_events() self._block() def _daemonize(self): @@ -842,7 +858,12 @@ class BaseCore(object): imd = self.metadata_cache.get(client_name, None) if not imd: self.logger.debug("Building metadata for %s" % client_name) - imd = self.metadata.get_initial_metadata(client_name) + try: + imd = self.metadata.get_initial_metadata(client_name) + except MetadataConsistencyError: + self.critical_error( + "Client metadata resolution error for %s: %s" % + (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: grps = conn.get_additional_groups(imd) @@ -1253,9 +1274,14 @@ class BaseCore(object): self.logger.info("Core: debug = %s" % debug) levels = self._loglevels[self.debug_flag] for handler in logging.root.handlers: - level = levels.get(handler.name, levels['default']) - self.logger.debug("Setting %s log handler to %s" % - (handler.name, logging.getLevelName(level))) + try: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + except AttributeError: + level = levels['default'] + self.logger.debug("Setting unknown log handler %s to %s" % + (handler, logging.getLevelName(level))) handler.setLevel(level) return self.debug_flag diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index c7b62f352..c0a3036a9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -230,7 +230,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if strict: raise PluginExecutionError(msg) else: - self.logger.warning(msg) + self.logger.info(msg) Index.__doc__ = StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 6934739a3..e8962d707 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -749,7 +749,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) def remove_client(self, client_name): - """Remove a bundle.""" + """Remove a client.""" if self._use_db: try: client = MetadataClientModel.objects.get(hostname=client_name) @@ -1142,9 +1142,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, require_public=False) profile = _add_group(pgroup) else: - msg = "Cannot add new client %s; no default group set" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError( + "Cannot add new client %s; no default group set" % client) for cgroup in self.clientgroups.get(client, []): if cgroup in groups: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..39c51f351 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -614,6 +614,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..55787681f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,6 +53,7 @@ The Yum Backend import os import re import sys +import stat import copy import errno import socket @@ -282,13 +283,15 @@ class YumCollection(Collection): #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + self.setup_data() else: self.cachefile = None @@ -924,6 +927,28 @@ class YumCollection(Collection): "output: %s" % err) raise + def _set_cache_writeable(self, writeable): + """ Set the writeability of the yum cache. + + :param writeable: If True, the cache will be made writeable. + If False, the cache will be made read-only. + :type writeable: bool + """ + fmode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + if writeable: + self.debug_log("Packages: Making cache %s writeable" % + self.cachefile) + fmode |= stat.S_IWUSR + else: + self.debug_log("Packages: Making cache %s read-only" % + self.cachefile) + dmode = fmode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + for root, dirs, files in os.walk(self.cachefile): + for dname in dirs: + os.chmod(os.path.join(root, dname), dmode) + for fname in files: + os.chmod(os.path.join(root, fname), fmode) + def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called when sources are loaded or reloaded by @@ -931,11 +956,11 @@ class YumCollection(Collection): If the builtin yum parsers are in use, this defers to :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`. - If using the yum Python libraries, this cleans up cached yum - metadata, regenerates the server-side yum config (in order to - catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + If using the yum Python libraries, this makes the cache + writeable, cleans up cached yum metadata, regenerates the + server-side yum config (in order to catch any new sources that + have been added to this server), regenerates the yum cache, + and then sets the cache back to read-only. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -945,24 +970,25 @@ class YumCollection(Collection): if not self.use_yum: return Collection.setup_data(self, force_update) + self._set_cache_writeable(True) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config + # clean up data from the old config try: self.call_helper("clean") except ValueError: # error reported by call_helper pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - try: - self.call_helper("clean") - except ValueError: - # error reported by call_helper - pass + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass + self._set_cache_writeable(False) class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f93bd0932..a021cda13 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,6 +23,52 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __str__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -535,20 +582,41 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ + return self.get_collection(metadata).get_additional_data + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 0974184b4..407cfc2d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -249,7 +249,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, ProbesDataModel.objects.filter( hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: @@ -264,7 +264,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() + group__in=self.cgroups[client.hostname]).delete() def load_data(self): """ Load probe data from the appropriate backend (probed.xml diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index e97f66675..89f2d21ff 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -223,7 +223,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): if strict: raise PluginExecutionError(msg) else: - LOGGER.warning(msg) + LOGGER.info(msg) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 9adfd66bf..6e718a079 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -26,6 +26,7 @@ DATABASE_USER = None DATABASE_PASSWORD = None DATABASE_HOST = None DATABASE_PORT = None +DATABASE_OPTIONS = None TIME_ZONE = None @@ -58,8 +59,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ - MEDIA_URL + DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DEBUG, \ + TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): @@ -86,7 +87,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): USER=setup['db_user'], PASSWORD=setup['db_password'], HOST=setup['db_host'], - PORT=setup['db_port']) + PORT=setup['db_port'], + OPTIONS=setup['db_options']) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] @@ -95,6 +97,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] DATABASE_HOST = DATABASES['default']['HOST'] DATABASE_PORT = DATABASES['default']['PORT'] + DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 12fc584fe..140fb6937 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.1" +__version__ = "1.3.2" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index aad89882f..7102d06a9 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -18,291 +18,167 @@ except ImportError: raise SystemExit(1) -class EncryptionChunkingError(Exception): - """ error raised when Encryptor cannot break a file up into chunks - to be encrypted, or cannot reassemble the chunks """ - pass +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ -class Encryptor(object): - """ Generic encryptor for all files """ - - def __init__(self, setup): +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + def __init__(self, filename, setup): self.setup = setup - self.passphrase = None - self.pname = None self.logger = logging.getLogger(self.__class__.__name__) + self.passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - def get_encrypted_filename(self, plaintext_filename): - """ get the name of the file encrypted data should be written to """ - return plaintext_filename - - def get_plaintext_filename(self, encrypted_filename): - """ get the name of the file decrypted data should be written to """ - return encrypted_filename - - def chunk(self, data): - """ generator to break the file up into smaller chunks that - will each be individually encrypted or decrypted """ - yield data - - def unchunk(self, data, original): # pylint: disable=W0613 - """ given chunks of a file, reassemble then into the whole file """ + self.filename = filename try: - return data[0] - except IndexError: - raise EncryptionChunkingError("No data to unchunk") + self.data = open(self.filename).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % (filename, + err)) + return False - def set_passphrase(self): - """ set the passphrase for the current file """ + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ if (not self.setup.cfp.has_section(Bcfg2.Encryption.CFG_SECTION) or len(Bcfg2.Encryption.get_passphrases(self.setup)) == 0): - self.logger.error("No passphrases available in %s" % - self.setup['configfile']) - return False - - if self.passphrase: - self.logger.debug("Using previously determined passphrase %s" % - self.pname) - return True + raise PassphraseError("No passphrases available in %s" % + self.setup['configfile']) + pname = None if self.setup['passphrase']: - self.pname = self.setup['passphrase'] + pname = self.setup['passphrase'] - if self.pname: + if pname: if self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - self.pname): - self.passphrase = \ - self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - self.pname) + pname): + passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, + pname) self.logger.debug("Using passphrase %s specified on command " - "line" % self.pname) - return True + "line" % pname) + return (pname, passphrase) else: - self.logger.error("Could not find passphrase %s in %s" % - (self.pname, self.setup['configfile'])) - return False + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, self.setup['configfile'])) else: pnames = Bcfg2.Encryption.get_passphrases(self.setup) if len(pnames) == 1: - self.pname = pnames.keys()[0] - self.passphrase = pnames[self.pname] - self.logger.info("Using passphrase %s" % self.pname) - return True + pname = pnames.keys()[0] + passphrase = pnames[pname] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) elif len(pnames) > 1: - self.logger.warning("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - self.logger.info("No passphrase could be determined") - return False - - def encrypt(self, fname): - """ encrypt the given file, returning the encrypted data """ + return (None, None) + raise PassphraseError("No passphrase could be determined") + + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename + + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) try: - plaintext = open(fname).read() + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True except IOError: err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) return False - if not self.set_passphrase(): - return False + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) - crypted = [] - try: - for chunk in self.chunk(plaintext): - try: - passphrase, pname = self.get_passphrase(chunk) - except TypeError: - return False - crypted.append(self._encrypt(chunk, passphrase, name=pname)) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting data to encrypt from %s: %s" % - (fname, err)) - return False - return self.unchunk(crypted, plaintext) +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError - # pylint: disable=W0613 - def _encrypt(self, plaintext, passphrase, name=None): - """ encrypt a single chunk of a file """ - return Bcfg2.Encryption.ssl_encrypt( - plaintext, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - # pylint: enable=W0613 - def decrypt(self, fname): - """ decrypt the given file, returning the plaintext data """ - try: - crypted = open(fname).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) - return False +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError - self.set_passphrase() - plaintext = [] - try: - for chunk in self.chunk(crypted): - try: - passphrase, pname = self.get_passphrase(chunk) - try: - plaintext.append(self._decrypt(chunk, passphrase)) - except Bcfg2.Encryption.EVPError: - self.logger.info("Could not decrypt %s with the " - "specified passphrase" % fname) - continue - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - continue - except TypeError: - pchunk = None - passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - for pname, passphrase in passphrases.items(): - self.logger.debug("Trying passphrase %s" % pname) - try: - pchunk = self._decrypt(chunk, passphrase) - break - except Bcfg2.Encryption.EVPError: - pass - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - if pchunk is not None: - plaintext.append(pchunk) - else: - self.logger.error("Could not decrypt %s with any " - "passphrase in %s" % - (fname, self.setup['configfile'])) - continue - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting encrypted data from %s: %s" % - (fname, err)) - return False +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ - try: - return self.unchunk(plaintext, crypted) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling plaintext data from %s: %s" % - (fname, err)) - return False + def __init__(self, filename, setup): + Encryptor.__init__(self, filename, setup) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) - def _decrypt(self, crypted, passphrase): - """ decrypt a single chunk """ - return Bcfg2.Encryption.ssl_decrypt( - crypted, passphrase, + def encrypt(self): + return Bcfg2.Encryption.ssl_encrypt( + self.data, self.passphrase, Bcfg2.Encryption.get_algorithm(self.setup)) - def write_encrypted(self, fname, data=None): - """ write encrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_encrypted_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote encrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling encrypted data from %s: %s" % - (fname, err)) - return False + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" - def write_decrypted(self, fname, data=None): - """ write decrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_plaintext_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote decrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - def get_passphrase(self, chunk): - """ get the passphrase for a chunk of a file """ - pname = self._get_passphrase(chunk) - if not self.pname: - if not pname: - self.logger.info("No passphrase given on command line or " - "found in file") +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return Bcfg2.Encryption.ssl_decrypt( + self.data, self.passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + self.logger.info("Could not decrypt %s with the " + "specified passphrase" % self.filename) return False - elif self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - pname) - else: - self.logger.error("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % + (self.filename, err)) return False + else: # no passphrase given, brute force + try: + return Bcfg2.Encryption.bruteforce_decrypt( + self.data, passphrases=self.passphrases.values(), + algorithm=Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + self.logger.info("Could not decrypt %s with any passphrase" % + self.filename) + + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] else: - pname = self.pname - passphrase = self.passphrase - if self.pname != pname: - self.logger.warning("Passphrase given on command line (%s) " - "differs from passphrase embedded in " - "file (%s), using command-line option" % - (self.pname, pname)) - return (passphrase, pname) - - def _get_passphrase(self, chunk): # pylint: disable=W0613 - """ get the passphrase for a chunk of a file """ - return None - - -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ - - def get_encrypted_filename(self, plaintext_filename): - return plaintext_filename + ".crypt" - - def get_plaintext_filename(self, encrypted_filename): - if encrypted_filename.endswith(".crypt"): - return encrypted_filename[:-6] - else: - return Encryptor.get_plaintext_filename(self, encrypted_filename) + return Decryptor.get_plaintext_filename(self, original_filename) -class PropertiesEncryptor(Encryptor): - """ encryptor class for Properties files """ - - def _encrypt(self, plaintext, passphrase, name=None): - # plaintext is an lxml.etree._Element - if name is None: - name = "true" - if plaintext.text and plaintext.text.strip(): - plaintext.text = Bcfg2.Encryption.ssl_encrypt( - plaintext.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - plaintext.set("encrypted", name) - return plaintext +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*' - def chunk(self, data): - xdata = lxml.etree.XML(data, parser=XMLParser) + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ if self.setup['xpath']: elements = xdata.xpath(self.setup['xpath']) if not elements: - raise EncryptionChunkingError("XPath expression %s matched no " - "elements" % self.setup['xpath']) + self.logger.warning("XPath expression %s matched no " + "elements" % self.setup['xpath']) else: - elements = xdata.xpath('//*[@encrypted]') + elements = xdata.xpath(self.default_xpath) if not elements: elements = list(xdata.getiterator(tag=lxml.etree.Element)) @@ -329,50 +205,85 @@ class PropertiesEncryptor(Encryptor): ans = input("Encrypt this element? [y/N] ") if not ans.lower().startswith("y"): elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in self.passphrases: + passphrase = self.passphrases[pname] + elif self.passphrase: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" + % (pname, self.setup['configfile'])) + passphrase = self.passphrase + pname = self.pname + else: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) + return (pname, passphrase) - # this is not a good use of a generator, but we need to - # generate the full list of elements in order to ensure that - # some exist before we know what to return - for elt in elements: - yield elt - - def unchunk(self, data, original): - # Properties elements are modified in-place, so we don't - # actually need to unchunk anything - xdata = data[0] - # find root element - while xdata.getparent() is not None: - xdata = xdata.getparent() - return lxml.etree.tostring(xdata, - xml_declaration=False, - pretty_print=True).decode('UTF-8') - - def _get_passphrase(self, chunk): - pname = chunk.get("encrypted") - if pname and pname.lower() != "true": - return pname - return None - - def _decrypt(self, crypted, passphrase): - # crypted is in lxml.etree._Element - if not crypted.text or not crypted.text.strip(): - self.logger.warning("Skipping empty element %s" % crypted.tag) - return crypted - decrypted = Bcfg2.Encryption.ssl_decrypt( - crypted.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - try: - crypted.text = decrypted.encode('ascii', 'xmlcharrefreplace') - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains content - # that can't even be encoded into xml entities. what - # probably happened here is that we coincidentally could - # decrypt a value encrypted with a different key, and - # wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - crypted.tag) - return crypted + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) + + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + elt.text = Bcfg2.Encryption.ssl_encrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + default_xpath = '//*[@encrypted]' + + def decrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + decrypted = Bcfg2.Encryption.ssl_decrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + try: + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) def main(): # pylint: disable=R0912,R0915 @@ -422,9 +333,6 @@ def main(): # pylint: disable=R0912,R0915 logger.error("--remove cannot be used with --properties, ignoring") setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - props_crypt = PropertiesEncryptor(setup) - cfg_crypt = CfgEncryptor(setup) - for fname in setup['args']: if not os.path.exists(fname): logger.error("%s does not exist, skipping" % fname) @@ -454,10 +362,10 @@ def main(): # pylint: disable=R0912,R0915 props = False if props: - encryptor = props_crypt if setup['remove']: logger.info("Cannot use --remove with Properties file %s, " "ignoring for this file" % fname) + tools = (PropertiesEncryptor, PropertiesDecryptor) else: if setup['xpath']: logger.info("Cannot use xpath with Cfg file %s, ignoring " @@ -465,31 +373,52 @@ def main(): # pylint: disable=R0912,R0915 if setup['interactive']: logger.info("Cannot use interactive mode with Cfg file %s, " "ignoring -I for this file" % fname) - encryptor = cfg_crypt + tools = (CfgEncryptor, CfgDecryptor) data = None + mode = None if setup['encrypt']: - xform = encryptor.encrypt - write = encryptor.write_encrypted + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" elif setup['decrypt']: - xform = encryptor.decrypt - write = encryptor.write_decrypted + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "decrypt" else: logger.info("Neither --encrypt nor --decrypt specified, " "determining mode") - data = encryptor.decrypt(fname) - if data: - write = encryptor.write_decrypted - else: - logger.info("Failed to decrypt %s, trying encryption" % fname) + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + + try: + data = tool.decrypt() + mode = "decrypt" + except: # pylint: disable=W0702 + pass + if data is False: data = None - xform = encryptor.encrypt - write = encryptor.write_encrypted + logger.info("Failed to decrypt %s, trying encryption" % fname) + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" if data is None: - data = xform(fname) - if not data: - logger.error("Failed to %s %s, skipping" % (xform.__name__, fname)) + data = getattr(tool, mode)() + if data is False: + logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if setup['crypt_stdout']: if len(setup['args']) > 1: @@ -498,10 +427,10 @@ def main(): # pylint: disable=R0912,R0915 if len(setup['args']) > 1: print("") else: - write(fname, data=data) + tool.write(data) if (setup['remove'] and - encryptor.get_encrypted_filename(fname) != fname): + tool.get_destination_filename(fname) != fname): try: os.unlink(fname) except IOError: diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 133e1ccb3..cc9b03b80 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -717,7 +717,7 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: self.load_plugins() - self.fam.handle_events_in_interval(1) + self.block_for_fam_events(handle_events=True) if args: self.onecmd(" ".join(args)) else: @@ -758,7 +758,8 @@ USAGE = build_usage() def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER) + interpreter=Bcfg2.Options.INTERPRETER, + command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index ab3b6450f..9ceb1dd04 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -73,7 +73,7 @@ def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index d7a1894f0..7c38a65d8 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -157,7 +157,7 @@ def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core @@ -298,8 +298,8 @@ def main(): for client in clients: yield ClientTest(core, client, ignore) - TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) + result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], + suite=LazySuite(generate_tests), exit=False) # block until all children have completed -- should be # immediate since we've already gotten all the results we @@ -308,7 +308,10 @@ def main(): child.join() core.shutdown() - os._exit(0) # pylint: disable=W0212 + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 if __name__ == "__main__": diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 4ef531d39..473214d89 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -193,6 +193,20 @@ class DepSolver(object): if not msg.startswith("0 "): self.logger.info(msg) + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + def main(): parser = OptionParser() @@ -233,6 +247,15 @@ def main(): sys.exc_info()[1], exc_info=1) print(json.dumps(False)) rv = 2 + elif cmd == "makecache": + try: + # this code copied from yumcommands.py + depsolver.populate_cache() + print json.dumps(True) + except yum.Errors.YumBaseError: + logger.error("Unexpected error creating cache: %s" % + sys.exc_info()[1], exc_info=1) + print json.dumps(False) elif cmd == "complete": try: data = json.loads(sys.stdin.read()) |