diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
39 files changed, 971 insertions, 187 deletions
diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 207106596..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -173,15 +173,33 @@ class Backup(AdminCmd): class Client(_ServerAdminCmd): - """ Create, delete, or list client entries """ + """ Create, modify, delete, or list client entries """ + __plugin_whitelist__ = ["Metadata"] options = _ServerAdminCmd.options + [ Bcfg2.Options.PositionalArgument( "mode", - choices=["add", "del", "list"]), - Bcfg2.Options.PositionalArgument("hostname", nargs='?')] - - __plugin_whitelist__ = ["Metadata"] + choices=["add", "del", "delete", "remove", "rm", "up", "update", + "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("attributes", metavar="KEY=VALUE", + nargs='*')] + + valid_attribs = ['profile', 'uuid', 'password', 'floating', 'secure', + 'address', 'auth'] + + def get_attribs(self, setup): + """ Get attributes for adding or updating a client from the command + line """ + attr_d = {} + for i in setup.attributes: + attr, val = i.split('=', 1) + if attr not in self.valid_attribs: + print("Attribute %s unknown. Valid attributes: %s" % + (attr, self.valid_attribs)) + raise SystemExit(1) + attr_d[attr] = val + return attr_d def run(self, setup): if setup.mode != 'list' and not setup.hostname: @@ -189,23 +207,32 @@ class Client(_ServerAdminCmd): elif setup.mode == 'list' and setup.hostname: self.logger.warning("<hostname> is not honored in list mode") - if setup.mode == 'add': - try: - self.metadata.add_client(setup.hostname) - except MetadataConsistencyError: - err = sys.exc_info()[1] - self.errExit("Error adding client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'del': + if setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + else: + include_attribs = True + if setup.mode == 'add': + func = self.metadata.add_client + action = "adding" + elif setup.mode in ['up', 'update']: + func = self.metadata.update_client + action = "updating" + elif setup.mode in ['del', 'delete', 'rm', 'remove']: + func = self.metadata.remove_client + include_attribs = False + action = "deleting" + + if include_attribs: + args = (setup.hostname, self.get_attribs(setup)) + else: + args = (setup.hostname,) try: - self.metadata.remove_client(setup.hostname) + func(*args) except MetadataConsistencyError: err = sys.exc_info()[1] - self.errExit("Error deleting client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'list': - for client in self.metadata.list_clients(): - print(client) + self.errExit("Error %s client %s: %s" % (setup.hostname, + action, err)) class Compare(AdminCmd): @@ -885,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -933,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 0023e9313..769addf55 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,8 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - ca=Bcfg2.Options.setup.ca) + ca=Bcfg2.Options.setup.ca, + protocol=Bcfg2.Options.setup.protocol) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 398053374..892f2832a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,14 +19,13 @@ import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics try: from django.core.exceptions import ImproperlyConfigured - from django.core import management import django.conf HAS_DJANGO = True except ImportError: @@ -74,6 +73,24 @@ def sort_xml(node, key=None): node[:] = sorted_children +def close_db_connection(func): + """ Decorator that closes the Django database connection at the end of + the function. This should decorate any exposed function that + might open a database connection. """ + @wraps(func) + def inner(self, *args, **kwargs): + """ The decorated function """ + rv = func(self, *args, **kwargs) + if self._database_available: # pylint: disable=W0212 + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() + return rv + + return inner + + class CoreInitError(Exception): """ Raised when the server core cannot be initialized. """ pass @@ -114,7 +131,8 @@ class Core(object): Bcfg2.Options.Common.repository, Bcfg2.Options.Common.filemonitor, Bcfg2.Options.BooleanOption( - cf=('server', 'fam_blocking'), default=False, + "--no-fam-blocking", cf=('server', 'fam_blocking'), + dest="fam_blocking", default=True, help='FAM blocks on startup until all events are processed'), Bcfg2.Options.BooleanOption( cf=('logging', 'performance'), dest="perflog", @@ -128,6 +146,10 @@ class Core(object): default='off', choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] + #: The name of this server core. This can be overridden by core + #: implementations to provide a more specific name. + name = "Core" + def __init__(self): # pylint: disable=R0912,R0915 """ .. automethod:: _run @@ -196,6 +218,12 @@ class Core(object): self.revision = '-1' atexit.register(self.shutdown) + #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, + #: then :mod:`atexit` calls it *again*, so it gets called + #: twice. This is potentially bad, so we use + #: :attr:`Bcfg2.Server.Core._running` as a flag to determine + #: if the core needs to be shutdown, and only do it once. + self._running = True #: Threading event to signal worker threads (e.g., #: :attr:`fam_thread`) to shutdown @@ -236,16 +264,16 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) + self.logger.error("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] self.logger.error("Updating database %s failed: %s" % - (Bcfg2.Options.setup.db_name, err)) + (Bcfg2.Options.setup.db_name, + sys.exc_info()[1])) def __str__(self): return self.__class__.__name__ @@ -332,7 +360,7 @@ class Core(object): This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ for plugin in Bcfg2.Options.setup.plugins: - if not plugin in self.plugins: + if plugin not in self.plugins: self.init_plugin(plugin) # Remove blacklisted plugins @@ -403,14 +431,22 @@ class Core(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.info("Shutting down core...") + if not self._running: + self.logger.debug("%s: Core already shut down" % self.name) + return + self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() - self.fam.shutdown() - self.logger.info("FAM shut down") - for plugin in list(self.plugins.values()): - plugin.shutdown() - self.logger.info("All plugins shut down") + self._running = False + self.fam.shutdown() + self.logger.info("%s: FAM shut down" % self.name) + for plugin in list(self.plugins.values()): + plugin.shutdown() + self.logger.info("%s: All plugins shut down" % self.name) + if self._database_available: + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() @property def metadata_cache_mode(self): @@ -601,9 +637,10 @@ class Core(object): del entry.attrib['realname'] return ret except: - self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('realname'), - entry.get('name'))) + self.logger.error( + "Failed binding entry %s:%s with altsrc %s: %s" % + (entry.tag, entry.get('realname'), entry.get('name'), + sys.exc_info()[1])) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -1052,6 +1089,7 @@ class Core(object): @exposed @track_statistics() + @close_db_connection def DeclareVersion(self, address, version): """ Declare the client version. @@ -1074,6 +1112,7 @@ class Core(object): return True @exposed + @close_db_connection def GetProbes(self, address): """ Fetch probes for the client. @@ -1099,6 +1138,7 @@ class Core(object): (client, err)) @exposed + @close_db_connection def RecvProbeData(self, address, probedata): """ Receive probe data from clients. @@ -1146,6 +1186,7 @@ class Core(object): return True @exposed + @close_db_connection def AssertProfile(self, address, profile): """ Set profile for a client. @@ -1165,6 +1206,7 @@ class Core(object): return True @exposed + @close_db_connection def GetConfig(self, address): """ Build config for a client by calling :func:`BuildConfiguration`. @@ -1184,6 +1226,7 @@ class Core(object): self.critical_error("Metadata consistency failure for %s" % client) @exposed + @close_db_connection def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. @@ -1199,6 +1242,7 @@ class Core(object): return True @exposed + @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. @@ -1326,8 +1370,16 @@ class NetworkCore(Core): daemonized, etc.""" options = Core.options + [ Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, - Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, - Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', + default="/etc/pki/tls/private/bcfg2.key"), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', + default="/etc/pki/tls/certs/bcfg2.crt"), Bcfg2.Options.BooleanOption( '--listen-all', cf=('server', 'listen_all'), default=False, help="Listen on all interfaces"), diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index f8b602d90..b60302871 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -173,6 +173,17 @@ def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): return b64encode("Salted__" + salt + crypted) + "\n" +def is_encrypted(val): + """ Make a best guess if the value is encrypted or not. This just + checks to see if ``val`` is a base64-encoded string whose content + starts with "Salted__", so it may have (rare) false positives. It + will not have false negatives. """ + try: + return b64decode(val).startswith("Salted__") + except: # pylint: disable=W0702 + return False + + def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): """ Convenience method to decrypt the given encrypted string by trying the given passphrases or all passphrases sequentially until @@ -233,6 +244,10 @@ class DecryptError(Exception): """ Exception raised when decryption fails. """ +class EncryptError(Exception): + """ Exception raised when encryption fails. """ + + class CryptoTool(object): """ Generic decryption/encryption interface base object """ @@ -319,6 +334,8 @@ class CfgEncryptor(Encryptor): Bcfg2.Options.setup.config) def encrypt(self): + if is_encrypted(self.data): + raise EncryptError("Data is alraedy encrypted") return ssl_encrypt(self.data, self.passphrase) def get_destination_filename(self, original_filename): @@ -355,7 +372,7 @@ class CfgDecryptor(Decryptor): class PropertiesCryptoMixin(object): """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*' + default_xpath = '//*[@encrypted]' def _get_elements(self, xdata): """ Get the list of elements to encrypt or decrypt """ @@ -425,11 +442,13 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): def encrypt(self): xdata = lxml.etree.XML(self.data, parser=XMLParser) for elt in self._get_elements(xdata): + if is_encrypted(elt.text): + raise EncryptError("Element is already encrypted: %s" % + print_xml(elt)) try: pname, passphrase = self._get_element_passphrase(elt) except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False + raise EncryptError(str(sys.exc_info()[1])) self.logger.debug("Encrypting %s" % print_xml(elt)) elt.text = ssl_encrypt(elt.text, passphrase).strip() elt.set("encrypted", pname) @@ -441,7 +460,6 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): """ decryptor class for Properties files """ - default_xpath = '//*[@encrypted]' def decrypt(self): decrypted_any = False @@ -640,9 +658,9 @@ class CLI(object): if data is None: try: data = getattr(tool, mode)() - except DecryptError: - self.logger.error("Failed to %s %s, skipping" % (mode, - fname)) + except (EncryptError, DecryptError): + self.logger.error("Failed to %s %s, skipping: %s" % + (mode, fname, sys.exc_info()[1])) continue if Bcfg2.Options.setup.stdout: if len(Bcfg2.Options.setup.files) > 1: diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 69463ab4c..b349d20fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -27,11 +27,11 @@ class GaminEvent(Event): class Gamin(FileMonitor): """ File monitor backend with `Gamin - <http://people.gnome.org/~veillard/gamin/>`_ support. """ + <http://people.gnome.org/~veillard/gamin/>`_ support. **Deprecated.** """ - #: The Gamin backend is fairly decent, particularly newer - #: releases, so it has a fairly high priority. - __priority__ = 90 + #: The Gamin backend is deprecated, but better than pseudo, so it + #: has a medium priority. + __priority__ = 50 def __init__(self): FileMonitor.__init__(self) @@ -46,6 +46,9 @@ class Gamin(FileMonitor): #: The queue used to record monitors that are added before #: :func:`start` has been called and :attr:`mon` is created. self.add_q = [] + + self.logger.warning("The Gamin file monitor backend is deprecated. " + "Please switch to a supported file monitor.") __init__.__doc__ = FileMonitor.__init__.__doc__ def start(self): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index b8eb06aa1..c4b34a469 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): - if self.notifier: + if self.started and self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py index 0caf4d7ed..aee15cb5d 100644 --- a/src/lib/Bcfg2/Server/Lint/Bundler.py +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -1,12 +1,12 @@ """ ``bcfg2-lint`` plugin for :ref:`Bundler -<server-plugins-structures-bundler-index>` """ +<server-plugins-structures-bundler>` """ from Bcfg2.Server.Lint import ServerPlugin class Bundler(ServerPlugin): """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ + <server-plugins-structures-bundler>` checks. """ def Run(self): self.missing_bundles() diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index e2d1ec597..fc4506c12 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -9,6 +9,7 @@ from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML @@ -76,6 +77,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Cheetah-templated Cfg files"), Bcfg2.Options.Option( + cf=("Comments", "jinja2_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "jinja2_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( cf=("Comments", "infoxml_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for info.xml files"), @@ -235,6 +244,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "cfg" elif isinstance(entry, CfgCheetahGenerator): rtype = "cheetah" + elif isinstance(entry, CfgJinja2Generator): + rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, entry.infoxml.pnode.data, diff --git a/src/lib/Bcfg2/Server/Lint/Crypto.py b/src/lib/Bcfg2/Server/Lint/Crypto.py new file mode 100644 index 000000000..53a54031c --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Crypto.py @@ -0,0 +1,61 @@ +""" Check for data that claims to be encrypted, but is not. """ + +import os +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin +from Bcfg2.Server.Encryption import is_encrypted + + +class Crypto(ServerlessPlugin): + """ Check for templated scripts or executables. """ + + def Run(self): + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + self.check_cfg() + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, + "Properties")): + self.check_properties() + # TODO: check all XML files + + @classmethod + def Errors(cls): + return {"unencrypted-cfg": "error", + "empty-encrypted-properties": "error", + "unencrypted-properties": "error"} + + def check_cfg(self): + """ Check for Cfg files that end in .crypt but aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".crypt"): + if not is_encrypted(open(fpath).read()): + self.LintError( + "unencrypted-cfg", + "%s is a .crypt file, but it is not encrypted" % + fpath) + + def check_properties(self): + """ Check for Properties data that has an ``encrypted`` attribute but + aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Properties")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".xml"): + xdata = lxml.etree.parse(fpath) + for elt in xdata.xpath('//*[@encrypted]'): + if not elt.text: + self.LintError( + "empty-encrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but no text content: %s" % + (fpath, self.RenderXML(elt))) + elif not is_encrypted(elt.text): + self.LintError( + "unencrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but is not encrypted: %s" % + (fpath, self.RenderXML(elt))) diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py new file mode 100755 index 000000000..333249cc2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py @@ -0,0 +1,41 @@ +""" Check Jinja2 templates for syntax errors. """ + +import sys +import Bcfg2.Server.Lint +from jinja2 import Template, TemplateSyntaxError +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator + + +class Jinja2(Bcfg2.Server.Lint.ServerPlugin): + """ Check Jinja2 templates for syntax errors. """ + + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + + @classmethod + def Errors(cls): + return {"jinja2-syntax-error": "error", + "unknown-jinja2-error": "error"} + + def check_template(self, entry): + """ Generic check for all jinja2 templates """ + try: + Template(entry.data.decode(entry.encoding)) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("jinja2-syntax-error", + "Jinja2 syntax error in %s: %s" % (entry.name, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-jinja2-error", + "Unknown Jinja2 error in %s: %s" % (entry.name, + err)) + + def check_cfg(self): + """ Check jinja2 templates in Cfg for syntax errors. """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, CfgJinja2Generator)): + self.check_template(entry) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 5d9e229fa..ebf4c4954 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unknown-entry-type": "error", + return {"missing-elements": "error", + "unknown-entry-type": "error", "unknown-entry-tag": "error", "required-attrs-missing": "error", "required-attr-format": "error", "extra-attrs": "warning"} + def check_default_acl(self, path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 0 + if path.xpath("ACL[@type='default' and @scope='user' and @user='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='group' and @group='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='other']"): + defaults += 1 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") + def check_packages(self): """ Check Packages sources for Source entries with missing attributes. """ @@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ Check bundles for BoundPath entries with missing + """ Check bundles for BoundPath and BoundPackage entries with missing attrs. """ if 'Bundler' not in self.core.plugins: return @@ -183,6 +201,25 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) + # ensure that abstract Path tags have either name + # or glob specified + for path in bundle.xdata.xpath("//Path"): + if ('name' not in path.attrib and + 'glob' not in path.attrib): + self.LintError( + "required-attrs-missing", + "Path tags require either a 'name' or 'glob' " + "attribute: \n%s" % self.RenderXML(path)) + # ensure that abstract Package tags have either name + # or group specified + for package in bundle.xdata.xpath("//Package"): + if ('name' not in package.attrib and + 'group' not in package.attrib): + self.LintError( + "required-attrs-missing", + "Package tags require either a 'name' or 'group' " + "attribute: \n%s" % self.RenderXML(package)) + def check_entry(self, entry, filename): """ Generic entry check. @@ -221,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): required_attrs['major'] = is_device_mode required_attrs['minor'] = is_device_mode + if tag == 'Path': + self.check_default_acl(entry) + if tag == 'ACL' and 'scope' in required_attrs: required_attrs[entry.get('scope')] = is_username diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py new file mode 100644 index 000000000..5a80a5884 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -0,0 +1,80 @@ +""" Check for templated scripts or executables. """ + +import os +import stat +import Bcfg2.Server.Lint +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.Plugin import default_path_metadata +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ + CfgEncryptedGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ + CfgEncryptedCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import \ + CfgEncryptedJinja2Generator + + +class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): + """ Check for templated scripts or executables. """ + templates = [CfgGenshiGenerator, CfgCheetahGenerator, CfgJinja2Generator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator, + CfgEncryptedJinja2Generator] + extensions = [".pl", ".py", ".sh", ".rb"] + + def Run(self): + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + any(isinstance(entry, t) for t in self.templates)): + self.check_template(entryset, entry) + + @classmethod + def Errors(cls): + return {"templated-script": "warning", + "templated-executable": "warning"} + + def check_template(self, entryset, entry): + """ Check a template to see if it's a script or an executable. """ + # first, check for a known script extension + ext = os.path.splitext(entryset.path)[1] + if ext in self.extensions: + self.LintError("templated-script", + "Templated script found: %s\n" + "File has a known script extension: %s\n" + "Template a config file for the script instead" % + (entry.name, ext)) + return + + # next, check for a shebang line + firstline = open(entry.name).readline() + if firstline.startswith("#!"): + self.LintError("templated-script", + "Templated script found: %s\n" + "File starts with a shebang: %s\n" + "Template a config file for the script instead" % + (entry.name, firstline)) + return + + # finally, check for executable permissions in info.xml + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + try: + mode = int( + pinfo.get("mode", + default_path_metadata()['mode']), 8) + except ValueError: + # LintError will be produced by RequiredAttrs plugin + self.logger.warning("Non-octal mode: %s" % mode) + continue + if mode & stat.S_IXUSR != 0: + self.LintError( + "templated-executable", + "Templated executable found: %s\n" + "Template a config file for the executable instead" + % entry.name) + return diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index fbd5a2893..a952da724 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -23,9 +23,11 @@ class TemplateHelper(ServerPlugin): def __init__(self, *args, **kwargs): ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - self.reserved_defaults = \ - self.core.plugins['TemplateHelper'].reserved_defaults + # we instantiate a dummy helper to discover which keywords and + # defaults are reserved + dummy = HelperModule("foo.py") + self.reserved_keywords = dir(dummy) + self.reserved_defaults = dummy.reserved_defaults def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index e38619355..0b3f1e24d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -113,9 +114,17 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): :type filename: string :returns: lxml.etree._ElementTree - the parsed data""" try: - return lxml.etree.parse(filename) - except SyntaxError: - result = self.cmd.run(["xmllint", filename]) + xdata = lxml.etree.parse(filename) + if self.files is None: + self._expand_wildcard_xincludes(xdata) + xdata.xinclude() + return xdata + except (lxml.etree.XIncludeError, SyntaxError): + cmd = ["xmllint", "--noout"] + if self.files is None: + cmd.append("--xinclude") + cmd.append(filename) + result = self.cmd.run(cmd) self.LintError("xml-failed-to-parse", "%s fails to parse:\n%s" % (filename, result.stdout + result.stderr)) @@ -125,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. @@ -146,6 +182,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if not schema: return False datafile = self.parse(filename) + if not datafile: + return False if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py new file mode 100644 index 000000000..6383a3c99 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -0,0 +1,72 @@ +"""Ensure that all JSON files in the Bcfg2 repository are +valid. Currently, the only plugins that uses JSON are Ohai and +Properties.""" + +import os +import sys +import glob +import fnmatch +import Bcfg2.Server.Lint + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + + +class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): + """Ensure that all JSON files in the Bcfg2 repository are + valid. Currently, the only plugins that uses JSON are Ohai and + Properties. """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A list of file globs that give the path to JSON files. The + #: globs are extended :mod:`fnmatch` globs that also support + #: ``**``, which matches any number of any characters, + #: including forward slashes. + self.globs = ["Properties/*.json", "Ohai/*.json"] + self.files = self.get_files() + + def Run(self): + for path in self.files: + self.logger.debug("Validating JSON in %s" % path) + try: + json.load(open(path)) + except ValueError: + self.LintError("json-failed-to-parse", + "%s does not contain valid JSON: %s" % + (path, sys.exc_info()[1])) + + @classmethod + def Errors(cls): + return {"json-failed-to-parse": "error"} + + def get_files(self): + """Return a list of all JSON files to validate, based on + :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, + os.path.join('*', p)) + else: + listfiles = lambda p: glob.glob( + os.path.join(Bcfg2.Options.setup.repository, p)) + + rv = [] + for path in self.globs: + if '/**/' in path: + if self.files is not None: + rv.extend(listfiles(path)) + else: # self.files is None + fpath, fname = path.split('/**/') + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, + fpath)): + rv.extend([os.path.join(root, f) + for f in files if f == fname]) + else: + rv.extend(listfiles(path)) + return rv diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 8a793fd94..9b3e6ece2 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -13,6 +13,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins +from Bcfg2.Compat import walk_packages def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -297,11 +298,10 @@ class LintPluginAction(Bcfg2.Options.ComponentAction): bases = ['Bcfg2.Server.Lint'] def __call__(self, parser, namespace, values, option_string=None): - for plugin in getattr(Bcfg2.Options.setup, "plugins", []): - module = sys.modules[plugin.__module__] - if hasattr(module, "%sLint" % plugin.name): - print("Adding lint plugin %s" % plugin) - values.append(plugin) + plugins = getattr(Bcfg2.Options.setup, "plugins", []) + for lint_plugin in walk_packages(path=__path__): + if lint_plugin[1] in plugins: + values.append(lint_plugin[1]) Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, option_string) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 294963669..724b34d8d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -275,6 +275,7 @@ class ChildCore(Core): @exposed def GetConfig(self, client): """ Render the configuration for a client """ + self.metadata.update_client_list() self.logger.debug("%s: Building configuration for %s" % (self.name, client)) return lxml.etree.tostring(self.BuildConfiguration(client)) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 1cb5a7b3e..559612d1e 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -18,7 +18,7 @@ from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ - PluginExecutionError + PluginExecutionError, PluginInitError try: import Bcfg2.Server.Encryption @@ -219,6 +219,18 @@ class DatabaseBacked(Plugin): .. private-include: _must_lock """ + def __init__(self, core): + Plugin.__init__(self, core) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) + if use_db and not HAS_DJANGO: + raise PluginInitError("%s is configured to use the database but " + "Django libraries are not found" % self.name) + elif use_db and not self.core.database_available: + raise PluginInitError("%s is configured to use the database but " + "the database is unavailable due to prior " + "errors" % self.name) + @property def _use_db(self): """ Whether or not this plugin is configured to use the @@ -227,11 +239,7 @@ class DatabaseBacked(Plugin): False) if use_db and HAS_DJANGO and self.core.database_available: return True - elif not use_db: - return False else: - self.logger.error("%s: use_database is true but django not found" % - self.name) return False @property @@ -267,7 +275,8 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: # pylint: disable=C0111,W0232 + class Meta(object): # pylint: disable=W0232 + """ Model metadata options """ app_label = "Server" @@ -638,7 +647,13 @@ class XMLFileBacked(FileBacked): if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): self.logger.debug(msg) else: - self.logger.warning(msg) + self.logger.error(msg) + # add a FAM monitor for this path. this isn't perfect + # -- if there's an xinclude of "*.xml", we'll watch + # the literal filename "*.xml". but for non-globbing + # filenames, it works fine. + if fpath not in self.extra_monitors: + self.add_monitor(fpath) parent = el.getparent() parent.remove(el) @@ -748,9 +763,6 @@ class StructFile(XMLFileBacked): err)) if HAS_CRYPTO and self.encryption: - lax_decrypt = self.xdata.get( - "lax_decryption", - str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -759,10 +771,14 @@ class StructFile(XMLFileBacked): self.logger.info("%s: Decrypted %s to gibberish, skipping" % (self.name, el.tag)) except Bcfg2.Server.Encryption.EVPError: + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == \ + "true" msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) if lax_decrypt: - self.logger.warning(msg) + self.logger.debug(msg) else: raise PluginExecutionError(msg) Index.__doc__ = XMLFileBacked.Index.__doc__ @@ -774,16 +790,11 @@ class StructFile(XMLFileBacked): passes = Bcfg2.Options.setup.passphrases try: passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Server.Encryption.ssl_decrypt(element.text, - passphrase) - except Bcfg2.Server.Encryption.EVPError: - # error is raised below - pass + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Server.Encryption.bruteforce_decrypt(element.text) + raise Bcfg2.Server.Encryption.EVPError("No passphrase named '%s'" % + element.get("encrypted")) raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") def _include_element(self, item, metadata, *args): @@ -818,7 +829,8 @@ class StructFile(XMLFileBacked): """ stream = self.template.generate( **get_xml_template_data(self, metadata)).filter(removecomment) - return lxml.etree.XML(stream.render('xml', strip_whitespace=False), + return lxml.etree.XML(stream.render('xml', + strip_whitespace=False).encode(), parser=Bcfg2.Server.XMLParser) def _match(self, item, metadata, *args): @@ -935,7 +947,7 @@ class InfoXML(StructFile): _include_tests = copy.copy(StructFile._include_tests) _include_tests['Path'] = lambda el, md, entry, *args: \ - entry.get("name") == el.get("name") + entry.get('realname', entry.get('name')) == el.get("name") def Match(self, metadata, entry): # pylint: disable=W0221 """ Implementation of diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 622b69c79..c45d6fa84 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -216,6 +216,10 @@ class Metadata(object): """ raise NotImplementedError + def update_client_list(self): + """ Re-read the cached list of clients """ + raise NotImplementedError + class Connector(object): """ Connector plugins augment client metadata instances with diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 8b9330c9b..41ee57b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -4,31 +4,30 @@ import os import re import sys import copy -import Bcfg2.Server -import Bcfg2.Server.Plugin +import fnmatch +import lxml.etree +from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ + StructureValidator, XMLDirectoryBacked, Generator from genshi.template import TemplateError -class BundleFile(Bcfg2.Server.Plugin.StructFile): +class BundleFile(StructFile): """ Representation of a bundle XML file """ bundle_name_re = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') def __init__(self, filename, should_monitor=False): - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, - should_monitor=should_monitor) + StructFile.__init__(self, filename, should_monitor=should_monitor) if self.name.endswith(".genshi"): self.logger.warning("Bundler: %s: Bundle filenames ending with " ".genshi are deprecated; add the Genshi XML " "namespace to a .xml bundle instead" % self.name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def Index(self): - Bcfg2.Server.Plugin.StructFile.Index(self) + StructFile.Index(self) if self.xdata.get("name"): self.logger.warning("Bundler: %s: Explicitly specifying bundle " "names is deprecated" % self.name) - Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ @property def bundle_name(self): @@ -37,9 +36,10 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): os.path.basename(self.name)).group("name") -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): +class Bundler(Plugin, + Structure, + StructureValidator, + XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' @@ -47,18 +47,30 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, patterns = re.compile(r'^.*\.(?:xml|genshi)$') def __init__(self, core): - Bcfg2.Server.Plugin.Plugin.__init__(self, core) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) + Plugin.__init__(self, core) + Structure.__init__(self) + StructureValidator.__init__(self) + XMLDirectoryBacked.__init__(self, self.data) #: Bundles by bundle name, rather than filename self.bundles = dict() def HandleEvent(self, event): - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - + XMLDirectoryBacked.HandleEvent(self, event) self.bundles = dict([(b.bundle_name, b) for b in self.entries.values()]) + def validate_structures(self, metadata, structures): + """ Translate <Path glob='...'/> entries into <Path name='...'/> + entries """ + for struct in structures: + for pathglob in struct.xpath("//Path[@glob]"): + for plugin in self.core.plugins_by_type(Generator): + for match in fnmatch.filter(plugin.Entries['Path'].keys(), + pathglob.get("glob")): + lxml.etree.SubElement(pathglob.getparent(), + "Path", name=match) + pathglob.getparent().remove(pathglob) + def BuildStructures(self, metadata): bundleset = [] bundles = copy.copy(metadata.bundles) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index e2a2f696a..849c75f70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,6 +1,7 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: @@ -25,7 +26,6 @@ class CfgEncryptedGenerator(CfgGenerator): CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") - __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): CfgGenerator.handle_event(self, event) @@ -35,11 +35,13 @@ class CfgEncryptedGenerator(CfgGenerator): try: self.data = bruteforce_decrypt(self.data) except EVPError: - raise PluginExecutionError("Failed to decrypt %s" % self.name) - handle_event.__doc__ = CfgGenerator.handle_event.__doc__ + msg = "Cfg: Failed to decrypt %s" % self.name + if Bcfg2.Options.setup.lax_decryption: + self.logger.debug(msg) + else: + raise PluginExecutionError(msg) def get_data(self, entry, metadata): if self.data is None: raise PluginExecutionError("Failed to decrypt %s" % self.name) return CfgGenerator.get_data(self, entry, metadata) - get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..c8da84ae0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py @@ -0,0 +1,25 @@ +""" Handle encrypted Jinja2 templates (.crypt.jinja2 or +.jinja2.crypt files)""" + +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator \ + import CfgEncryptedGenerator + + +class CfgEncryptedJinja2Generator(CfgJinja2Generator, CfgEncryptedGenerator): + """ CfgEncryptedJinja2Generator lets you encrypt your Jinja2 + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.jinja2 or .jinja2.crypt files + __extensions__ = ['jinja2.crypt', 'crypt.jinja2'] + + #: Override low priority from parent class + __priority__ = 0 + + def handle_event(self, event): + CfgEncryptedGenerator.handle_event(self, event) + handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__ + + def get_data(self, entry, metadata): + return CfgJinja2Generator.get_data(self, entry, metadata) + get_data.__doc__ = CfgJinja2Generator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py new file mode 100644 index 000000000..e36ee78aa --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py @@ -0,0 +1,52 @@ +""" The CfgJinja2Generator allows you to use the `Jinja2 +<http://jinja.pocoo.org/>`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +try: + from jinja2 import Template + HAS_JINJA2 = True +except ImportError: + HAS_JINJA2 = False + + +class DefaultJinja2DataProvider(DefaultTemplateDataProvider): + """ Template data provider for Jinja2 templates. Jinja2 and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + +class CfgJinja2Generator(CfgGenerator): + """ The CfgJinja2Generator allows you to use the `Jinja2 + <http://jinja.pocoo.org/>`_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .jinja2 files + __extensions__ = ['jinja2'] + + #: Low priority to avoid matching host- or group-specific + #: .crypt.jinja2 files + __priority__ = 50 + + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) + if not HAS_JINJA2: + raise PluginExecutionError("Jinja2 is not available") + __init__.__doc__ = CfgGenerator.__init__.__doc__ + + def get_data(self, entry, metadata): + template = Template(self.data.decode(Bcfg2.Options.setup.encoding)) + return template.render( + get_template_data(entry, metadata, self.name, + default=DefaultJinja2DataProvider())) + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index e9698f526..8cc3f7b21 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -34,7 +34,6 @@ class CfgPrivateKeyCreator(XMLCfgCreator): pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) self.cmd = Executor() - __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index d2b982349..5dc3d98eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -872,8 +872,7 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, """ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client - machines. - """ + machines. """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 3d3ef8f8c..b30a9acea 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -31,4 +31,4 @@ class Decisions(Bcfg2.Server.Plugin.Plugin, self.blacklist = DecisionFile(os.path.join(self.data, "blacklist.xml")) def GetDecisions(self, metadata, mode): - return getattr(self, mode).get_decision(metadata) + return getattr(self, mode).get_decisions(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 78f86f28e..1d15656af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -674,6 +674,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if attribs is None: attribs = dict() if self._use_db: + if attribs: + msg = "Metadata does not support setting client attributes " +\ + "with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: @@ -681,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = MetadataClientModel(hostname=client_name) # pylint: enable=E1102 client.save() - self.clients = self.list_clients() + self.update_client_list() return client else: try: @@ -734,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs, alias=True) def list_clients(self): - """ List all clients in client database """ + """ List all clients in client database. + + Making ``self.clients`` a property and reading the client list + dynamically from the database on every call to + ``self.clients`` can result in very high rates of database + reads, so we cache the ``list_clients()`` results to reduce + the database load. When the database is in use, the client + list is reread periodically with + :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: @@ -785,13 +798,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() - self.clients = self.list_clients() + self.update_client_list() else: return self._remove_xdata(self.clients_xml, "Client", client_name) def _handle_clients_xml_event(self, _): # pylint: disable=R0912 """ handle all events for clients.xml and files xincluded from clients.xml """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the clients.xml data + self.states['clients.xml'] = False + xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} @@ -853,9 +871,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.clientgroups[clname].append(profile) except KeyError: self.clientgroups[clname] = [profile] + self.update_client_list() + self.cache.expire() self.states['clients.xml'] = True - if self._use_db: - self.clients = self.list_clients() def _get_condition(self, element): """ Return a predicate that returns True if a client meets @@ -883,7 +901,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the groups.xml data + self.states['groups.xml'] = False + self.groups = {} + self.group_membership = dict() + self.negated_groups = dict() + self.ordered_groups = [] # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of @@ -908,10 +934,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if grp.get('default', 'false') == 'true': self.default = grp.get('name') - self.group_membership = dict() - self.negated_groups = dict() - self.ordered_groups = [] - # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ @@ -944,6 +966,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership.setdefault(gname, []) self.group_membership[gname].append( self._aggregate_conditions(conditions)) + self.cache.expire() self.states['groups.xml'] = True def HandleEvent(self, event): @@ -1447,6 +1470,32 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 + def update_client_list(self): + """ Re-read the client list from the database (if the database is in + use) """ + if self._use_db: + self.logger.debug("Metadata: Re-reading client list from database") + old = set(self.clients) + self.clients = self.list_clients() + + # we could do this with set.symmetric_difference(), but we + # want detailed numbers of added/removed clients for + # logging + new = set(self.clients) + added = new - old + removed = old - new + self.logger.debug("Metadata: Added %s clients: %s" % + (len(added), added)) + self.logger.debug("Metadata: Removed %s clients: %s" % + (len(removed), removed)) + + for client in added.union(removed): + self.cache.expire(client) + + def start_client_run(self, metadata): + """ Hook to reread client list if the database is in use """ + self.update_client_list() + def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ba7baab11..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index dba56eed2..3d5c68e3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -69,12 +69,11 @@ class AptSource(Source): else: return ["%sPackages.gz" % self.rawurl] - def read_files(self): + def read_files(self): # pylint: disable=R0912 bdeps = dict() + brecs = dict() bprov = dict() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') + self.essentialpkgs = set() for fname in self.files: if not self.rawurl: barch = [x @@ -86,6 +85,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -100,9 +100,10 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -113,17 +114,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 8b20df58a..004e27874 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -299,9 +299,14 @@ class Collection(list, Debuggable): :type package: string :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + for source in self: if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + return [] def get_essential(self): @@ -465,7 +470,8 @@ class Collection(list, Debuggable): return list(complete.difference(initial)) @track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + def complete(self, packagelist, # pylint: disable=R0912,R0914 + recommended=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -529,7 +535,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py new file mode 100644 index 000000000..e393cabfe --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -0,0 +1,86 @@ +""" pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +import lzma +import tarfile + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class PkgngCollection(Collection): + """ Handle collections of pkgng sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class PkgngSource(Source): + """ Handle pkgng sources """ + + #: PkgngSource sets the ``type`` on Package entries to "pkgng" + ptype = 'pkgng' + + @property + def urls(self): + """ A list of URLs to the base metadata file for each + repository described by this source. """ + if not self.rawurl: + rv = [] + for part in self.components: + for arch in self.arches: + rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % + (self.url, self.version, arch, part)) + return rv + else: + return ["%s/packagesite.txz" % self.rawurl] + + def read_files(self): + bdeps = dict() + for fname in self.files: + if not self.rawurl: + abi = [x + for x in fname.split('@') + if x.startswith('freebsd:')][0][8:] + barch = ':'.join(abi.split(':')[1:]) + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + if barch not in bdeps: + bdeps[barch] = dict() + try: + tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) + reader = tar.extractfile('packagesite.yaml') + except: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') + pkg = json.loads(line) + pkgname = pkg['name'] + self.pkgnames.add(pkgname) + if 'deps' in pkg: + bdeps[barch][pkgname] = pkg['deps'].keys() + self.process_files(bdeps, dict()) + read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 4b6130f72..24db2963d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -246,6 +246,10 @@ class Source(Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ``<package name>`` -> ``<list of recommended + #: symbols>``. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -310,7 +314,7 @@ class Source(Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -318,7 +322,7 @@ class Source(Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() @track_statistics() @@ -513,13 +517,14 @@ class Source(Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, # pylint: disable=R0912,W0102 + provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -531,14 +536,20 @@ class Source(Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -548,6 +559,18 @@ class Source(Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = \ + recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -655,17 +678,24 @@ class Source(Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index b98d3f419..f26ded4c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -63,6 +63,7 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor +from distutils.spawn import find_executable # pylint: disable=E0611 # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -89,7 +90,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -340,25 +343,21 @@ class YumCollection(Collection): @property def helper(self): - """ The full path to :file:`bcfg2-yum-helper`. First, we - check in the config file to see if it has been explicitly - specified; next we see if it's in $PATH (which we do by making - a call to it; I wish there was a way to do this without - forking, but apparently not); finally we check in /usr/sbin, - the default location. """ + """The full path to :file:`bcfg2-yum-helper`. First, we check in the + config file to see if it has been explicitly specified; next + we see if it's in $PATH; finally we default to /usr/sbin, the + default location. """ + # pylint: disable=W0212 if not self._helper: - # pylint: disable=W0212 self.__class__._helper = Bcfg2.Options.setup.yum_helper if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH - try: - self.debug_log("Checking for bcfg2-yum-helper in $PATH") - self.cmd.run(['bcfg2-yum-helper']) - self.__class__._helper = 'bcfg2-yum-helper' - except OSError: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + self.__class__._helper = find_executable('bcfg2-yum-helper') + if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" - # pylint: enable=W0212 - return self._helper + return self.__class__._helper + # pylint: enable=W0212 @property def use_yum(self): @@ -417,6 +416,25 @@ class YumCollection(Collection): yumconf.write(open(self.cfgfile, 'w')) + def get_arch(self): + """ If 'arch' for each source is the same, return that arch, otherwise + None. + + This helps bcfg2-yum-helper when the client arch is + incompatible with the bcfg2 server's arch. + + In case multiple arches are found, punt back to the default behavior. + """ + arches = set() + for source in self: + for url_map in source.url_map: + if url_map['arch'] in self.metadata.groups: + arches.add(url_map['arch']) + if len(arches) == 1: + return arches.pop() + else: + return None + def get_config(self, raw=False): # pylint: disable=W0221 """ Get the yum configuration for this collection. @@ -839,7 +857,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist): + def complete(self, packagelist, recommended=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -857,7 +875,7 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 @@ -872,10 +890,12 @@ class YumCollection(Collection): if packagelist: try: - result = self.call_helper( - "complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) + helper_dict = dict(packages=list(packagelist), + groups=list(self.get_relevant_groups())) + arch = self.get_arch() + if arch is not None: + helper_dict['arch'] = arch + result = self.call_helper("complete", helper_dict) except ValueError: # error reported by call_helper() return set(), packagelist diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 49f64bdf3..d11ac60fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -101,7 +101,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "backends"), dest="packages_backends", help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, - action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + action=PackagesBackendAction, + default=['Yum', 'Apt', 'Pac', 'Pkgng']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -319,8 +320,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, structures.append(indep) @track_statistics() - def _build_packages(self, metadata, independent, structures, - collection=None): + def _build_packages(self, metadata, independent, # pylint: disable=R0914 + structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of @@ -357,10 +358,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -399,7 +405,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base) + pcache[pkey] = collection.complete(base, recommended) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 9f2375fcd..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,7 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode # pylint: disable=W0622 +from Bcfg2.Compat import unicode, any # pylint: disable=W0622 import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics @@ -51,8 +51,10 @@ def load_django_models(): try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -431,7 +433,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, options = [ Bcfg2.Options.BooleanOption( cf=('probes', 'use_database'), dest="probes_db", - help="Use database capabilities of the Probes plugin")] + help="Use database capabilities of the Probes plugin"), + Bcfg2.Options.Option( + cf=('probes', 'allowed_groups'), dest="probes_allowed_groups", + help="Whitespace-separated list of group name regexps to which " + "probes can assign a client", + default=[re.compile('.*')], + type=Bcfg2.Options.Types.anchored_regex_list)] options_parsed_hook = staticmethod(load_django_models) def __init__(self, core): @@ -480,7 +488,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, for line in dlines[:]: match = self.groupline_re.match(line) if match: - groups.append(match.group("groupname")) + newgroup = match.group("groupname") + if self._group_allowed(newgroup): + groups.append(newgroup) + else: + self.logger.warning( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) return (groups, ProbeData("\n".join(dlines))) @@ -489,3 +503,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, def get_additional_data(self, metadata): return self.probestore.get_data(metadata.hostname) + + def _group_allowed(self, group): + """ Determine if the named group can be set as a probe group + by checking the regexes listed in the [probes] groups_allowed + setting """ + return any(r.match(group) + for r in Bcfg2.Options.setup.probes_allowed_groups) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 87cee7029..28400f6d2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -161,7 +163,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) PropertyFile.__init__(self, name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def _write(self): open(self.name, "wb").write( @@ -169,7 +170,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): xml_declaration=False, pretty_print=True).decode('UTF-8')) return True - _write.__doc__ = PropertyFile._write.__doc__ def validate_data(self): """ ensure that the data in this object validates against the @@ -192,7 +192,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): self.name) else: return True - validate_data.__doc__ = PropertyFile.validate_data.__doc__ def get_additional_data(self, metadata): if Bcfg2.Options.setup.automatch: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 8b8ada852..282de8247 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -54,7 +54,7 @@ class Reporting(Statistics, Threaded, PullSource): self.logger.error(msg) raise PluginInitError(msg) - def start_threads(self): + # This must be loaded here for bcfg2-admin try: self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: @@ -63,6 +63,10 @@ class Reporting(Statistics, Threaded, PullSource): if self.debug_flag: self.transport.set_debug(self.debug_flag) + def start_threads(self): + """Nothing to do here""" + pass + def set_debug(self, debug): rv = Statistics.set_debug(self, debug) if self.transport is not None: diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index b2a16e52e..b752650f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -20,8 +20,8 @@ class Svn(Bcfg2.Server.Plugin.Version): Bcfg2.Options.Option( cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", type=lambda v: v.replace("-", "_"), - choices=dir(pysvn.wc_conflict_choice), - default=pysvn.wc_conflict_choice.postpone, + choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 + default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 help="SVN conflict resolution method"), Bcfg2.Options.Option( cf=("svn", "user"), dest="svn_user", help="SVN username"), diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 5e6846a44..6ad5b5635 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -72,7 +72,7 @@ class SSLServer(SocketServer.TCPServer, object): def __init__(self, listen_all, server_address, RequestHandlerClass, keyfile=None, certfile=None, reqCert=False, ca=None, - timeout=None, protocol='xmlrpc/ssl'): + timeout=None, protocol='xmlrpc/tlsv1'): """ :param listen_all: Listen on all interfaces :type listen_all: bool @@ -333,7 +333,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, """ Component XMLRPCServer. """ def __init__(self, listen_all, server_address, RequestHandlerClass=None, - keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/tlsv1', timeout=10, logRequests=False, register=True, allow_none=True, encoding=None): """ |