diff options
31 files changed, 448 insertions, 379 deletions
diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 62c4ac1a8..6c801ff1e 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -267,7 +267,7 @@ revision information out of your repository for reporting purposes. Ldap Plugin +++++++++++ -The Ldap plugin makes it possible to fetch data from an LDAP directory, +The Ldap plugin makes it possible to fetch data from a LDAP directory, process it and attach it to your metadata. Metadata Plugin @@ -718,6 +718,11 @@ Reporting options Maximum number of children for the reporting collector. Use 0 to disable the limit. (default is 0) + django_settings + Arbitrary options for the Django installation. The value expected + is a literal python dictionary, that is merged with the already set + django settings. + See Also -------- diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt index 9ad64db4e..1ed2ae70f 100644 --- a/doc/releases/1.4.0pre2.txt +++ b/doc/releases/1.4.0pre2.txt @@ -32,6 +32,20 @@ backwards-incompatible user-facing changes This fixes potentially long client runs when comparing files that have diverged significantly. +* The database options in the config (options and reporting_options in database + section) now have to be literal python dictionaries. + + This allows to set arbitrary options with nested settings. + +* The Ldap plugin changed significantly. The configuration interface was + simplified and new configuration options for the number of retries and the + delay in between were added. + + You have to register your ldap queries in the global list, there is no + distinction between LdapQueries and LdapSubQueries anymore, the names of + your queries default to the class names and the Ldap plugin expires + the metadata caches if the config file changes. + Thanks ------ diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index c991f20c9..7e7a7c72e 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -275,9 +275,7 @@ Several variables are pre-defined inside templates: | | <server-plugins-grouping-metadata-clientmetadata>` | +-------------+--------------------------------------------------------+ | name | The value of the ``name`` attribute as specified in | -| | the Path entry in Bcfg2. If an :ref:`altsrc | -| | <server-plugins-structures-altsrc>` attribute is used, | -| | then ``name`` will be the value of that attribute. | +| | the Path entry in Bcfg2. | +-------------+--------------------------------------------------------+ | source_path | The path to the template file on the filesystem | +-------------+--------------------------------------------------------+ diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt index 90590a272..96e224761 100644 --- a/doc/server/plugins/grouping/ldap.txt +++ b/doc/server/plugins/grouping/ldap.txt @@ -33,39 +33,38 @@ next section. Configuration ------------- -As processing LDAP search results can get pretty complex, the configuration has +As processing LDAP search results can get pretty complex, the configuration has to be written in Python. Here is a minimal example to get you started:: - from Bcfg2.Server.Plugins.Ldap import LdapConnection, LdapQuery, LdapSubQuery, register_query - - conn_default = LdapConnection() - conn_default.binddn = "uid=example,ou=People,dc=example,dc=com" - conn_default.bindpw = "foobat" - - @register_query + from Bcfg2.Server.Plugins.Ldap import LdapConnection, LdapQuery + + __queries__ = ['ExampleQuery'] + + conn_default = LdapConnection( + binddn="uid=example,ou=People,dc=example,dc=com", + bindpw = "foobat") + class ExampleQuery(LdapQuery): - name = "example" base = "ou=People,dc=example,dc=com" scope = "one" attrs = ["cn", "uid"] connection = conn_default - + def prepare_query(self, metadata): self.filter = "(personalServer=" + metadata.hostname + ")" - + def process_result(self, metadata): if not self.result: admin_uid = None admin_name = "This server has no admin." - return { + return { "admin_uid" : self.result[0][1]["uid"], "admin_name" : self.result[0][1]["cn"] } -The first line provides three classes for dealing with connections and queries -(details below) and a decorator function for registering your queries with the plugin. +The first line provides the two required classes for dealing with connections and queries. In this example our LDAP directory has a number of user objects in it. Each of those may have a personal server they administer. Whenever metadata for this machine is being @@ -73,7 +72,20 @@ generated by the Bcfg2 server, the UID and name of the admin are retrieved from In your bundles and config templates, you can access this data via the metadata object:: - ${metadata.Ldap["example"]["admin_name"]} + ${metadata.Ldap["ExampleQuery"]["admin_name"]} + +Connection retry +++++++++++++++++ + +If the LDAP server is down during a request, the LDAP plugin tries to reconnect after a +short delay. By default, it waits 3 seconds during the retries and tries to reconnect +up to three times. + +If you wish, you could customize these values in your ``bcfg2.conf``:: + + [ldap] + retries = 3 + retry_delay = 3.0 Class reference --------------- @@ -83,23 +95,23 @@ LdapConnection .. class:: LdapConnection - This class represents an LDAP connection. Every query must be associated with exactly + This class represents an LDAP connection. Every query must be associated with exactly one connection. - -.. attribute:: LdapConnection.binddn - + +.. attribute:: LdapConnection.binddn + DN used to authenticate against LDAP (required). - + .. attribute:: LdapConnection.bindpw - + Password for the previously mentioned **binddn** (required). - + .. attribute:: LdapConnection.host - + Hostname of host running the LDAP server (defaults to "localhost"). .. attribute:: LdapConnection.port - + Port where LDAP server is listening (defaults to 389). You may pass any of these attributes as keyword arguments when creating the connection object. @@ -108,143 +120,140 @@ LdapQuery +++++++++ .. class:: LdapQuery - + This class defines a single query that may adapt itself depending on the current metadata. .. attribute:: LdapQuery.attrs - + Can be used to retrieve only a certain subset of attributes. May either be a list of strings (attribute names) or ``None``, meaning all attributes (defaults to ``None``). .. attribute:: LdapQuery.base - - This is the search base. Only LDAP entries below this DN will be included in your + + This is the search base. Only LDAP entries below this DN will be included in your search results (required). - + .. attribute:: LdapQuery.connection - + Set this to an instance of the LdapConnection class (required). .. attribute:: LdapQuery.filter - + LDAP search filter used to narrow down search results (defaults to ``(objectClass=*)``). .. attribute:: LdapQuery.name - + This will be used as the dictionary key that provides access to the query results from - the metadata object (``metadata.Ldap["NAMEGOESHERE"]``) (required). + the metadata object: ``metadata.Ldap["NAMEGOESHERE"]`` (defaults to the class name). .. attribute:: LdapQuery.scope - - Set this to one of "base", "one" or "sub" to specify LDAP search depth (defaults to "sub"). + + Set this to one of "base", "one" or "sub" to specify LDAP search depth (defaults to "sub"). .. method:: LdapQuery.is_applicable(self, metadata) - + You can override this method to indicate whether this query makes sense for a given set of metadata (e.g. you need a query only for a certain bundle or group). - + (defaults to returning True) - -.. method:: LdapQuery.prepare_query(self, metadata) - + +.. method:: LdapQuery.prepare_query(self, metadata, \**kwargs) + Override this method to alter the query prior to execution. This is useful if your filter depends on the current metadata, e.g.:: - + self.filter = "(cn=" + metadata.hostname + ")" - + (defaults to doing nothing) -.. method:: LdapQuery.process_result(self, metadata) - +.. method:: LdapQuery.process_result(self, metadata, \**kwargs) + You will probably override this method in every query to reformat the results from LDAP. The raw result is stored in ``self.result``, you must return the altered data. Note that LDAP search results are presented in this structure:: - + ( ("DN of first entry returned", { "firstAttribute" : 1, "secondAttribute" : 2, - } + } ), ("DN of second entry returned", { "firstAttribute" : 1, "secondAttribute" : 2, - } + } ), ) - + Therefore, to return just the value of the firstAttribute of the second object returned, you'd write:: - + return self.result[1][1][0] - + (defaults to returning ``self.result`` unaltered) -LdapSubQuery -++++++++++++ - -.. class:: LdapSubQuery - - Sometimes you need more than one query to obtain the data you need (e.g. use the first - query to return all websites running on metadata.hostname and another query to find all - customers that should have access to those sites). - - LdapSubQueries are the same as LdapQueries, except for that the methods - - * ``get_result()`` - * ``prepare_query()`` - * ``process_result()`` - - allow any additional keyword arguments that may contain additional data as needed. Note - that ``get_result()`` will call ``prepare_query()`` and ``process_result()`` for you, - so you shouldn't ever need to invoke these yourself, just override them. - -Here is another example that uses LdapSubQuery:: - - class WebSitesQuery(LdapSubQuery): - name = "web_sites" +.. method:: LdapQuery.get_result(self, metadata, \**kwargs) + + This executes the query. First it will call ``prepare_query() for you, then it will try + to execute the query with the specified connection and last it will call ``process_result()`` + and return that return value. + +If you use a LdapQuery class by yourself, you could pass additional keyword arguments to +``get_result()``. It will call ``prepare_query()`` and ``process_result()`` for you and +also supply this additional arguments to this methods. + +Here is an example:: + + __queries__ = ['WebPackageQuery'] + + class WebSitesQuery(LdapQuery): filter = "(objectClass=webHostingSite)" attrs = ["dc"] connection = conn_default - + def prepare_query(self, metadata, base_dn): self.base = base_dn - - def process_result(self, metadata): + + def process_result(self, metadata, **kwargs): [...] # build sites dict from returned dc attributes return sites - - @register_query + class WebPackagesQuery(LdapQuery): - name = "web_packages" base = "dc=example,dc=com" attrs = ["customerId"] connection = conn_default - + def prepare_query(self, metadata): self.filter = "(&(objectClass=webHostingPackage)(cn:dn:=" + metadata.hostname + "))" - + def process_result(self, metadata): customers = {} for customer in self.result: dn = customer[0] cid = customer[1]["customerId"][0] - customers[cid]["sites"] = WebSitesQuery().get_result(metadata, base_dn = dn) + customers[cid]["sites"] = WebSitesQuery().get_result(metadata, base_dn=dn) return customers This example assumes that we have a number of webhosting packages that contain various -sites. We need a first query ("web_packages") to get a list of the packages our customers -have and another query for each of those to find out what sites are contained in each -package. The magic happens in the second class where ``WebSitesQuery.get_result()`` is -called with the additional ``base_dn`` parameter that allows our LdapSubQuery to only +sites. We need the ``WebPackagesQuery`` to get a list of the packages our customers +have and another query for each of those to find out what sites are contained in each +package. The magic happens in the second class where ``WebSitesQuery.get_result()`` is +called with the additional ``base_dn`` parameter that allows our LdapQuery to only search below that DN. -.. warning:: - Do NOT apply the ``register_query`` decorator to LdapSubQueries. +You do not need to add all LdapQueries to the ``__queries__`` list. Only add those to +that list, that should be called automatically and whose results should be added to the +client metadata. Known Issues ------------ * At this point there is no support for SSL/TLS. +* This module could not know, if a value changed on the LDAP server. So it could not + expire the client metadata cache sanely. + If you are using aggressive caching mode, this plugin will expire the metadata cache + for a single client at the start of a client run. If you are using LDAP data from + another client in a template, you will probably get the cached values from the last + client run of that other client. diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 13000a719..43a28bad0 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -286,7 +286,7 @@ a Mercurial version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. .SS Ldap Plugin .sp -The Ldap plugin makes it possible to fetch data from an LDAP directory, +The Ldap plugin makes it possible to fetch data from a LDAP directory, process it and attach it to your metadata. .SS Metadata Plugin .sp @@ -778,6 +778,11 @@ Turn on Django debugging. .B max_children Maximum number of children for the reporting collector. Use 0 to disable the limit. (default is 0) +.TP +.B django_settings +Arbitrary options for the Django installation. The value expected +is a literal python dictionary, that is merged with the already set +django settings. .UNINDENT .UNINDENT .UNINDENT diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index fc445e07c..1f1772d46 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -8,6 +8,7 @@ import tempfile import Bcfg2.Options from Bcfg2.Client.Tools.POSIX.base import POSIXTool from Bcfg2.Compat import unicode, b64encode, b64decode # pylint: disable=W0622 +import Bcfg2.Utils class POSIXFile(POSIXTool): @@ -17,21 +18,6 @@ class POSIXFile(POSIXTool): def fully_specified(self, entry): return entry.text is not None or entry.get('empty', 'false') == 'true' - def _is_string(self, strng, encoding): - """ Returns true if the string contains no ASCII control - characters and can be decoded from the specified encoding. """ - for char in strng: - if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: - return False - if not hasattr(strng, "decode"): - # py3k - return True - try: - strng.decode(encoding) - return True - except: # pylint: disable=W0702 - return False - def _get_data(self, entry): """ Get a tuple of (<file data>, <is binary>) for the given entry """ is_binary = entry.get('encoding', 'ascii') == 'base64' @@ -181,8 +167,8 @@ class POSIXFile(POSIXTool): (entry.get("name"), sys.exc_info()[1])) return False if not is_binary: - is_binary |= not self._is_string(content, - Bcfg2.Options.setup.encoding) + is_binary |= not Bcfg2.Utils.is_string( + content, Bcfg2.Options.setup.encoding) if is_binary: # don't compute diffs if the file is binary prompt.append('Binary file, no printable diff') diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index f7e5b1b0b..bfcc69475 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -84,7 +84,7 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): else: if entry.get('status') == 'on': cmd = self.get_svc_command(entry, 'start') - else: + elif entry.get('status') == 'off': cmd = self.get_svc_command(entry, 'stop') if cmd: diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 5a83c25c3..3e101405d 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -123,6 +123,9 @@ def finalize_django_config(opts=None, silent=False): opts.web_prefix.rstrip('/') + \ settings['MEDIA_URL'] + if opts.django_settings: + settings.update(opts.django_settings) + logger = logging.getLogger() logger.debug("Finalizing Django settings: %s" % settings) @@ -229,7 +232,7 @@ class _OptionContainer(object): dest='db_schema', default='public'), Bcfg2.Options.Option( cf=('database', 'options'), help='Database options', - dest='db_opts', type=Bcfg2.Options.Types.comma_dict, + dest='db_opts', type=Bcfg2.Options.Types.literal_dict, default=dict()), # reporting database options Bcfg2.Options.Option( @@ -258,14 +261,18 @@ class _OptionContainer(object): Bcfg2.Options.Option( cf=('database', 'reporting_options'), help='Reporting database options', dest='reporting_db_opts', - type=Bcfg2.Options.Types.comma_dict, default=dict()), + type=Bcfg2.Options.Types.literal_dict, default=dict()), # Django options Bcfg2.Options.Option( cf=('reporting', 'time_zone'), help='Django timezone'), Bcfg2.Options.BooleanOption( cf=('reporting', 'web_debug'), help='Django debug'), Bcfg2.Options.Option( - cf=('reporting', 'web_prefix'), help='Web prefix')] + cf=('reporting', 'web_prefix'), help='Web prefix'), + Bcfg2.Options.Option( + cf=('reporting', 'django_settings'), + help='Additional django settings', + type=Bcfg2.Options.Types.literal_dict, default=dict())] @staticmethod def component_parsed_hook(opts): diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index d146e3aa2..b72a495f1 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -141,6 +141,9 @@ class Parser(argparse.ArgumentParser): self.option_list.extend(option.list_options()) option.add_to_parser(self) + for opt in option.list_options(): + opt.default_from_config(self._cfp) + self._defaults_set.append(opt) def add_component(self, component): """ Add a component (and all of its options) to the diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py index ac099e135..ad2e04f10 100644 --- a/src/lib/Bcfg2/Options/Types.py +++ b/src/lib/Bcfg2/Options/Types.py @@ -5,6 +5,7 @@ import os import re import pwd import grp +from Bcfg2.Compat import literal_eval _COMMA_SPLIT_RE = re.compile(r'\s*,\s*') @@ -32,28 +33,10 @@ def colon_list(value): return value.split(':') -def comma_dict(value): - """ Split an option string on commas, optionally surrounded by - whitespace, and split the resulting items again on equals signs, - returning a dict """ - result = dict() - if value: - items = comma_list(value) - for item in items: - if '=' in item: - key, value = item.split(r'=', 1) - if value in ["true", "yes", "on"]: - result[key] = True - elif value in ["false", "no", "off"]: - result[key] = False - else: - try: - result[key] = int(value) - except ValueError: - result[key] = value - else: - result[item] = True - return result +def literal_dict(value): + """ literally evaluate the option in order to allow for arbitrarily nested + dictionaries """ + return literal_eval(value) def anchored_regex_list(value): diff --git a/src/lib/Bcfg2/Reporting/templates/base-timeview.html b/src/lib/Bcfg2/Reporting/templates/base-timeview.html index 9a5ef651c..28a9fa0f4 100644 --- a/src/lib/Bcfg2/Reporting/templates/base-timeview.html +++ b/src/lib/Bcfg2/Reporting/templates/base-timeview.html @@ -17,7 +17,7 @@ function bcfg2_check_date() { } document.write(getCalendarStyles()); </script> -{% if not timestamp %}Rendered at {% now "Y-m-d H:i" %} | {% else %}View as of {{ timestamp|date:"Y-m-d H:i" }} | {% endif %}{% spaceless %} +{% if not timestamp %}Rendered at {% now "SHORT_DATETIME_FORMAT" %} | {% else %}View as of {{ timestamp|date:"SHORT_DATETIME_FORMAT" }} | {% endif %}{% spaceless %} <a id='cal_link' name='cal_link' href='#' onclick='showCalendar(); return false;' >[change]</a> <form method='post' action='{{ path }}' id='cal_form' name='cal_form'> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 6732bb8c9..6809dcc2d 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -39,7 +39,7 @@ span.history_links a { <select id="quick" name="quick" onchange="javascript:pageJump('quick');"> <option value="" selected="selected">--- Time ---</option> {% for i in client.interactions.all|slice:":25" %} - <option value="{% url "reports_client_detail_pk" hostname=client.name pk=i.id %}">{{i.timestamp|date:"c"}}</option> + <option value="{% url "reports_client_detail_pk" hostname=client.name pk=i.id %}">{{i.timestamp|date:"DATETIME_FORMAT"}}</option> {% endfor %} </select></span> </div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 6a314bd88..cf778504a 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -32,7 +32,9 @@ This is needed for Django versions less than 1.5 <td class='right_column_narrow'>{{ entry.bad_count }}</td> <td class='right_column_narrow'>{{ entry.modified_count }}</td> <td class='right_column_narrow'>{{ entry.extra_count }}</td> - <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> + <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %} style="white-space: nowrap;"> + {{ entry.timestamp|date:"SHORT_DATETIME_FORMAT"|safe }} + </span></td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index e3befb0eb..8a5d93690 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -22,7 +22,9 @@ This is needed for Django versions less than 1.5 {% for inter in inters %} <tr class='{% cycle listview,listview_alt %}'> <td><a href='{% url "reports_client_detail" hostname=inter.client.name %}'>{{inter.client.name}}</a></td> - <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.pk %}' style="white-space: nowrap;"> + {{inter.timestamp|date:"SHORT_DATETIME_FORMAT"|safe}} + </a></td> <td>{{ item.get_state_display }}</td> <td style='white-space: nowrap'><a href='{% url "reports_item" entry_type=item.class_name pk=item.pk %}'>({{item.pk}}) {{item.short_list|join:","}}</a></td> </tr> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index c6e6df020..2e2fd36fa 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -130,7 +130,7 @@ div.entry_list h3 { <div class='entry_list'> <div class='entry_list_head'> - <h3>Occurences on {{ timestamp|date:"Y-m-d" }}</h3> + <h3>Occurences on {{ timestamp|date:"SHORT_DATE_FORMAT" }}</h3> </div> {% if associated_list %} <table class="entry_list" cellpadding="3"> diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc index 30ed2fd3e..444d05e1e 100644 --- a/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc +++ b/src/lib/Bcfg2/Reporting/templates/widgets/interaction_list.inc @@ -15,7 +15,9 @@ </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='left_column'><a href='{% url reports_client_detail_pk hostname=entry.client.name, pk=entry.id %}'>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</a></td> + <td class='left_column'><a href='{% url reports_client_detail_pk hostname=entry.client.name, pk=entry.id %}' class="white-space: nowrap;"> + {{ entry.timestamp|date:"SHORT_DATETIME_FORMAT"|safe }} + </a></td> {% if not client %} <td class='right_column_wide'><a href='{% add_url_filter hostname=entry.client.name %}'>{{ entry.client.name }}</a></td> {% endif %} diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index e138c57e4..dc5cc46fb 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -34,7 +34,8 @@ class BuiltinCore(NetworkCore): daemon_args = dict(uid=Bcfg2.Options.setup.daemon_uid, gid=Bcfg2.Options.setup.daemon_gid, umask=int(Bcfg2.Options.setup.umask, 8), - detach_process=True) + detach_process=True, + files_preserve=self._logfilehandles()) if Bcfg2.Options.setup.daemon: daemon_args['pidfile'] = TimeoutPIDLockFile( Bcfg2.Options.setup.daemon, acquire_timeout=5) @@ -44,6 +45,24 @@ class BuiltinCore(NetworkCore): self.context = daemon.DaemonContext(**daemon_args) __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] + def _logfilehandles(self, logger=None): + """ Get a list of all filehandles logger, that have to be handled + with DaemonContext.files_preserve to keep looging working. + + :param logger: The logger to get the file handles of. By default, + self.logger is used. + :type logger: logging.Logger + """ + if logger is None: + logger = self.logger + + handles = [handler.stream.fileno() + for handler in logger.handlers + if hasattr(handler, 'stream')] + if logger.parent: + handles += self._logfilehandles(logger.parent) + return handles + def _dispatch(self, method, args, dispatch_dict): """ Dispatch XML-RPC method calls diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 8e6a926ae..3a6251594 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -6,6 +6,7 @@ import copy from difflib import SequenceMatcher import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from Bcfg2.Utils import is_string def threshold(val): @@ -50,6 +51,8 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): for filename, entryset in self.core.plugins['Cfg'].entries.items(): candidates = dict([(f, e) for f, e in entryset.entries.items() if (isinstance(e, CfgGenerator) and + is_string(e.data, + Bcfg2.Options.setup.encoding) and f not in ignore and not f.endswith(".crypt"))]) similar, identical = self.get_similar(candidates) diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index 9d05516f1..ce6fdca74 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -4,8 +4,8 @@ import sys import imp from Bcfg2.Server.Lint import ServerPlugin -from Bcfg2.Server.Plugins.TemplateHelper import HelperModule, MODULE_RE, \ - safe_module_name +from Bcfg2.Server.Plugins.TemplateHelper import HelperModule, MODULE_RE +from Bcfg2.Utils import safe_module_name class TemplateHelper(ServerPlugin): @@ -44,7 +44,8 @@ class TemplateHelper(ServerPlugin): module_name = MODULE_RE.search(helper).group(1) try: - module = imp.load_source(safe_module_name(module_name), helper) + module = imp.load_source( + safe_module_name('TemplateHelper', module_name), helper) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("templatehelper-import-error", diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py index cff9ff61e..e4df9e09d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py @@ -12,14 +12,14 @@ from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: from jinja2 import Environment, FileSystemLoader HAS_JINJA2 = True -except ImportError: - HAS_JINJA2 = False + class RelEnvironment(Environment): + """Override join_path() to enable relative template paths.""" + def join_path(self, template, parent): + return os.path.join(os.path.dirname(parent), template) -class RelEnvironment(Environment): - """Override join_path() to enable relative template paths.""" - def join_path(self, template, parent): - return os.path.join(os.path.dirname(parent), template) +except ImportError: + HAS_JINJA2 = False class DefaultJinja2DataProvider(DefaultTemplateDataProvider): @@ -42,15 +42,16 @@ class CfgJinja2Generator(CfgGenerator): #: Handle .jinja2 files __extensions__ = ['jinja2'] - #: ``__loader_cls__`` is the class that will be instantiated to - #: load the template files. It must implement one public function, - #: ``load()``, as :class:`genshi.template.TemplateLoader`. - __loader_cls__ = FileSystemLoader + if HAS_JINJA2: + #: ``__loader_cls__`` is the class that will be instantiated to + #: load the template files. It must implement one public function, + #: ``load()``, as :class:`genshi.template.TemplateLoader`. + __loader_cls__ = FileSystemLoader - #: ``__environment_cls__`` is the class that will be instantiated to - #: store the jinja2 environment. It must implement one public function, - #: ``get_template()``, as :class:`jinja2.Environment`. - __environment_cls__ = RelEnvironment + #: ``__environment_cls__`` is the class that will be instantiated to + #: store the jinja2 environment. It must implement one public + #: function, ``get_template()``, as :class:`jinja2.Environment`. + __environment_cls__ = RelEnvironment #: Ignore ``.jinja2_include`` files so they can be used with the #: Jinja2 ``{% include ... %}`` directive without raising warnings. diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 553ddbc47..66f317c20 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -1,120 +1,132 @@ +""" A plugin to fetch data from a LDAP directory """ + import imp -import logging +import os import sys import time import traceback +import Bcfg2.Options import Bcfg2.Server.Plugin - -logger = logging.getLogger('Bcfg2.Plugins.Ldap') +from Bcfg2.Logger import Debuggable +from Bcfg2.Utils import ClassName, safe_module_name try: import ldap + HAS_LDAP = True except ImportError: - logger.error("Unable to load ldap module. Is python-ldap installed?") - raise ImportError - -# time in seconds between retries after failed LDAP connection -RETRY_DELAY = 5 -# how many times to try reaching the LDAP server if a connection is broken -# at the very minimum, one retry is needed to handle a restarted LDAP daemon -RETRY_COUNT = 3 - -SCOPE_MAP = { - "base": ldap.SCOPE_BASE, - "one": ldap.SCOPE_ONELEVEL, - "sub": ldap.SCOPE_SUBTREE, -} - -LDAP_QUERIES = [] - - -def register_query(query): - LDAP_QUERIES.append(query) + HAS_LDAP = False class ConfigFile(Bcfg2.Server.Plugin.FileBacked): - """ - Config file for the Ldap plugin - - The config file cannot be 'parsed' in the traditional sense as we would - need some serious type checking ugliness to just get the LdapQuery - subclasses. The alternative would be to have the user create a list with - a predefined name that contains all queries. - The approach implemented here is having the user call a registering - decorator that updates a global variable in this module. - """ - def __init__(self, filename): - self.filename = filename - Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename) - self.fam.AddMonitor(self.filename, self) + """ Config file for the Ldap plugin """ + + def __init__(self, name, core): + Bcfg2.Server.Plugin.FileBacked.__init__(self, name) + self.core = core + self.queries = list() + self.fam.AddMonitor(name, self) def Index(self): - """ - Reregisters the queries in the config file + """ Get the queries from the config file """ + try: + module = imp.load_source(safe_module_name('Ldap', self.name), + self.name) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.logger.error("Ldap: Failed to import %s: %s" % + (self.name, err)) + return + + if not hasattr(module, "__queries__"): + self.logger.error("Ldap: %s has no __queries__ list" % self.name) + return + + self.queries = list() + for query in module.__queries__: + try: + self.queries.append(getattr(module, query)) + except AttributeError: + self.logger.warning( + "Ldap: %s exports %s, but has no such attribute" % + (self.name, query)) - The config will take care of actually registering the queries, - so we just load it once and don't keep it. - """ - global LDAP_QUERIES - LDAP_QUERIES = [] - imp.load_source("ldap_cfg", self.filename) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() -class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ - The Ldap plugin allows adding data from an LDAP server to your metadata. - """ - name = "Ldap" +class Ldap(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ The Ldap plugin allows adding data from an LDAP server + to your metadata. """ + experimental = True - debug_flag = False + + options = [ + Bcfg2.Options.Option( + cf=('ldap', 'retries'), type=int, default=3, + help='The number of times to retry reaching the ' + 'LDAP server if a connection is broken'), + Bcfg2.Options.Option( + cf=('ldap', 'retry_delay'), type=float, default=5.0, + help='The time in seconds betreen retries')] def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = ConfigFile(self.data + "/config.py") - def debug_log(self, message, flag=None): - if (flag is None) and self.debug_flag or flag: - self.logger.error(message) + if not HAS_LDAP: + msg = "Python ldap module is required for Ldap plugin" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + + self.config = ConfigFile(os.path.join(self.data, 'config.py')) def get_additional_data(self, metadata): query = None try: data = {} - self.debug_log("LdapPlugin debug: found queries " + - str(LDAP_QUERIES)) - for QueryClass in LDAP_QUERIES: - query = QueryClass() + self.debug_log("Found queries %s" % self.config.queries) + for query_class in self.config.queries: + query = query_class() if query.is_applicable(metadata): - self.debug_log("LdapPlugin debug: processing query '" + - query.name + "'") + self.debug_log("Processing query '%s'" % query.name) data[query.name] = query.get_result(metadata) else: - self.debug_log("LdapPlugin debug: query '" + query.name + - "' not applicable to host '" + - metadata.hostname + "'") + self.debug_log("query '%s' not applicable to host '%s'" % + (query.name, metadata.hostname)) return data - except Exception: + except: # pylint: disable=W0702 if hasattr(query, "name"): - logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + - str(query.name) + - "', query results will be empty" + - " and may cause bind failures") - for line in traceback.format_exception(sys.exc_info()[0], - sys.exc_info()[1], - sys.exc_info()[2]): - logger.error("LdapPlugin error: " + - line.replace("\n", "")) + self.logger.error( + "Exception during processing of query named '%s', query " + "results will be empty and may cause bind failures" % + query.name) + for line in traceback.format_exc().split('\n'): + self.logger.error(line) return {} + def start_client_run(self, metadata): + if self.core.metadata_cache_mode == 'aggressive': + self.logger.warning("Ldap is incompatible with aggressive " + "client metadata caching, try 'cautious' " + "or 'initial'") + self.core.metadata_cache.expire(metadata.hostname) + + +class LdapConnection(Debuggable): + """ Connection to an LDAP server. """ + + __scopes__ = { + 'base': ldap.SCOPE_BASE, + 'one': ldap.SCOPE_ONELEVEL, + 'sub': ldap.SCOPE_SUBTREE, + } + + def __init__(self, host="localhost", port=389, binddn=None, + bindpw=None): + Debuggable.__init__(self) -class LdapConnection(object): - """ - Connection to an LDAP server. - """ - def __init__(self, host="localhost", port=389, - binddn=None, bindpw=None): self.host = host self.port = port self.binddn = binddn @@ -122,48 +134,62 @@ class LdapConnection(object): self.conn = None def __del__(self): + """ Disconnection if the instance is destroyed. """ + self.disconnect() + + def disconnect(self): + """ If a connection to an LDAP server is available, disconnect it. """ if self.conn: - self.conn.unbind() + self.conn.unbund() + self.conn = None - def init_conn(self): + def connect(self): + """ Open a connection to the configured LDAP server, and do a simple + bind ff both binddn and bindpw are set. """ + self.disconnect() self.conn = ldap.initialize(self.url) if self.binddn is not None and self.bindpw is not None: self.conn.simple_bind_s(self.binddn, self.bindpw) def run_query(self, query): - result = None - for attempt in range(RETRY_COUNT + 1): - if attempt >= 1: - logger.error("LdapPlugin error: " + - "LDAP server down (retry " + str(attempt) + "/" + - str(RETRY_COUNT) + ")") + """ Connect to the server and execute the query. If the server is + down, wait the configured amount and try to reconnect. + + :param query: The query to execute on the LDAP server. + :type query: Bcfg.Server.Plugins.Ldap.LdapQuery + """ + for attempt in range(Bcfg2.Options.setup.ldap_retries + 1): try: if not self.conn: - self.init_conn() - result = self.conn.search_s( - query.base, - SCOPE_MAP[query.scope], - query.filter.replace("\\", "\\\\"), - query.attrs, - ) - break + self.connect() + + return self.conn.search_s( + query.base, self.__scopes__[query.scope], + query.filter.replace('\\', '\\\\'), query.attrs) + except ldap.SERVER_DOWN: self.conn = None - time.sleep(RETRY_DELAY) - return result + self.logger.error( + "LdapConnection: Server %s down. Retry %d/%d in %.2fs." % + (self.url, attempt + 1, Bcfg2.Options.setup.ldap_retries, + Bcfg2.Options.setup.ldap_retry_delay)) + time.sleep(Bcfg2.Options.setup.ldap_retry_delay) + + return None @property def url(self): - return "ldap://" + self.host + ":" + str(self.port) + """ The URL of the LDAP server. """ + return "ldap://%s:%d" % (self.host, self.port) class LdapQuery(object): - """ - Query referencing an LdapConnection and providing several - methods for query manipulation. - """ + """ Query referencing an LdapConnection and providing several + methods for query manipulation. """ + + #: Name of the Query, used to register it in additional data. + name = ClassName() - name = "unknown" base = "" scope = "sub" filter = "(objectClass=*)" @@ -172,80 +198,48 @@ class LdapQuery(object): result = None def __unicode__(self): - return "LdapQuery:" + self.name + return "LdapQuery: %s" % self.name - def is_applicable(self, metadata): - """ - Overrideable method to determine if the query is to be executed for - the given metadata object. - Defaults to true. - """ - return True + def is_applicable(self, metadata): # pylint: disable=W0613 + """ Check is the query should be executed for a given metadata + object. - def prepare_query(self, metadata): + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - - In most cases, you will do something like + return True - self.filter = "(cn=" + metadata.hostname + ")" + def prepare_query(self, metadata, **kwargs): # pylint: disable=W0613 + """ Prepares the query based on the client metadata. You can + for example modify the filter based on the client hostname. - here. + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ pass - def process_result(self, metadata): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. - """ - return self.result - - def get_result(self, metadata): - """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): - self.prepare_query(metadata) - self.result = self.connection.run_query(self) - self.result = self.process_result(metadata) - return self.result - else: - logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None - - -class LdapSubQuery(LdapQuery): - """ - SubQueries are meant for internal use only and are not added - to the metadata object. They are useful for situations where - you need to run more than one query to obtain some data. - """ - def prepare_query(self, metadata, **kwargs): - """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - """ - pass + def process_result(self, metadata, **kwargs): # pylint: disable=W0613 + """ Post-process the query result. - def process_result(self, metadata, **kwargs): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ return self.result def get_result(self, metadata, **kwargs): + """ Handle the perparation, execution and processing of the query. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): + + if self.connection is not None: self.prepare_query(metadata, **kwargs) self.result = self.connection.run_query(self) - return self.process_result(metadata, **kwargs) + self.result = self.process_result(metadata, **kwargs) else: - logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None + raise Bcfg2.Server.Plugin.PluginExecutionError( + 'No connection defined for %s' % self.name) + + return self.result diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index cec2de297..b0b8d0061 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -7,18 +7,11 @@ import lxml.etree from Bcfg2.Server.Plugin import Plugin, Connector, DirectoryBacked, \ TemplateDataProvider, DefaultTemplateDataProvider from Bcfg2.Logger import Debuggable +from Bcfg2.Utils import safe_module_name MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') -def safe_module_name(module): - """ Munge the name of a TemplateHelper module to avoid collisions - with other Python modules. E.g., if someone has a helper named - 'ldap.py', it should not be added to ``sys.modules`` as ``ldap``, - but rather as something more obscure. """ - return '__TemplateHelper_%s' % module - - class HelperModule(Debuggable): """ Representation of a TemplateHelper module """ @@ -52,8 +45,9 @@ class HelperModule(Debuggable): return try: - module = imp.load_source(safe_module_name(self._module_name), - self.name) + module = imp.load_source( + safe_module_name('TemplateHelper', self._module_name), + self.name) except: # pylint: disable=W0702 # this needs to be a blanket except because the # imp.load_source() call can raise literally any error, diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 10057b63e..2fdc0c3e0 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -321,6 +321,15 @@ def safe_input(msg): return input(msg) +def safe_module_name(prefix, module): + """ Munge the name of a module with prefix to avoid collisions + with other Python modules. E.g., if you want to import user + defined helper modules and someone has a helper named 'ldap.py', + it should not be added to ``sys.modules`` as ``ldap``, but rather + as something more obscure. """ + return '__%s_%s' % (prefix, module) + + class classproperty(object): # pylint: disable=C0103 """ Decorator that can be used to create read-only class properties. """ @@ -330,3 +339,19 @@ class classproperty(object): # pylint: disable=C0103 def __get__(self, instance, owner): return self.getter(owner) + + +def is_string(strng, encoding): + """ Returns true if the string contains no ASCII control + characters and can be decoded from the specified encoding. """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + if not hasattr(strng, "decode"): + # py3k + return True + try: + strng.decode(encoding) + return True + except: # pylint: disable=W0702 + return False diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 69dd562be..47d3b84ed 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -37,22 +37,6 @@ class TestPOSIXFile(TestPOSIXTool): entry.text = "text" self.assertTrue(ptool.fully_specified(entry)) - def test_is_string(self): - ptool = self.get_obj() - - for char in list(range(8)) + list(range(14, 32)): - self.assertFalse(ptool._is_string("foo" + chr(char) + "bar", - 'UTF-8')) - for char in list(range(9, 14)) + list(range(33, 128)): - self.assertTrue(ptool._is_string("foo" + chr(char) + "bar", - 'UTF-8')) - ustr = 'é' - self.assertTrue(ptool._is_string(ustr, 'UTF-8')) - if not inPy3k: - self.assertFalse(ptool._is_string("foo" + chr(128) + "bar", - 'ascii')) - self.assertFalse(ptool._is_string(ustr, 'ascii')) - def test_get_data(self): orig_entry = lxml.etree.Element("Path", name="/test", type="file") Bcfg2.Options.setup.encoding = "ascii" @@ -216,7 +200,8 @@ class TestPOSIXFile(TestPOSIXTool): mock_unlink.assert_called_with(newfile) @patch("%s.open" % builtins) - def test__get_diffs(self, mock_open): + @patch("Bcfg2.Utils") + def test__get_diffs(self, mock_utils, mock_open): orig_entry = lxml.etree.Element("Path", name="/test", type="file", mode='0644', owner='root', group='root') @@ -226,16 +211,15 @@ class TestPOSIXFile(TestPOSIXTool): ptool = self.get_obj() ptool._get_data = Mock() ptool._diff = Mock() - ptool._is_string = Mock() def reset(): - ptool._is_string.reset_mock() + mock_utils.is_string.reset_mock() ptool._get_data.reset_mock() ptool._diff.reset_mock() mock_open.reset_mock() return copy.deepcopy(orig_entry) - ptool._is_string.return_value = True + mock_utils.is_string.return_value = True ptool._get_data.return_value = (orig_entry.text, False) mock_open.return_value.read.return_value = ondisk ptool._diff.return_value = ["-test2", "+test"] @@ -250,7 +234,7 @@ class TestPOSIXFile(TestPOSIXTool): # binary data on disk entry = reset() - ptool._is_string.return_value = False + mock_utils.is_string.return_value = False ptool._get_diffs(entry, content=ondisk) self.assertFalse(mock_open.called) self.assertFalse(ptool._diff.called) @@ -258,7 +242,7 @@ class TestPOSIXFile(TestPOSIXTool): # sensitive, non-interactive -- do nothing entry = reset() - ptool._is_string.return_value = True + mock_utils.is_string.return_value = True ptool._get_diffs(entry, sensitive=True, interactive=False) self.assertFalse(mock_open.called) self.assertFalse(ptool._diff.called) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py b/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py index 61b87de2a..b1ed4cb2b 100644 --- a/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py +++ b/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py @@ -182,7 +182,8 @@ class TestImportComponentOptions(OptionTestCase): """test cases for component loading.""" def setUp(self): - self.options = [Option("--cls", action=ImportComponentAction), + self.options = [Option("--cls", cf=("config", "cls"), + action=ImportComponentAction), Option("--module", action=ImportModuleAction)] self.result = argparse.Namespace() @@ -227,3 +228,10 @@ class TestImportComponentOptions(OptionTestCase): self.assertRaises(SystemExit, self.parser.parse, ["-C", config_file, "--cls", "Bcfg2.No.Such.Thing"]) + + @make_config({"config": {"test": "foo", "cls": "Two"}}) + def test_default_from_config_for_component_options(self, config_file): + """use default value from config file for options added by dynamic loaded component.""" + self.parser.parse(["-C", config_file]) + self.assertEqual(self.result.cls, Two.Two) + self.assertEqual(self.result.test, "foo") diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py b/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py index 404d67fdc..0b67db38a 100644 --- a/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py +++ b/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py @@ -46,21 +46,23 @@ class TestOptionTypes(Bcfg2TestCase): self.assertItemsEqual(self._test_options(["--test", "one:two three"]), ["one", "two three"]) - def test_comma_dict(self): - """parse comma-dict values.""" - self.options = [Option("--test", type=Types.comma_dict)] + def test_literal_dict(self): + """parse literal-dict values.""" + self.options = [Option("--test", type=Types.literal_dict)] expected = { "one": True, "two": 2, "three": "three", - "four": False} + "four": False, + "five": { + "a": 1, + "b": 2 + }} self.assertDictEqual( self._test_options(["--test", - "one=yes, two=2 , three=three,four=no"]), - expected) - - self.assertDictEqual( - self._test_options(["--test", "one,two=2,three=three,four=off"]), + '''{ "one": True, "two": 2, + "three": "three", "four": False, + "five": { "a": 1, "b": 2 }}''']), expected) def test_anchored_regex_list(self): diff --git a/testsuite/Testsrc/Testlib/TestOptions/Two.py b/testsuite/Testsrc/Testlib/TestOptions/Two.py index 189e0817f..0120e8b77 100644 --- a/testsuite/Testsrc/Testlib/TestOptions/Two.py +++ b/testsuite/Testsrc/Testlib/TestOptions/Two.py @@ -1,6 +1,7 @@ """Test module for component loading.""" +from Bcfg2.Options import Option class Two(object): """Test class for component loading.""" - pass + options = [Option('--test', cf=("config", "test"), dest="test", default="bar")] diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py index 128d6cae5..1c2fa23a2 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py @@ -2,6 +2,7 @@ import os import sys import Bcfg2.Server.Plugin from mock import Mock, MagicMock, patch +from Bcfg2.Utils import safe_module_name from Bcfg2.Server.Plugins.TemplateHelper import * # add all parent testsuite directories to sys.path to allow (most) @@ -39,8 +40,9 @@ class TestHelperModule(Bcfg2TestCase): mock_load_source.side_effect = ImportError attrs = dir(hm) hm.HandleEvent() - mock_load_source.assert_called_with(safe_module_name(hm._module_name), - hm.name) + mock_load_source.assert_called_with( + safe_module_name('TemplateHelper', hm._module_name), + hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) @@ -51,8 +53,9 @@ class TestHelperModule(Bcfg2TestCase): mock_load_source.return_value = Mock() attrs = dir(hm) hm.HandleEvent() - mock_load_source.assert_called_with(safe_module_name(hm._module_name), - hm.name) + mock_load_source.assert_called_with( + safe_module_name('TemplateHelper', hm._module_name), + hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) @@ -63,8 +66,9 @@ class TestHelperModule(Bcfg2TestCase): mock_load_source.return_value = module attrs = dir(hm) hm.HandleEvent() - mock_load_source.assert_called_with(safe_module_name(hm._module_name), - hm.name) + mock_load_source.assert_called_with( + safe_module_name('TemplateHelper', hm._module_name), + hm.name) self.assertEqual(attrs, dir(hm)) self.assertEqual(hm._attrs, []) @@ -74,8 +78,9 @@ class TestHelperModule(Bcfg2TestCase): mock_load_source.reset() mock_load_source.return_value = module hm.HandleEvent() - mock_load_source.assert_called_with(safe_module_name(hm._module_name), - hm.name) + mock_load_source.assert_called_with( + safe_module_name('TemplateHelper', hm._module_name), + hm.name) self.assertTrue(hasattr(hm, "foo")) self.assertTrue(hasattr(hm, "bar")) self.assertTrue(hasattr(hm, "baz")) @@ -87,8 +92,9 @@ class TestHelperModule(Bcfg2TestCase): mock_load_source.reset() mock_load_source.return_value = module hm.HandleEvent() - mock_load_source.assert_called_with(safe_module_name(hm._module_name), - hm.name) + mock_load_source.assert_called_with( + safe_module_name('TemplateHelper', hm._module_name), + hm.name) self.assertTrue(hasattr(hm, "foo")) self.assertTrue(hasattr(hm, "bar")) self.assertTrue(hasattr(hm, "quux")) diff --git a/testsuite/Testsrc/Testlib/TestUtils.py b/testsuite/Testsrc/Testlib/TestUtils.py index 4bed67248..a37f2ecbe 100644 --- a/testsuite/Testsrc/Testlib/TestUtils.py +++ b/testsuite/Testsrc/Testlib/TestUtils.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os import sys from Bcfg2.Utils import * @@ -42,3 +43,17 @@ class TestPackedDigitRange(Bcfg2TestCase): for test in exc: self.assertNotIn(test, rng) self.assertFalse(rng.includes(test)) + + +class TestIsString(Bcfg2TestCase): + def test_is_string(self): + for char in list(range(8)) + list(range(14, 32)): + self.assertFalse(is_string("foo" + chr(char) + "bar", 'UTF-8')) + for char in list(range(9, 14)) + list(range(33, 128)): + self.assertTrue(is_string("foo" + chr(char) + "bar", 'UTF-8')) + + ustr = 'é' + self.assertTrue(is_string(ustr, 'UTF-8')) + if not inPy3k: + self.assertFalse(is_string("foo" + chr(128) + "bar", 'ascii')) + self.assertFalse(is_string(ustr, 'ascii')) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index ba4b19d1c..6d3c4c714 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -66,7 +66,6 @@ error_checks = { "VCS.py", "YUM24.py"], "lib/Bcfg2/Server/Plugins": ["Deps.py", - "Ldap.py", "Pkgmgr.py"] } diff --git a/testsuite/common.py b/testsuite/common.py index bdf0c01cb..45cfcbf0b 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -64,6 +64,7 @@ try: set_setup_default("time_zone") set_setup_default("web_debug", False) set_setup_default("web_prefix") + set_setup_default("django_settings") import Bcfg2.DBSettings Bcfg2.DBSettings.finalize_django_config() |