From 4e7df5b7a747c7f1656678ae0362fb7701e96695 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 8 Aug 2013 15:03:43 -0400 Subject: Metadata: catch dns lookup errors better --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 03323d64b..2f8f49a9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1069,7 +1069,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if cname in self.aliases: return self.aliases[cname] return cname - except socket.herror: + except (socket.gaierror, socket.herror): err = "Address resolution error for %s: %s" % (address, sys.exc_info()[1]) self.logger.error(err) -- cgit v1.2.3-1-g7c22 From f59d96dd483c0101eb29474bdf5384dd7aedd928 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 8 Aug 2013 15:05:59 -0400 Subject: testsuite: perform pylint tests against MultiprocessingCore when appropriate --- testsuite/Testsrc/test_code_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 22db0b331..e1214a942 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -79,7 +79,7 @@ no_checks = { "TCheetah.py", "TGenshi.py"], } -if sys.version_info > (2, 5): +if sys.version_info < (2, 6): # multiprocessing core requires py2.6 no_checks['lib/Bcfg2/Server'].append('MultiprocessingCore.py') -- cgit v1.2.3-1-g7c22 From bb3445f5e474fc3d5cba019d522ec43ff10aaa6c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 8 Aug 2013 17:24:24 -0500 Subject: Reporting: Fix typo Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 598e1c6ec..fc9523067 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -88,7 +88,7 @@ class InteractionManager(models.Manager): Returns the ids of most recent interactions for clients as of a date. Arguments: - maxdate -- datetime object. Most recent date to pull. (dafault None) + maxdate -- datetime object. Most recent date to pull. (default None) """ from django.db import connection -- cgit v1.2.3-1-g7c22 From 37f828a8cce5b5e051f5b5b3c1311ae38eed95a1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 08:52:07 -0400 Subject: Core: fixed py3k incompatibility Also abstracted getting the list of objects that may register RMI calls into a separate function. --- src/lib/Bcfg2/Server/Core.py | 10 ++++++++-- src/lib/Bcfg2/Server/MultiprocessingCore.py | 6 ++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6357baae4..da278e4ed 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -973,11 +973,17 @@ class BaseCore(object): raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, "Critical failure: %s" % message) + def _get_rmi_objects(self): + """ Get a dict (name: object) of all objects that may have RMI + calls. Currently, that includes all plugins and the FAM. """ + rv = {self.fam.__class__.__name__: self.fam} + rv.update(self.plugins) + return rv + def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - for pname, pinst in self.plugins.items() + \ - [(self.fam.__class__.__name__, self.fam)]: + for pname, pinst in self._get_rmi_objects(): for mname in pinst.__rmi__: rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) return rmi diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e79207291..e7518ca18 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -298,8 +298,7 @@ class ChildCore(BaseCore): def _get_rmi(self): rmi = dict() - for pname, pinst in self.plugins.items() + \ - [(self.fam.__class__.__name__, self.fam)]: + for pname, pinst in self._get_rmi_objects(): for crmi in pinst.__child_rmi__: if isinstance(crmi, tuple): mname = crmi[1] @@ -413,8 +412,7 @@ class Core(BuiltinCore): def _get_rmi(self): child_rmi = dict() - for pname, pinst in self.plugins.items() + \ - [(self.fam.__class__.__name__, self.fam)]: + for pname, pinst in self._get_rmi_objects(): for crmi in pinst.__child_rmi__: if isinstance(crmi, tuple): parentname, childname = crmi -- cgit v1.2.3-1-g7c22 From 8eb46ebcba381c79ff2e35eb91f35d28bd870016 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 09:41:54 -0400 Subject: Core: iterate over RMI objects properly --- src/lib/Bcfg2/Server/Core.py | 2 +- src/lib/Bcfg2/Server/MultiprocessingCore.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index da278e4ed..a8601fd08 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -983,7 +983,7 @@ class BaseCore(object): def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - for pname, pinst in self._get_rmi_objects(): + for pname, pinst in self._get_rmi_objects().items(): for mname in pinst.__rmi__: rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) return rmi diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e7518ca18..c0c8fd9e1 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -298,7 +298,7 @@ class ChildCore(BaseCore): def _get_rmi(self): rmi = dict() - for pname, pinst in self._get_rmi_objects(): + for pname, pinst in self._get_rmi_objects().items(): for crmi in pinst.__child_rmi__: if isinstance(crmi, tuple): mname = crmi[1] @@ -412,7 +412,7 @@ class Core(BuiltinCore): def _get_rmi(self): child_rmi = dict() - for pname, pinst in self._get_rmi_objects(): + for pname, pinst in self._get_rmi_objects().items(): for crmi in pinst.__child_rmi__: if isinstance(crmi, tuple): parentname, childname = crmi -- cgit v1.2.3-1-g7c22 From e456076478ba127e71fa9ef699488d126efe9b5d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 09:44:05 -0400 Subject: MultiprocessingCore: added missing docstring --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index c0c8fd9e1..3cc308b1c 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -435,6 +435,8 @@ class Core(BuiltinCore): (i.e., in the parent process). """ @wraps(parent_rmi) def inner(*args, **kwargs): + """ Function that dispatches an RMI call to child + processes and to the (original) parent function. """ self.logger.debug("Dispatching RMI call to %s to children: %s" % (method, child_rmi)) self.rpc_q.publish(child_rmi, args=args, kwargs=kwargs) -- cgit v1.2.3-1-g7c22 From ff4fc8d1a569a0cde58e198152c80d342c40332b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 9 Aug 2013 08:51:01 -0500 Subject: Metadata: Fix bundle visualization Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 2f8f49a9c..245051976 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1449,7 +1449,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, the graph""" return not clientmeta or bundle in clientmeta.bundles - bundles = list(set(bund.get('name')) + bundles = list(bund.get('name') for bund in self.groups_xml.xdata.findall('.//Bundle') if include_bundle(bund.get('name'))) bundles.sort() -- cgit v1.2.3-1-g7c22 From 97a5cfd668ec7e0ff64c03acbf1076ca43daa169 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 10:01:51 -0400 Subject: Packages: always add collections to group/package cache This fixes cases where the collection itself would not be cached because a host had no sources, or multiple sources of different types; we still want to cache package group results and package sets (both of which will be empty). --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4096153a3..3cdcdc162 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -573,8 +573,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - return Collection(metadata, [], self.cachepath, self.data, - self.core.fam) + collection = Collection(metadata, [], self.cachepath, self.data, + self.core.fam) + ckey = collection.cachekey + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) + return collection if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -611,8 +615,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): -- cgit v1.2.3-1-g7c22 From 4d6dbca9a04cc2766d9fa8968318af9a8dbb42c0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 10:15:53 -0400 Subject: Viz: distinguish categories from hosts --- src/lib/Bcfg2/Server/Admin/Viz.py | 1 + src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index 1d9d25f16..2cbd7eaf6 100644 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -102,6 +102,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): dotpipe.stdin.write('\tcolor="lightblue";\n') dotpipe.stdin.write('\tBundle [ shape="septagon" ];\n') dotpipe.stdin.write('\tGroup [shape="ellipse"];\n') + dotpipe.stdin.write('\tGroup Category [shape="trapezium"];\n') dotpipe.stdin.write('\tProfile [style="bold", shape="ellipse"];\n') dotpipe.stdin.write('\tHblock [label="Host1|Host2|Host3", ' 'shape="record"];\n') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 245051976..0c00dcb50 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1406,7 +1406,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, viz_str.extend(self._viz_groups(egroups, bundles, clientmeta)) if key: for category in categories: - viz_str.append('"%s" [label="%s", shape="record", ' + viz_str.append('"%s" [label="%s", shape="trapezium", ' 'style="filled", fillcolor="%s"];' % (category, category, categories[category])) return "\n".join("\t" + s for s in viz_str) -- cgit v1.2.3-1-g7c22 From e599de4753a79b9da09d4d85d1bfaeb2a40dc0e5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 10:21:39 -0400 Subject: Viz: exclude duplicate bundles for a given host --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 0c00dcb50..e7589d3c2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1449,9 +1449,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, the graph""" return not clientmeta or bundle in clientmeta.bundles - bundles = list(bund.get('name') - for bund in self.groups_xml.xdata.findall('.//Bundle') - if include_bundle(bund.get('name'))) + bundles = \ + list(set(bund.get('name') + for bund in self.groups_xml.xdata.findall('.//Bundle') + if include_bundle(bund.get('name')))) bundles.sort() return ['"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, bundle) -- cgit v1.2.3-1-g7c22 From 4201e3b68f392869359493ab9462706f4956e387 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 9 Aug 2013 10:37:13 -0400 Subject: Viz: include clients that should be included, not vice-versa --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index e7589d3c2..bb0fad011 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1421,7 +1421,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] for client in list(self.clients): - if include_client(client): + if not include_client(client): continue if client in self.clientgroups: grps = self.clientgroups[client] -- cgit v1.2.3-1-g7c22