From 94ba31279869d7052ba001e38927f9eecd0a636f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 10 Dec 2013 20:58:55 -0500 Subject: Augeas improvements: * Added ability to specify initial content for a file that doesn't exist, to avoid a messy situation where you'd have to probe for file existence and either use a Path type="file" or Path type="augeas" depending, and run Bcfg2 twice. * All commands in an Augeas path are run if *any* of them fail to verify. Previously, only commands that hadn't been run would be installed, but that had issues, particularly with the Clear command, which could pass verification but then be required during the installation phase anyway. * Miscellaneous bug fixes. --- src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py | 31 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index 81c948d0d..187b4d77c 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -4,6 +4,7 @@ import sys import Bcfg2.Client.XML from augeas import Augeas from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Client.Tools.POSIX.File import POSIXFile class AugeasCommand(object): @@ -187,13 +188,14 @@ class Insert(AugeasCommand): class POSIXAugeas(POSIXTool): """ Handle entries. See :ref:`client-tools-augeas`. """ - - __handles__ = [('Path', 'augeas')] - __req__ = {'Path': ['type', 'name', 'setting', 'value']} + __req__ = ['name', 'mode', 'owner', 'group'] def __init__(self, logger, setup, config): POSIXTool.__init__(self, logger, setup, config) self._augeas = dict() + # file tool for setting initial values of files that don't + # exist + self.filetool = POSIXFile(logger, setup, config) def get_augeas(self, entry): """ Get an augeas object for the given entry. """ @@ -214,9 +216,9 @@ class POSIXAugeas(POSIXTool): return self._augeas[entry.get("name")] def fully_specified(self, entry): - return entry.text is not None + return len(entry.getchildren()) != 0 - def get_commands(self, entry, unverified=False): + def get_commands(self, entry): """ Get a list of commands to verify or install. @param entry: The entry to get commands from. @@ -229,7 +231,7 @@ class POSIXAugeas(POSIXTool): """ rv = [] for cmd in entry.iterchildren(): - if unverified and cmd.get("verified", "false") != "false": + if cmd.tag == "Initial": continue if cmd.tag in globals(): rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), @@ -266,7 +268,17 @@ class POSIXAugeas(POSIXTool): def install(self, entry): rv = True - for cmd in self.get_commands(entry, unverified=True): + if entry.get("current_exists", "true") == "false": + initial = entry.find("Initial") + if initial is not None: + self.logger.debug("Augeas: Setting initial data for %s" % + entry.get("name")) + file_entry = Bcfg2.Client.XML.Element("Path", **entry.attrib) + file_entry.text = initial.text + self.filetool.install(file_entry) + # re-parse the file + self.get_augeas(entry).load() + for cmd in self.get_commands(entry): try: cmd.install() except: # pylint: disable=W0702 @@ -277,8 +289,7 @@ class POSIXAugeas(POSIXTool): try: self.get_augeas(entry).save() except: # pylint: disable=W0702 - self.logger.error( - "Failure saving Augeas changes to %s: %s" % - (entry.get("name"), sys.exc_info()[1])) + self.logger.error("Failure saving Augeas changes to %s: %s" % + (entry.get("name"), sys.exc_info()[1])) rv = False return POSIXTool.install(self, entry) and rv -- cgit v1.2.3-1-g7c22 From 3ac796ffdd6777814e85cf8cffdb1c733c69feb0 Mon Sep 17 00:00:00 2001 From: Dan Foster Date: Sun, 29 Dec 2013 17:01:08 +0000 Subject: Reset essential package list to an empty set when Packages is refreshed --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index a82a183d8..27a725f23 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -77,6 +77,7 @@ class AptSource(Source): def read_files(self): bdeps = dict() bprov = dict() + self.essentialpkgs = set() depfnames = ['Depends', 'Pre-Depends'] if self.recommended: depfnames.append('Recommends') -- cgit v1.2.3-1-g7c22 From cfcbc4af38fdbc75c55410a71bd764771160bdfd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 31 Dec 2013 14:10:03 -0500 Subject: bcfg2-crypt: actually skip an element when we say we are --- src/sbin/bcfg2-crypt | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index f3b96909e..5641732cd 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -307,6 +307,7 @@ class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): except (Bcfg2.Encryption.EVPError, TypeError): self.logger.error("Could not decrypt %s, skipping" % print_xml(elt)) + continue try: elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') elt.set("encrypted", pname) -- cgit v1.2.3-1-g7c22 From 334d210d6be5b262578e8b60e722647a25e799be Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 31 Dec 2013 14:10:24 -0500 Subject: bcfg2-lint: expand wildcard xincludes when validating XML --- src/lib/Bcfg2/Server/Lint/Validate.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index ced16770b..3efcc890d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -84,6 +84,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -109,6 +110,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: xdata = lxml.etree.parse(filename) if self.files is None: + self._expand_wildcard_xincludes(xdata) xdata.xinclude() return xdata except (lxml.etree.XIncludeError, SyntaxError): @@ -127,6 +129,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. -- cgit v1.2.3-1-g7c22 From 7520748d3644d3b92f784e6142a899dc9a3182fd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 2 Jan 2014 09:40:03 -0500 Subject: Encryption: better error message when base64 decode fails --- src/lib/Bcfg2/Encryption.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index 6d16748d5..a09d652d2 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -3,6 +3,7 @@ handling encryption in Bcfg2. See :ref:`server-encryption` for more details. """ import os +import sys from M2Crypto import Rand from M2Crypto.EVP import Cipher, EVPError from Bcfg2.Compat import StringIO, md5, b64encode, b64decode @@ -114,7 +115,15 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): :returns: string - The decrypted data """ # base64-decode the data - data = b64decode(data) + try: + data = b64decode(data) + except TypeError: + # we do not include the data in the error message, because one + # of the common causes of this is data that claims to be + # encrypted but is not. we don't want to include a plaintext + # secret in the error logs. + raise TypeError("Could not decode base64 data: %s" % + (data, sys.exc_info()[1])) salt = data[8:16] # pylint: disable=E1101,E1121 hashes = [md5(passwd + salt).digest()] -- cgit v1.2.3-1-g7c22 From 053a87c6c496665a6dacab3efdecd4d7ebc2234d Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Sun, 5 Jan 2014 19:51:19 +0000 Subject: Fixed group blacklist issue #150 --- src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index bf23aca6b..161145a14 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -147,6 +147,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() if entry.get("name") in g[3] and g[0] != entry.get("group")] + and self._in_managed_range('POSIXGroup', g[2] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ -- cgit v1.2.3-1-g7c22 From 293fcb0b601fd8ce46c34b49b4b1b661f34a968f Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Sun, 5 Jan 2014 20:05:04 +0000 Subject: Fixed syntatical error in previous --- src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 161145a14..1bae94d28 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -146,8 +146,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group")] - and self._in_managed_range('POSIXGroup', g[2] + if entry.get("name") in g[3] and g[0] != entry.get("group") + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ -- cgit v1.2.3-1-g7c22 From f3154b9f0471faaef0e64f89b669cd01614fef20 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Sun, 5 Jan 2014 20:33:38 +0000 Subject: fixed indentation in previous --- src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 1bae94d28..6d18cd176 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -147,7 +147,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() if entry.get("name") in g[3] and g[0] != entry.get("group") - and self._in_managed_range('POSIXGroup', g[2])] + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ -- cgit v1.2.3-1-g7c22 From deef870d5c8ce3a874878a16abbfb4b126dca4dc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 7 Jan 2014 11:57:23 -0500 Subject: Encryption: fixed base64 decode failure message --- src/lib/Bcfg2/Encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index a09d652d2..69d40ea37 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -123,7 +123,7 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): # encrypted but is not. we don't want to include a plaintext # secret in the error logs. raise TypeError("Could not decode base64 data: %s" % - (data, sys.exc_info()[1])) + sys.exc_info()[1]) salt = data[8:16] # pylint: disable=E1101,E1121 hashes = [md5(passwd + salt).digest()] -- cgit v1.2.3-1-g7c22 From 73979d1482a73da24104950be4bad8f3d50c0754 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 10 Jan 2014 11:59:14 -0500 Subject: Reporting: occasionally reap child threads I noticed that, at least on Python 2.4, memory for threads doesn't get freed until the threads are joined. This patch causes the collector to periodically go through and reap those threads. Tested in production for ~1 month, no reported issues. --- src/lib/Bcfg2/Reporting/Collector.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index b42364d8d..52700f917 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -63,6 +63,8 @@ class ReportingCollector(object): self.encoding = setup['encoding'] self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if setup['debug']: level = logging.DEBUG @@ -110,6 +112,7 @@ class ReportingCollector(object): self.terminate = threading.Event() atexit.register(self.shutdown) self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if self.setup['daemon']: self.logger.debug("Daemonizing") @@ -133,6 +136,13 @@ class ReportingCollector(object): store_thread = ReportingStoreThread(interaction, self.storage) store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -152,3 +162,16 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist -- cgit v1.2.3-1-g7c22 From e3e1936fedb8d5845c576124c46677c90ca7fff2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jan 2014 10:36:49 -0500 Subject: FAM: Get full traceback when event handling errors --- src/lib/Bcfg2/Server/FileMonitor/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index e430e3160..52c3906fa 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -237,7 +237,8 @@ class FileMonitor(Debuggable): except: # pylint: disable=W0702 err = sys.exc_info()[1] LOGGER.error("Error in handling of event %s for %s: %s" % - (event.code2str(), event.filename, err)) + (event.code2str(), event.filename, err), + exc_info=1) def handle_event_set(self, lock=None): """ Handle all pending events. -- cgit v1.2.3-1-g7c22 From f37833e5ea103796d5177a24901befd9b0f7ab28 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jan 2014 10:44:17 -0500 Subject: POSIX: Properly stringify ACLs with no user/group specified This is just a workaround to avoid a traceback; the real fix will involve making the POSIX tool properly handle ACLs with no user/group given, which refer to the current user/group of the file they apply to. --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 85da3576b..3243bbf50 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -395,7 +395,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl_str.append("user") elif scope == posix1e.ACL_GROUP: acl_str.append("group") - acl_str.append(qualifier) + if qualifier is None: + acl_str.append('') + else: + acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) -- cgit v1.2.3-1-g7c22 From 9f16500047a7985ac1533e3dc0e1c0e3da23502f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 22 Jan 2014 08:49:09 -0500 Subject: Don't parse unicode XML with encoding Fix another place where a unicode XML string with an encoding declaration may be read. Cf. 0f8d403d1a86cfbfe8222662dc445e16e8f7eff9 --- src/lib/Bcfg2/Client/XML.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 91d4ac5c6..1f1b7df7d 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,9 +5,18 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, XML, tostring + from lxml.etree import Element, SubElement, tostring from lxml.etree import XMLSyntaxError as ParseError + from lxml.etree import XML as _XML driver = 'lxml' + + def XML(val, **kwargs): + try: + return _XML(val, **kwargs) + except ValueError: + # unicode strings w/encoding declaration are not supported + # in recent lxml.etree + return _XML(val.encode(), **kwargs) except ImportError: # lxml not available from xml.parsers.expat import ExpatError as ParseError -- cgit v1.2.3-1-g7c22 From e957f89d17e431cd4210f08def2f62dfe8fcf91f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 22 Jan 2014 09:18:00 -0500 Subject: added missing docstring --- src/lib/Bcfg2/Client/XML.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 1f1b7df7d..561cd25a8 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -8,14 +8,17 @@ try: from lxml.etree import Element, SubElement, tostring from lxml.etree import XMLSyntaxError as ParseError from lxml.etree import XML as _XML + from Bcfg2.Compat import wraps driver = 'lxml' + @wraps(_XML) def XML(val, **kwargs): + """ unicode strings w/encoding declaration are not supported in + recent lxml.etree, so we try to read XML, and if it fails we try + encoding the string. """ try: return _XML(val, **kwargs) except ValueError: - # unicode strings w/encoding declaration are not supported - # in recent lxml.etree return _XML(val.encode(), **kwargs) except ImportError: # lxml not available -- cgit v1.2.3-1-g7c22 From 79f5fce2a387f5dfc13e10146b36cb620a5bf166 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 28 Jan 2014 15:52:42 -0500 Subject: Client: let lxml.etree XML implementation parse very large documents --- src/lib/Bcfg2/Client/XML.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 561cd25a8..e35ce4497 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,17 +5,25 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, tostring + from lxml.etree import Element, SubElement, tostring, XMLParser from lxml.etree import XMLSyntaxError as ParseError from lxml.etree import XML as _XML from Bcfg2.Compat import wraps driver = 'lxml' + try: + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html + _parser = XMLParser(huge_tree=True) + except TypeError: + _parser = XMLParser() + @wraps(_XML) def XML(val, **kwargs): """ unicode strings w/encoding declaration are not supported in recent lxml.etree, so we try to read XML, and if it fails we try encoding the string. """ + kwargs.setdefault('parser', _parser) try: return _XML(val, **kwargs) except ValueError: -- cgit v1.2.3-1-g7c22 From 2defc9cf5155f21a988855c31049fad3e40ecdd1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 29 Jan 2014 07:43:06 -0500 Subject: load correct JSON library on py2.4 --- src/lib/Bcfg2/Server/Lint/ValidateJSON.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Ohai.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Probes.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Properties.py | 3 ++- src/sbin/bcfg2-yum-helper | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index c4a82a5d2..31bbadeb2 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -10,7 +10,8 @@ import Bcfg2.Server.Lint try: import json -except ImportError: + json.loads # py2.4 json library is structured differently +except (ImportError, AttributeError): import simplejson as json diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 1ec3cbd60..f5bde1820 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,8 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + json.loads # py2.4 json library is structured differently +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 66f8e9dbe..43a54471c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -87,7 +87,8 @@ try: import yum try: import json - except ImportError: + json.loads # py2.4 json library is structured differently + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index fdc047283..471ebfdaf 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -34,8 +34,9 @@ except ImportError: try: import json + json.loads # py2.4 json library is structured differently HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 89f2d21ff..b6090c4d1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -17,8 +17,9 @@ except ImportError: try: import json + json.loads # py2.4 json library is structured differently HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 49baeb9c3..132f9efce 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -15,7 +15,8 @@ from lockfile import FileLock, LockTimeout from optparse import OptionParser try: import json -except ImportError: + json.loads # py2.4 json library is structured differently +except (ImportError, AttributeError): import simplejson as json -- cgit v1.2.3-1-g7c22 From 9be8b7fb9eef3aa9cc7d44adb565dad7625ede4f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 29 Jan 2014 16:33:33 -0500 Subject: Augeas: ensure that entry.attrib is always a dict --- src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index 187b4d77c..8506f4bc7 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -273,7 +273,8 @@ class POSIXAugeas(POSIXTool): if initial is not None: self.logger.debug("Augeas: Setting initial data for %s" % entry.get("name")) - file_entry = Bcfg2.Client.XML.Element("Path", **entry.attrib) + file_entry = Bcfg2.Client.XML.Element("Path", + **dict(entry.attrib)) file_entry.text = initial.text self.filetool.install(file_entry) # re-parse the file -- cgit v1.2.3-1-g7c22 From 8ac001a3dffebf8e7d3fa7db20f13e58b90dd5b4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 30 Jan 2014 07:30:18 -0500 Subject: fixed pylint tests --- src/lib/Bcfg2/Client/XML.py | 4 ++-- src/lib/Bcfg2/Server/Lint/ValidateJSON.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Ohai.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Probes.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Properties.py | 3 ++- 6 files changed, 12 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index e35ce4497..4ba06abae 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -11,9 +11,9 @@ try: from Bcfg2.Compat import wraps driver = 'lxml' + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html try: - # libxml2 2.9.0+ doesn't parse 10M+ documents by default: - # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html _parser = XMLParser(huge_tree=True) except TypeError: _parser = XMLParser() diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index 31bbadeb2..1f55962eb 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -10,7 +10,8 @@ import Bcfg2.Server.Lint try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 except (ImportError, AttributeError): import simplejson as json diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index f5bde1820..0853ea993 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,8 @@ import Bcfg2.Server.Plugin try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 except (ImportError, AttributeError): import simplejson as json diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 43a54471c..f038ec9c0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -87,7 +87,8 @@ try: import yum try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 except (ImportError, AttributeError): import simplejson as json HAS_YUM = True diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 471ebfdaf..48be1ac26 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -34,7 +34,8 @@ except ImportError: try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True except (ImportError, AttributeError): try: diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index b6090c4d1..8c6cf799a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -17,7 +17,8 @@ except ImportError: try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True except (ImportError, AttributeError): try: -- cgit v1.2.3-1-g7c22 From 29b16ebd00742cddca94f740917d1c0191667fc6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 30 Jan 2014 15:59:43 -0500 Subject: fixed more pylint tests --- src/sbin/bcfg2-yum-helper | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 132f9efce..227d977de 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -15,7 +15,8 @@ from lockfile import FileLock, LockTimeout from optparse import OptionParser try: import json - json.loads # py2.4 json library is structured differently + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 except (ImportError, AttributeError): import simplejson as json -- cgit v1.2.3-1-g7c22 From d19c0ced56af17132e902e1409129dd4ad96afb5 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 14:53:13 +0000 Subject: Support ACLs without a specific user/group for default owner/owning-group ACLs on directories --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 80 +++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 3243bbf50..82e7c8fd8 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -213,21 +213,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): # clear ACLs out so we start fresh -- way easier than trying # to add/remove/modify ACLs for aclentry in acl: - if aclentry.tag_type in [posix1e.ACL_USER, posix1e.ACL_GROUP]: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_GROUP, + posix1e.ACL_OTHER]: acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) - if not defacl.valid(): - # when a default ACL is queried on a directory that - # has no default ACL entries at all, you get an empty - # ACL, which is not valid. in this circumstance, we - # just copy the access ACL to get a base valid ACL - # that we can add things to. - defacl = posix1e.ACL(acl=acl) - else: + if defacl.valid(): for aclentry in defacl: if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_GROUP]: + posix1e.ACL_USER_OBJ, + posix1e.ACL_GROUP, + posix1e.ACL_GROUP_OBJ, + posix1e.ACL_OTHER]: defacl.delete_entry(aclentry) else: defacl = None @@ -253,11 +251,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): aclentry.tag_type = scope try: if scope == posix1e.ACL_USER: - scopename = "user" - aclentry.qualifier = self._norm_uid(qualifier) + if qualifier: + scopename = "user" + aclentry.qualifier = self._norm_uid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_USER_OBJ elif scope == posix1e.ACL_GROUP: - scopename = "group" - aclentry.qualifier = self._norm_gid(qualifier) + if qualifier: + scopename = "group" + aclentry.qualifier = self._norm_gid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_GROUP_OBJ except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Could not resolve %s %s: %s" % @@ -358,7 +362,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: # single octal digit rv = int(perms) - if rv > 0 and rv < 8: + if rv >= 0 and rv < 8: return rv else: self.logger.error("POSIX: Permissions digit out of range in " @@ -388,17 +392,18 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get a string representation of the given ACL. aclkey must be a tuple of (, , ) """ atype, scope, qualifier = aclkey + if not qualifier: + qualifier = '' acl_str = [] if atype == 'default': acl_str.append(atype) - if scope == posix1e.ACL_USER: + if scope == posix1e.ACL_USER or scope == posix1e.ACL_USER_OBJ: acl_str.append("user") - elif scope == posix1e.ACL_GROUP: + elif scope == posix1e.ACL_GROUP or scope == posix1e.ACL_GROUP_OBJ: acl_str.append("group") - if qualifier is None: - acl_str.append('') - else: - acl_str.append(qualifier) + elif scope == posix1e.ACL_OTHER: + acl_str.append("other") + acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) @@ -564,9 +569,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): wanted = dict() for acl in entry.findall("ACL"): if acl.get("scope") == "user": - scope = posix1e.ACL_USER + if acl.get("user"): + scope = posix1e.ACL_USER + else: + scope = posix1e.ACL_USER_OBJ elif acl.get("scope") == "group": - scope = posix1e.ACL_GROUP + if acl.get("group"): + scope = posix1e.ACL_GROUP + else: + scope = posix1e.ACL_GROUP_OBJ + elif acl.get("scope") == "other": + scope = posix1e.ACL_OTHER else: self.logger.error("POSIX: Unknown ACL scope %s" % acl.get("scope")) @@ -575,7 +588,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.error("POSIX: No permissions set for ACL: %s" % Bcfg2.Client.XML.tostring(acl)) continue - wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \ + qual = acl.get(acl.get("scope")) + if not qual: + qual = '' + wanted[(acl.get("type"), scope, qual)] = \ self._norm_acl_perms(acl.get('perms')) return wanted @@ -589,11 +605,14 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Given an ACL object, process it appropriately and add it to the return value """ try: + qual = '' if acl.tag_type == posix1e.ACL_USER: qual = pwd.getpwuid(acl.qualifier)[0] elif acl.tag_type == posix1e.ACL_GROUP: qual = grp.getgrgid(acl.qualifier)[0] - else: + elif atype == "access": + return + elif acl.tag_type == posix1e.ACL_MASK: return except (OSError, KeyError): err = sys.exc_info()[1] @@ -655,15 +674,20 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): atype, scope, qual = aclkey aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, perms=str(perms)) - if scope == posix1e.ACL_USER: + if (scope == posix1e.ACL_USER or + scope == posix1e.ACL_USER_OBJ): aclentry.set("scope", "user") - elif scope == posix1e.ACL_GROUP: + elif (scope == posix1e.ACL_GROUP or + scope == posix1e.ACL_GROUP_OBJ): aclentry.set("scope", "group") + elif scope == posix1e.ACL_OTHER: + aclentry.set("scope", "other") else: self.logger.debug("POSIX: Unknown ACL scope %s on %s" % (scope, path)) continue - aclentry.set(aclentry.get("scope"), qual) + if scope != posix1e.ACL_OTHER: + aclentry.set(aclentry.get("scope"), qual) entry.append(aclentry) for aclkey, perms in existing.items(): -- cgit v1.2.3-1-g7c22 From a40c7fe2457688fd574558de7b8e31e9c30afb96 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 15:52:48 +0000 Subject: Minor changes to default ACL code --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 82e7c8fd8..4ef4ae3f5 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -251,14 +251,14 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): aclentry.tag_type = scope try: if scope == posix1e.ACL_USER: + scopename = "user" if qualifier: - scopename = "user" aclentry.qualifier = self._norm_uid(qualifier) else: aclentry.tag_type = posix1e.ACL_USER_OBJ elif scope == posix1e.ACL_GROUP: + scopename = "group" if qualifier: - scopename = "group" aclentry.qualifier = self._norm_gid(qualifier) else: aclentry.tag_type = posix1e.ACL_GROUP_OBJ @@ -610,9 +610,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): qual = pwd.getpwuid(acl.qualifier)[0] elif acl.tag_type == posix1e.ACL_GROUP: qual = grp.getgrgid(acl.qualifier)[0] - elif atype == "access": - return - elif acl.tag_type == posix1e.ACL_MASK: + elif atype == "access" or acl.tag_type == posix1e.ACL_MASK: return except (OSError, KeyError): err = sys.exc_info()[1] @@ -686,6 +684,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.debug("POSIX: Unknown ACL scope %s on %s" % (scope, path)) continue + if scope != posix1e.ACL_OTHER: aclentry.set(aclentry.get("scope"), qual) entry.append(aclentry) -- cgit v1.2.3-1-g7c22 From d208eed80e048ea2081165c7aaaa92c558c38b25 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 16:03:17 +0000 Subject: fix for "Too many branches" in _verify_acls --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 43 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 4ef4ae3f5..4fb3c7b34 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -643,26 +643,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): def _verify_acls(self, entry, path=None): """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ - if not HAS_ACLS: - if entry.findall("ACL"): - self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " - "library installed" % entry.get('name')) - return True - - if path is None: - path = entry.get("name") - - # create lists of normalized representations of the ACLs we want - # and the ACLs we have. this will make them easier to compare - # than trying to mine that data out of the ACL objects and XML - # objects and compare it at the same time. - wanted = self._list_entry_acls(entry) - existing = self._list_file_acls(path) - - missing = [] - extra = [] - wrong = [] - for aclkey, perms in wanted.items(): + def _verify_acl(aclkey, perms): if aclkey not in existing: missing.append(self._acl2string(aclkey, perms)) elif existing[aclkey] != perms: @@ -689,6 +670,28 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): aclentry.set(aclentry.get("scope"), qual) entry.append(aclentry) + if not HAS_ACLS: + if entry.findall("ACL"): + self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " + "library installed" % entry.get('name')) + return True + + if path is None: + path = entry.get("name") + + # create lists of normalized representations of the ACLs we want + # and the ACLs we have. this will make them easier to compare + # than trying to mine that data out of the ACL objects and XML + # objects and compare it at the same time. + wanted = self._list_entry_acls(entry) + existing = self._list_file_acls(path) + + missing = [] + extra = [] + wrong = [] + for aclkey, perms in wanted.items(): + _verify_acl(aclkey, perms) + for aclkey, perms in existing.items(): if aclkey not in wanted: extra.append(self._acl2string(aclkey, perms)) -- cgit v1.2.3-1-g7c22 From 310ec389a7542b8f7aeb807c6ed71ae7a82515d0 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 16:07:54 +0000 Subject: Fixed continue not in a loop error in previous --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 4fb3c7b34..4089e9f22 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -664,7 +664,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: self.logger.debug("POSIX: Unknown ACL scope %s on %s" % (scope, path)) - continue + return if scope != posix1e.ACL_OTHER: aclentry.set(aclentry.get("scope"), qual) -- cgit v1.2.3-1-g7c22 From b30cdb5330940c4606acbb83b057335be480552b Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 16:20:24 +0000 Subject: Removed redundant condition for empty default ACL --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 4089e9f22..d230a5b3a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -219,14 +219,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) - if defacl.valid(): - for aclentry in defacl: - if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_USER_OBJ, - posix1e.ACL_GROUP, - posix1e.ACL_GROUP_OBJ, - posix1e.ACL_OTHER]: - defacl.delete_entry(aclentry) + for aclentry in defacl: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_USER_OBJ, + posix1e.ACL_GROUP, + posix1e.ACL_GROUP_OBJ, + posix1e.ACL_OTHER]: + defacl.delete_entry(aclentry) else: defacl = None -- cgit v1.2.3-1-g7c22 From bfdf4e9cac0c5d93cf7ce63513557847f005d5d2 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 16:59:44 +0000 Subject: Except _verify_acls from pylint branches check --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index d230a5b3a..d4051ba7f 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -639,7 +639,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): _process_acl(acl, "default") return existing - def _verify_acls(self, entry, path=None): + def _verify_acls(self, entry, path=None): # pylint: disable=R0912 """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ def _verify_acl(aclkey, perms): -- cgit v1.2.3-1-g7c22 From 86f4766fd357c79956cd95b0506d1c85da96416d Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 4 Feb 2014 20:05:24 +0000 Subject: docstring for new sub-method --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index d4051ba7f..0eb9239c5 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -643,6 +643,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ def _verify_acl(aclkey, perms): + """ Given ACL data, process it appropriately and add it to + missing or wrong lists if appropriate """ if aclkey not in existing: missing.append(self._acl2string(aclkey, perms)) elif existing[aclkey] != perms: -- cgit v1.2.3-1-g7c22 From ea25166d42aaf5d8a48ef7a172a3f8e1f87767d9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 31 Jan 2014 14:39:33 -0500 Subject: Metadata: fix check for Groups with options in duplicate groups test --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 343e14162..f734c98d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1657,8 +1657,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "client") def duplicate_groups(self): - """ Check for groups that are defined more than once. There - are two ways this can happen: + """ Check for groups that are defined more than once. There are two + ways this can happen: 1. The group is listed twice with contradictory options. 2. The group is listed with no options *first*, and then with @@ -1674,7 +1674,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): grpname = grp.get("name") if grpname in duplicates: duplicates[grpname].append(grp) - elif len(grp.attrib) > 1: # group has options + elif set(grp.attrib.keys()).difference(['negate', 'name']): + # group has options if grpname in groups: duplicates[grpname] = [grp, groups[grpname]] else: -- cgit v1.2.3-1-g7c22 From 75e1233c4ac292a554562b51475e972c647bd2b5 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Mon, 10 Feb 2014 17:32:57 +0000 Subject: Don't strip other entries from default ACL, these are defined by mode number. --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 0eb9239c5..e593e0a0a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -213,9 +213,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): # clear ACLs out so we start fresh -- way easier than trying # to add/remove/modify ACLs for aclentry in acl: - if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_GROUP, - posix1e.ACL_OTHER]: + if aclentry.tag_type in [posix1e.ACL_USER, posix1e.ACL_GROUP]: acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) -- cgit v1.2.3-1-g7c22 From 5ca328cf8a4eb4f82b134b292c858fe9f20d4548 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 13 Feb 2014 07:40:59 -0500 Subject: Core: explicitly close database connections at the end of each client run --- src/lib/Bcfg2/Server/Core.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index c2cf6b7a4..587afefe0 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -726,6 +726,11 @@ class BaseCore(object): self.validate_goals(meta, config) self.client_run_hook("end_client_run", meta) + if self._database_available: + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() sort_xml(config, key=lambda e: e.get('name')) -- cgit v1.2.3-1-g7c22 From 9ac25c247afc348c90197f33039c066d2a9d4247 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Fri, 14 Feb 2014 12:04:43 +0000 Subject: Lint checking for invalid default ACLs --- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 33 +++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index e49779a10..77934d720 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -119,6 +119,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): POSIXUser={None: dict(name=is_username)}) def Run(self): + self.check_default_acls() self.check_packages() if "Defaults" in self.core.plugins: self.logger.info("Defaults plugin enabled; skipping required " @@ -129,12 +130,42 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unknown-entry-type": "error", + return {"missing-elements": "error", + "unknown-entry-type": "error", "unknown-entry-tag": "error", "required-attrs-missing": "error", "required-attr-format": "error", "extra-attrs": "warning"} + def check_default_acls(self): + """ Check Path entries have valid default ACLs """ + def check_acl(path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 1 if len(path.xpath( + "/ACL[@type='default' and @scope='user']")) else 0 + defaults += 1 if len(path.xpath( + "/ACL[@type='default' and @scope='user']")) else 0 + defaults += 1 if len(path.xpath( + "/ACL[@type='default' and @scope='user']")) else 0 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") + + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = bundle.pnode.data + for path in xdata.xpath("//BoundPath"): + check_acl(path) + if 'Rules' in self.core.plugins: + for rules in self.core.plugins['Rules'].entries.values(): + xdata = rules.pnode.data + for path in xdata.xpath("//Path"): + check_acl(path) + def check_packages(self): """ Check Packages sources for Source entries with missing attributes. """ -- cgit v1.2.3-1-g7c22 From 06bc91f8a8c919e5e552f46386841a75fcc3619a Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Fri, 14 Feb 2014 12:10:04 +0000 Subject: Correct XML source for bundles in default ACL Lint --- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 77934d720..fce90154e 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -157,7 +157,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = bundle.pnode.data + xdata = lxml.etree.XML(bundle.data) for path in xdata.xpath("//BoundPath"): check_acl(path) if 'Rules' in self.core.plugins: -- cgit v1.2.3-1-g7c22 From 7522e96a978188e080d68beaeb808cfcb2f4f37e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 14 Feb 2014 14:08:17 -0500 Subject: settings: Make it possible to use ibm_db_django engine This is a forward-port of 49362b6d633a7784f77650d5218d0e629d50e4fb --- src/lib/Bcfg2/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 82a3bdb2a..aa784336d 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -82,8 +82,13 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): if repo is None: repo = setup['repo'] + if setup['db_engine'] == 'ibm_db_django': + db_engine = setup['db_engine'] + else: + db_engine = "django.db.backends.%s" % setup['db_engine'] + DATABASES['default'] = \ - dict(ENGINE="django.db.backends.%s" % setup['db_engine'], + dict(ENGINE=db_engine, NAME=setup['db_name'], USER=setup['db_user'], PASSWORD=setup['db_password'], -- cgit v1.2.3-1-g7c22 From e4b2b05de382743883ee613236d4647c588d811d Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Fri, 14 Feb 2014 23:29:48 +0000 Subject: Working lint check for invalid default ACLs --- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 47 ++++++++++++------------------ 1 file changed, 18 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index fce90154e..bb0d6956a 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -119,7 +119,6 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): POSIXUser={None: dict(name=is_username)}) def Run(self): - self.check_default_acls() self.check_packages() if "Defaults" in self.core.plugins: self.logger.info("Defaults plugin enabled; skipping required " @@ -137,34 +136,21 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "required-attr-format": "error", "extra-attrs": "warning"} - def check_default_acls(self): - """ Check Path entries have valid default ACLs """ - def check_acl(path): - """ Check that a default ACL contains either no entries or minimum - required entries """ - defaults = 1 if len(path.xpath( - "/ACL[@type='default' and @scope='user']")) else 0 - defaults += 1 if len(path.xpath( - "/ACL[@type='default' and @scope='user']")) else 0 - defaults += 1 if len(path.xpath( - "/ACL[@type='default' and @scope='user']")) else 0 - if defaults > 0 and defaults < 3: - self.LintError( - "missing-elements", - "A Path must have either no default ACLs or at" - " least default:user::, default:group:: and" - " default:other::") - - if 'Bundler' in self.core.plugins: - for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = lxml.etree.XML(bundle.data) - for path in xdata.xpath("//BoundPath"): - check_acl(path) - if 'Rules' in self.core.plugins: - for rules in self.core.plugins['Rules'].entries.values(): - xdata = rules.pnode.data - for path in xdata.xpath("//Path"): - check_acl(path) + def check_default_acl(self, path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 1 if path.xpath( + "ACL[@type='default' and @scope='user' and @user='']") else 0 + defaults += 1 if path.xpath( + "ACL[@type='default' and @scope='group' and @group='']") else 0 + defaults += 1 if path.xpath( + "ACL[@type='default' and @scope='other']") else 0 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") def check_packages(self): """ Check Packages sources for Source entries with missing @@ -265,6 +251,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): required_attrs['major'] = is_device_mode required_attrs['minor'] = is_device_mode + if tag == 'Path': + self.check_default_acl(entry) + if tag == 'ACL' and 'scope' in required_attrs: required_attrs[entry.get('scope')] = is_username -- cgit v1.2.3-1-g7c22 From 2de76a7b44c148f2ce9e851060c16513581174ff Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 18 Feb 2014 08:53:35 -0500 Subject: ensure that DB connections are always closed at thread/process exit --- src/lib/Bcfg2/Reporting/Collector.py | 9 ++++++--- src/lib/Bcfg2/Server/Core.py | 18 +++++++++++------- src/sbin/bcfg2-reports | 3 +++ 3 files changed, 20 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 52700f917..0493c907b 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -56,7 +56,7 @@ class ReportingCollector(object): """The collecting process for reports""" def __init__(self, setup): - """Setup the collector. This may be called by the daemon or though + """Setup the collector. This may be called by the daemon or though bcfg2-admin""" self.setup = setup self.datastore = setup['repo'] @@ -99,12 +99,12 @@ class ReportingCollector(object): raise ReportingError try: - self.logger.debug("Validating storage %s" % + self.logger.debug("Validating storage %s" % self.storage.__class__.__name__) self.storage.validate() except: self.logger.error("Storage backed %s failed to validate: %s" % - (self.storage.__class__.__name__, + (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) def run(self): @@ -162,6 +162,9 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() def reap_children(self): """Join any non-live threads""" diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 587afefe0..cc9270a17 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -414,7 +414,7 @@ class BaseCore(object): :type plugin: string :returns: None """ - self.logger.debug("Loading plugin %s" % plugin) + self.logger.debug("%s: Loading plugin %s" % (self.name, plugin)) try: mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % (plugin)).Server.Plugins, plugin) @@ -450,14 +450,18 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.info("Shutting down core...") + self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() - self.fam.shutdown() - self.logger.info("FAM shut down") - for plugin in list(self.plugins.values()): - plugin.shutdown() - self.logger.info("All plugins shut down") + self.fam.shutdown() + self.logger.info("%s: FAM shut down" % self.name) + for plugin in list(self.plugins.values()): + plugin.shutdown() + self.logger.info("%s: All plugins shut down" % self.name) + if self._database_available: + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() @property def metadata_cache_mode(self): diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index b0c170b1b..2a8447ae4 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -24,6 +24,7 @@ sys.path.pop() os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name from Bcfg2.Reporting.models import (Client, BaseEntry) +from django import db def hosts_by_entry_type(clients, etype, entryspec): result = [] @@ -293,6 +294,8 @@ def main(): if not client.expiration: print_fields(fields, client, fmt, extra=edata.get(client, None)) + db.close_connection() + if __name__ == "__main__": sys.exit(main()) -- cgit v1.2.3-1-g7c22 From f9efbc106c454bcef4c24708958dedceb56ba242 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 18 Feb 2014 09:16:41 -0500 Subject: Core: add default name for base Core implementation --- src/lib/Bcfg2/Server/Core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index cc9270a17..702b66983 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1,4 +1,4 @@ -""" Bcfg2.Server.Core provides the base core object that server core +y""" Bcfg2.Server.Core provides the base core object that server core implementations inherit from. """ import os @@ -86,6 +86,7 @@ class BaseCore(object): """ The server core is the container for all Bcfg2 server logic and modules. All core implementations must inherit from ``BaseCore``. """ + name = "core" def __init__(self, setup): # pylint: disable=R0912,R0915 """ -- cgit v1.2.3-1-g7c22 From a9f17d383460d0894e3a101c133be472f300ba94 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 18 Feb 2014 09:32:51 -0500 Subject: Fixed typo. Need coffee. --- src/lib/Bcfg2/Server/Core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 702b66983..38daa870e 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1,4 +1,4 @@ -y""" Bcfg2.Server.Core provides the base core object that server core +""" Bcfg2.Server.Core provides the base core object that server core implementations inherit from. """ import os -- cgit v1.2.3-1-g7c22 From 304cf13f4988312a4ec6ac14fff79bc74737e3ee Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Tue, 18 Feb 2014 14:46:49 +0000 Subject: support python 2.4 for default ACL checking in Lint --- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index bb0d6956a..1d12ee461 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -139,12 +139,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): def check_default_acl(self, path): """ Check that a default ACL contains either no entries or minimum required entries """ - defaults = 1 if path.xpath( - "ACL[@type='default' and @scope='user' and @user='']") else 0 - defaults += 1 if path.xpath( - "ACL[@type='default' and @scope='group' and @group='']") else 0 - defaults += 1 if path.xpath( - "ACL[@type='default' and @scope='other']") else 0 + defaults = 0 + if path.xpath("ACL[@type='default' and @scope='user' and @user='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='group' and @group='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='other']"): + defaults += 1 if defaults > 0 and defaults < 3: self.LintError( "missing-elements", -- cgit v1.2.3-1-g7c22 From 92d94a3f6164f74d7ff48703b002e9127d7d7821 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 19 Feb 2014 08:08:28 -0500 Subject: FAM: Only shut down inotify notifier once --- src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 2cdf27ed8..bce7ce7c2 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): - if self.notifier: + if self.started and self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ -- cgit v1.2.3-1-g7c22 From 8a8a47998438707f988838b505b98b99a11ce687 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 19 Feb 2014 08:08:37 -0500 Subject: core: only shut down core once --- src/lib/Bcfg2/Server/Core.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 38daa870e..03bc7bff4 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -189,6 +189,12 @@ class BaseCore(object): self.setup = setup atexit.register(self.shutdown) + #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, + #: then :mod:`atexit` calls it *again*, so it gets called + #: twice. This is potentially bad, so we use + #: :attr:`Bcfg2.Server.Core._running` as a flag to determine + #: if the core needs to be shutdown, and only do it once. + self._running = True #: Threading event to signal worker threads (e.g., #: :attr:`fam_thread`) to shutdown @@ -451,9 +457,13 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ + if not self._running: + self.logger.debug("%s: Core already shut down" % self.name) + return self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() + self._running = False self.fam.shutdown() self.logger.info("%s: FAM shut down" % self.name) for plugin in list(self.plugins.values()): -- cgit v1.2.3-1-g7c22 From 58cee8566fba7b48d127227d96c98549b7db3028 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 10 Feb 2014 09:24:56 -0500 Subject: testsuite: Fixed several pylint 1.0 issues --- src/lib/Bcfg2/Client/Tools/VCS.py | 10 +++++++--- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index aca5dbbc7..4fa2fb5e2 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -161,15 +161,19 @@ class VCS(Bcfg2.Client.Tools.Tool): def Verifysvn(self, entry, _): """Verify svn repositories""" - headrev = pysvn.Revision( pysvn.opt_revision_kind.head ) + # pylint: disable=E1101 + headrev = pysvn.Revision(pysvn.opt_revision_kind.head) + # pylint: enable=E1101 client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) - server = client.info2(entry.get('sourceurl'), headrev, recurse=False) + server = client.info2(entry.get('sourceurl'), headrev, + recurse=False) if server: server_rev = str(server[0][1].rev.number) except: - self.logger.info("Repository %s does not exist" % entry.get('name')) + self.logger.info("Repository %s does not exist" % + entry.get('name')) return False if entry.get('revision') == 'latest' and cur_rev == server_rev: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index f038ec9c0..7b5475ecc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -323,7 +323,7 @@ class YumCollection(Collection): self.fam) @property - def disableMetaData(self): + def disableMetaData(self): # pylint: disable=C0103 """ Report whether or not metadata processing is enabled. This duplicates code in Packages/__init__.py, and can probably be removed in Bcfg2 1.4 when we have a module-level setup diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 479138ef1..4b58c0fdb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -188,7 +188,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ @property - def disableResolver(self): + def disableResolver(self): # pylint: disable=C0103 """ Report the state of the resolver. This can be disabled in the configuration. Note that disabling metadata (see :attr:`disableMetaData`) implies disabling the resolver. @@ -213,7 +213,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, default="enabled").lower() == "disabled" @property - def disableMetaData(self): + def disableMetaData(self): # pylint: disable=C0103 """ Report whether or not metadata processing is enabled. This property cannot be set. """ -- cgit v1.2.3-1-g7c22 From cae2fcc0135c26811b1ce353ea28e4a93900c138 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 10 Feb 2014 09:02:16 -0500 Subject: POSIX: Fix verification of symlinks * Stat the link itself, not its target * Get SELinux context from the link, not the target * Don't get ACLs at all; symlinks don't have their own ACLs The first issue listed wasn't actually a bug, because none of the information queried from the target by the stat call was actually used in verification, but it's been fixed for completeness. --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index e593e0a0a..12f7f8a56 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -419,7 +419,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get data on the existing state of -- e.g., whether or not it exists, owner, group, permissions, etc. """ try: - ondisk = os.stat(path) + ondisk = os.lstat(path) except OSError: self.logger.debug("POSIX: %s does not exist" % path) return (False, None, None, None, None, None) @@ -456,7 +456,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if HAS_SELINUX: try: - secontext = selinux.getfilecon(path)[1].split(":")[2] + secontext = selinux.lgetfilecon(path)[1].split(":")[2] except (OSError, KeyError): err = sys.exc_info()[1] self.logger.debug("POSIX: Could not get current SELinux " @@ -465,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS: + if HAS_ACLS and not stat.S_ISLNK(ondisk): acls = self._list_file_acls(path) else: acls = None -- cgit v1.2.3-1-g7c22 From c9e234dc15b527c930bbb5a1e7e0e7d9212168b1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 21 Feb 2014 08:39:18 -0500 Subject: Yum: only fork to find bcfg2-yum-helper once, for real --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7b5475ecc..71c6d003c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -354,7 +354,7 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - if not self._helper: + if not self.__class___helper: # pylint: disable=W0212 try: self.__class__._helper = self.setup.cfp.get("packages:yum", @@ -368,7 +368,7 @@ class YumCollection(Collection): except OSError: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" # pylint: enable=W0212 - return self._helper + return self.__class__._helper @property def use_yum(self): -- cgit v1.2.3-1-g7c22 From 673c947a552ab601b159e7a95e3506870d837328 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 21 Feb 2014 08:53:09 -0500 Subject: Yum: fix pylint tests --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 71c6d003c..a149ccfe2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -354,8 +354,8 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ + # pylint: disable=W0212 if not self.__class___helper: - # pylint: disable=W0212 try: self.__class__._helper = self.setup.cfp.get("packages:yum", "helper") @@ -367,8 +367,8 @@ class YumCollection(Collection): self.__class__._helper = 'bcfg2-yum-helper' except OSError: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" - # pylint: enable=W0212 return self.__class__._helper + # pylint: enable=W0212 @property def use_yum(self): -- cgit v1.2.3-1-g7c22 From 9d12865d7598544bbbe2bb4b40c085c101b3d42e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 21 Feb 2014 09:09:51 -0500 Subject: Fixed typo Note to self: do not push code before 9 am --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index a149ccfe2..67ff05ca1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -355,7 +355,7 @@ class YumCollection(Collection): forking, but apparently not); finally we check in /usr/sbin, the default location. """ # pylint: disable=W0212 - if not self.__class___helper: + if not self.__class__._helper: try: self.__class__._helper = self.setup.cfp.get("packages:yum", "helper") -- cgit v1.2.3-1-g7c22 From c6a1de59fbd49b3e1b24a7fb4e55f109e8a54e2e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Sun, 23 Feb 2014 15:55:03 -0500 Subject: Metadata: reread client list from database This fixes two related bugs: One causes Metadata to use an out-of-date cached list of clients when a client is deleted or added with bcfg2-admin; the other causes child worker processes to use an out-of-date cached list of clients when a client is added with a Bcfg2 run when the multiprocessing core is in use. --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 1 + src/lib/Bcfg2/Server/Plugin/interfaces.py | 4 +++ src/lib/Bcfg2/Server/Plugins/Metadata.py | 41 +++++++++++++++++++++++++---- 3 files changed, 41 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 6d41bbcbb..2cb3adae3 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -303,6 +303,7 @@ class ChildCore(BaseCore): @exposed def GetConfig(self, client): """ Render the configuration for a client """ + self.metadata.update_client_list() self.logger.debug("%s: Building configuration for %s" % (self.name, client)) return lxml.etree.tostring(self.BuildConfiguration(client)) diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 33f6d338c..07717a710 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -213,6 +213,10 @@ class Metadata(object): """ raise NotImplementedError + def update_client_list(self): + """ Re-read the cached list of clients """ + raise NotImplementedError + class Connector(object): """ Connector plugins augment client metadata instances with diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index f734c98d0..d6febcff6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -668,7 +668,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, except MetadataClientModel.DoesNotExist: client = MetadataClientModel(hostname=client_name) client.save() - self.clients = self.list_clients() + self.update_client_list() return client else: try: @@ -721,7 +721,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs, alias=True) def list_clients(self): - """ List all clients in client database """ + """ List all clients in client database. + + Making ``self.clients`` a property and reading the client list + dynamically from the database on every call to + ``self.clients`` can result in very high rates of database + reads, so we cache the ``list_clients()`` results to reduce + the database load. When the database is in use, the client + list is reread periodically with + :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: @@ -772,7 +780,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() - self.clients = self.list_clients() + self.update_client_list() else: return self._remove_xdata(self.clients_xml, "Client", client_name) @@ -841,8 +849,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, except KeyError: self.clientgroups[clname] = [profile] self.states['clients.xml'] = True - if self._use_db: - self.clients = self.list_clients() + self.update_client_list() def _get_condition(self, element): """ Return a predicate that returns True if a client meets @@ -1434,6 +1441,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 + def update_client_list(self): + """ Re-read the client list from the database (if the database is in + use) """ + if self._use_db: + self.logger.debug("Metadata: Re-reading client list from database") + old = set(self.clients) + self.clients = self.list_clients() + new = set(self.clients) + added = new - old + removed = old - new + self.logger.debug("Metadata: Added %s clients: %s" % + (len(added), added)) + self.logger.debug("Metadata: Removed %s clients: %s" % + (len(removed), removed)) + # we could do this with set.symmetric_difference(), but we + # want detailed numbers of added/removed clients for + # logging + for client in added.union(removed): + self.expire_cache(client) + + def start_client_run(self, metadata): + """ Hook to reread client list if the database is in use """ + self.update_client_list() + def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, -- cgit v1.2.3-1-g7c22 From 1f0515a25551eee4dce6af96210aef8f6fdc0e8c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Sun, 23 Feb 2014 23:54:28 -0500 Subject: bcfg2-lint: Verify abstract Package tags --- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 1d12ee461..ce8b237b9 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -214,6 +214,16 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) + # ensure that abstract Package tags have either name + # or group specified + for package in xdata.xpath("//Package"): + if ('name' not in package.attrib and + 'group' not in package.attrib): + self.LintError( + "required-attrs-missing", + "Package tags require either a 'name' or 'group' " + "attribute: \n%s" % self.RenderXML(package)) + def check_entry(self, entry, filename): """ Generic entry check. -- cgit v1.2.3-1-g7c22 From 7f93c994854fb6876a4008baf172cdb432eed612 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 25 Feb 2014 14:08:56 -0500 Subject: helpers: better error message when failing to read FileBacked --- src/lib/Bcfg2/Server/Plugin/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 3e7d68cd8..de02adc4e 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1145,7 +1145,8 @@ class SpecificData(object): except UnicodeDecodeError: self.data = open(self.name, mode='rb').read() except: # pylint: disable=W0201 - LOGGER.error("Failed to read file %s" % self.name) + LOGGER.error("Failed to read file %s: %s" % (self.name, + sys.exc_info()[1])) class EntrySet(Debuggable): -- cgit v1.2.3-1-g7c22 From 74fd7a8e65ac9737a6ab39afa3378ca63fbe4e21 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 25 Feb 2014 13:25:16 -0600 Subject: Version bump to 1.3.4 Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/templates/base.html | 2 +- src/lib/Bcfg2/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7edf3a949..ef6799c2b 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 35d4cfa0a..ae82724f3 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.3" +__version__ = "1.3.4" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 -- cgit v1.2.3-1-g7c22 From c51850b13f54d6f46e6c671e5ee1d3f0cacef727 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 25 Feb 2014 14:33:41 -0500 Subject: POSIX: fixed test to only apply ACLs to non-symlinks --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 12f7f8a56..5fd3f2668 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -1,4 +1,4 @@ -""" Base class for tools that handle POSIX (Path) entries """ +git """ Base class for tools that handle POSIX (Path) entries """ import os import sys @@ -465,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS and not stat.S_ISLNK(ondisk): + if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): acls = self._list_file_acls(path) else: acls = None -- cgit v1.2.3-1-g7c22 From 38c8800814d5e09d40d7afb5aac4852e22776606 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 25 Feb 2014 14:42:56 -0500 Subject: Revert "POSIX: fixed test to only apply ACLs to non-symlinks" Massive typo. This reverts commit c51850b13f54d6f46e6c671e5ee1d3f0cacef727. --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 5fd3f2668..12f7f8a56 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -1,4 +1,4 @@ -git """ Base class for tools that handle POSIX (Path) entries """ +""" Base class for tools that handle POSIX (Path) entries """ import os import sys @@ -465,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): + if HAS_ACLS and not stat.S_ISLNK(ondisk): acls = self._list_file_acls(path) else: acls = None -- cgit v1.2.3-1-g7c22 From 12edf6186b236006963b83ac8ffe174665e097b9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 25 Feb 2014 14:43:50 -0500 Subject: POSIX: fixed test to only apply ACLs to non-symlinks --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 12f7f8a56..3d1358ce0 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -465,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS and not stat.S_ISLNK(ondisk): + if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): acls = self._list_file_acls(path) else: acls = None -- cgit v1.2.3-1-g7c22 From e7d8c70958e80924408c0e31fa06f1ad06607f99 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 28 Feb 2014 11:25:40 -0500 Subject: Reporting: properly close db connection Close the db connection at the end of each DjangoORM import, not when the reporting collector shuts down. The collector may not have even opened a connection, in the case of a storage backend other than DjangoORM. Fixes #157 --- src/lib/Bcfg2/Reporting/Collector.py | 3 --- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 0493c907b..2859eca28 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -162,9 +162,6 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() - from django import db - self.logger.info("%s: Closing database connection" % self.name) - db.close_connection() def reap_children(self): """Join any non-live threads""" diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index ef1e92103..e2e565b1d 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -18,6 +18,7 @@ from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache +from django import db #Used by GetCurrentEntry import difflib @@ -370,6 +371,11 @@ class DjangoORM(StorageBase): except: self.logger.error("Failed to import interaction: %s" % traceback.format_exc().splitlines()[-1]) + finally: + self.logger.info("%s: Closing database connection" % + self.__class__.__name__) + db.close_connection() + def validate(self): """Validate backend storage. Should be called once when loaded""" @@ -451,4 +457,3 @@ class DjangoORM(StorageBase): else: ret.append(None) return ret - -- cgit v1.2.3-1-g7c22 From a6ae7149bd9ba423fb55eacc6eb8f0c1f49c2dc7 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 3 Mar 2014 09:03:46 -0500 Subject: Core: close all database connections at the end of XML-RPC requests --- src/lib/Bcfg2/Server/Core.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 03bc7bff4..a8069ff1b 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -18,7 +18,7 @@ import Bcfg2.Server.FileMonitor from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain -from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics @@ -67,6 +67,24 @@ def sort_xml(node, key=None): node[:] = sorted_children +def close_db_connection(func): + """ Decorator that closes the Django database connection at the end of + the function. This should decorate any exposed function that + might open a database connection. """ + @wraps(func) + def inner(self, *args, **kwargs): + """ The decorated function """ + rv = func(self, *args, **kwargs) + if self._database_available: # pylint: disable=W0212 + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() + return rv + + return inner + + class CoreInitError(Exception): """ Raised when the server core cannot be initialized. """ pass @@ -741,11 +759,6 @@ class BaseCore(object): self.validate_goals(meta, config) self.client_run_hook("end_client_run", meta) - if self._database_available: - from django import db - self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) - db.close_connection() sort_xml(config, key=lambda e: e.get('name')) @@ -1089,6 +1102,7 @@ class BaseCore(object): @exposed @track_statistics() + @close_db_connection def DeclareVersion(self, address, version): """ Declare the client version. @@ -1111,6 +1125,7 @@ class BaseCore(object): return True @exposed + @close_db_connection def GetProbes(self, address): """ Fetch probes for the client. @@ -1136,6 +1151,7 @@ class BaseCore(object): (client, err)) @exposed + @close_db_connection def RecvProbeData(self, address, probedata): """ Receive probe data from clients. @@ -1183,6 +1199,7 @@ class BaseCore(object): return True @exposed + @close_db_connection def AssertProfile(self, address, profile): """ Set profile for a client. @@ -1202,6 +1219,7 @@ class BaseCore(object): return True @exposed + @close_db_connection def GetConfig(self, address): """ Build config for a client by calling :func:`BuildConfiguration`. @@ -1221,6 +1239,7 @@ class BaseCore(object): self.critical_error("Metadata consistency failure for %s" % client) @exposed + @close_db_connection def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. @@ -1258,6 +1277,7 @@ class BaseCore(object): address) @exposed + @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. -- cgit v1.2.3-1-g7c22 From b1b2b53f484d266089d9f043042729e29245382a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 3 Mar 2014 09:33:57 -0500 Subject: cleared pylint error on PluginDatabaseModel metadata options --- src/lib/Bcfg2/Server/Plugin/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index de02adc4e..170af50ac 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -199,7 +199,8 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: # pylint: disable=C0111,W0232 + class Meta(object): # pylint: disable=W0232 + """ Model metadata options """ app_label = "Server" -- cgit v1.2.3-1-g7c22 From f76ee4bae4d8a45cc2050b8219cdf7f2e8c6533d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 4 Mar 2014 07:46:46 -0500 Subject: bcfg2-report-collector: better error messages when failing to daemonize --- src/lib/Bcfg2/Reporting/Collector.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 2859eca28..8ca145f16 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import sys import atexit import daemon import logging @@ -6,6 +7,7 @@ import traceback import threading # pylint: disable=E0611 +from lockfile import LockFailed, LockTimeout try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -119,6 +121,17 @@ class ReportingCollector(object): try: self.context.pidfile = PIDLockFile(self.setup['daemon']) self.context.open() + except LockFailed: + self.logger.error("Failed to daemonize: %s" % + sys.exc_info()[1]) + self.shutdown() + return + except LockTimeout: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % traceback.format_exc().splitlines()[-1]) -- cgit v1.2.3-1-g7c22 From 16072ab1b2c3785b7a35c54b1f28a7142563330b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 14 Mar 2014 09:46:03 -0400 Subject: Core: better error messages when altsrc bind fails --- src/lib/Bcfg2/Server/Core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index a8069ff1b..4375512e9 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -681,9 +681,10 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('realname'), - entry.get('name'))) + self.logger.error( + "Failed binding entry %s:%s with altsrc %s: %s" % + (entry.tag, entry.get('realname'), entry.get('name'), + sys.exc_info()[1])) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) -- cgit v1.2.3-1-g7c22 From bb873c4ab0848bc820db445938d4ff530173a062 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 25 Mar 2014 14:14:25 -0500 Subject: Revert "Systemd: systemd is a replacement for chkconfig" This reverts commit 690a18b5bb61516e5c11f6da3d788332373c196b. While systemd is meant to replace chkconfig, it appears that RHEL7 has both and does not provide systemd alternatives for certain SYSV init scripts by default. --- src/lib/Bcfg2/Client/Tools/Systemd.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 20a172d3d..027d91c71 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,8 +13,6 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} - conflicts = ['Chkconfig'] - def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) -- cgit v1.2.3-1-g7c22 From 24a261f842a4bc1d4dc125fad0f43343d5d4c9d8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 8 Apr 2014 15:03:54 -0400 Subject: Correctly upgrade or downgrade yum packages Formerly, yum did an 'update' to install the correct version of a package, even if the desired package was older than the installed package. This is wrong; it needs to do a downgrade. This changes it to downgrade when the desired package is older, and upgrade if it is newer. There is still the possibility of upgrading a package that should be downgraded if the desired package is only partially specified, but this should be very rare. --- src/lib/Bcfg2/Client/Tools/YUM.py | 51 +++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index c30c0a13a..b4ca32847 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -588,34 +588,38 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = True stat['version_fail'] = True # Just chose the first pkg for the error message + current_pkg = all_pkg_objs[0] if virt_pkg: provides = \ - [p for p in all_pkg_objs[0].provides + [p for p in current_pkg.provides if p[0] == entry.get("name")][0] - entry.set('current_version', "%s:%s-%s" % provides[2]) + current_evr = provides[2] self.logger.info( " %s: Wrong version installed. " "Want %s, but %s provides %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]), + nevra2string(current_pkg), yum.misc.prco_tuple_to_string(provides))) else: - entry.set('current_version', "%s:%s-%s.%s" % - (all_pkg_objs[0].epoch, - all_pkg_objs[0].version, - all_pkg_objs[0].release, - all_pkg_objs[0].arch)) + current_evr = (current_pkg.epoch, + current_pkg.version, + current_pkg.release) self.logger.info(" %s: Wrong version installed. " "Want %s, but have %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]))) - entry.set('version', "%s:%s-%s.%s" % - (nevra.get('epoch', 'any'), - nevra.get('version', 'any'), - nevra.get('release', 'any'), - nevra.get('arch', 'any'))) + nevra2string(current_pkg))) + wanted_evr = (nevra.get('epoch', 'any'), + nevra.get('version', 'any'), + nevra.get('release', 'any')) + entry.set('current_version', "%s:%s-%s" % current_evr) + entry.set('version', "%s:%s-%s" % wanted_evr) + if yum.compareEVR(current_evr, wanted_evr) == 1: + entry.set("package_fail_action", "downgrade") + else: + entry.set("package_fail_action", "update") + qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue @@ -887,6 +891,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_pkgs = [] gpg_keys = [] upgrade_pkgs = [] + downgrade_pkgs = [] reinstall_pkgs = [] def queue_pkg(pkg, inst, queue): @@ -929,7 +934,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if not status.get('installed', False) and self.do_install: queue_pkg(pkg, inst, install_pkgs) elif status.get('version_fail', False) and self.do_upgrade: - queue_pkg(pkg, inst, upgrade_pkgs) + if pkg.get("package_fail_action") == "downgrade": + queue_pkg(pkg, inst, downgrade_pkgs) + else: + queue_pkg(pkg, inst, upgrade_pkgs) elif status.get('verify_fail', False) and self.do_reinst: queue_pkg(pkg, inst, reinstall_pkgs) else: @@ -992,6 +1000,19 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Error upgrading package %s: %s" % (pkg_arg, yume)) + if len(downgrade_pkgs) > 0: + self.logger.info("Attempting to downgrade packages") + + for inst in downgrade_pkgs: + pkg_arg = self.instance_status[inst].get('pkg').get('name') + self.logger.debug("Downgrading %s" % pkg_arg) + try: + self.yumbase.downgrade(**build_yname(pkg_arg, inst)) + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] + self.logger.error("Error downgrading package %s: %s" % + (pkg_arg, yume)) + if len(reinstall_pkgs) > 0: self.logger.info("Attempting to reinstall packages") for inst in reinstall_pkgs: -- cgit v1.2.3-1-g7c22