diff options
28 files changed, 825 insertions, 568 deletions
diff --git a/src/lib/Bcfg2/Cache.py b/src/lib/Bcfg2/Cache.py index 9a828e2c9..842098eda 100644 --- a/src/lib/Bcfg2/Cache.py +++ b/src/lib/Bcfg2/Cache.py @@ -2,11 +2,13 @@ doesn't provide many features, but more (time-based expiration, etc.) can be added as necessary. """ + class Cache(dict): """ an implementation of a simple memory-backed cache """ + def expire(self, key=None): + """ expire all items, or a specific item, from the cache """ if key is None: self.clear() elif key in self: del self[key] - diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 2fb81d6ba..ef61940eb 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,13 +1,12 @@ -""" -Frame is the Client Framework that verifies and -installs entries, and generates statistics. -""" +""" Frame is the Client Framework that verifies and installs entries, +and generates statistics. """ -import logging import sys import time +import fnmatch +import logging import Bcfg2.Client.Tools -from Bcfg2.Compat import input +from Bcfg2.Compat import input, any, all # pylint: disable=W0622 def cmpent(ent1, ent2): @@ -19,35 +18,36 @@ def cmpent(ent1, ent2): def matches_entry(entryspec, entry): - # both are (tag, name) + """ Determine if the Decisions-style entry specification matches + the entry. Both are tuples of (tag, name). The entryspec can + handle the wildcard * in either position. """ if entryspec == entry: return True - else: - for i in [0, 1]: - if entryspec[i] == entry[i]: - continue - elif entryspec[i] == '*': - continue - elif '*' in entryspec[i]: - starpt = entryspec[i].index('*') - if entry[i].startswith(entryspec[i][:starpt]): - continue - return False - return True + return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) def matches_white_list(entry, whitelist): - return True in [matches_entry(we, (entry.tag, entry.get('name'))) - for we in whitelist] + """ Return True if (<entry tag>, <entry name>) is in the given + whitelist. """ + return any(matches_entry(we, (entry.tag, entry.get('name'))) + for we in whitelist) def passes_black_list(entry, blacklist): - return True not in [matches_entry(be, (entry.tag, entry.get('name'))) - for be in blacklist] + """ Return True if (<entry tag>, <entry name>) is not in the given + blacklist. """ + return not any(matches_entry(be, (entry.tag, entry.get('name'))) + for be in blacklist) + + +# pylint: disable=W0702 +# in frame we frequently want to catch all exceptions, regardless of +# type, so disable the pylint rule that catches that. class Frame(object): """Frame is the container for all Tool objects and state information.""" + def __init__(self, config, setup, times, drivers, dryrun): self.config = config self.times = times @@ -92,8 +92,9 @@ class Frame(object): for tool in self.tools[:]: for conflict in getattr(tool, 'conflicts', []): - [self.tools.remove(item) for item in self.tools \ - if item.name == conflict] + for item in self.tools: + if item.name == conflict: + self.tools.remove(item) self.logger.info("Loaded tool drivers:") self.logger.info([tool.name for tool in self.tools]) @@ -104,7 +105,8 @@ class Frame(object): if entry not in self.handled] if self.unhandled: - self.logger.error("The following entries are not handled by any tool:") + self.logger.error("The following entries are not handled by any " + "tool:") for entry in self.unhandled: self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), entry.get('name'))) @@ -118,10 +120,12 @@ class Frame(object): if pkgs: self.logger.debug("The following packages are specified in bcfg2:") self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == None]) - self.logger.debug("The following packages are prereqs added by Packages:") + self.logger.debug("The following packages are prereqs added by " + "Packages:") self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) def find_dups(self, config): + """ Find duplicate entries and warn about them """ entries = dict() for struct in config: for entry in struct: @@ -134,7 +138,8 @@ class Frame(object): entries[pkey] = 1 multi = [e for e, c in entries.items() if c > 1] if multi: - self.logger.debug("The following entries are included multiple times:") + self.logger.debug("The following entries are included multiple " + "times:") for entry in multi: self.logger.debug(entry) @@ -179,7 +184,8 @@ class Frame(object): non-whitelisted/blacklisted 'important' entries from being installed prior to determining the decision mode on the client. """ - # Need to process decision stuff early so that dryrun mode works with it + # Need to process decision stuff early so that dryrun mode + # works with it self.whitelist = [entry for entry in self.states \ if not self.states[entry]] if not self.setup['file']: @@ -188,17 +194,23 @@ class Frame(object): w_to_rem = [e for e in self.whitelist \ if not matches_white_list(e, dwl)] if w_to_rem: - self.logger.info("In whitelist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) + self.logger.info("In whitelist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in w_to_rem]) self.whitelist = [x for x in self.whitelist \ if x not in w_to_rem] elif self.setup['decision'] == 'blacklist': - b_to_rem = [e for e in self.whitelist \ - if not passes_black_list(e, self.setup['decision_list'])] + b_to_rem = \ + [e for e in self.whitelist + if not passes_black_list(e, self.setup['decision_list'])] if b_to_rem: - self.logger.info("In blacklist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + self.logger.info("In blacklist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in b_to_rem] # take care of important entries first if not self.dryrun: @@ -216,22 +228,22 @@ class Frame(object): (parent.tag == "Independent" and (self.setup['bundle'] or self.setup['skipindep']))): continue - tl = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] - if tl: - if self.setup['interactive'] and not \ - self.promptFilter("Install %s: %s? (y/N):", [cfile]): + tools = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] + if tools: + if (self.setup['interactive'] and not + self.promptFilter("Install %s: %s? (y/N):", [cfile])): self.whitelist.remove(cfile) continue try: - self.states[cfile] = tl[0].InstallPath(cfile) + self.states[cfile] = tools[0].InstallPath(cfile) if self.states[cfile]: - tl[0].modified.append(cfile) + tools[0].modified.append(cfile) except: self.logger.error("Unexpected tool failure", exc_info=1) cfile.set('qtext', '') - if tl[0].VerifyPath(cfile, []): + if tools[0].VerifyPath(cfile, []): self.whitelist.remove(cfile) def Inventory(self): @@ -249,9 +261,10 @@ class Frame(object): try: tool.Inventory(self.states) except: - self.logger.error("%s.Inventory() call failed:" % tool.name, exc_info=1) + self.logger.error("%s.Inventory() call failed:" % tool.name, + exc_info=1) - def Decide(self): + def Decide(self): # pylint: disable=R0912 """Set self.whitelist based on user interaction.""" prompt = "Install %s: %s? (y/N): " rprompt = "Remove %s: %s? (y/N): " @@ -270,12 +283,14 @@ class Frame(object): if self.dryrun: if self.whitelist: - self.logger.info("In dryrun mode: suppressing entry installation for:") + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) for entry in self.whitelist]) self.whitelist = [] if self.removal: - self.logger.info("In dryrun mode: suppressing entry removal for:") + self.logger.info("In dryrun mode: " + "suppressing entry removal for:") self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) for entry in self.removal]) self.removal = [] @@ -301,7 +316,7 @@ class Frame(object): if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = filter(lambda b: \ + bundles = filter(lambda b: b.get('name') not in self.setup['skipbundle'], bundles) if self.setup['skipindep']: @@ -314,7 +329,8 @@ class Frame(object): for bundle in bundles[:]: if bundle.tag != 'Bundle': continue - bmodified = len([item for item in bundle if item in self.whitelist]) + bmodified = len([item for item in bundle + if item in self.whitelist]) actions = [a for a in bundle.findall('./Action') if (a.get('timing') != 'post' and (bmodified or a.get('when') == 'always'))] @@ -335,7 +351,8 @@ class Frame(object): (bundle.get('name'))) self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_remv]) - [self.whitelist.remove(ent) for ent in b_to_remv] + for ent in b_to_remv: + self.whitelist.remove(ent) if self.setup['interactive']: self.whitelist = self.promptFilter(prompt, self.whitelist) @@ -354,7 +371,8 @@ class Frame(object): try: tool.Install(handled, self.states) except: - self.logger.error("%s.Install() call failed:" % tool.name, exc_info=1) + self.logger.error("%s.Install() call failed:" % tool.name, + exc_info=1) def Install(self): """Install all entries.""" @@ -373,9 +391,12 @@ class Frame(object): try: tool.Inventory(self.states, [bundle]) except: - self.logger.error("%s.Inventory() call failed:" % tool.name, exc_info=1) + self.logger.error("%s.Inventory() call failed:" % + tool.name, + exc_info=1) clobbered = [entry for bundle in mbundles for entry in bundle \ - if not self.states[entry] and entry not in self.blacklist] + if (not self.states[entry] and + entry not in self.blacklist)] if clobbered: self.logger.debug("Found clobbered entries:") self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) \ @@ -395,18 +416,20 @@ class Frame(object): else: tool.BundleNotUpdated(bundle, self.states) except: - self.logger.error("%s.BundleNotUpdated() call failed:" % \ - (tool.name), exc_info=1) + self.logger.error("%s.BundleNotUpdated() call failed:" % + tool.name, exc_info=1) def Remove(self): """Remove extra entries.""" for tool in self.tools: - extras = [entry for entry in self.removal if tool.handlesEntry(entry)] + extras = [entry for entry in self.removal + if tool.handlesEntry(entry)] if extras: try: tool.Remove(extras) except: - self.logger.error("%s.Remove() failed" % tool.name, exc_info=1) + self.logger.error("%s.Remove() failed" % tool.name, + exc_info=1) def CondDisplayState(self, phase): """Conditionally print tracing information.""" @@ -420,8 +443,8 @@ class Frame(object): if not self.states[entry]: etype = entry.get('type') if etype: - self.logger.info( "%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) else: self.logger.info(" %s:%s" % (entry.tag, entry.get('name'))) @@ -432,8 +455,8 @@ class Frame(object): for entry in self.extra: etype = entry.get('type') if etype: - self.logger.info( "%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) else: self.logger.info(" %s:%s" % (entry.tag, entry.get('name'))) @@ -467,24 +490,26 @@ class Frame(object): def GenerateStats(self): """Generate XML summary of execution statistics.""" feedback = Bcfg2.Client.XML.Element("upload-statistics") - stats = Bcfg2.Client.XML.SubElement(feedback, - 'Statistics', - total=str(len(self.states)), - version='2.0', - revision=self.config.get('revision', '-1')) + stats = Bcfg2.Client.XML.SubElement( + feedback, + 'Statistics', + total=str(len(self.states)), + version='2.0', + revision=self.config.get('revision', '-1')) good_entries = [key for key, val in list(self.states.items()) if val] good = len(good_entries) stats.set('good', str(good)) - if len([key for key, val in list(self.states.items()) if not val]) == 0: - stats.set('state', 'clean') - else: + if any(not val for val in list(self.states.values())): stats.set('state', 'dirty') + else: + stats.set('state', 'clean') # List bad elements of the configuration - for (data, ename) in [(self.modified, 'Modified'), (self.extra, "Extra"), \ + for (data, ename) in [(self.modified, 'Modified'), + (self.extra, "Extra"), (good_entries, "Good"), - ([entry for entry in self.states if not \ - self.states[entry]], "Bad")]: + ([entry for entry in self.states + if not self.states[entry]], "Bad")]: container = Bcfg2.Client.XML.SubElement(stats, ename) for item in data: item.set('qtext', '') diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index d6bbd3b72..720416724 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -2,7 +2,7 @@ # library will use lxml, then builtin xml.etree, then ElementTree -# pylint: disable=F0401,E0611 +# pylint: disable=F0401,E0611,W0611,W0613,C0103 try: from lxml.etree import Element, SubElement, XML, tostring @@ -16,8 +16,11 @@ except ImportError: Element = xml.etree.ElementTree.Element SubElement = xml.etree.ElementTree.SubElement XML = xml.etree.ElementTree.XML - def tostring(e, encoding=None, xml_declaration=None): - return xml.etree.ElementTree.tostring(e, encoding=encoding) + + def tostring(el, encoding=None, xml_declaration=None): + """ tostring implementation compatible with lxml """ + return xml.etree.ElementTree.tostring(el, encoding=encoding) + driver = 'etree-py' except ImportError: try: @@ -28,10 +31,12 @@ except ImportError: Element = elementtree.ElementTree.Element SubElement = elementtree.ElementTree.SubElement XML = elementtree.ElementTree.XML - def tostring(e, encoding=None, xml_declaration=None): - return elementtree.ElementTree.tostring(e) + + def tostring(el, encoding=None, xml_declaration=None): + """ tostring implementation compatible with lxml """ + return elementtree.ElementTree.tostring(el) except ImportError: - print("Failed to load lxml, xml.etree and elementtree.ElementTree") + print("Failed to load lxml, xml.etree or elementtree.ElementTree") print("Cannot continue") raise SystemExit(1) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 9aeda6d36..f466b8e03 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -3,7 +3,7 @@ Python 2.4 and such-like """ import sys -# pylint: disable=F0401,E0611 +# pylint: disable=F0401,E0611,W0611,W0622,C0103 try: from email.Utils import formatdate @@ -75,8 +75,9 @@ if sys.hexversion >= 0x03000000: else: unicode = unicode -# print to file compatibility + def u_str(string, encoding=None): + """ print to file compatibility """ if sys.hexversion >= 0x03000000: if encoding is not None: return string.encode(encoding) @@ -90,7 +91,7 @@ def u_str(string, encoding=None): try: unicode = unicode -except: +except NameError: unicode = str # base64 compat @@ -103,7 +104,7 @@ else: try: input = raw_input -except: +except NameError: input = input try: @@ -117,24 +118,25 @@ except ImportError: from UserDict import DictMixin as MutableMapping -# in py3k __cmp__ is no longer magical, so we define a mixin that can -# be used to define the rich comparison operators from __cmp__ class CmpMixin(object): + """ in py3k __cmp__ is no longer magical, so this mixin can be + used to define the rich comparison operators from __cmp__ """ + def __lt__(self, other): return self.__cmp__(other) < 0 - + def __gt__(self, other): return self.__cmp__(other) > 0 - + def __eq__(self, other): return self.__cmp__(other) == 0 - + def __ne__(self, other): return not self.__eq__(other) - + def __ge__(self, other): return self.__gt__(other) or self.__eq__(other) - + def __le__(self, other): return self.__lt__(other) or self.__eq__(other) @@ -145,11 +147,16 @@ except ImportError: from pkgutil import iter_modules # iter_modules was added in python 2.5; use it to get an exact # re-implementation of walk_packages if possible + def walk_packages(path=None, prefix='', onerror=None): - def seen(p, m={}): - if p in m: + """ Implementation of walk_packages for python 2.5 """ + def seen(path, seenpaths={}): # pylint: disable=W0102 + """ detect if a path has been 'seen' (i.e., considered + for inclusion in the generator). tracks what has been + seen through the magic of python default arguments """ + if path in seenpaths: return True - m[p] = True + seenpaths[path] = True for importer, name, ispkg in iter_modules(path, prefix): yield importer, name, ispkg @@ -179,7 +186,7 @@ except ImportError: def walk_packages(path=None, prefix='', onerror=None): """ imperfect, incomplete implementation of walk_packages() for python 2.4. Differences: - + * requires a full path, not a path relative to something in sys.path. anywhere we care about that shouldn't be an issue @@ -187,14 +194,8 @@ except ImportError: * the first element of each tuple is None instead of an importer object """ - def seen(p, m={}): - if p in m: - return True - m[p] = True - if path is None: path = sys.path - rv = [] for mpath in path: for fname in os.listdir(mpath): fpath = os.path.join(mpath, fname) @@ -227,12 +228,14 @@ try: any = any except NameError: def all(iterable): + """ implementation of builtin all() for python 2.4 """ for element in iterable: if not element: return False return True def any(iterable): + """ implementation of builtin any() for python 2.4 """ for element in iterable: if element: return True @@ -247,5 +250,6 @@ except ImportError: try: from functools import wraps except ImportError: - def wraps(wrapped): + def wraps(wrapped): # pylint: disable=W0613 + """ implementation of functools.wraps() for python 2.4 """ return lambda f: f diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index 5eb7ffe8e..eb2841bb5 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -33,6 +33,8 @@ Rand.rand_seed(os.urandom(1024)) def _cipher_filter(cipher, instr): + """ M2Crypto reads and writes file-like objects, so this uses + StringIO to pass data through it """ inbuf = StringIO(instr) outbuf = StringIO() while 1: @@ -161,6 +163,7 @@ def get_algorithm(setup): return setup.cfp.get("encryption", "algorithm", default=ALGORITHM).lower().replace("-", "_") + def get_passphrases(setup): """ Get all candidate encryption passphrases from the config file. diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 1f4ba7dd0..06379ce7b 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -29,7 +29,7 @@ class TermiosFormatter(logging.Formatter): "\000" * 8))[1] if self.width == 0: self.width = 80 - except: + except: # pylint: disable=W0702 self.width = 80 else: # output to a pipe @@ -44,22 +44,24 @@ class TermiosFormatter(logging.Formatter): if len(line) <= line_len: returns.append(line) else: - inner_lines = int(math.floor(float(len(line)) / line_len)) + 1 - for i in range(inner_lines): - returns.append("%s" % (line[i * line_len:(i + 1) * line_len])) + inner_lines = \ + int(math.floor(float(len(line)) / line_len)) + 1 + for msgline in range(inner_lines): + returns.append( + line[msgline * line_len:(msgline + 1) * line_len]) elif isinstance(record.msg, list): if not record.msg: return '' record.msg.sort() msgwidth = self.width - columnWidth = max([len(item) for item in record.msg]) - columns = int(math.floor(float(msgwidth) / (columnWidth + 2))) + col_width = max([len(item) for item in record.msg]) + columns = int(math.floor(float(msgwidth) / (col_width + 2))) lines = int(math.ceil(float(len(record.msg)) / columns)) - for lineNumber in range(lines): - indices = [idx for idx in [(colNum * lines) + lineNumber + for lineno in range(lines): + indices = [idx for idx in [(colNum * lines) + lineno for colNum in range(columns)] if idx < len(record.msg)] - retformat = (len(indices) * (" %%-%ds " % columnWidth)) + retformat = (len(indices) * (" %%-%ds " % col_width)) returns.append(retformat % tuple([record.msg[idx] for idx in indices])) else: @@ -99,13 +101,13 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): else: msgs = [record] for newrec in msgs: - msg = '<%d>%s\000' % (self.encodePriority(self.facility, - newrec.levelname.lower()), - self.format(newrec)) + msg = '<%d>%s\000' % \ + (self.encodePriority(self.facility, newrec.levelname.lower()), + self.format(newrec)) try: self.socket.send(msg.encode('ascii')) except socket.error: - for i in range(10): + for i in range(10): # pylint: disable=W0612 try: if isinstance(self.address, tuple): self.socket = socket.socket(socket.AF_INET, @@ -124,12 +126,13 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): logging.WARNING), self.format(reconn))) self.socket.send(msg) - except: + except: # pylint: disable=W0702 # If we still fail then drop it. Running # bcfg2-server as non-root can trigger permission # denied exceptions. pass + def add_console_handler(level=logging.DEBUG): """Add a logging handler that logs at a level to sys.stdout.""" console = logging.StreamHandler(sys.stdout) @@ -138,6 +141,7 @@ def add_console_handler(level=logging.DEBUG): console.setFormatter(TermiosFormatter()) logging.root.addHandler(console) + def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): """Add a logging handler that logs as procname to syslog_facility.""" try: @@ -150,20 +154,24 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): ('localhost', 514), syslog_facility) syslog.setLevel(level) - syslog.setFormatter(logging.Formatter('%(name)s[%(process)d]: %(message)s')) + syslog.setFormatter( + logging.Formatter('%(name)s[%(process)d]: %(message)s')) logging.root.addHandler(syslog) except socket.error: logging.root.error("failed to activate syslogging") except: print("Failed to activate syslogging") + def add_file_handler(to_file, level=logging.DEBUG): """Add a logging handler that logs to to_file.""" filelog = logging.FileHandler(to_file) filelog.setLevel(level) - filelog.setFormatter(logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) + filelog.setFormatter( + logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) logging.root.addHandler(filelog) + def setup_logging(procname, to_console=True, to_syslog=True, syslog_facility='daemon', level=0, to_file=None): """Setup logging for Bcfg2 software.""" diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index ff7c3ce70..a5436dbd0 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -8,12 +8,12 @@ import re import shlex import sys import Bcfg2.Client.Tools -# Compatibility imports from Bcfg2.Compat import ConfigParser from Bcfg2.version import __version__ class OptionFailure(Exception): + """ raised when malformed Option objects are instantiated """ pass DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' @@ -21,6 +21,9 @@ DEFAULT_INSTALL_PREFIX = '/usr' class DefaultConfigParser(ConfigParser.ConfigParser): + """ A config parser that can be used to query options with default + values in the event that the option is not found """ + def __init__(self, *args, **kwargs): """Make configuration options case sensitive""" ConfigParser.ConfigParser.__init__(self, *args, **kwargs) @@ -59,7 +62,11 @@ class DefaultConfigParser(ConfigParser.ConfigParser): class Option(object): - def __init__(self, desc, default, cmd=False, odesc=False, + """ a single option, which might be read from the command line, + environment, or config file """ + + # pylint: disable=C0103,R0913 + def __init__(self, desc, default, cmd=None, odesc=False, env=False, cf=False, cook=False, long_arg=False, deprecated_cf=None): self.desc = desc @@ -69,7 +76,7 @@ class Option(object): if not self.long: if cmd and (cmd[0] != '-' or len(cmd) != 2): raise OptionFailure("Poorly formed command %s" % cmd) - elif cmd and (not cmd.startswith('--')): + elif cmd and not cmd.startswith('--'): raise OptionFailure("Poorly formed command %s" % cmd) self.odesc = odesc self.env = env @@ -79,8 +86,13 @@ class Option(object): if not odesc and not cook and isinstance(self.default, bool): self.boolean = True self.cook = cook + self.value = None + # pylint: enable=C0103,R0913 def get_cooked_value(self, value): + """ get the value of this option after performing any option + munging specified in the 'cook' keyword argument to the + constructor """ if self.boolean: return True if self.cook: @@ -112,6 +124,7 @@ class Option(object): return "".join(rv) def buildHelpMessage(self): + """ build the help message for this option """ vals = [] if not self.cmd: return '' @@ -121,11 +134,13 @@ class Option(object): else: vals.append("%s %s" % (self.cmd, self.odesc)) else: - vals.append(self.cmd) + vals.append(self.cmd) vals.append(self.desc) return " %-28s %s\n" % tuple(vals) def buildGetopt(self): + """ build a string suitable for describing this short option + to getopt """ gstr = '' if self.long: return gstr @@ -136,12 +151,18 @@ class Option(object): return gstr def buildLongGetopt(self): + """ build a string suitable for describing this long option to + getopt """ if self.odesc: return self.cmd[2:] + '=' else: return self.cmd[2:] def parse(self, opts, rawopts, configparser=None): + """ parse a single option. try parsing the data out of opts + (the results of getopt), rawopts (the raw option string), the + environment, and finally the config parser. either opts or + rawopts should be provided, but not both """ if self.cmd and opts: # Processing getopted data optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] @@ -170,7 +191,8 @@ class Option(object): pass if self.deprecated_cf: try: - self.value = self.get_cooked_value(configparser.get(*self.deprecated_cf)) + self.value = self.get_cooked_value( + configparser.get(*self.deprecated_cf)) print("Warning: [%s] %s is deprecated, use [%s] %s instead" % (self.deprecated_cf[0], self.deprecated_cf[1], self.cf[0], self.cf[1])) @@ -184,9 +206,13 @@ class Option(object): class OptionSet(dict): + """ a set of Option objects that interfaces with getopt and + DefaultConfigParser to populate a dict of <option name>:<value> + """ + def __init__(self, *args, **kwargs): dict.__init__(self, *args) - self.hm = self.buildHelpMessage() + self.hm = self.buildHelpMessage() # pylint: disable=C0103 if 'configfile' in kwargs: self.cfile = kwargs['configfile'] else: @@ -203,27 +229,34 @@ class OptionSet(dict): if sys.argv[1] == 'init': return else: - print("Warning! Unable to read specified configuration file: %s" - % self.cfile) + print("Warning! Unable to read specified configuration file: " + "%s" % self.cfile) def buildGetopt(self): + """ build a short option description string suitable for use + by getopt.getopt """ return ''.join([opt.buildGetopt() for opt in list(self.values())]) def buildLongGetopt(self): + """ build a list of long options suitable for use by + getopt.getopt """ return [opt.buildLongGetopt() for opt in list(self.values()) if opt.long] def buildHelpMessage(self): + """ Build the help mesage for this option set, or use self.hm + if it is set """ if hasattr(self, 'hm'): return self.hm hlist = [] # list of _non-empty_ help messages for opt in list(self.values()): - hm = opt.buildHelpMessage() - if hm: - hlist.append(hm) + helpmsg = opt.buildHelpMessage() + if helpmsg: + hlist.append(helpmsg) return ''.join(hlist) def helpExit(self, msg='', code=1): + """ print help and exit """ if msg: print(msg) print("Usage:") @@ -231,6 +264,7 @@ class OptionSet(dict): raise SystemExit(code) def versionExit(self, code=0): + """ print the version of bcfg2 and exit """ print("%s %s on Python %s" % (os.path.basename(sys.argv[0]), __version__, @@ -269,42 +303,43 @@ class OptionSet(dict): def list_split(c_string): + """ split an option string on commas, optionally surrounded by + whitespace, returning a list """ if c_string: return re.split("\s*,\s*", c_string) return [] def colon_split(c_string): + """ split an option string on colons, returning a list """ if c_string: return c_string.split(':') return [] -def get_bool(s): +def get_bool(val): + """ given a string value of a boolean configuration option, return + an actual bool (True or False) """ # these values copied from ConfigParser.RawConfigParser.getboolean # with the addition of True and False truelist = ["1", "yes", "True", "true", "on"] falselist = ["0", "no", "False", "false", "off"] - if s in truelist: + if val in truelist: return True - elif s in falselist: + elif val in falselist: return False else: raise ValueError - -""" -Options: - Accepts keyword argument list with the following values: +# Options accepts keyword argument list with the following values: +# default: default value for the option +# cmd: command line switch +# odesc: option description +# cf: tuple containing section/option +# cook: method for parsing option +# long_arg: (True|False) specifies whether cmd is a long argument - default: default value for the option - cmd: command line switch - odesc: option description - cf: tuple containing section/option - cook: method for parsing option - long_arg: (True|False) specifies whether cmd is a long argument -""" # General options CFILE = \ Option('Specify configuration file', @@ -428,7 +463,8 @@ SERVER_REPOSITORY = \ SERVER_PLUGINS = \ Option('Server plugin list', # default server plugins - default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', 'SSHbase'], + default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', + 'SSHbase'], cf=('server', 'plugins'), cook=list_split) SERVER_MCONNECT = \ @@ -444,7 +480,7 @@ SERVER_FILEMONITOR = \ SERVER_FAM_IGNORE = \ Option('File globs to ignore', default=['*~', '*#', '.#*', '*.swp', '.*.swx', 'SCCS', '.svn', - '4913', '.gitignore',], + '4913', '.gitignore'], cf=('server', 'ignore_files'), cook=list_split) SERVER_LISTEN_ALL = \ @@ -1066,13 +1102,18 @@ class OptionParser(OptionSet): argv = sys.argv[1:] # the bootstrap is always quiet, since it's running with a # default config file and so might produce warnings otherwise - self.Bootstrap = OptionSet([('configfile', CFILE)], quiet=True) - self.Bootstrap.parse(argv, do_getopt=False) - OptionSet.__init__(self, args, configfile=self.Bootstrap['configfile'], + self.bootstrap = OptionSet([('configfile', CFILE)], quiet=True) + self.bootstrap.parse(argv, do_getopt=False) + OptionSet.__init__(self, args, configfile=self.bootstrap['configfile'], quiet=quiet) self.optinfo = copy.copy(args) + # these will be set by parse() and then used by reparse() + self.argv = [] + self.do_getopt = True def reparse(self): + """ parse the options again, taking any changes (e.g., to the + config file) into account """ for key, opt in self.optinfo.items(): self[key] = opt if "args" not in self.optinfo: @@ -1085,6 +1126,7 @@ class OptionParser(OptionSet): OptionSet.parse(self, self.argv, do_getopt=self.do_getopt) def add_option(self, name, opt): + """ Add an option to the parser """ self[name] = opt self.optinfo[name] = opt diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py index aef74add4..9c6031be9 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Fam.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -7,21 +7,24 @@ import logging from time import time from Bcfg2.Server.FileMonitor import FileMonitor -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class Fam(FileMonitor): + """ file monitor with support for FAM """ + __priority__ = 90 def __init__(self, ignore=None, debug=False): FileMonitor.__init__(self, ignore=ignore, debug=debug) - self.fm = _fam.open() + self.filemonitor = _fam.open() self.users = {} def fileno(self): """Return fam file handle number.""" - return self.fm.fileno() + return self.filemonitor.fileno() - def handle_event_set(self, _): + def handle_event_set(self, _=None): self.Service() def handle_events_in_interval(self, interval): @@ -30,13 +33,13 @@ class Fam(FileMonitor): if self.Service(): now = time() - def AddMonitor(self, path, obj): + def AddMonitor(self, path, obj, _=None): """Add a monitor to path, installing a callback to obj.HandleEvent.""" mode = os.stat(path)[stat.ST_MODE] if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) + handle = self.filemonitor.monitorDirectory(path, None) else: - handle = self.fm.monitorFile(path, None) + handle = self.filemonitor.monitorFile(path, None) self.handles[handle.requestID()] = handle if obj != None: self.users[handle.requestID()] = obj @@ -50,10 +53,10 @@ class Fam(FileMonitor): start = time() now = time() while (time() - now) < interval: - if self.fm.pending(): - while self.fm.pending(): + if self.filemonitor.pending(): + while self.filemonitor.pending(): count += 1 - rawevents.append(self.fm.nextEvent()) + rawevents.append(self.filemonitor.nextEvent()) now = time() unique = [] bookkeeping = [] @@ -73,10 +76,10 @@ class Fam(FileMonitor): if event.requestID in self.users: try: self.users[event.requestID].HandleEvent(event) - except: - logger.error("Handling event for file %s" % event.filename, + except: # pylint: disable=W0702 + LOGGER.error("Handling event for file %s" % event.filename, exc_info=1) end = time() - logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % - (count, (end - start), collapsed)) + LOGGER.info("Processed %s fam events in %03.03f seconds. " + "%s coalesced" % (count, (end - start), collapsed)) return count diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 9d4330e89..d0ba59cd8 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -2,15 +2,12 @@ import os import stat -import logging # pylint: disable=F0401 from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ GAMChanged, GAMDeleted # pylint: enable=F0401 from Bcfg2.Server.FileMonitor import Event, FileMonitor -logger = logging.getLogger(__name__) - class GaminEvent(Event): """ @@ -28,6 +25,7 @@ class GaminEvent(Event): class Gamin(FileMonitor): + """ file monitor with gamin support """ __priority__ = 10 def __init__(self, ignore=None, debug=False): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 75eff3bc5..5a8a1e1c6 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -1,17 +1,18 @@ """ Inotify driver for file alteration events """ import os -import sys import logging import pyinotify # pylint: disable=F0401 -from Bcfg2.Compat import reduce +from Bcfg2.Compat import reduce # pylint: disable=W0622 from Bcfg2.Server.FileMonitor import Event from Bcfg2.Server.FileMonitor.Pseudo import Pseudo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class Inotify(Pseudo, pyinotify.ProcessEvent): + """ file monitor with inotify support """ + __priority__ = 1 # pylint: disable=E1101 action_map = {pyinotify.IN_CREATE: 'created', @@ -24,18 +25,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): def __init__(self, ignore=None, debug=False): Pseudo.__init__(self, ignore=ignore, debug=debug) - pyinotify.ProcessEvent(self) + pyinotify.ProcessEvent.__init__(self) self.event_filter = dict() self.watches_by_path = dict() # these are created in start() after the server is done forking self.notifier = None - self.wm = None + self.watchmgr = None self.add_q = [] def start(self): Pseudo.start(self) - self.wm = pyinotify.WatchManager() - self.notifier = pyinotify.ThreadedNotifier(self.wm, self) + self.watchmgr = pyinotify.WatchManager() + self.notifier = pyinotify.ThreadedNotifier(self.watchmgr, self) self.notifier.start() for monitor in self.add_q: self.AddMonitor(*monitor) @@ -43,7 +44,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): def fileno(self): if self.started: - return self.wm.get_fd() + return self.watchmgr.get_fd() else: return None @@ -54,9 +55,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): action = aname break try: - watch = self.wm.watches[ievent.wd] + watch = self.watchmgr.watches[ievent.wd] except KeyError: - logger.error("Error handling event for %s: Watch %s not found" % + LOGGER.error("Error handling event for %s: Watch %s not found" % (ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent @@ -87,7 +88,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): ievent.pathname in self.event_filter[ievent.wd]): self.events.append(evt) - def AddMonitor(self, path, obj): + def AddMonitor(self, path, obj, handleID=None): # strip trailing slashes path = path.rstrip("/") @@ -116,18 +117,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): # see if this path is already being watched try: - wd = self.watches_by_path[watch_path] + watchdir = self.watches_by_path[watch_path] except KeyError: - wd = self.wm.add_watch(watch_path, self.mask, - quiet=False)[watch_path] - self.watches_by_path[watch_path] = wd + watchdir = self.watchmgr.add_watch(watch_path, self.mask, + quiet=False)[watch_path] + self.watches_by_path[watch_path] = watchdir produce_exists = True if not is_dir: - if wd not in self.event_filter: - self.event_filter[wd] = [path] - elif path not in self.event_filter[wd]: - self.event_filter[wd].append(path) + if watchdir not in self.event_filter: + self.event_filter[watchdir] = [path] + elif path not in self.event_filter[watchdir]: + self.event_filter[watchdir].append(path) else: # we've been asked to watch a file that we're already # watching, so we don't need to produce 'exists' diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py index 089d4cf0f..9062cbfd8 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -1,12 +1,13 @@ """ Pseudo provides static monitor support for file alteration events """ import os -import logging from Bcfg2.Server.FileMonitor import FileMonitor, Event -logger = logging.getLogger(__name__) class Pseudo(FileMonitor): + """ file monitor that only produces events on server startup and + doesn't actually monitor at all """ + __priority__ = 99 def AddMonitor(self, path, obj, handleID=None): @@ -15,9 +16,9 @@ class Pseudo(FileMonitor): handleID = len(list(self.handles.keys())) self.events.append(Event(handleID, path, 'exists')) if os.path.isdir(path): - dirList = os.listdir(path) - for includedFile in dirList: - self.events.append(Event(handleID, includedFile, 'exists')) + dirlist = os.listdir(path) + for fname in dirlist: + self.events.append(Event(handleID, fname, 'exists')) self.events.append(Event(handleID, path, 'endExist')) if obj != None: diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index fd0cb66f1..1b12ab703 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -4,12 +4,13 @@ import os import sys import fnmatch import logging -import pkgutil from time import sleep, time -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class Event(object): + """ Base class for all FAM events """ def __init__(self, request_id, filename, code): self.requestID = request_id self.filename = filename @@ -53,33 +54,40 @@ class FileMonitor(object): self.started = True def debug_log(self, msg): + """ log a debug message """ if self.debug: - logger.info(msg) + LOGGER.info(msg) def should_ignore(self, event): + """ returns true if an event should be ignored """ for pattern in self.ignore: - if (fnmatch.fnmatch(event.filename, pattern) or + if (fnmatch.fnmatch(event.filename, pattern) or fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): self.debug_log("Ignoring %s" % event) return True return False def pending(self): + """ returns True if there are pending events """ return bool(self.events) def get_event(self): + """ get the oldest pending event """ return self.events.pop(0) def fileno(self): + """ get the file descriptor of the file monitor thread """ return 0 def handle_one_event(self, event): + """ handle the given event by dispatching it to the object + that handles events for the path """ if not self.started: self.start() if self.should_ignore(event): return if event.requestID not in self.handles: - logger.info("Got event for unexpected id %s, file %s" % + LOGGER.info("Got event for unexpected id %s, file %s" % (event.requestID, event.filename)) return self.debug_log("Dispatching event %s %s to obj %s" % @@ -87,12 +95,13 @@ class FileMonitor(object): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] - logger.error("Error in handling of event %s for %s: %s" % + LOGGER.error("Error in handling of event %s for %s: %s" % (event.code2str(), event.filename, err)) def handle_event_set(self, lock=None): + """ Handle all pending events """ if not self.started: self.start() count = 1 @@ -100,19 +109,18 @@ class FileMonitor(object): start = time() if lock: lock.acquire() - try: - self.handle_one_event(event) - while self.pending(): - self.handle_one_event(self.get_event()) - count += 1 - except: - pass + self.handle_one_event(event) + while self.pending(): + self.handle_one_event(self.get_event()) + count += 1 if lock: lock.release() end = time() - logger.info("Handled %d events in %.03fs" % (count, (end - start))) + LOGGER.info("Handled %d events in %.03fs" % (count, (end - start))) def handle_events_in_interval(self, interval): + """ handle events for the specified period of time (in + seconds) """ if not self.started: self.start() end = time() + interval @@ -124,10 +132,15 @@ class FileMonitor(object): sleep(0.5) def shutdown(self): + """ shutdown the monitor """ self.started = False + def AddMonitor(self, path, obj, handleID=None): + """ watch the specified path, alerting obj to events """ + raise NotImplementedError + -available = dict() +available = dict() # pylint: disable=C0103 # todo: loading the monitor drivers should be automatic from Bcfg2.Server.FileMonitor.Pseudo import Pseudo diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index eea205b75..a59214048 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -10,6 +10,9 @@ import fcntl import termios import struct from Bcfg2.Server import XI_NAMESPACE +from Bcfg2.Compat import walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -99,6 +102,7 @@ class Plugin(object): class ErrorHandler (object): """ a class to handle errors for bcfg2-lint plugins """ + def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -114,32 +118,26 @@ class ErrorHandler (object): else: self._wrapper = lambda s: [s] - self._handlers = {} + self.errors = dict() if config is not None: - for err, action in config.items(): - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug + self.RegisterErrors(config.items()) def RegisterErrors(self, errors): """ Register a dict of errors (name: default level) that a plugin may raise """ for err, action in errors.items(): - if err not in self._handlers: + if err not in self.errors: if "warn" in action: - self._handlers[err] = self.warn + self.errors[err] = self.warn elif "err" in action: - self._handlers[err] = self.error + self.errors[err] = self.error else: - self._handlers[err] = self.debug + self.errors[err] = self.debug def dispatch(self, err, msg): """ Dispatch an error to the correct handler """ - if err in self._handlers: - self._handlers[err](msg) + if err in self.errors: + self.errors[err](msg) self.logger.debug(" (%s)" % err) else: # assume that it's an error, but complain diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 468d1f190..477f88b82 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -14,20 +14,20 @@ import Bcfg2.Server import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor -from Bcfg2.Compat import MutableMapping, all # pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: from django.db import models - has_django = True + HAS_DJANGO = True except ImportError: - has_django = False + HAS_DJANGO = False -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) def locked(fd): - """Aquire a lock on a file""" + """ Acquire a lock on a file """ try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: @@ -35,13 +35,17 @@ def locked(fd): return False -if has_django: +if HAS_DJANGO: class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ django model for storing clients in the database """ hostname = models.CharField(max_length=255, primary_key=True) version = models.CharField(max_length=31, null=True) - class ClientVersions(MutableMapping): + class ClientVersions(MutableMapping, object): + """ dict-like object to make it easier to access client bcfg2 + versions from the database """ + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version @@ -77,7 +81,7 @@ if has_django: def __contains__(self, key): try: - client = MetadataClientModel.objects.get(hostname=key) + MetadataClientModel.objects.get(hostname=key) return True except MetadataClientModel.DoesNotExist: return False @@ -85,6 +89,7 @@ if has_django: class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" + def __init__(self, metadata, watch_clients, basefile): # we tell XMLFileBacked _not_ to add a monitor for this file, # because the main Metadata plugin has already added one. @@ -105,18 +110,23 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): Bcfg2.Server.FileMonitor.Pseudo) def _get_xdata(self): + """ getter for xdata property """ if not self.data: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) return self.data def _set_xdata(self, val): + """ setter for xdata property. in practice this should only be + used by the test suite """ self.data = val xdata = property(_get_xdata, _set_xdata) @property def base_xdata(self): + """ property to get the data of the base file (without any + xincludes processed) """ if not self.basedata: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) @@ -160,7 +170,6 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): newcontents = lxml.etree.tostring(dataroot, xml_declaration=False, pretty_print=True).decode('UTF-8') - fd = datafile.fileno() while locked(fd) == True: pass @@ -198,7 +207,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): 'xmltree': self.basedata, 'xquery': cli} else: - """Try to find the data in included files""" + # Try to find the data in included files for included in self.extras: try: xdata = lxml.etree.parse(included, @@ -217,7 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): if self.fam and self.should_monitor: self.fam.AddMonitor(fpath, self.metadata) - def HandleEvent(self, event): + def HandleEvent(self, event=None): """Handle fam events""" filename = os.path.basename(event.filename) if event.filename in self.extras: @@ -233,7 +242,8 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): class ClientMetadata(object): """This object contains client metadata.""" - def __init__(self, client, profile, groups, bundles, aliases, addresses, + # pylint: disable=R0913 + def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): self.hostname = client self.profile = profile @@ -248,15 +258,18 @@ class ClientMetadata(object): self.version = version try: self.version_info = Bcfg2VersionInfo(version) - except: + except (ValueError, AttributeError): self.version_info = None self.query = query + # pylint: enable=R0913 def inGroup(self, group): """Test to see if client is a member of group.""" return group in self.groups def group_in_category(self, category): + """ return the group in the given category that the client is + a member of, or the empty string """ for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp @@ -264,6 +277,9 @@ class ClientMetadata(object): class MetadataQuery(object): + """ object supplied to client metadata to allow client metadata + objects to query metadata without being able to modify it """ + def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later @@ -275,37 +291,52 @@ class MetadataQuery(object): self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): - # it's a common mistake to call by_groups, etc., in templates with - # a single string argument instead of a list. that doesn't cause - # errors because strings are iterables. this decorator warns - # about that usage. + """ decorator to warn that a MetadataQuery function that + expects a list has been called with a single string argument + instead. this is a common mistake in templates, and it + doesn't cause errors because strings are iterables """ + + # pylint: disable=C0111 + @wraps(func) def inner(arg): if isinstance(arg, str): - logger.warning("%s: %s takes a list as argument, not a string" % - (self.__class__.__name__, func.__name__)) + LOGGER.warning("%s: %s takes a list as argument, not a string" + % (self.__class__.__name__, func.__name__)) return func(arg) + # pylint: enable=C0111 + return inner def by_groups(self, groups): + """ get a list of ClientMetadata objects that are in all given + groups """ # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): + """ get a list of ClientMetadata objects that are in any of + the given profiles """ # don't need to decorate this with _warn_string because # names_by_profiles is decorated - return [self.by_name(name) for name in self.names_by_profiles(profiles)] + return [self.by_name(name) + for name in self.names_by_profiles(profiles)] def all(self): + """ get a list of all ClientMetadata objects """ return [self.by_name(name) for name in self.all_clients()] class MetadataGroup(tuple): + """ representation of a metadata group. basically just a named tuple """ + + # pylint: disable=R0913,W0613 def __new__(cls, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): if bundles is None: bundles = set() return tuple.__new__(cls, (bundles, category)) + # pylint: enable=W0613 def __init__(self, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): @@ -320,6 +351,7 @@ class MetadataGroup(tuple): self.is_private = is_private # record which clients we've warned about category suppression self.warned = [] + # pylint: enable=R0913 def __str__(self): return repr(self) @@ -347,7 +379,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.watch_clients = watch_clients self.states = dict() self.extra = dict() - self.handlers = [] + self.handlers = dict() self.groups_xml = self._handle_file("groups.xml") if (self._use_db and os.path.exists(os.path.join(self.data, "clients.xml"))): @@ -407,6 +439,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, "w").write(kwargs[aname]) def _handle_file(self, fname): + """ set up the necessary magic for handling a metadata file + (clients.xml or groups.xml, e.g.) """ if self.watch_clients: try: self.core.fam.AddMonitor(os.path.join(self.data, fname), self) @@ -416,13 +450,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) self.states[fname] = False - aname = re.sub(r'[^A-z0-9_]', '_', fname) xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) - self.handlers.append(xmlcfg.HandleEvent) + aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) + self.handlers[xmlcfg.HandleEvent] = getattr(self, + "_handle_%s_event" % aname) self.extra[fname] = [] return xmlcfg def _search_xdata(self, tag, name, tree, alias=False): + """ Generic method to find XML data (group, client, etc.) """ for node in tree.findall("//%s" % tag): if node.get("name") == name: return node @@ -442,9 +478,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self._search_xdata("Bundle", bundle_name, tree) def search_client(self, client_name, tree): + """ find a client in the given XML tree """ return self._search_xdata("Client", client_name, tree, alias=True) def _add_xdata(self, config, tag, name, attribs=None, alias=False): + """ Generic method to add XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: self.logger.error("%s \"%s\" already exists" % (tag, name)) @@ -491,6 +529,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs=attribs, alias=True) def _update_xdata(self, config, tag, name, attribs, alias=False): + """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) @@ -534,7 +573,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, else: return self.clients - def _remove_xdata(self, config, tag, name, alias=False): + def _remove_xdata(self, config, tag, name): + """ Generic method to remove XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) @@ -561,10 +601,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def remove_bundle(self, bundle_name): """Remove a bundle.""" if self._use_db: - msg = "Metadata does not support removing bundles with " + \ - "use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support removing bundles with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) @@ -582,7 +622,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, else: return self._remove_xdata(self.clients_xml, "Client", client_name) - def _handle_clients_xml_event(self, event): + def _handle_clients_xml_event(self, _): + """ handle all events for clients.xml and files xincluded from + clients.xml """ xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} @@ -640,12 +682,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() - def _handle_groups_xml_event(self, event): + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 + """ re-read groups.xml on any event on it """ self.groups = {} # these three functions must be separate functions in order to # ensure that the scope is right for the closures they return def get_condition(element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ negate = element.get('negate', 'false').lower() == 'true' pname = element.get("name") if element.tag == 'Group': @@ -654,7 +700,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return lambda c, g, _: negate != (pname == c) def get_category_condition(category, gname): - def in_cat(client, groups, categories): + """ get a predicate that returns False if a client is + already a member of a group in the given category, True + otherwise """ + def in_cat(client, groups, categories): # pylint: disable=W0613 + """ return True if the client is already a member of a + group in the category given in the enclosing function, + False otherwise """ if category in categories: if (gname not in self.groups or client not in self.groups[gname].warned): @@ -670,6 +722,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return in_cat def aggregate_conditions(conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ return lambda client, groups, cats: \ all(cond(client, groups, cats) for cond in conditions) @@ -735,14 +789,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def HandleEvent(self, event): """Handle update events for data files.""" - for hdlr in self.handlers: - aname = re.sub(r'[^A-z0-9_]', '_', - os.path.basename(event.filename)) - if hdlr(event): + for handles, event_handler in self.handlers.items(): + if handles(event): # clear the entire cache when we get an event for any # metadata file self.core.metadata_cache.expire() - getattr(self, "_handle_%s_event" % aname)(event) + event_handler(event) if False not in list(self.states.values()) and self.debug_flag: # check that all groups are real and complete. this is @@ -759,18 +811,19 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Group %s set as nonexistent group %s" % (gname, group)) - def set_profile(self, client, profile, addresspair, force=False): + def set_profile(self, client, profile, addresspair): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) if False in list(self.states.values()): - raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") - if not force and profile not in self.groups: + raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " + "been read yet") + if profile not in self.groups: msg = "Profile group %s does not exist" % profile self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not force and not group.is_public: + if not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -788,8 +841,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, (client, profiles, profile)) self.update_client(client, dict(profile=profile)) if client in self.clientgroups: - for p in profiles: - self.clientgroups[client].remove(p) + for prof in profiles: + self.clientgroups[client].remove(prof) self.clientgroups[client].append(profile) else: self.clientgroups[client] = [profile] @@ -872,7 +925,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ set group membership based on the contents of groups.xml and initial group membership of this client. Returns a tuple of (allgroups, categories)""" - numgroups = -1 # force one initial pass + numgroups = -1 # force one initial pass if categories is None: categories = dict() while numgroups != len(groups): @@ -893,10 +946,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, del categories[group.category] return (groups, categories) - def get_initial_metadata(self, client): + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): - raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") + raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " + "been read yet") client = client.lower() if client in self.core.metadata_cache: @@ -904,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client in self.aliases: client = self.aliases[client] - + groups = set() categories = dict() profile = None @@ -917,7 +971,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None), force=True) + self.set_profile(client, pgroup, (None, None)) groups.add(pgroup) category = self.groups[pgroup].category if category: @@ -958,8 +1012,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: bundles.update(self.groups[group].bundles) except KeyError: - self.logger.warning("%s: %s is a member of undefined group %s" % - (self.name, client, group)) + self.logger.warning("%s: %s is a member of undefined group %s" + % (self.name, client, group)) aliases = self.raliases.get(client, set()) addresses = self.raddresses.get(client, set()) @@ -989,6 +1043,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return rv def get_all_group_names(self): + """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) all_groups.update([g.name for g in self.group_membership.values()]) @@ -998,10 +1053,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return all_groups def get_all_groups_in_category(self, category): + """ return a list of names of groups in the given category """ return set([g.name for g in self.groups.values() if g.category == category]) def get_client_names_by_profiles(self, profiles): + """ return a list of names of clients in the given profile groups """ rv = [] for client in list(self.clients): mdata = self.get_initial_metadata(client) @@ -1010,13 +1067,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return rv def get_client_names_by_groups(self, groups): - mdata = [self.core.build_metadata(client) - for client in list(self.clients)] + """ return a list of names of clients in the given groups """ + mdata = [self.core.build_metadata(client) for client in self.clients] return [md.hostname for md in mdata if md.groups.issuperset(groups)] def get_client_names_by_bundles(self, bundles): - mdata = [self.core.build_metadata(client) - for client in list(self.clients.keys())] + """ given a list of bundles, return a list of names of clients + that use those bundles """ + mdata = [self.core.build_metadata(client) for client in self.clients] return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] def merge_additional_groups(self, imd, groups): @@ -1071,6 +1129,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error("Resolved to %s" % resolved) return False + # pylint: disable=R0911,R0912 def AuthenticateConnection(self, cert, user, password, address): """This function checks auth creds.""" if not isinstance(user, str): @@ -1121,8 +1180,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client not in self.passwords: if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" - % address[0]) + self.logger.error("Client %s in secure mode but has no " + "password" % address[0]) return False if password != self.password: self.logger.error("Client %s used incorrect global password" % @@ -1147,9 +1206,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if user != 'root': self.session_cache[address] = (time.time(), client) return True + # pylint: enable=R0911,R0912 def process_statistics(self, meta, _): - """Hook into statistics interface to toggle clients in bootstrap mode.""" + """ Hook into statistics interface to toggle clients in + bootstrap mode """ client = meta.hostname if client in self.auth and self.auth[client] == 'bootstrap': self.update_client(client, dict(auth='cert')) @@ -1159,23 +1220,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if only_client: clientmeta = self.core.build_metadata(only_client) - def include_client(client): - return not only_client or client != only_client - - def include_bundle(bundle): - return not only_client or bundle in clientmeta.bundles - - def include_group(group): - return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"), - parser=Bcfg2.Server.XMLParser) - try: - groups_tree.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for groups.xml: %s" % - sys.exc_info()[1]) - groups = groups_tree.getroot() + groups = self.groups_xml.xdata.getroot() categories = {'default': 'grey83'} viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') @@ -1186,37 +1231,72 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if None in categories: del categories[None] if hosts: - instances = {} - for client in list(self.clients): - if include_client(client): - continue - if client in self.clientgroups: - groups = self.clientgroups[client] - elif self.default: - groups = [self.default] - else: - continue - for group in groups: - try: - instances[group].append(client) - except KeyError: - instances[group] = [client] - for group, clist in list(instances.items()): - clist.sort() - viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % - (group, '|'.join(clist))) - viz_str.append('"%s-instances" -> "group-%s";' % - (group, group)) + viz_str.extend(self._viz_hosts(only_client)) if bundles: - bundles = [] - [bundles.append(bund.get('name')) \ - for bund in groups.findall('.//Bundle') \ - if bund.get('name') not in bundles \ - and include_bundle(bund.get('name'))] - bundles.sort() - for bundle in bundles: - viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % - (bundle, bundle)) + viz_str.extend(self._viz_bundles(bundles, clientmeta)) + viz_str.extend(self._viz_groups(egroups, bundles, clientmeta)) + if key: + for category in categories: + viz_str.append('"%s" [label="%s", shape="record", ' + 'style="filled", fillcolor="%s"];' % + (category, category, categories[category])) + return "\n".join("\t" + s for s in viz_str) + + def _viz_hosts(self, only_client): + """ add hosts to the viz graph """ + def include_client(client): + """ return True if the given client should be included in + the graph""" + return not only_client or client != only_client + + instances = {} + rv = [] + for client in list(self.clients): + if include_client(client): + continue + if client in self.clientgroups: + grps = self.clientgroups[client] + elif self.default: + grps = [self.default] + else: + continue + for group in grps: + try: + instances[group].append(client) + except KeyError: + instances[group] = [client] + for group, clist in list(instances.items()): + clist.sort() + rv.append('"%s-instances" [ label="%s", shape="record" ];' % + (group, '|'.join(clist))) + rv.append('"%s-instances" -> "group-%s";' % (group, group)) + return rv + + def _viz_bundles(self, bundles, clientmeta): + """ add bundles to the viz graph """ + + def include_bundle(bundle): + """ return True if the given bundle should be included in + the graph""" + return not clientmeta or bundle in clientmeta.bundles + + bundles = list(set(bund.get('name')) + for bund in self.groups_xml.xdata.findall('.//Bundle') + if include_bundle(bund.get('name'))) + bundles.sort() + return ['"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, + bundle) + for bundle in bundles] + + def _viz_groups(self, egroups, bundles, clientmeta): + """ add groups to the viz graph """ + + def include_group(group): + """ return True if the given group should be included in + the graph """ + return not clientmeta or group in clientmeta.groups + + rv = [] gseen = [] for group in egroups: if group.get('profile', 'false') == 'true': @@ -1225,31 +1305,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, style = "filled" gseen.append(group.get('name')) if include_group(group.get('name')): - viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % - (group.get('name'), group.get('name'), style, - group.get('color'))) + rv.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' + % (group.get('name'), group.get('name'), style, + group.get('color'))) if bundles: for bundle in group.findall('Bundle'): - viz_str.append('"group-%s" -> "bundle-%s";' % - (group.get('name'), bundle.get('name'))) + rv.append('"group-%s" -> "bundle-%s";' % + (group.get('name'), bundle.get('name'))) gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' for group in egroups: for parent in group.findall('Group'): - if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str.append(gfmt % (parent.get('name'), - parent.get('name'))) + if (parent.get('name') not in gseen and + include_group(parent.get('name'))): + rv.append(gfmt % (parent.get('name'), + parent.get('name'))) gseen.append(parent.get("name")) if include_group(group.get('name')): - viz_str.append('"group-%s" -> "group-%s";' % - (group.get('name'), parent.get('name'))) - if key: - for category in categories: - viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % - (category, category, categories[category])) - return "\n".join("\t" + s for s in viz_str) + rv.append('"group-%s" -> "group-%s";' % + (group.get('name'), parent.get('name'))) class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): + """ bcfg2-lint plugin for Metadata """ + def Run(self): self.nested_clients() self.deprecated_options() @@ -1260,6 +1338,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "deprecated-clients-options": "warning"} def deprecated_options(self): + """ check for the location='floating' option, which has been + deprecated in favor of floating='true' """ clientdata = self.metadata.clients_xml.xdata for el in clientdata.xpath("//Client"): loc = el.get("location") @@ -1271,9 +1351,11 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("deprecated-clients-options", "The location='%s' option is deprecated. " "Please use floating='%s' instead: %s" % - (loc, floating, self.RenderXML(el))) + (loc, floating, self.RenderXML(el))) def nested_clients(self): + """ check for a Client tag inside a Client tag, which doesn't + make any sense """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 49e3b5e63..700c5e2e8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -132,6 +132,12 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): encoding) fam.AddMonitor(path, self) + def HandleEvent(self, event): + """ handle events on everything but probed.xml """ + if (event.filename != self.path and + not event.filename.endswith("probed.xml")): + return self.handle_event(event) + def get_probe_data(self, metadata): """ Get an XML description of all probes for a client suitable for sending to that client. diff --git a/src/lib/Bcfg2/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index f79b51dd3..3eb300a98 100644 --- a/src/lib/Bcfg2/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -6,8 +6,8 @@ __all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins", "Hostbase", "Reports", "Snapshots", "XMLParser", "XI", "XI_NAMESPACE"] -XMLParser = lxml.etree.XMLParser(remove_blank_text=True) - XI = 'http://www.w3.org/2001/XInclude' XI_NAMESPACE = '{%s}' % XI +# pylint: disable=C0103 +XMLParser = lxml.etree.XMLParser(remove_blank_text=True) diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index bae6497a9..0328c6bea 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,15 +1,18 @@ +""" Django database models for all plugins """ + import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins from django.db import models -from Bcfg2.Compat import ConfigParser -logger = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger('Bcfg2.Server.models') MODELS = [] + def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): + """ load models from plugins specified in the config """ global MODELS if plugins is None: @@ -19,9 +22,10 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): plugin_opt = Bcfg2.Options.SERVER_PLUGINS plugin_opt.default = Bcfg2.Server.Plugins.__all__ - setup = Bcfg2.Options.OptionParser(dict(plugins=plugin_opt, - configfile=Bcfg2.Options.CFILE), - quiet=quiet) + setup = \ + Bcfg2.Options.OptionParser(dict(plugins=plugin_opt, + configfile=Bcfg2.Options.CFILE), + quiet=quiet) setup.parse([Bcfg2.Options.CFILE.cmd, cfile]) plugins = setup['plugins'] @@ -42,7 +46,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): try: err = sys.exc_info()[1] mod = __import__(plugin) - except: + except: # pylint: disable=W0702 if plugins != Bcfg2.Server.Plugins.__all__: # only produce errors if the default plugin list # was not used -- i.e., if the config file was set @@ -50,7 +54,8 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # all plugins, IOW. the error from the first # attempt to import is probably more accurate than # the second attempt. - logger.error("Failed to load plugin %s: %s" % (plugin, err)) + LOGGER.error("Failed to load plugin %s: %s" % (plugin, + err)) continue for sym in dir(mod): obj = getattr(mod, sym) @@ -62,16 +67,16 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # and thus that this module will always work. load_models(quiet=True) -# Monitor our internal db version + class InternalDatabaseVersion(models.Model): - """Object that tell us to witch version is the database.""" + """ Object that tell us to which version the database is """ version = models.IntegerField() updated = models.DateTimeField(auto_now_add=True) def __str__(self): - return "version %d updated the %s" % (self.version, self.updated.isoformat()) + return "version %d updated the %s" % (self.version, + self.updated.isoformat()) - class Meta: + class Meta: # pylint: disable=C0111,W0232 app_label = "reports" get_latest_by = "version" - diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py index bee90bbf4..2379dd1c8 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py @@ -1,4 +1,10 @@ +""" module for tracking execution time statistics from the bcfg2 +server core """ + + class Statistic(object): + """ a single named statistic, tracking minimum, maximum, and + average execution time, and number of invocations """ def __init__(self, name, initial_value): self.name = name self.min = float(initial_value) @@ -7,24 +13,29 @@ class Statistic(object): self.count = 1 def add_value(self, value): + """ add a value to the statistic """ self.min = min(self.min, value) self.max = max(self.max, value) self.ave = (((self.ave * (self.count - 1)) + value) / self.count) self.count += 1 def get_value(self): + """ get a tuple of all the stats tracked on this named item """ return (self.name, (self.min, self.max, self.ave, self.count)) class Statistics(object): + """ A collection of named statistics """ def __init__(self): self.data = dict() def add_value(self, name, value): + """ add a value to the named statistic """ if name not in self.data: self.data[name] = Statistic(name, value) else: self.data[name].add_value(value) def display(self): + """ return a dict of all statistics """ return dict([value.get_value() for value in list(self.data.values())]) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 7e18fde09..a904bead5 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -1,12 +1,14 @@ +""" Django settings for the Bcfg2 server """ + import os import sys import Bcfg2.Options try: import django - has_django = True -except: - has_django = False + HAS_DJANGO = True +except ImportError: + HAS_DJANGO = False DATABASES = dict() @@ -25,9 +27,11 @@ TEMPLATE_DEBUG = DEBUG MEDIA_URL = '/site_media' -# default config file is /etc/bcfg2-web.conf, UNLESS /etc/bcfg2.conf -# exists AND /etc/bcfg2-web.conf does not exist. + def _default_config(): + """ get the default config file. returns /etc/bcfg2-web.conf, + UNLESS /etc/bcfg2.conf exists AND /etc/bcfg2-web.conf does not + exist. """ optinfo = dict(cfile=Bcfg2.Options.CFILE, web_cfile=Bcfg2.Options.WEB_CFILE) setup = Bcfg2.Options.OptionParser(optinfo, quiet=True) @@ -40,16 +44,20 @@ def _default_config(): DEFAULT_CONFIG = _default_config() + def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): + """ read the config file and set django settings based on it """ + # pylint: disable=W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ MEDIA_URL + # pylint: enable=W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): - print("%s does not exist, using %s for database configuration" % + print("%s does not exist, using %s for database configuration" % (cfile, DEFAULT_CONFIG)) cfile = DEFAULT_CONFIG - + optinfo = Bcfg2.Options.DATABASE_COMMON_OPTIONS optinfo['repo'] = Bcfg2.Options.SERVER_REPOSITORY # when setting a different config file, it has to be set in either @@ -71,7 +79,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): HOST=setup['db_host'], PORT=setup['db_port']) - if has_django and django.VERSION[0] == 1 and django.VERSION[1] < 2: + if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] DATABASE_NAME = DATABASES['default']['NAME'] DATABASE_USER = DATABASES['default']['USER'] @@ -166,7 +174,7 @@ TEMPLATE_DIRS = ( ) # TODO - sanitize this -if has_django and django.VERSION[0] == 1 and django.VERSION[1] < 2: +if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: TEMPLATE_CONTEXT_PROCESSORS = ( 'django.core.context_processors.auth', 'django.core.context_processors.debug', @@ -182,4 +190,3 @@ else: 'django.core.context_processors.media', 'django.core.context_processors.request' ) - diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index cd622eafc..09b8e5153 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -1,8 +1,14 @@ +""" bcfg2 version declaration and handling """ + import re __version__ = "1.3.0pre1" + class Bcfg2VersionInfo(tuple): + """ object to make granular version operations (particularly + comparisons) easier """ + v_re = re.compile(r'(\d+)(\w+)(\d+)') def __new__(cls, vstr): @@ -17,30 +23,31 @@ class Bcfg2VersionInfo(tuple): return tuple.__new__(cls, [int(major), int(minor), int(micro), releaselevel, int(serial)]) - def __init__(self, vstr): + def __init__(self, vstr): # pylint: disable=W0613 tuple.__init__(self) self.major, self.minor, self.micro, self.releaselevel, self.serial = \ tuple(self) - + def __repr__(self): return "(major=%s, minor=%s, micro=%s, releaselevel=%s, serial=%s)" % \ tuple(self) - def _release_cmp(self, r1, r2): - if r1 == r2: + def _release_cmp(self, rel1, rel2): # pylint: disable=R0911 + """ compare two release numbers """ + if rel1 == rel2: return 0 - elif r1 == "final": + elif rel1 == "final": return -1 - elif r2 == "final": + elif rel2 == "final": return 1 - elif r1 == "rc": + elif rel1 == "rc": return -1 - elif r2 == "rc": + elif rel2 == "rc": return 1 # should never get to anything past this point - elif r1 == "pre": + elif rel1 == "pre": return -1 - elif r2 == "pre": + elif rel2 == "pre": return 1 else: # wtf? @@ -54,19 +61,12 @@ class Bcfg2VersionInfo(tuple): return True try: for i in range(3): - if self[i] > version[i]: - return True - elif self[i] < version[i]: - return False + if self[i] != version[i]: + return self[i] > version[i] rel = self._release_cmp(self[3], version[3]) - if rel < 0: - return True - elif rel > 0: - return False - if self[4] > version[4]: - return True - else: - return False + if rel != 0: + return rel < 0 + return self[4] > version[4] except TypeError: return self > Bcfg2VersionInfo(version) @@ -78,19 +78,12 @@ class Bcfg2VersionInfo(tuple): return False try: for i in range(3): - if self[i] < version[i]: - return True - elif self[i] > version[i]: - return False + if self[i] != version[i]: + return self[i] < version[i] rel = self._release_cmp(self[3], version[3]) - if rel > 0: - return True - elif rel < 0: - return False - if self[4] < version[4]: - return True - else: - return False + if rel != 0: + return rel > 0 + return self[4] < version[4] except TypeError: return self < Bcfg2VersionInfo(version) diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index bae4ad8ef..904f77611 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -9,59 +9,48 @@ import lxml.etree import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Server import XMLParser -from Bcfg2.Compat import input +from Bcfg2.Compat import input # pylint: disable=W0622 try: import Bcfg2.Encryption except ImportError: - err = sys.exc_info()[1] - print("Could not import %s. Is M2Crypto installed?" % err) + print("Could not import %s. Is M2Crypto installed?" % sys.exc_info()[1]) raise SystemExit(1) -LOGGER = None - -def get_logger(verbose=0): - """ set up logging according to the verbose level given on the - command line """ - global LOGGER - if LOGGER is None: - LOGGER = logging.getLogger(sys.argv[0]) - stderr = logging.StreamHandler() - if verbose: - level = logging.DEBUG - else: - level = logging.WARNING - LOGGER.setLevel(level) - LOGGER.addHandler(stderr) - syslog = logging.handlers.SysLogHandler("/dev/log") - syslog.setFormatter(logging.Formatter("%(name)s: %(message)s")) - LOGGER.addHandler(syslog) - return LOGGER - class EncryptionChunkingError(Exception): + """ error raised when Encryptor cannot break a file up into chunks + to be encrypted, or cannot reassemble the chunks """ pass class Encryptor(object): + """ Generic encryptor for all files """ + def __init__(self, setup): self.setup = setup - self.logger = get_logger() self.passphrase = None self.pname = None + self.logger = logging.getLogger(self.__class__.__name__) def get_encrypted_filename(self, plaintext_filename): + """ get the name of the file encrypted data should be written to """ return plaintext_filename def get_plaintext_filename(self, encrypted_filename): + """ get the name of the file decrypted data should be written to """ return encrypted_filename def chunk(self, data): + """ generator to break the file up into smaller chunks that + will each be individually encrypted or decrypted """ yield data - def unchunk(self, data, original): + def unchunk(self, data, original): # pylint: disable=W0613 + """ given chunks of a file, reassemble then into the whole file """ return data[0] def set_passphrase(self): + """ set the passphrase for the current file """ if (not self.setup.cfp.has_section("encryption") or self.setup.cfp.options("encryption") == 0): self.logger.error("No passphrases available in %s" % @@ -98,6 +87,7 @@ class Encryptor(object): return False def encrypt(self, fname): + """ encrypt the given file, returning the encrypted data """ try: plaintext = open(fname).read() except IOError: @@ -124,12 +114,16 @@ class Encryptor(object): return False return self.unchunk(crypted, plaintext) + # pylint: disable=W0613 def _encrypt(self, plaintext, passphrase, name=None): + """ encrypt a single chunk of a file """ return Bcfg2.Encryption.ssl_encrypt( plaintext, passphrase, Bcfg2.Encryption.get_algorithm(self.setup)) + # pylint: enable=W0613 def decrypt(self, fname): + """ decrypt the given file, returning the plaintext data """ try: crypted = open(fname).read() except IOError: @@ -149,12 +143,12 @@ class Encryptor(object): except Bcfg2.Encryption.EVPError: self.logger.info("Could not decrypt %s with the " "specified passphrase" % fname) - return False + continue except: err = sys.exc_info()[1] self.logger.error("Error decrypting %s: %s" % (fname, err)) - return False + continue except TypeError: pchunk = None for pname in self.setup.cfp.options('encryption'): @@ -175,7 +169,7 @@ class Encryptor(object): self.logger.error("Could not decrypt %s with any " "passphrase in %s" % (fname, self.setup['configfile'])) - return False + continue except EncryptionChunkingError: err = sys.exc_info()[1] self.logger.error("Error getting encrypted data from %s: %s" % @@ -190,7 +184,14 @@ class Encryptor(object): (fname, err)) return False + def _decrypt(self, crypted, passphrase): + """ decrypt a single chunk """ + return Bcfg2.Encryption.ssl_decrypt( + crypted, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)) + def write_encrypted(self, fname, data=None): + """ write encrypted data to disk """ if data is None: data = self.decrypt(fname) new_fname = self.get_encrypted_filename(fname) @@ -210,6 +211,7 @@ class Encryptor(object): return False def write_decrypted(self, fname, data=None): + """ write decrypted data to disk """ if data is None: data = self.decrypt(fname) new_fname = self.get_plaintext_filename(fname) @@ -224,6 +226,7 @@ class Encryptor(object): return False def get_passphrase(self, chunk): + """ get the passphrase for a chunk of a file """ pname = self._get_passphrase(chunk) if not self.pname: if not pname: @@ -246,16 +249,14 @@ class Encryptor(object): (self.pname, pname)) return (passphrase, pname) - def _get_passphrase(self, chunk): + def _get_passphrase(self, chunk): # pylint: disable=W0613 + """ get the passphrase for a chunk of a file """ return None - def _decrypt(self, crypted, passphrase): - return Bcfg2.Encryption.ssl_decrypt( - crypted, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ + def get_encrypted_filename(self, plaintext_filename): return plaintext_filename + ".crypt" @@ -267,6 +268,8 @@ class CfgEncryptor(Encryptor): class PropertiesEncryptor(Encryptor): + """ encryptor class for Properties files """ + def _encrypt(self, plaintext, passphrase, name=None): # plaintext is an lxml.etree._Element if name is None: @@ -347,7 +350,7 @@ class PropertiesEncryptor(Encryptor): return crypted -def main(): +def main(): # pylint: disable=R0912,R0915 optinfo = dict(interactive=Bcfg2.Options.INTERACTIVE) optinfo.update(Bcfg2.Options.CRYPT_OPTIONS) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) @@ -360,35 +363,40 @@ def main(): print(setup.hm) raise SystemExit(1) + log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) + if setup['verbose']: + log_args['to_console'] = logging.DEBUG + Bcfg2.Logger.setup_logging('bcfg2-crypt', **log_args) + logger = logging.getLogger('bcfg2-crypt') + if setup['decrypt']: if setup['encrypt']: - print("You cannot specify both --encrypt and --decrypt") + logger.error("You cannot specify both --encrypt and --decrypt") raise SystemExit(1) elif setup['remove']: - print("--remove cannot be used with --decrypt, ignoring") + logger.error("--remove cannot be used with --decrypt, ignoring") setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default elif setup['interactive']: - print("Cannot decrypt interactively") + logger.error("Cannot decrypt interactively") setup['interactive'] = False if setup['cfg']: if setup['properties']: - print("You cannot specify both --cfg and --properties") + logger.error("You cannot specify both --cfg and --properties") raise SystemExit(1) if setup['xpath']: - print("Specifying --xpath with --cfg is nonsensical, ignoring " - "--xpath") + logger.error("Specifying --xpath with --cfg is nonsensical, " + "ignoring --xpath") setup['xpath'] = Bcfg2.Options.CRYPT_XPATH.default if setup['interactive']: - print("You cannot use interactive mode with --cfg, ignoring -I") + logger.error("You cannot use interactive mode with --cfg, " + "ignoring -I") setup['interactive'] = False elif setup['properties']: if setup['remove']: - print("--remove cannot be used with --properties, ignoring") + logger.error("--remove cannot be used with --properties, ignoring") setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - logger = get_logger(setup['verbose']) - props_crypt = PropertiesEncryptor(setup) cfg_crypt = CfgEncryptor(setup) @@ -456,7 +464,7 @@ def main(): if data is None: data = xform(fname) if not data: - print("Failed to %s %s, skipping" % (xform.__name__, fname)) + logger.error("Failed to %s %s, skipping" % (xform.__name__, fname)) continue if setup['crypt_stdout']: if len(setup['args']) > 1: diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index f1f91b7f4..0321d3045 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -9,108 +9,89 @@ import Bcfg2.Logger import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Lint -# Compatibility imports from Bcfg2.Compat import ConfigParser -logger = logging.getLogger('bcfg2-lint') +LOGGER = logging.getLogger('bcfg2-lint') -def run_serverless_plugins(plugins, config=None, setup=None, errorhandler=None): - logger.debug("Running serverless plugins") + +def run_serverless_plugins(plugins, setup=None, errorhandler=None, files=None): + """ Run serverless plugins """ + LOGGER.debug("Running serverless plugins") for plugin_name, plugin in list(plugins.items()): run_plugin(plugin, plugin_name, errorhandler=errorhandler, - setup=setup, config=config, files=files) + setup=setup, files=files) + -def run_server_plugins(plugins, config=None, setup=None, errorhandler=None): +def run_server_plugins(plugins, setup=None, errorhandler=None, files=None): + """ run plugins that require a running server to run """ core = load_server(setup) - logger.debug("Running server plugins") + LOGGER.debug("Running server plugins") for plugin_name, plugin in list(plugins.items()): run_plugin(plugin, plugin_name, args=[core], errorhandler=errorhandler, - setup=setup, config=config, files=files) + setup=setup, files=files) + def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, - args=None, config=None, files=None): - logger.debug(" Running %s" % plugin_name) + args=None, files=None): + """ run a single plugin, server-ful or serverless. """ + LOGGER.debug(" Running %s" % plugin_name) if args is None: args = [] if errorhandler is None: - errorhandler = get_errorhandler(config) + errorhandler = get_errorhandler(setup) - if config is not None and config.has_section(plugin_name): + if setup is not None and setup.cfp.has_section(plugin_name): arg = setup - for key, val in config.items(plugin_name): + for key, val in setup.cfp.items(plugin_name): arg[key] = val args.append(arg) else: args.append(setup) - - # older versions of python do not support mixing *-magic and - # non-*-magic (e.g., "plugin(*args, files=files)", so we do this - # all with *-magic - kwargs = dict(files=files, errorhandler=errorhandler) - - return plugin(*args, **kwargs).Run() - -def get_errorhandler(config): + + return plugin(*args, files=files, errorhandler=errorhandler).Run() + + +def get_errorhandler(setup): """ get a Bcfg2.Server.Lint.ErrorHandler object """ - if config.has_section("errors"): - conf = dict(config.items("errors")) + if setup.cfp.has_section("errors"): + conf = dict(setup.cfp.items("errors")) else: conf = None return Bcfg2.Server.Lint.ErrorHandler(config=conf) + def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) core.fam.handle_events_in_interval(4) return core + def load_plugin(module, obj_name=None): + """ load a single plugin """ parts = module.split(".") if obj_name is None: obj_name = parts[-1] mod = __import__(module) - for p in parts[1:]: - mod = getattr(mod, p) + for part in parts[1:]: + mod = getattr(mod, part) return getattr(mod, obj_name) -if __name__ == '__main__': - optinfo = dict(config=Bcfg2.Options.LINT_CONFIG, - showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, - stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, - schema=Bcfg2.Options.SCHEMA_PATH, - plugins=Bcfg2.Options.SERVER_PLUGINS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-info', **log_args) - - config = ConfigParser.SafeConfigParser() - config.read(setup['configfile']) - config.read(setup['config']) - # get list of plugins to run +def load_plugins(setup): + """ get list of plugins to run """ if setup['args']: plugin_list = setup['args'] elif "bcfg2-repo-validate" in sys.argv[0]: plugin_list = 'Duplicates,RequiredAttrs,Validate'.split(',') else: try: - plugin_list = config.get('lint', 'plugins').split(',') + plugin_list = setup.cfp.get('lint', 'plugins').split(',') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): plugin_list = Bcfg2.Server.Lint.__all__ - if setup['stdin']: - files = [s.strip() for s in sys.stdin.readlines()] - else: - files = None - allplugins = dict() for plugin in plugin_list: try: @@ -121,12 +102,12 @@ if __name__ == '__main__': load_plugin("Bcfg2.Server.Plugins." + plugin, obj_name=plugin + "Lint") except (ImportError, AttributeError): - err = sys.exc_info()[1] - logger.error("Failed to load plugin %s: %s" % - (plugin + "Lint", err)) + err = sys.exc_info()[1] + LOGGER.error("Failed to load plugin %s: %s" % + (plugin + "Lint", err)) except AttributeError: err = sys.exc_info()[1] - logger.error("Failed to load plugin %s: %s" % (plugin, err)) + LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) serverplugins = dict() serverlessplugins = dict() @@ -136,21 +117,47 @@ if __name__ == '__main__': serverplugins[plugin_name] = plugin else: serverlessplugins[plugin_name] = plugin + return (serverlessplugins, serverplugins) + + +def main(): + optinfo = dict(lint_config=Bcfg2.Options.LINT_CONFIG, + showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, + stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, + schema=Bcfg2.Options.SCHEMA_PATH, + plugins=Bcfg2.Options.SERVER_PLUGINS) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) + setup.parse(sys.argv[1:]) + + log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) + if setup['verbose']: + log_args['to_console'] = logging.DEBUG + Bcfg2.Logger.setup_logging('bcfg2-info', **log_args) - errorhandler = get_errorhandler(config) + setup.cfp.read(setup['lint_config']) + + if setup['stdin']: + files = [s.strip() for s in sys.stdin.readlines()] + else: + files = None + + (serverlessplugins, serverplugins) = load_plugins(setup) + + errorhandler = get_errorhandler(setup) if setup['showerrors']: for plugin in serverplugins.values() + serverlessplugins.values(): errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) print("%-35s %-35s" % ("Error name", "Handler")) - for err, handler in errorhandler._handlers.items(): + for err, handler in errorhandler.errors.items(): print("%-35s %-35s" % (err, handler.__name__)) raise SystemExit(0) - run_serverless_plugins(serverlessplugins, - errorhandler=errorhandler, - config=config, setup=setup) + run_serverless_plugins(serverlessplugins, errorhandler=errorhandler, + setup=setup, files=files) if serverplugins: if errorhandler.errors: @@ -166,7 +173,7 @@ if __name__ == '__main__': "plugins") else: run_server_plugins(serverplugins, errorhandler=errorhandler, - config=config, setup=setup) + setup=setup, files=files) if errorhandler.errors or errorhandler.warnings or setup['verbose']: print("%d errors" % errorhandler.errors) @@ -176,3 +183,6 @@ if __name__ == '__main__': raise SystemExit(2) elif errorhandler.warnings: raise SystemExit(3) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 8323eeb22..815d2740c 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -32,6 +32,8 @@ class ClientTest(TestCase): self.ignore = ignore def ignore_entry(self, tag, name): + """ return True if an error on a given entry should be ignored + """ if tag in self.ignore: if name in self.ignore[tag]: return True @@ -43,6 +45,7 @@ class ClientTest(TestCase): return False def runTest(self): + """ run this individual test """ config = self.bcfg2_core.BuildConfiguration(self.client) failures = [] @@ -86,6 +89,7 @@ def main(): ignore[tag] = [name] def run_tests(): + """ Run the test suite """ core.fam.handle_events_in_interval(0.1) if setup['args']: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 2aebb0705..13e514e77 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -96,11 +96,11 @@ def get_metadata_object(core=None, watch_clients=False, use_db=False): class TestMetadataDB(DBModelTestCase): - if has_django: + if HAS_DJANGO: models = [MetadataClientModel] -if has_django or can_skip: +if HAS_DJANGO or can_skip: class TestClientVersions(Bcfg2TestCase): test_clients = dict(client1="1.2.0", client2="1.2.2", @@ -109,7 +109,7 @@ if has_django or can_skip: client5=None, client6=None) - @skipUnless(has_django, "Django not found") + @skipUnless(HAS_DJANGO, "Django not found") def setUp(self): syncdb(TestMetadataDB) for client, version in self.test_clients.items(): @@ -421,7 +421,7 @@ class TestMetadata(_TestMetadata, TestStatistics, TestDatabaseBacked): return get_metadata_object(core=core, watch_clients=watch_clients, use_db=self.use_db) - @skipUnless(has_django, "Django not found") + @skipUnless(HAS_DJANGO, "Django not found") def test__use_db(self): # with the way we've set up our metadata tests, it's unweildy # to test _use_db. however, given the way get_obj works, if @@ -1184,7 +1184,7 @@ class TestMetadataBase(TestMetadata): __test__ = False use_db = True - @skipUnless(has_django, "Django not found") + @skipUnless(HAS_DJANGO, "Django not found") def setUp(self): syncdb(TestMetadataDB) @@ -1291,7 +1291,7 @@ class TestMetadata_NoClientsXML(TestMetadataBase): # have django. otherwise they'll all get run because our fake # skipping decorators for python < 2.7 won't work when they # decorate setUp() - if can_skip or has_django: + if can_skip or HAS_DJANGO: __test__ = True def load_groups_data(self, metadata=None, xdata=None): @@ -1456,7 +1456,7 @@ class TestMetadata_ClientsXML(TestMetadataBase): # have django. otherwise they'll all get run because our fake # skipping decorators for python < 2.7 won't work when they # decorate setUp() - if can_skip or has_django: + if can_skip or HAS_DJANGO: __test__ = True def load_clients_data(self, metadata=None, xdata=None): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 34b2a0f0e..0ad92ca72 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -25,6 +25,7 @@ from TestPlugin import TestEntrySet, TestProbing, TestConnector, \ # test data for JSON and YAML tests test_data = dict(a=1, b=[1, 2, 3], c="test") + class FakeList(list): pass @@ -32,7 +33,7 @@ class FakeList(list): class TestProbesDB(DBModelTestCase): if HAS_DJANGO: models = [ProbesGroupsModel, ProbesDataModel] - + class TestClientProbeDataSet(Bcfg2TestCase): def test__init(self): @@ -40,11 +41,12 @@ class TestClientProbeDataSet(Bcfg2TestCase): self.assertLessEqual(ds.timestamp, time.time()) self.assertIsInstance(ds, dict) self.assertNotIn("timestamp", ds) - + ds = ClientProbeDataSet(timestamp=123) self.assertEqual(ds.timestamp, 123) self.assertNotIn("timestamp", ds) + class TestProbeData(Bcfg2TestCase): def test_str(self): # a value that is not valid XML, JSON, or YAML @@ -108,10 +110,32 @@ class TestProbeSet(TestEntrySet): fam.AddMonitor.assert_called_with(datastore, ps) TestEntrySet.test__init(self) + def test_HandleEvent(self): + ps = self.get_obj() + ps.handle_event = Mock() + + # test that events on the data store itself are skipped + evt = Mock() + evt.filename = datastore + ps.HandleEvent(evt) + self.assertFalse(ps.handle_event.called) + + # test that events on probed.xml are skipped + evt.reset_mock() + evt.filename = "probed.xml" + ps.HandleEvent(evt) + self.assertFalse(ps.handle_event.called) + + # test that other events are processed appropriately + evt.reset_mock() + evt.filename = "fooprobe" + ps.HandleEvent(evt) + ps.handle_event.assert_called_with(evt) + @patch("%s.list" % builtins, FakeList) def test_get_probe_data(self): ps = self.get_obj() - + # build some fairly complex test data for this. in the end, # we want the probe data to include only the most specific # version of a given probe, and by basename only, not full @@ -209,7 +233,7 @@ text return {"foo.example.com": ["group", "group with spaces", "group-with-dashes"], "bar.example.com": []} - + def get_probes_object(self, use_db=False, load_data=None): core = Mock() core.setup.cfp.getboolean = Mock() @@ -229,7 +253,7 @@ text return Probes(core, datastore) return inner() - + def test__init(self): mock_load_data = Mock() probes = self.get_probes_object(load_data=mock_load_data) @@ -272,7 +296,7 @@ text probes.probedata = self.get_test_probedata() probes.cgroups = self.get_test_cgroups() probes._write_data_xml(None) - + mock_open.assert_called_with(os.path.join(datastore, probes.name, "probed.xml"), "w") data = lxml.etree.XML(mock_open.return_value.write.call_args[0][0]) @@ -333,7 +357,7 @@ text client = Mock() client.hostname = cname probes._write_data_db(client) - + pdata = ProbesDataModel.objects.filter(hostname=cname).all() self.assertEqual(len(pdata), len(probes.probedata[cname])) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 41a91caff..ac739f4e5 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -39,43 +39,20 @@ except OSError: # perform a full range of code checks on the listed files. full_checks = { + "lib/Bcfg2": ["*.py"], "lib/Bcfg2/Server": ["Lint", "Plugin", - "BuiltinCore.py", - "CherryPyCore.py", - "Core.py"], - "lib/Bcfg2/Server/Plugins": ["Bundler.py", - "Bzr.py", - "Cfg", - "Cvs.py", - "DBStats.py", - "Darcs.py", - "Defaults.py", - "FileProbes.py", - "Fossil.py", - "Git.py", - "GroupPatterns.py", - "Guppy.py", - "Hg.py", - "Ohai.py", - "Packages", - "Probes.py", - "Properties.py", - "PuppetENC.py", - "Rules.py", - "SEModules.py", - "ServiceCompat.py", - "Svn.py", - "Svn2.py", - "TemplateHelper.py", - "Trigger.py", - ], + "FileMonitor", + "*.py"], + "lib/Bcfg2/Server/Plugins": ["Cfg", "Packages", "*.py"], + "lib/Bcfg2/Client": ["*.py"], "lib/Bcfg2/Client/Tools": ["POSIX"], } # perform full code checks on the listed executables sbin_checks = { - "sbin": ["bcfg2-server", "bcfg2-yum-helper"] + "sbin": ["bcfg2-server", "bcfg2-yum-helper", "bcfg2-crypt", "bcfg2-test", + "bcfg2-lint"] } # perform limited, django-safe checks on the listed files @@ -83,12 +60,42 @@ django_checks = { "lib/Bcfg2/Server": ["Reports", "models.py"] } +# perform only error checking on the listed files +error_checks = { + "lib/Bcfg2": ["Proxy.py", "SSLServer.py"], + "lib/Bcfg2/Server/Plugins": ["Decisions.py", + "Deps.py", + "Ldap.py", + "NagiosGen.py", + "Pkgmgr.py", + "SSHbase.py", + "SSLCA.py"] + } + # perform no checks at all on the listed files no_checks = { "lib/Bcfg2/Client/Tools": ["APT.py", "RPMng.py", "rpmtools.py"], - "lib/Bcfg2/Server": ["Snapshots", "Hostbase"] + "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], + "lib/Bcfg2": ["manage.py"], + "lib/Bcfg2/Server/Reports": ["manage.py"], + "lib/Bcfg2/Server/Plugins": ["Account.py", + "Base.py", + "Editor.py", + "Hostbase.py", + "Snapshots.py", + "Statistics.py", + "TCheetah.py", + "TGenshi.py"], } +def expand_path_dict(pathdict): + """ given a path dict as above, return a list of all the paths """ + rv = [] + for parent, modules in pathdict.items(): + for mod in modules: + rv.extend(glob.glob(os.path.join(srcpath, parent, mod))) + return rv + class TestPylint(Bcfg2TestCase): pylint_cmd = ["pylint", "--rcfile", rcfile] @@ -97,21 +104,18 @@ class TestPylint(Bcfg2TestCase): error_re = re.compile(r':\d+:\s+\[[EF]\d{4}') # build the blacklist - blacklist = [] - for parent, modules in no_checks.items(): - blacklist.extend([os.path.join(srcpath, parent, m) for m in modules]) + blacklist = expand_path_dict(no_checks) - def _get_paths(self, pathlist): - paths = [] - for parent, modules in pathlist.items(): - paths.extend([os.path.join(srcpath, parent, m) for m in modules]) - return list(set(paths) - set(self.blacklist)) + def _get_paths(self, pathdict): + return list(set(expand_path_dict(pathdict)) - set(self.blacklist)) @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) @skipUnless(HAS_PYLINT, "pylint not found, skipping") def test_lib_full(self): - self._pylint_full(self._get_paths(full_checks)) + full_list = list(set(self._get_paths(full_checks)) - + set(expand_path_dict(error_checks))) + self._pylint_full(full_list) @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @skipIf(not os.path.exists(rcfile), "%s does not exist" % rcfile) @@ -126,7 +130,6 @@ class TestPylint(Bcfg2TestCase): if extra_args is None: extra_args = [] args = self.pylint_cmd + extra_args + \ - ["-f", "parseable"] + \ [os.path.join(srcpath, p) for p in paths] pylint = Popen(args, stdout=PIPE, stderr=STDOUT) print(pylint.communicate()[0]) diff --git a/testsuite/install.sh b/testsuite/install.sh index 565e158df..c8312bce9 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -10,8 +10,9 @@ if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then if [[ $PYVER == "2.5" ]]; then # markdown 2.2.0 is broken on py2.5, so until 2.2.1 is released use 2.1 pip install --use-mirrors 'markdown<2.2' + pip install --use-mirrors simplejson fi - pip install --use-mirrors genshi cheetah 'django<1.4' M2Crypto + pip install --use-mirrors genshi cheetah 'django<1.4' M2Crypto yaml else # python < 2.6 requires M2Crypto for SSL communication, not just # for encryption support diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 2c56a3e81..2e4279e75 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -33,7 +33,7 @@ load-plugins= # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). -disable=W0142,W0511,W0603,R0201,R0901,R0902,R0903,R0904,R0921,R0922,C0302,I0011 +disable=W0142,W0511,W0603,W1201,R0201,R0901,R0902,R0903,R0904,R0921,R0922,C0302,I0011 [REPORTS] @@ -121,7 +121,7 @@ zope=no # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent,objects,DoesNotExist +generated-members=objects,DoesNotExist,isoformat [MISCELLANEOUS] @@ -136,7 +136,7 @@ notes=FIXME,XXX,TODO required-attributes= # List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input +bad-functions=map,apply # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ @@ -156,20 +156,20 @@ function-rgx=[a-z_][a-z0-9_]{2,30}$ method-rgx=[A-z_][A-z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names -attr-rgx=(Entries|[a-z_][a-z0-9_]{2,30})$ +attr-rgx=(Entries|[a-z_][a-z0-9_]{2,30}(ID)?)$ # Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +argument-rgx=[a-z_][a-z0-9_]{2,30}(ID)?$ # Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +variable-rgx=[a-z_][a-z0-9_]{2,30}(ID)?$ # Regular expression which should only match correct list comprehension / # generator expression variable names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma -good-names=_,rv,el,fd,ca,re +good-names=_,rv,el,fd,ca,re,i,j,iv # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata @@ -200,7 +200,7 @@ int-import-graph= [DESIGN] # Maximum number of arguments for function / method -max-args=6 +max-args=8 # Argument names that match this expression will be ignored. Default to name # with leading underscore |