diff options
Diffstat (limited to 'src/lib')
26 files changed, 388 insertions, 147 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index a95c0a7a6..bc6bd4d4c 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,14 +1,12 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ -import os -import sys import time -import select import fnmatch import logging import Bcfg2.Client.Tools -from Bcfg2.Compat import input, any, all # pylint: disable=W0622 +from Bcfg2.Client import prompt +from Bcfg2.Compat import any, all # pylint: disable=W0622 def cmpent(ent1, ent2): @@ -154,7 +152,7 @@ class Frame(object): for entry in multi: self.logger.debug(entry) - def promptFilter(self, prompt, entries): + def promptFilter(self, msg, entries): """Filter a supplied list based on user input.""" ret = [] entries.sort(cmpent) @@ -165,20 +163,9 @@ class Frame(object): if 'qtext' in entry.attrib: iprompt = entry.get('qtext') else: - iprompt = prompt % (entry.tag, entry.get('name')) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - try: - ans = input(iprompt.encode(sys.stdout.encoding, 'replace')) - if ans in ['y', 'Y']: - ret.append(entry) - except EOFError: - # python 2.4.3 on CentOS doesn't like ^C for some reason - break - except: - print("Error while reading input") - continue + iprompt = msg % (entry.tag, entry.get('name')) + if prompt(iprompt): + ret.append(entry) return ret def __getattr__(self, name): @@ -281,7 +268,7 @@ class Frame(object): def Decide(self): # pylint: disable=R0912 """Set self.whitelist based on user interaction.""" - prompt = "Install %s: %s? (y/N): " + iprompt = "Install %s: %s? (y/N): " rprompt = "Remove %s: %s? (y/N): " if self.setup['remove']: if self.setup['remove'] == 'all': @@ -354,7 +341,7 @@ class Frame(object): (bmodified or a.get('when') == 'always'))] # now we process all "always actions" if self.setup['interactive']: - self.promptFilter(prompt, actions) + self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) # need to test to fail entries in whitelist @@ -377,7 +364,7 @@ class Frame(object): if b.get("name"))) if self.setup['interactive']: - self.whitelist = self.promptFilter(prompt, self.whitelist) + self.whitelist = self.promptFilter(iprompt, self.whitelist) self.removal = self.promptFilter(rprompt, self.removal) for entry in candidates: @@ -474,7 +461,8 @@ class Frame(object): len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) if phase == 'final' and self.setup['extra']: - for entry in self.extra: + for entry in sorted(self.extra, key=lambda e: e.tag + ":" + + e.get('name')): etype = entry.get('type') if etype: self.logger.info("%s:%s:%s" % (entry.tag, etype, diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py index 896ca5f49..64a0b1e15 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py @@ -12,5 +12,4 @@ class POSIXHardlink(POSIXLinkTool): return os.path.samefile(entry.get('name'), entry.get('to')) def _link(self, entry): - ## TODO: set permissions return os.link(entry.get('to'), entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index b867fa3d8..f46875743 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -687,7 +687,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if path is None: path = entry.get("name") cur = path - while cur != '/': + while cur and cur != '/': if not os.path.exists(cur): created.append(cur) cur = os.path.dirname(cur) diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 08d943251..451495be2 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -360,7 +360,7 @@ class SELinuxEntryHandler(object): """ find extra entries of this entry type """ specified = [self._key(e) for e in self.tool.getSupportedEntries() - if e.get("type") == self.etype] + if e.tag == "SE%s" % self.etype.title()] try: records = self.custom_records except ValueError: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 1fe275c2c..c9fae7fc7 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -123,7 +123,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): ('Package', 'rpm'), ('Path', 'ignore')] - __req__ = {'Package': ['name'], + __req__ = {'Package': ['type'], 'Path': ['type']} conflicts = ['YUM24', 'RPM', 'RPMng', 'YUMng'] @@ -287,6 +287,17 @@ class YUM(Bcfg2.Client.Tools.PkgTool): return self.yumbase.rpmdb.returnGPGPubkeyPackages() return self.yumbase.rpmdb.searchNevra(name='gpg-pubkey') + def missing_attrs(self, entry): + """ Implementing from superclass to check for existence of either + name or group attribute for Package entry in the case of a YUM + group. """ + missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) + + if entry.get('name', None) == None and \ + entry.get('group', None) == None: + missing += ['name', 'group'] + return missing + def _verifyHelper(self, pkg_obj): """ _verifyHelper primarly deals with a yum bug where the pkg_obj.verify() method does not properly take into count multilib @@ -409,8 +420,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if entry.get('version', False) == 'auto': self._fixAutoVersion(entry) - self.logger.debug("Verifying package instances for %s" % - entry.get('name')) + if entry.get('group'): + self.logger.debug("Verifying packages for group %s" % + entry.get('group')) + else: + self.logger.debug("Verifying package instances for %s" % + entry.get('name')) self.verify_cache = dict() # Used for checking multilib packages self.modlists[entry] = modlist @@ -423,14 +438,58 @@ class YUM(Bcfg2.Client.Tools.PkgTool): entry.get('pkg_checks', 'true').lower() == 'true' pkg_verify = self.pkg_verify and \ entry.get('pkg_verify', 'true').lower() == 'true' + yum_group = False if entry.get('name') == 'gpg-pubkey': all_pkg_objs = self._getGPGKeysAsPackages() pkg_verify = False # No files here to verify + elif entry.get('group'): + entry.set('name', 'group:%s' % entry.get('group')) + yum_group = True + all_pkg_objs = [] + instances = [] + if self.yumbase.comps.has_group(entry.get('group')): + group = self.yumbase.comps.return_group(entry.get('group')) + group_packages = [p + for p, d in group.mandatory_packages.items() + if d] + group_type = entry.get('choose', 'default') + if group_type in ['default', 'optional', 'all']: + group_packages += [p + for p, d in + group.default_packages.items() + if d] + if group_type in ['optional', 'all']: + group_packages += [p + for p, d in + group.optional_packages.items() + if d] + if len(group_packages) == 0: + self.logger.error("No packages found for group %s" % + entry.get("group")) + for pkg in group_packages: + # create package instances for each package in yum group + instance = Bcfg2.Client.XML.SubElement(entry, 'Package') + instance.attrib['name'] = pkg + instance.attrib['type'] = 'yum' + try: + newest = \ + self.yumbase.pkgSack.returnNewestByName(pkg)[0] + instance.attrib['version'] = newest['version'] + instance.attrib['epoch'] = newest['epoch'] + instance.attrib['release'] = newest['release'] + except: # pylint: disable=W0702 + self.logger.info("Error finding newest package " + "for %s" % + pkg) + instance.attrib['version'] = 'any' + instances.append(instance) + else: + self.logger.error("Group not found: %s" % entry.get("group")) else: all_pkg_objs = \ self.yumbase.rpmdb.searchNevra(name=entry.get('name')) - if len(all_pkg_objs) == 0: + if len(all_pkg_objs) == 0 and yum_group != True: # Some sort of virtual capability? Try to resolve it all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name')) if len(all_pkg_objs) > 0: @@ -441,7 +500,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.info(" %s" % pkg) for inst in instances: - nevra = build_yname(entry.get('name'), inst) + if yum_group: + # the entry is not the name of the package + nevra = build_yname(inst.get('name'), inst) + all_pkg_objs = \ + self.yumbase.rpmdb.searchNevra(name=inst.get('name')) + else: + nevra = build_yname(entry.get('name'), inst) if nevra in pkg_cache: continue # Ignore duplicate instances else: @@ -455,7 +520,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): stat['version_fail'] = False stat['verify'] = {} stat['verify_fail'] = False - stat['pkg'] = entry + if yum_group: + stat['pkg'] = inst + else: + stat['pkg'] = entry stat['modlist'] = modlist if inst.get('verify_flags'): # this splits on either space or comma @@ -624,7 +692,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: install_only = False - if virt_pkg or (install_only and not self.setup['kevlar']): + if virt_pkg or \ + (install_only and not self.setup['kevlar']) or \ + yum_group: # virtual capability supplied, we are probably dealing # with multiple packages of different names. This check # doesn't make a lot of since in this case. diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd86a2a4b..a4a68ea3b 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -3,10 +3,10 @@ import os import sys import stat -import select +import Bcfg2.Client import Bcfg2.Client.XML from Bcfg2.Utils import Executor, ClassName -from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 +from Bcfg2.Compat import walk_packages # pylint: disable=W0622 __all__ = [m[1] for m in walk_packages(path=__path__)] @@ -113,25 +113,34 @@ class Tool(object): #: A list of all entries handled by this tool self.handled = [] - for struct in config: + self._analyze_config() + self._check_execs() + + def _analyze_config(self): + """ Analyze the config at tool initialization-time for + important and handled entries """ + for struct in self.config: for entry in struct: if (entry.tag == 'Path' and entry.get('important', 'false').lower() == 'true'): self.__important__.append(entry.get('name')) - if self.handlesEntry(entry): - self.handled.append(entry) + self.handled = self.getSupportedEntries() + + def _check_execs(self): + """ Check all executables used by this tool to ensure that + they exist and are executable """ for filename in self.__execs__: try: mode = stat.S_IMODE(os.stat(filename)[stat.ST_MODE]) - if mode & stat.S_IEXEC != stat.S_IEXEC: - raise ToolInstantiationError("%s: %s not executable" % - (self.name, filename)) except OSError: raise ToolInstantiationError(sys.exc_info()[1]) except: raise ToolInstantiationError("%s: Failed to stat %s" % - (self.name, filename), - exc_info=1) + (self.name, filename)) + if not mode & stat.S_IEXEC: + raise ToolInstantiationError("%s: %s not executable" % + (self.name, filename)) + def BundleUpdated(self, bundle, states): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -185,11 +194,13 @@ class Tool(object): if self.canVerify(entry): try: func = getattr(self, "Verify%s" % entry.tag) - states[entry] = func(entry, mods) except AttributeError: self.logger.error("%s: Cannot verify %s entries" % (self.name, entry.tag)) - except: + continue + try: + states[entry] = func(entry, mods) + except: # pylint: disable=W0702 self.logger.error("%s: Unexpected failure verifying %s" % (self.name, self.primarykey(entry)), @@ -213,14 +224,16 @@ class Tool(object): :returns: None """ for entry in entries: try: - func = getattr(self, "Install%s" % (entry.tag)) - states[entry] = func(entry) - if states[entry]: - self.modified.append(entry) + func = getattr(self, "Install%s" % entry.tag) except AttributeError: self.logger.error("%s: Cannot install %s entries" % (self.name, entry.tag)) - except: + continue + try: + states[entry] = func(entry) + if states[entry]: + self.modified.append(entry) + except: # pylint: disable=W0702 self.logger.error("%s: Unexpected failure installing %s" % (self.name, self.primarykey(entry)), exc_info=1) @@ -409,6 +422,19 @@ class PkgTool(Tool): """ raise NotImplementedError + def _get_package_command(self, packages): + """ Get the command to install the given list of packages. + + :param packages: The Package entries to install + :type packages: list of lxml.etree._Element + :returns: string - the command to run + """ + pkgargs = " ".join(self.pkgtool[1][0] % + tuple(pkg.get(field) + for field in self.pkgtool[1][1]) + for pkg in packages) + return self.pkgtool[0] % pkgargs + def Install(self, packages, states): """ Run a one-pass install where all required packages are installed with a single command, followed by single package @@ -422,12 +448,9 @@ class PkgTool(Tool): self.logger.info("Trying single pass package install for pkgtype %s" % self.pkgtype) - data = [tuple([pkg.get(field) for field in self.pkgtool[1][1]]) - for pkg in packages] - pkgargs = " ".join([self.pkgtool[1][0] % datum for datum in data]) - - self.logger.debug("Installing packages: %s" % pkgargs) - if self.cmd.run(self.pkgtool[0] % pkgargs): + pkgcmd = self._get_package_command(packages) + self.logger.debug("Running command: %s" % pkgcmd) + if self.cmd.run(pkgcmd): self.logger.info("Single Pass Succeded") # set all package states to true and flush workqueues pkgnames = [pkg.get('name') for pkg in packages] @@ -436,7 +459,7 @@ class PkgTool(Tool): and entry.get('type') == self.pkgtype and entry.get('name') in pkgnames): self.logger.debug('Setting state to true for pkg %s' % - (entry.get('name'))) + entry.get('name')) states[entry] = True self.RefreshPackages() else: @@ -452,18 +475,13 @@ class PkgTool(Tool): else: self.logger.info("Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) - if self.cmd.run( - self.pkgtool[0] % - (self.pkgtool[1][0] % - tuple([pkg.get(field) - for field in self.pkgtool[1][1]]))): + if self.cmd.run(self._get_package_command([pkg])): states[pkg] = True else: self.logger.error("Failed to install package %s" % - (pkg.get('name'))) + pkg.get('name')) self.RefreshPackages() - for entry in [ent for ent in packages if states[ent]]: - self.modified.append(entry) + self.modified.extend(entry for entry in packages if states[entry]) def RefreshPackages(self): """ Refresh the internal representation of the package @@ -557,11 +575,13 @@ class SvcTool(Tool): if self.setup['servicemode'] == 'disabled': return - for entry in [ent for ent in bundle if self.handlesEntry(ent)]: - restart = entry.get("restart", "true") - if (restart.lower() == "false" or - (restart.lower() == "interactive" and - not self.setup['interactive'])): + for entry in bundle: + if not self.handlesEntry(entry): + continue + + restart = entry.get("restart", "true").lower() + if (restart == "false" or + (restart == "interactive" and not self.setup['interactive'])): continue success = False @@ -570,14 +590,8 @@ class SvcTool(Tool): success = self.stop_service(entry) elif entry.get('name') not in self.restarted: if self.setup['interactive']: - prompt = ('Restart service %s?: (y/N): ' % - entry.get('name')) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input(prompt) - if ans not in ['y', 'Y']: + if not Bcfg2.Client.prompt('Restart service %s? (y/N) ' + % entry.get('name')): continue success = self.restart_service(entry) if success: @@ -593,8 +607,8 @@ class SvcTool(Tool): install_entries = [] for entry in entries: if entry.get('install', 'true').lower() == 'false': - self.logger.info("Service %s installation is false. Skipping " - "installation." % (entry.get('name'))) + self.logger.info("Installation is false for %s:%s, skipping" % + (entry.tag, entry.get('name'))) else: install_entries.append(entry) return Tool.Install(self, install_entries, states) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index c03021f14..dd5ae1e83 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -1,3 +1,31 @@ """This contains all Bcfg2 Client modules""" __all__ = ["Frame", "Tools", "XML", "Client"] + +import os +import sys +import select +from Bcfg2.Compat import input # pylint: disable=W0622 + + +def prompt(msg): + """ Helper to give a yes/no prompt to the user. Flushes input + buffers, handles exceptions, etc. Returns True if the user + answers in the affirmative, False otherwise. + + :param msg: The message to show to the user. The message is not + altered in any way for display; i.e., it should + contain "[y/N]" if desired, etc. + :type msg: string + :returns: bool - True if yes, False if no """ + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + try: + ans = input(msg.encode(sys.stdout.encoding, 'replace')) + return ans in ['y', 'Y'] + except EOFError: + # python 2.4.3 on CentOS doesn't like ^C for some reason + return False + except: + print("Error while reading input: %s" % sys.exc_info()[1]) + return False diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 4bcc76e8f..44c76303c 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -19,12 +19,13 @@ except ImportError: # urllib imports try: + from urllib import quote_plus from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ urlopen, HTTPError, URLError except ImportError: - from urllib.parse import urljoin, urlparse + from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen from urllib.error import HTTPError, URLError @@ -262,3 +263,10 @@ def oct_mode(mode): :type mode: int :returns: string """ return oct(mode).replace('o', '') + + +try: + long = long +except NameError: + # longs are just ints in py3k + long = int diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 618d0f2cd..8f7bb14f8 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -105,7 +105,11 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): (self.encodePriority(self.facility, newrec.levelname.lower()), self.format(newrec)) try: - self.socket.send(msg.encode('ascii')) + try: + encoded = msg.encode('utf-8') + except UnicodeDecodeError: + encoded = msg + self.socket.send(encoded) except socket.error: for i in range(10): # pylint: disable=W0612 try: diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 8e7540c22..7c91ca3cc 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -642,17 +642,19 @@ WEB_CFILE = \ default="/etc/bcfg2-web.conf", cmd='-W', odesc='<conffile>', - cf=('statistics', 'config'),) + cf=('reporting', 'config'), + deprecated_cf=('statistics', 'web_prefix'),) DJANGO_TIME_ZONE = \ Option('Django timezone', default=None, - cf=('statistics', 'time_zone'),) + cf=('reporting', 'time_zone'), + deprecated_cf=('statistics', 'web_prefix'),) DJANGO_DEBUG = \ Option('Django debug', default=None, - cf=('statistics', 'web_debug'), + cf=('reporting', 'web_debug'), + deprecated_cf=('statistics', 'web_prefix'), cook=get_bool,) -# Django options DJANGO_WEB_PREFIX = \ Option('Web prefix', default=None, @@ -1107,7 +1109,7 @@ CRYPT_STDOUT = \ cmd='--stdout', long_arg=True) CRYPT_PASSPHRASE = \ - Option('Encryption passphrase (name or passphrase)', + Option('Encryption passphrase name', default=None, cmd='-p', odesc='<passphrase>') diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index 3aefed5d1..62b83d0b4 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -1,6 +1,6 @@ -import logging import re import socket +import logging # The ssl module is provided by either Python 2.6 or a separate ssl # package that works on older versions of Python (see @@ -20,7 +20,7 @@ import sys import time # Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse +from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) @@ -352,7 +352,8 @@ def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, if user and password: method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, user, password, path) + newurl = "%s://%s:%s@%s" % (method, quote_plus(user, ''), + quote_plus(password, ''), path) else: newurl = url ssl_trans = XMLRPCTransport(key, cert, ca, diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 533dcc79e..c73339911 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -88,7 +88,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.0rc2</span> + <span>Bcfg2 Version 1.3.1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html index 759415507..bb4f650d1 100644 --- a/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html +++ b/src/lib/Bcfg2/Reporting/templates/widgets/filter_bar.html @@ -16,7 +16,7 @@ <label for="id_group">Group filter:</label> <select id="id_group" name="group" onchange="javascript:url=document.forms['filter_form'].group.value; if(url) { location.href=url }"> {% for group, group_url, selected in groups %} - <option label="{{group}}" value="{{group_url}}" {% if selected %}selected {% endif %}/> + <option value="{{group_url}}" {% if selected %}selected {% endif %}>{{group}}</option> {% endfor %} </select> {% endif %} diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 5e3c6232a..a149b676f 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -403,12 +403,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, name = instance.name except AttributeError: name = "unknown" - if hasattr(instance, 'plugins'): - for pname, pinst in list(instance.plugins.items()): - for mname in pinst.__rmi__: - xmname = "%s.%s" % (pname, mname) - fn = getattr(pinst, mname) - self.register_function(fn, name=xmname) + if hasattr(instance, '_get_rmi'): + for fname, func in instance._get_rmi().items(): + self.register_function(func, name=fname) self.logger.info("serving %s at %s" % (name, self.url)) def serve_forever(self): diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 14065980d..4b8d65597 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -13,6 +13,7 @@ import subprocess import Bcfg2.Server.Admin import Bcfg2.Server.Plugin import Bcfg2.Options +import Bcfg2.Server.Plugins.Metadata from Bcfg2.Compat import input # pylint: disable=W0622 # default config file @@ -174,8 +175,6 @@ class Init(Bcfg2.Server.Admin.Mode): self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - # Parse options opts = Bcfg2.Options.OptionParser(self.options) opts.parse(args) @@ -214,7 +213,7 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the repository path.""" while True: newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) + self.data['repopath']) if newrepo != '': self.data['repopath'] = os.path.abspath(newrepo) if os.path.isdir(self.data['repopath']): @@ -292,7 +291,7 @@ class Init(Bcfg2.Server.Admin.Mode): "created [%s]: " % self.data['keypath']) if keypath: self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created" + certpath = safe_input("Path where Bcfg2 server cert will be created " "[%s]: " % self.data['certpath']) if certpath: self.data['certpath'] = certpath @@ -320,6 +319,16 @@ class Init(Bcfg2.Server.Admin.Mode): def init_repo(self): """Setup a new repo and create the content of the configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + confdata = CONFIG % (self.data['repopath'], ','.join(self.plugins), self.data['sendmail'], @@ -335,13 +344,3 @@ class Init(Bcfg2.Server.Admin.Mode): create_key(self.data['shostname'], self.data['keypath'], self.data['certpath'], self.data['country'], self.data['state'], self.data['location']) - - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 9be71e2e2..0ded7ac26 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -2,13 +2,14 @@ implementations inherit from. """ import os -import atexit -import logging -import select import sys -import threading import time +import atexit +import select +import signal +import logging import inspect +import threading import lxml.etree import Bcfg2.settings import Bcfg2.Server @@ -302,6 +303,14 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = setup['ca'] + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + self.shutdown() + os._exit(1) # pylint: disable=W0212 + + signal.signal(signal.SIGINT, hdlr) + #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -904,10 +913,12 @@ class BaseCore(object): def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - if self.plugins: - for pname, pinst in list(self.plugins.items()): - for mname in pinst.__rmi__: - rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + for pname, pinst in list(self.plugins.items()): + for mname in pinst.__rmi__: + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + famname = self.fam.__class__.__name__ + for mname in self.fam.__rmi__: + rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname) return rmi def _resolve_exposed_method(self, method_name): @@ -1177,12 +1188,15 @@ class BaseCore(object): return self.set_core_debug(address, not self.debug_flag) @exposed - def toggle_fam_debug(self, _): + def toggle_fam_debug(self, address): """ Toggle debug status of the FAM :returns: bool - The new debug state of the FAM """ - return self.fam.toggle_debug() + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.toggle_debug() @exposed def set_debug(self, address, debug): @@ -1227,7 +1241,7 @@ class BaseCore(object): return self.debug_flag @exposed - def set_fam_debug(self, _, debug): + def set_fam_debug(self, address, debug): """ Explicitly set debug status of the FAM :param debug: The new debug status of the FAM. This can @@ -1239,4 +1253,7 @@ class BaseCore(object): """ if debug not in [True, False]: debug = debug.lower() == "true" - return self.fam.set_debug(debug) + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 178a47b1a..cdd52dbb9 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -2,6 +2,7 @@ support. """ import os +import errno import logging import pyinotify from Bcfg2.Compat import reduce # pylint: disable=W0622 @@ -15,6 +16,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): """ File monitor backend with `inotify <http://inotify.aiken.cz/>`_ support. """ + __rmi__ = Pseudo.__rmi__ + ["list_watches", "list_paths"] + #: Inotify is the best FAM backend, so it gets a very high #: priority __priority__ = 99 @@ -182,6 +185,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): try: watchdir = self.watches_by_path[watch_path] except KeyError: + if not os.path.exists(watch_path): + raise OSError(errno.ENOENT, + "No such file or directory: '%s'" % path) watchdir = self.watchmgr.add_watch(watch_path, self.mask, quiet=False)[watch_path] self.watches_by_path[watch_path] = watchdir @@ -211,3 +217,20 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ + + def list_watches(self): + """ XML-RPC that returns a list of current inotify watches for + debugging purposes. """ + return list(self.watches_by_path.keys()) + + def list_paths(self): + """ XML-RPC that returns a list of paths that are handled for + debugging purposes. Because inotify doesn't like watching + files, but prefers to watch directories, this will be + different from + :func:`Bcfg2.Server.FileMonitor.Inotify.Inotify.ListWatches`. For + instance, if a plugin adds a monitor to + ``/var/lib/bcfg2/Plugin/foo.xml``, :func:`ListPaths` will + return ``/var/lib/bcfg2/Plugin/foo.xml``, while + :func:`ListWatches` will return ``/var/lib/bcfg2/Plugin``. """ + return list(self.handles.keys()) diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 54d35e38d..e430e3160 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -116,6 +116,9 @@ class FileMonitor(Debuggable): #: should have higher priorities. __priority__ = -1 + #: List of names of methods to be exposed as XML-RPC functions + __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] + def __init__(self, ignore=None, debug=False): """ :param ignore: A list of filename globs describing events that @@ -312,6 +315,15 @@ class FileMonitor(Debuggable): """ raise NotImplementedError + def list_event_handlers(self): + """ XML-RPC that returns + :attr:`Bcfg2.Server.FileMonitor.FileMonitor.handles` for + debugging purposes. """ + rv = dict() + for watch, handler in self.handles.items(): + rv[watch] = getattr(handler, "name", handler.__class__.__name__) + return rv + #: A dict of all available FAM backends. Keys are the human-readable #: names of the backends, which are used in bcfg2.conf to select a diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 61b737a82..2a10da417 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -43,7 +43,7 @@ def is_octal_mode(val): def is_username(val): """ Return True if val is a string giving either a positive integer uid, or a valid Unix username """ - return re.match(r'^([a-z]\w{0,30}|\d+)$', val) + return re.match(r'^([A-z][-_A-z0-9]{0,30}|\d+)$', val) def is_device_mode(val): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 4fa2fb894..c2e5afbad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -101,6 +101,10 @@ class CfgGenshiGenerator(CfgGenerator): __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): + if self.template is None: + raise PluginExecutionError("Failed to load template %s" % + self.name) + fname = entry.get('realname', entry.get('name')) stream = \ self.template.generate(name=fname, diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 09ecfaf82..a81139b5d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -139,7 +139,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): self.logger.error('Failed to parse %s' % self.basefile) return self.extras = [] - self.basedata = copy.copy(xdata) + self.basedata = copy.deepcopy(xdata) self._follow_xincludes(xdata=xdata) if self.extras: try: diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 0dd42c9cb..490ee6f20 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -15,6 +15,11 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" + if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + # do not care about a client that is _any_ 1.3.0 release + # (including prereleases and RCs) + return + for goal in goals: for entry in goal.getchildren(): if entry.tag == 'Path' and 'mode' in entry.keys(): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 46231c636..4cd938651 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -313,9 +313,7 @@ class YumCollection(Collection): @property def __package_groups__(self): - """ YumCollections support package groups only if - :attr:`use_yum` is True """ - return self.use_yum + return True @property def helper(self): @@ -665,11 +663,6 @@ class YumCollection(Collection): In this implementation the packages may be strings or tuples. See :ref:`yum-pkg-objects` for more information. """ - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported " - "by Bcfg2's internal Yum dependency generator") - return dict() - if not grouplist: return dict() @@ -680,8 +673,16 @@ class YumCollection(Collection): if not ptype: ptype = "default" gdicts.append(dict(group=group, type=ptype)) - - return self.call_helper("get_groups", inputdata=gdicts) + + if self.use_yum: + return self.call_helper("get_groups", inputdata=gdicts) + else: + pkgs = dict() + for gdict in gdicts: + pkgs[gdict['group']] = Collection.get_group(self, + gdict['group'], + gdict['type']) + return pkgs def _element_to_pkg(self, el, name): """ Convert a Package or Instance element to a package tuple """ @@ -991,6 +992,7 @@ class YumSource(Source): for x in ['global'] + self.arches]) self.needed_paths = set() self.file_to_arch = dict() + self.yumgroups = dict() __init__.__doc__ = Source.__init__.__doc__ @property @@ -1008,7 +1010,8 @@ class YumSource(Source): if not self.use_yum: cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) + self.filemap, self.url_map, + self.yumgroups), cache, 2) cache.close() def load_state(self): @@ -1018,7 +1021,7 @@ class YumSource(Source): if not self.use_yum: data = open(self.cachefile) (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) + self.filemap, self.url_map, self.yumgroups) = cPickle.load(data) @property def urls(self): @@ -1073,7 +1076,7 @@ class YumSource(Source): urls = [] for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: + if elt.get('type') in ['filelists', 'primary', 'group']: floc = elt.find(RPO + 'location') fullurl = url + floc.get('href') urls.append(fullurl) @@ -1090,11 +1093,14 @@ class YumSource(Source): # we have to read primary.xml first, and filelists.xml afterwards; primaries = list() filelists = list() + groups = list() for fname in self.files: if fname.endswith('primary.xml.gz'): primaries.append(fname) elif fname.endswith('filelists.xml.gz'): filelists.append(fname) + elif fname.find('comps'): + groups.append(fname) for fname in primaries: farch = self.file_to_arch[fname] @@ -1104,6 +1110,9 @@ class YumSource(Source): farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) + for fname in groups: + fdata = lxml.etree.parse(fname).getroot() + self.parse_group(fdata) # merge data sdata = list(self.packages.values()) @@ -1167,6 +1176,35 @@ class YumSource(Source): self.provides[arch][prov] = list() self.provides[arch][prov].append(pkgname) + @Bcfg2.Server.Plugin.track_statistics() + def parse_group(self, data): + """ parse comps.xml.gz data """ + for group in data.getchildren(): + if not group.tag.endswith('group'): + continue + try: + groupid = group.xpath('id')[0].text + self.yumgroups[groupid] = {'mandatory': list(), + 'default': list(), + 'optional': list(), + 'conditional': list()} + except IndexError: + continue + try: + packagelist = group.xpath('packagelist')[0] + except IndexError: + continue + for pkgreq in packagelist.getchildren(): + pkgtype = pkgreq.get('type', None) + if pkgtype == 'mandatory': + self.yumgroups[groupid]['mandatory'].append(pkgreq.text) + elif pkgtype == 'default': + self.yumgroups[groupid]['default'].append(pkgreq.text) + elif pkgtype == 'optional': + self.yumgroups[groupid]['optional'].append(pkgreq.text) + elif pkgtype == 'conditional': + self.yumgroups[groupid]['conditional'].append(pkgreq.text) + def is_package(self, metadata, package): arch = [a for a in self.arches if a in metadata.groups] if not arch: @@ -1246,3 +1284,27 @@ class YumSource(Source): return self.pulp_id else: return Source.get_repo_name(self, url_map) + + def get_group(self, metadata, group, ptype=None): # pylint: disable=W0613 + """ Get the list of packages of the given type in a package + group. + + :param group: The name of the group to query + :type group: string + :param ptype: The type of packages to get, for backends that + support multiple package types in package groups + (e.g., "recommended," "optional," etc.) + :type ptype: string + :returns: list of strings - package names + """ + try: + yumgroup = self.yumgroups[group] + except KeyError: + return [] + packages = yumgroup['conditional'] + yumgroup['mandatory'] + if ptype in ['default', 'optional', 'all']: + packages += yumgroup['default'] + if ptype in ['optional', 'all']: + packages += yumgroup['optional'] + return packages + diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f112c65cd..c3eadc6bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -310,20 +310,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if self.disableResolver: # Config requests no resolver + for struct in structures: + for pkg in struct.xpath('//Package | //BoundPackage'): + if pkg.get("group"): + if pkg.get("type"): + pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) - # base is the set of initial packages -- explicitly - # given in the specification, from expanded package groups, - # and essential to the distribution - base = set() + initial = set() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - base.update(collection.packages_from_entry(pkg)) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -335,6 +337,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pkg, xml_declaration=False).decode('UTF-8')) + # base is the set of initial packages explicitly given in the + # specification, packages from expanded package groups, and + # packages essential to the distribution + base = set(initial) + # remove package groups for el in to_remove: el.getparent().remove(el) @@ -350,7 +357,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = collection.get_new_packages(base, packages) + newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 0328c6bea..1f64111e7 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,6 +1,7 @@ """ Django database models for all plugins """ import sys +import copy import logging import Bcfg2.Options import Bcfg2.Server.Plugins @@ -19,7 +20,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - plugin_opt = Bcfg2.Options.SERVER_PLUGINS + plugin_opt = copy.deepcopy(Bcfg2.Options.SERVER_PLUGINS) plugin_opt.default = Bcfg2.Server.Plugins.__all__ setup = \ diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 8223d7543..6f3ba3e49 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.0rc2" +__version__ = "1.3.1" class Bcfg2VersionInfo(tuple): |