diff options
Diffstat (limited to 'src/sbin')
| -rwxr-xr-x | src/sbin/bcfg2-admin | 4 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-crypt | 585 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-info | 31 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-lint | 9 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-reports | 10 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-test | 27 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-yum-helper | 161 |
7 files changed, 476 insertions, 351 deletions
diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 31e49c00b..14d188342 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -83,7 +83,7 @@ def main(): raise SystemExit(1) mode = mode_cls(setup) try: - mode(setup['args'][1:]) + return mode(setup['args'][1:]) finally: mode.shutdown() else: @@ -93,6 +93,6 @@ def main(): if __name__ == '__main__': try: - main() + sys.exit(main()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index aad89882f..5641732cd 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -18,291 +18,199 @@ except ImportError: raise SystemExit(1) -class EncryptionChunkingError(Exception): - """ error raised when Encryptor cannot break a file up into chunks - to be encrypted, or cannot reassemble the chunks """ - pass - - -class Encryptor(object): - """ Generic encryptor for all files """ - - def __init__(self, setup): +def print_xml(element, keep_text=False): + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element for + display + :type keep_text: boolean + """ + xml = None + if len(element) or element.text: + el = copy.copy(element) + if el.text and not keep_text: + el.text = '...' + for child in el.iterchildren(): + el.remove(child) + xml = lxml.etree.tostring( + el, + xml_declaration=False).decode("UTF-8").strip() + else: + xml = lxml.etree.tostring( + element, + xml_declaration=False).decode("UTF-8").strip() + return "%s (line %s)" % (xml, element.sourceline) + + +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ + + +class DecryptError(Exception): + """ Exception raised when decryption fails. """ + + +class EncryptError(Exception): + """ Exception raised when encryption fails. """ + + +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + def __init__(self, filename, setup): self.setup = setup - self.passphrase = None - self.pname = None self.logger = logging.getLogger(self.__class__.__name__) + self.passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - def get_encrypted_filename(self, plaintext_filename): - """ get the name of the file encrypted data should be written to """ - return plaintext_filename - - def get_plaintext_filename(self, encrypted_filename): - """ get the name of the file decrypted data should be written to """ - return encrypted_filename - - def chunk(self, data): - """ generator to break the file up into smaller chunks that - will each be individually encrypted or decrypted """ - yield data - - def unchunk(self, data, original): # pylint: disable=W0613 - """ given chunks of a file, reassemble then into the whole file """ + self.filename = filename try: - return data[0] - except IndexError: - raise EncryptionChunkingError("No data to unchunk") + self.data = open(self.filename).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % (filename, + err)) + return False - def set_passphrase(self): - """ set the passphrase for the current file """ + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ if (not self.setup.cfp.has_section(Bcfg2.Encryption.CFG_SECTION) or len(Bcfg2.Encryption.get_passphrases(self.setup)) == 0): - self.logger.error("No passphrases available in %s" % - self.setup['configfile']) - return False - - if self.passphrase: - self.logger.debug("Using previously determined passphrase %s" % - self.pname) - return True + raise PassphraseError("No passphrases available in %s" % + self.setup['configfile']) + pname = None if self.setup['passphrase']: - self.pname = self.setup['passphrase'] + pname = self.setup['passphrase'] - if self.pname: + if pname: if self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - self.pname): - self.passphrase = \ - self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - self.pname) + pname): + passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, + pname) self.logger.debug("Using passphrase %s specified on command " - "line" % self.pname) - return True + "line" % pname) + return (pname, passphrase) else: - self.logger.error("Could not find passphrase %s in %s" % - (self.pname, self.setup['configfile'])) - return False + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, self.setup['configfile'])) else: pnames = Bcfg2.Encryption.get_passphrases(self.setup) if len(pnames) == 1: - self.pname = pnames.keys()[0] - self.passphrase = pnames[self.pname] - self.logger.info("Using passphrase %s" % self.pname) - return True + pname = pnames.keys()[0] + passphrase = pnames[pname] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) elif len(pnames) > 1: - self.logger.warning("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - self.logger.info("No passphrase could be determined") - return False - - def encrypt(self, fname): - """ encrypt the given file, returning the encrypted data """ - try: - plaintext = open(fname).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) - return False + return (None, None) + raise PassphraseError("No passphrase could be determined") - if not self.set_passphrase(): - return False + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename - crypted = [] + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) try: - for chunk in self.chunk(plaintext): - try: - passphrase, pname = self.get_passphrase(chunk) - except TypeError: - return False - - crypted.append(self._encrypt(chunk, passphrase, name=pname)) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting data to encrypt from %s: %s" % - (fname, err)) - return False - return self.unchunk(crypted, plaintext) - - # pylint: disable=W0613 - def _encrypt(self, plaintext, passphrase, name=None): - """ encrypt a single chunk of a file """ - return Bcfg2.Encryption.ssl_encrypt( - plaintext, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - # pylint: enable=W0613 - - def decrypt(self, fname): - """ decrypt the given file, returning the plaintext data """ - try: - crypted = open(fname).read() + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True except IOError: err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) return False - self.set_passphrase() + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) - plaintext = [] - try: - for chunk in self.chunk(crypted): - try: - passphrase, pname = self.get_passphrase(chunk) - try: - plaintext.append(self._decrypt(chunk, passphrase)) - except Bcfg2.Encryption.EVPError: - self.logger.info("Could not decrypt %s with the " - "specified passphrase" % fname) - continue - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - continue - except TypeError: - pchunk = None - passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - for pname, passphrase in passphrases.items(): - self.logger.debug("Trying passphrase %s" % pname) - try: - pchunk = self._decrypt(chunk, passphrase) - break - except Bcfg2.Encryption.EVPError: - pass - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - if pchunk is not None: - plaintext.append(pchunk) - else: - self.logger.error("Could not decrypt %s with any " - "passphrase in %s" % - (fname, self.setup['configfile'])) - continue - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting encrypted data from %s: %s" % - (fname, err)) - return False - try: - return self.unchunk(plaintext, crypted) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling plaintext data from %s: %s" % - (fname, err)) - return False +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError - def _decrypt(self, crypted, passphrase): - """ decrypt a single chunk """ - return Bcfg2.Encryption.ssl_decrypt( - crypted, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - def write_encrypted(self, fname, data=None): - """ write encrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_encrypted_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote encrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling encrypted data from %s: %s" % - (fname, err)) - return False +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError - def write_decrypted(self, fname, data=None): - """ write decrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_plaintext_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote decrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - def get_passphrase(self, chunk): - """ get the passphrase for a chunk of a file """ - pname = self._get_passphrase(chunk) - if not self.pname: - if not pname: - self.logger.info("No passphrase given on command line or " - "found in file") - return False - elif self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - pname) - else: - self.logger.error("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) - return False - else: - pname = self.pname - passphrase = self.passphrase - if self.pname != pname: - self.logger.warning("Passphrase given on command line (%s) " - "differs from passphrase embedded in " - "file (%s), using command-line option" % - (self.pname, pname)) - return (passphrase, pname) +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ - def _get_passphrase(self, chunk): # pylint: disable=W0613 - """ get the passphrase for a chunk of a file """ - return None + def __init__(self, filename, setup): + Encryptor.__init__(self, filename, setup) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) + def encrypt(self): + return Bcfg2.Encryption.ssl_encrypt( + self.data, self.passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)) -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" - def get_encrypted_filename(self, plaintext_filename): - return plaintext_filename + ".crypt" - def get_plaintext_filename(self, encrypted_filename): - if encrypted_filename.endswith(".crypt"): - return encrypted_filename[:-6] +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return Bcfg2.Encryption.ssl_decrypt( + self.data, self.passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + raise DecryptError("Could not decrypt %s with the " + "specified passphrase" % self.filename) + except: + raise DecryptError("Error decrypting %s: %s" % + (self.filename, sys.exc_info()[1])) + else: # no passphrase given, brute force + try: + return Bcfg2.Encryption.bruteforce_decrypt( + self.data, passphrases=self.passphrases.values(), + algorithm=Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + raise DecryptError("Could not decrypt %s with any passphrase" % + self.filename) + + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] else: - return Encryptor.get_plaintext_filename(self, encrypted_filename) + return Decryptor.get_plaintext_filename(self, original_filename) -class PropertiesEncryptor(Encryptor): - """ encryptor class for Properties files """ +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*' - def _encrypt(self, plaintext, passphrase, name=None): - # plaintext is an lxml.etree._Element - if name is None: - name = "true" - if plaintext.text and plaintext.text.strip(): - plaintext.text = Bcfg2.Encryption.ssl_encrypt( - plaintext.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - plaintext.set("encrypted", name) - return plaintext - - def chunk(self, data): - xdata = lxml.etree.XML(data, parser=XMLParser) + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ if self.setup['xpath']: elements = xdata.xpath(self.setup['xpath']) if not elements: - raise EncryptionChunkingError("XPath expression %s matched no " - "elements" % self.setup['xpath']) + self.logger.warning("XPath expression %s matched no " + "elements" % self.setup['xpath']) else: - elements = xdata.xpath('//*[@encrypted]') + elements = xdata.xpath(self.default_xpath) if not elements: elements = list(xdata.getiterator(tag=lxml.etree.Element)) @@ -329,50 +237,96 @@ class PropertiesEncryptor(Encryptor): ans = input("Encrypt this element? [y/N] ") if not ans.lower().startswith("y"): elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in self.passphrases: + passphrase = self.passphrases[pname] + elif self.passphrase: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" + % (pname, self.setup['configfile'])) + passphrase = self.passphrase + pname = self.pname + else: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) + return (pname, passphrase) + + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) - # this is not a good use of a generator, but we need to - # generate the full list of elements in order to ensure that - # some exist before we know what to return - for elt in elements: - yield elt - - def unchunk(self, data, original): - # Properties elements are modified in-place, so we don't - # actually need to unchunk anything - xdata = data[0] - # find root element - while xdata.getparent() is not None: - xdata = xdata.getparent() - return lxml.etree.tostring(xdata, - xml_declaration=False, - pretty_print=True).decode('UTF-8') - - def _get_passphrase(self, chunk): - pname = chunk.get("encrypted") - if pname and pname.lower() != "true": - return pname - return None - - def _decrypt(self, crypted, passphrase): - # crypted is in lxml.etree._Element - if not crypted.text or not crypted.text.strip(): - self.logger.warning("Skipping empty element %s" % crypted.tag) - return crypted - decrypted = Bcfg2.Encryption.ssl_decrypt( - crypted.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - try: - crypted.text = decrypted.encode('ascii', 'xmlcharrefreplace') - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains content - # that can't even be encoded into xml entities. what - # probably happened here is that we coincidentally could - # decrypt a value encrypted with a different key, and - # wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - crypted.tag) - return crypted + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + raise EncryptError(str(sys.exc_info()[1])) + self.logger.debug("Encrypting %s" % print_xml(elt)) + elt.text = Bcfg2.Encryption.ssl_encrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + default_xpath = '//*[@encrypted]' + + def decrypt(self): + decrypted_any = False + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + raise DecryptError(str(sys.exc_info()[1])) + self.logger.debug("Decrypting %s" % print_xml(elt)) + try: + decrypted = Bcfg2.Encryption.ssl_decrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + decrypted_any = True + except (Bcfg2.Encryption.EVPError, TypeError): + self.logger.error("Could not decrypt %s, skipping" % + print_xml(elt)) + continue + try: + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + if decrypted_any: + return xdata + else: + raise DecryptError("Failed to decrypt any data in %s" % + self.filename) + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) def main(): # pylint: disable=R0912,R0915 @@ -422,9 +376,6 @@ def main(): # pylint: disable=R0912,R0915 logger.error("--remove cannot be used with --properties, ignoring") setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - props_crypt = PropertiesEncryptor(setup) - cfg_crypt = CfgEncryptor(setup) - for fname in setup['args']: if not os.path.exists(fname): logger.error("%s does not exist, skipping" % fname) @@ -454,10 +405,10 @@ def main(): # pylint: disable=R0912,R0915 props = False if props: - encryptor = props_crypt if setup['remove']: logger.info("Cannot use --remove with Properties file %s, " "ignoring for this file" % fname) + tools = (PropertiesEncryptor, PropertiesDecryptor) else: if setup['xpath']: logger.info("Cannot use xpath with Cfg file %s, ignoring " @@ -465,32 +416,52 @@ def main(): # pylint: disable=R0912,R0915 if setup['interactive']: logger.info("Cannot use interactive mode with Cfg file %s, " "ignoring -I for this file" % fname) - encryptor = cfg_crypt + tools = (CfgEncryptor, CfgDecryptor) data = None + mode = None if setup['encrypt']: - xform = encryptor.encrypt - write = encryptor.write_encrypted + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" elif setup['decrypt']: - xform = encryptor.decrypt - write = encryptor.write_decrypted + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "decrypt" else: logger.info("Neither --encrypt nor --decrypt specified, " "determining mode") - data = encryptor.decrypt(fname) - if data: - write = encryptor.write_decrypted - else: + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + + try: + data = tool.decrypt() + mode = "decrypt" + except DecryptError: logger.info("Failed to decrypt %s, trying encryption" % fname) - data = None - xform = encryptor.encrypt - write = encryptor.write_encrypted + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" if data is None: - data = xform(fname) - if not data: - logger.error("Failed to %s %s, skipping" % (xform.__name__, fname)) - continue + try: + data = getattr(tool, mode)() + except (EncryptError, DecryptError): + logger.error("Failed to %s %s, skipping: %s" % + (mode, fname, sys.exc_info()[1])) + continue if setup['crypt_stdout']: if len(setup['args']) > 1: print("----- %s -----" % fname) @@ -498,10 +469,10 @@ def main(): # pylint: disable=R0912,R0915 if len(setup['args']) > 1: print("") else: - write(fname, data=data) + tool.write(data) if (setup['remove'] and - encryptor.get_encrypted_filename(fname) != fname): + tool.get_destination_filename(fname) != fname): try: os.unlink(fname) except IOError: diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 6aafd24d1..6008f8896 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -231,10 +231,14 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): print("Refusing to write files outside of /tmp without -f " "option") return - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) + try: + lxml.etree.ElementTree(self.BuildConfiguration(client)).write( + ofile, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write File %s: %s" % (ofile, err)) else: print(self._get_usage(self.do_build)) @@ -433,7 +437,7 @@ Bcfg2 client itself.""") pname, client = alist automatch = self.setup.cfp.getboolean("properties", "automatch", default=False) - pfile = self.plugins['Properties'].store.entries[pname] + pfile = self.plugins['Properties'].entries[pname] if (not force and not automatch and pfile.xdata.get("automatch", "false").lower() != "true"): @@ -469,7 +473,6 @@ Bcfg2 client itself.""") ('Path Bcfg2 repository', self.setup['repo']), ('Plugins', self.setup['plugins']), ('Password', self.setup['password']), - ('Server Metadata Connector', self.setup['mconnect']), ('Filemonitor', self.setup['filemonitor']), ('Server address', self.setup['location']), ('Path to key', self.setup['key']), @@ -479,6 +482,17 @@ Bcfg2 client itself.""") ('Logging', self.setup['logging'])] print_tabular(output) + def do_expirecache(self, args): + """ expirecache [<hostname> [<hostname> ...]]- Expire the + metadata cache """ + alist = args.split() + if len(alist): + for client in self._get_client_list(alist): + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + else: + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + def do_probes(self, args): """ probes [-p] <hostname> - Get probe list for the given host, in XML (the default) or human-readable pretty (with -p) @@ -714,7 +728,7 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: self.load_plugins() - self.fam.handle_events_in_interval(1) + self.block_for_fam_events(handle_events=True) if args: self.onecmd(" ".join(args)) else: @@ -755,7 +769,8 @@ USAGE = build_usage() def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER) + interpreter=Bcfg2.Options.INTERPRETER, + command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 9a98eaaaa..9ceb1dd04 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -3,6 +3,7 @@ """This tool examines your Bcfg2 specifications for errors.""" import sys +import time import inspect import logging import Bcfg2.Logger @@ -52,7 +53,11 @@ def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, args.append(setup) # python 2.5 doesn't support mixing *magic and keyword arguments - return plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + start = time.time() + rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, + time.time() - start)) + return rv def get_errorhandler(setup): @@ -68,7 +73,7 @@ def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index bb45e0009..b0c170b1b 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -53,15 +53,15 @@ def print_fields(fields, client, fmt, extra=None): else: fdata.append("dirty") elif field == 'total': - fdata.append(client.current_interaction.totalcount) + fdata.append(client.current_interaction.total_count) elif field == 'good': - fdata.append(client.current_interaction.goodcount) + fdata.append(client.current_interaction.good_count) elif field == 'modified': - fdata.append(client.current_interaction.modified_entry_count()) + fdata.append(client.current_interaction.modified_count) elif field == 'extra': - fdata.append(client.current_interaction.extra_entry_count()) + fdata.append(client.current_interaction.extra_count) elif field == 'bad': - fdata.append((client.current_interaction.badcount())) + fdata.append(client.current_interaction.bad_count) else: try: fdata.append(getattr(client, field)) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index c33143a04..7c38a65d8 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -5,6 +5,7 @@ without failures""" import os import sys +import signal import fnmatch import logging import Bcfg2.Logger @@ -156,7 +157,7 @@ def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core @@ -190,9 +191,23 @@ def run_child(setup, clients, queue): core.shutdown() +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + def parse_args(): """ Parse command line arguments. """ optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -246,6 +261,7 @@ def main(): setup = parse_args() logger = logging.getLogger(sys.argv[0]) core = get_core(setup) + signal.signal(signal.SIGINT, get_sigint_handler(core)) if setup['args']: clients = setup['args'] @@ -282,8 +298,8 @@ def main(): for client in clients: yield ClientTest(core, client, ignore) - TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) + result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], + suite=LazySuite(generate_tests), exit=False) # block until all children have completed -- should be # immediate since we've already gotten all the results we @@ -292,7 +308,10 @@ def main(): child.join() core.shutdown() - os._exit(0) # pylint: disable=W0212 + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 if __name__ == "__main__": diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 7dbdad16b..227d977de 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -10,10 +10,14 @@ import sys import yum import logging import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout from optparse import OptionParser try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json @@ -42,8 +46,8 @@ def pkgtup_to_string(package): return ''.join(str(e) for e in rv) -class DepSolver(object): - """ Yum dependency solver """ +class YumHelper(object): + """ Yum helper base object """ def __init__(self, cfgfile, verbose=1): self.cfgfile = cfgfile @@ -57,6 +61,16 @@ class DepSolver(object): self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 self._groups = None def get_groups(self): @@ -181,6 +195,45 @@ class DepSolver(object): packages.add(txmbr.pkgtup) return list(packages), list(unknown) + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock def clean_cache(self): """ clean the yum cache """ for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -193,6 +246,27 @@ class DepSolver(object): if not msg.startswith("0 "): self.logger.info(msg) + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + def main(): parser = OptionParser() @@ -221,29 +295,70 @@ def main(): logger.error("Config file %s not found" % options.config) return 1 - depsolver = DepSolver(options.config, options.verbose) + # pylint: disable=W0702 + rv = 0 if cmd == "clean": - depsolver.clean_cache() - print(json.dumps(True)) + cachemgr = CacheManager(options.config, options.verbose) + try: + cachemgr.clean_cache() + print(json.dumps(True)) + except: + logger.error("Unexpected error cleaning cache: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(False)) + rv = 2 + elif cmd == "makecache": + cachemgr = CacheManager(options.config, options.verbose) + try: + # this code copied from yumcommands.py + cachemgr.populate_cache() + print(json.dumps(True)) + except yum.Errors.YumBaseError: + logger.error("Unexpected error creating cache: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(False)) elif cmd == "complete": - data = json.loads(sys.stdin.read()) - depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete([pkg_to_tuple(p) - for p in data['packages']]) - print(json.dumps(dict(packages=list(packages), - unknown=list(unknown)))) + depsolver = DepSolver(options.config, options.verbose) + try: + data = json.loads(sys.stdin.read()) + except: + logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + rv = 2 + try: + depsolver.groups = data['groups'] + (packages, unknown) = depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + print(json.dumps(dict(packages=list(packages), + unknown=list(unknown)))) + except: + logger.error("Unexpected error completing package set: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(dict(packages=[], unknown=data['packages']))) + rv = 2 elif cmd == "get_groups": - data = json.loads(sys.stdin.read()) - rv = dict() - for gdata in data: - if "type" in gdata: - packages = depsolver.get_group(gdata['group'], - ptype=gdata['type']) - else: - packages = depsolver.get_group(gdata['group']) - rv[gdata['group']] = list(packages) - print(json.dumps(rv)) - + depsolver = DepSolver(options.config, options.verbose) + try: + data = json.loads(sys.stdin.read()) + rv = dict() + for gdata in data: + if "type" in gdata: + packages = depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + print(json.dumps(rv)) + except: + logger.error("Unexpected error getting groups: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(dict())) + rv = 2 + else: + logger.error("Unknown command %s" % cmd) + print(json.dumps(None)) + rv = 2 + return rv if __name__ == '__main__': sys.exit(main()) |