diff options
Diffstat (limited to 'src/sbin')
| -rwxr-xr-x | src/sbin/bcfg2 | 27 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-admin | 90 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-build-reports | 306 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-crypt | 439 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-info | 803 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-lint | 209 | ||||
| l--------- | src/sbin/bcfg2-repo-validate | 1 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-report-collector | 14 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-server | 86 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-test | 317 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-yum-helper | 354 |
11 files changed, 57 insertions, 2589 deletions
diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 444e86a7c..eca7c3395 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -2,28 +2,9 @@ """Bcfg2 Client""" import sys -import signal -import Bcfg2.Options -from Bcfg2.Client.Client import Client - - -def cb_sigint_handler(signum, frame): - """ Exit upon CTRL-C. """ - raise SystemExit(1) - - -def main(): - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - if setup['args']: - print("Bcfg2 takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - - signal.signal(signal.SIGINT, cb_sigint_handler) - return Client(setup).run() +from Bcfg2.Options import get_parser +from Bcfg2.Client import Client if __name__ == '__main__': - sys.exit(main()) + get_parser("Bcfg2 client", components=[Client]).parse() + sys.exit(Client().run()) diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 14d188342..d57cd8b35 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -2,97 +2,11 @@ """ bcfg2-admin is a script that helps to administer a Bcfg2 deployment. """ -import re import sys -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Admin -from Bcfg2.Compat import StringIO - - -def mode_import(modename): - """Load Bcfg2.Server.Admin.<mode>.""" - modname = modename.capitalize() - mod = getattr(__import__("Bcfg2.Server.Admin.%s" % - (modname)).Server.Admin, modname) - return getattr(mod, modname) - - -def get_modes(): - """Get all available modes, except for the base mode.""" - return [x.lower() for x in Bcfg2.Server.Admin.__all__ if x != 'mode'] - - -def create_description(): - """Create the description string from the list of modes.""" - modes = get_modes() - description = StringIO() - description.write("Available modes are:\n\n") - for mode in modes: - try: - doc = re.sub(r'\s{2,}', ' ', mode_import(mode).__doc__.strip()) - except (ImportError, SystemExit): - continue - description.write((" %-15s %s\n" % (mode, doc))) - return description.getvalue() - - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - # override default help message to include description of all modes - setup.hm = "Usage:\n\n%s\n%s" % (setup.buildHelpMessage(), - create_description()) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-admin', to_syslog=setup['syslog'], - level=level) - - log = logging.getLogger('bcfg2-admin') - - # Provide help if requested or no args were specified - if (not setup['args'] or len(setup['args']) < 1 or - setup['args'][0] == 'help' or setup['help']): - if len(setup['args']) > 1: - # Get help for a specific mode by passing it the help argument - setup['args'] = [setup['args'][1], setup['args'][0]] - else: - # Print short help for all modes - print(setup.hm) - raise SystemExit(0) - - if setup['args'][0] in get_modes(): - modname = setup['args'][0].capitalize() - if len(setup['args']) > 1 and setup['args'][1] == 'help': - mode_cls = mode_import(modname) - mode_cls.usage(rv=0) - try: - mode_cls = mode_import(modname) - except ImportError: - err = sys.exc_info()[1] - log.error("Failed to load admin mode %s: %s" % (modname, err)) - raise SystemExit(1) - mode = mode_cls(setup) - try: - return mode(setup['args'][1:]) - finally: - mode.shutdown() - else: - log.error("Error: Unknown mode '%s'\n" % setup['args'][0]) - print(create_description()) - raise SystemExit(1) +from Bcfg2.Server.Admin import CLI if __name__ == '__main__': try: - sys.exit(main()) + sys.exit(CLI().run()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports deleted file mode 100755 index 1c9e9ad97..000000000 --- a/src/sbin/bcfg2-build-reports +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python - -""" -bcfg2-build-reports generates & distributes reports of statistic -information for Bcfg2.""" - -import copy -import getopt -import re -import os -import socket -import sys -from time import asctime, strptime -from lxml.etree import XML, XSLT, parse, Element, ElementTree, SubElement, tostring, XMLSyntaxError -# Compatibility imports -from Bcfg2.Compat import ConfigParser, cmp - -def generatereport(rspec, nrpt): - """ - generatereport creates and returns an ElementTree representation - of a report adhering to the XML spec for intermediate reports. - """ - reportspec = copy.deepcopy(rspec) - nodereprt = copy.deepcopy(nrpt) - - reportgood = reportspec.get("good", default = 'Y') - reportmodified = reportspec.get("modified", default = 'Y') - current_date = asctime()[:10] - - """Build regex of all the nodes we are reporting about.""" - pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) - - for node in nodereprt.findall('Node'): - if not (node.findall("Statistics") and pattern.match(node.get('name'))): - # Don't know enough about node. - nodereprt.remove(node) - continue - - # Reduce to most recent Statistics entry. - statisticslist = node.findall('Statistics') - # This line actually sorts from most recent to oldest. - statisticslist.sort(lambda y, x: cmp(strptime(x.get("time")), strptime(y.get("time")))) - stats = statisticslist[0] - - [node.remove(item) for item in node.findall('Statistics')] - - # Add a good tag if node is good and we wnat to report such. - if reportgood == 'Y' and stats.get('state') == 'clean': - SubElement(stats,"Good") - - [stats.remove(item) for item in stats.findall("Bad") + stats.findall("Modified") if \ - item.getchildren() == []] - [stats.remove(item) for item in stats.findall("Modified") if reportmodified == 'N'] - - # Test for staleness -if stale add Stale tag. - if stats.get("time").find(current_date) == -1: - SubElement(stats,"Stale") - node.append(stats) - return nodereprt - -def mail(mailbody, confi): - """mail mails a previously generated report.""" - - try: - mailer = confi.get('statistics', 'sendmailpath') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - mailer = "/usr/sbin/sendmail" - # Open a pipe to the mail program and - # write the data to the pipe. - pipe = os.popen("%s -t" % mailer, 'w') - pipe.write(mailbody) - exitcode = pipe.close() - if exitcode: - print("Exit code: %s" % exitcode) - -def rss(reportxml, delivery, report): - """rss appends a new report to the specified rss file - keeping the last 9 articles. - """ - # Check and see if rss file exists. - for destination in delivery.findall('Destination'): - try: - fil = open(destination.attrib['address'], 'r') - olddoc = XML(fil.read()) - - # Defines the number of recent articles to keep. - items = olddoc.find("channel").findall("item")[0:9] - fil.close() - fil = open(destination.attrib['address'], 'w') - except (IOError, XMLSyntaxError): - fil = open(destination.attrib['address'], 'w') - items = [] - - rssdata = Element("rss") - channel = SubElement(rssdata, "channel") - rssdata.set("version", "2.0") - chantitle = SubElement(channel, "title") - chantitle.text = report.attrib['name'] - chanlink = SubElement(channel, "link") - - # This can later link to WWW report if one gets published - # simultaneously? - chanlink.text = "http://www.mcs.anl.gov/cobalt/bcfg2" - chandesc = SubElement(channel, "description") - chandesc.text = "Information regarding the 10 most recent bcfg2 runs." - - channel.append(XML(reportxml)) - - if items != []: - for item in items: - channel.append(item) - - tree = tostring(rssdata, xml_declaration=False).decode('UTF-8') - fil.write(tree) - fil.close() - -def www(reportxml, delivery): - """www outputs report to.""" - - # This can later link to WWW report if one gets published - # simultaneously? - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def fileout(reportxml, delivery): - """Outputs to plain text file.""" - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def pretty_print(element, level=0): - """Produce a pretty-printed text representation of element.""" - if element.text: - fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())])), - element.text, element.tag) - if element._children: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(element._children) * "%s") + "%s</%%s>\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())]),) - data += tuple([pretty_print(entry, level+2) for entry in element._children]) + (element.tag, ) - else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())])) - return fmt % data - - -if __name__ == '__main__': - all=False - if '-C' in sys.argv: - cfpath = sys.argv[sys.argv.index('-C') + 1] - else: - cfpath = '/etc/bcfg2.conf' - c = ConfigParser.ConfigParser() - c.read([cfpath]) - configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') - statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') - clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') - try: - prefix = c.get('server', 'prefix') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - prefix = '/usr' - - transformpath = "/%s/share/bcfg2/xsl-transforms/" % (prefix) - #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" - - try: - opts, args = getopt.getopt(sys.argv[1:], "C:hAc:Ns:", ["help", "all", "config=", "stats="]) - except getopt.GetoptError: - mesg = sys.exc_info()[1] - # Print help information and exit: - print("%s\nUsage:\nbcfg2-build-reports [-h][-A (include ALL clients)] [-c <configuration-file>] [-s <statistics-file>]" % (mesg)) - raise SystemExit(2) - for o, a in opts: - if o in ("-h", "--help"): - print("Usage:\nbcfg2-build-reports [-h] [-c <configuration-file>] [-s <statistics-file>]") - raise SystemExit - if o in ("-A", "--all"): - all=True - if o in ("-c", "--config"): - configpath = a - if o in ("-s", "--stats"): - statpath = a - - - """Reads data & config files.""" - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(statpath)) - raise SystemExit(1) - try: - configdata = XML(open(configpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(configpath)) - raise SystemExit(1) - try: - clientsdata = XML(open(clientsdatapath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(clientsdatapath)) - raise SystemExit(1) - - # Merge data from three sources. - nodereport = Element("Report", attrib={"time" : asctime()}) - # Should all of the other info in Metadata be appended? - # What about all of the package stuff for other types of reports? - for client in clientsdata.findall("Client"): - nodel = Element("Node", attrib={"name" : client.get("name")}) - nodel.append(client) - for nod in statsdata.findall("Node"): - if client.get('name').find(nod.get('name')) == 0: - for statel in nod.findall("Statistics"): - nodel.append(statel) - nodereport.append(nodel) - - if all: - for nod in statsdata.findall("Node"): - for client in clientsdata.findall("Client"): - if client.get('name').find(nod.get('name')) == 0: - break - else: - nodel = Element("Node", attrib={"name" : nod.get("name")}) - client = Element("Client", attrib={"name" : nod.get("name"), "profile" : "default"}) - nodel.append(client) - for statel in nod.findall("Statistics"): - nodel.append(statel) - nodereport.append(nodel) - - - for reprt in configdata.findall('Report'): - nodereport.set("name", reprt.get("name", default="BCFG Report")) - - if reprt.get('refresh-time') != None: - nodereport.set("refresh-time", reprt.get("refresh-time", default="600")) - - procnodereport = generatereport(reprt, nodereport) - - for deliv in reprt.findall('Delivery'): - # Is a deepcopy of procnodereport necessary? - - delivtype = deliv.get('type', default='nodes-digest') - deliverymechanism = deliv.get('mechanism', default='www') - - # Apply XSLT, different ones based on report type, and options - if deliverymechanism == 'null-operator': # Special Cases - fileout(tostring(ElementTree(procnodereport).getroot(), xml_declaration=False).decode('UTF-8'), deliv) - break - transform = delivtype + '-' + deliverymechanism + '.xsl' - - try: # Make sure valid stylesheet is selected. - os.stat(transformpath + transform) - except: - print("bcfg2-build-reports: Invalid report type or delivery mechanism.\n Can't find: "\ - + transformpath + transform) - raise SystemExit(1) - - try: # Try to parse stylesheet. - stylesheet = XSLT(parse(transformpath + transform)) - except: - print("bcfg2-build-reports: invalid XSLT transform file.") - raise SystemExit(1) - - if deliverymechanism == 'mail': - if delivtype == 'nodes-individual': - reportdata = copy.deepcopy(procnodereport) - for noden in reportdata.findall("Node"): - [reportdata.remove(y) for y in reportdata.findall("Node")] - reportdata.append(noden) - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - # Prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - - else: - reportdata = copy.deepcopy(procnodereport) - - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - # Prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - else: - outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), xml_declaration=False).decode('UTF-8') - if deliverymechanism == 'rss': - rss(outputstring, deliv, reprt) - else: # Must be deliverymechanism == 'www': - www(outputstring, deliv) diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 0ba84fa0a..26d5eedf1 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -1,443 +1,8 @@ #!/usr/bin/env python """ helper for encrypting/decrypting Cfg and Properties files """ -import os import sys -import copy -import select -import logging -import lxml.etree -import Bcfg2.Logger -import Bcfg2.Options -from Bcfg2.Server import XMLParser -from Bcfg2.Compat import input # pylint: disable=W0622 -try: - import Bcfg2.Encryption -except ImportError: - print("Could not import %s. Is M2Crypto installed?" % sys.exc_info()[1]) - raise SystemExit(1) - - -class PassphraseError(Exception): - """ Exception raised when there's a problem determining the - passphrase to encrypt or decrypt with """ - - -class CryptoTool(object): - """ Generic decryption/encryption interface base object """ - def __init__(self, filename, setup): - self.setup = setup - self.logger = logging.getLogger(self.__class__.__name__) - self.passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - - self.filename = filename - try: - self.data = open(self.filename).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (filename, - err)) - return False - - self.pname, self.passphrase = self._get_passphrase() - - def _get_passphrase(self): - """ get the passphrase for the current file """ - if (not self.setup.cfp.has_section(Bcfg2.Encryption.CFG_SECTION) or - len(Bcfg2.Encryption.get_passphrases(self.setup)) == 0): - raise PassphraseError("No passphrases available in %s" % - self.setup['configfile']) - - pname = None - if self.setup['passphrase']: - pname = self.setup['passphrase'] - - if pname: - if self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - pname) - self.logger.debug("Using passphrase %s specified on command " - "line" % pname) - return (pname, passphrase) - else: - raise PassphraseError("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) - else: - pnames = Bcfg2.Encryption.get_passphrases(self.setup) - if len(pnames) == 1: - pname = pnames.keys()[0] - passphrase = pnames[pname] - self.logger.info("Using passphrase %s" % pname) - return (pname, passphrase) - elif len(pnames) > 1: - return (None, None) - raise PassphraseError("No passphrase could be determined") - - def get_destination_filename(self, original_filename): - """ Get the filename where data should be written """ - return original_filename - - def write(self, data): - """ write data to disk """ - new_fname = self.get_destination_filename(self.filename) - try: - self._write(new_fname, data) - self.logger.info("Wrote data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing data from %s to %s: %s" % - (self.filename, new_fname, err)) - return False - - def _write(self, filename, data): - """ Perform the actual write of data. This is separate from - :func:`CryptoTool.write` so it can be easily - overridden. """ - open(filename, "wb").write(data) - - -class Decryptor(CryptoTool): - """ Decryptor interface """ - def decrypt(self): - """ decrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class Encryptor(CryptoTool): - """ encryptor interface """ - def encrypt(self): - """ encrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ - - def __init__(self, filename, setup): - Encryptor.__init__(self, filename, setup) - if self.passphrase is None: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - - def encrypt(self): - return Bcfg2.Encryption.ssl_encrypt( - self.data, self.passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - - def get_destination_filename(self, original_filename): - return original_filename + ".crypt" - - -class CfgDecryptor(Decryptor): - """ Decrypt Cfg files """ - - def decrypt(self): - """ decrypt the given file, returning the plaintext data """ - if self.passphrase: - try: - return Bcfg2.Encryption.ssl_decrypt( - self.data, self.passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - except Bcfg2.Encryption.EVPError: - self.logger.info("Could not decrypt %s with the " - "specified passphrase" % self.filename) - return False - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (self.filename, err)) - return False - else: # no passphrase given, brute force - try: - return Bcfg2.Encryption.bruteforce_decrypt( - self.data, passphrases=self.passphrases.values(), - algorithm=Bcfg2.Encryption.get_algorithm(self.setup)) - except Bcfg2.Encryption.EVPError: - self.logger.info("Could not decrypt %s with any passphrase" % - self.filename) - return False - - def get_destination_filename(self, original_filename): - if original_filename.endswith(".crypt"): - return original_filename[:-6] - else: - return Decryptor.get_plaintext_filename(self, original_filename) - - -class PropertiesCryptoMixin(object): - """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*' - - def _get_elements(self, xdata): - """ Get the list of elements to encrypt or decrypt """ - if self.setup['xpath']: - elements = xdata.xpath(self.setup['xpath']) - if not elements: - self.logger.warning("XPath expression %s matched no " - "elements" % self.setup['xpath']) - else: - elements = xdata.xpath(self.default_xpath) - if not elements: - elements = list(xdata.getiterator(tag=lxml.etree.Element)) - - # filter out elements without text data - for el in elements[:]: - if not el.text: - elements.remove(el) - - if self.setup['interactive']: - for element in elements[:]: - if len(element): - elt = copy.copy(element) - for child in elt.iterchildren(): - elt.remove(child) - else: - elt = element - print(lxml.etree.tostring( - elt, - xml_declaration=False).decode("UTF-8").strip()) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Encrypt this element? [y/N] ") - if not ans.lower().startswith("y"): - elements.remove(element) - return elements - - def _get_element_passphrase(self, element): - """ Get the passphrase to use to encrypt or decrypt a given - element """ - pname = element.get("encrypted") - if pname in self.passphrases: - passphrase = self.passphrases[pname] - elif self.passphrase: - if pname: - self.logger.warning("Passphrase %s not found in %s, " - "using passphrase given on command line" - % (pname, self.setup['configfile'])) - passphrase = self.passphrase - pname = self.pname - else: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - return (pname, passphrase) - - def _write(self, filename, data): - """ Write the data """ - data.getroottree().write(filename, - xml_declaration=False, - pretty_print=True) - - -class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): - """ encryptor class for Properties files """ - - def encrypt(self): - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False - elt.text = Bcfg2.Encryption.ssl_encrypt( - elt.text, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - elt.set("encrypted", pname) - return xdata - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): - """ decryptor class for Properties files """ - default_xpath = '//*[@encrypted]' - - def decrypt(self): - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False - decrypted = Bcfg2.Encryption.ssl_decrypt( - elt.text, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - try: - elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') - elt.set("encrypted", pname) - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains - # content that can't even be encoded into xml - # entities. what probably happened here is that we - # coincidentally could decrypt a value encrypted with - # a different key, and wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - elt.tag) - return xdata - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -def main(): # pylint: disable=R0912,R0915 - optinfo = dict(interactive=Bcfg2.Options.INTERACTIVE) - optinfo.update(Bcfg2.Options.CRYPT_OPTIONS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = " bcfg2-crypt [options] <filename>\nOptions:\n%s" % \ - setup.buildHelpMessage() - setup.parse(sys.argv[1:]) - - if not setup['args']: - print(setup.hm) - raise SystemExit(1) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-crypt', **log_args) - logger = logging.getLogger('bcfg2-crypt') - - if setup['decrypt']: - if setup['encrypt']: - logger.error("You cannot specify both --encrypt and --decrypt") - raise SystemExit(1) - elif setup['remove']: - logger.error("--remove cannot be used with --decrypt, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - elif setup['interactive']: - logger.error("Cannot decrypt interactively") - setup['interactive'] = False - - if setup['cfg']: - if setup['properties']: - logger.error("You cannot specify both --cfg and --properties") - raise SystemExit(1) - if setup['xpath']: - logger.error("Specifying --xpath with --cfg is nonsensical, " - "ignoring --xpath") - setup['xpath'] = Bcfg2.Options.CRYPT_XPATH.default - if setup['interactive']: - logger.error("You cannot use interactive mode with --cfg, " - "ignoring -I") - setup['interactive'] = False - elif setup['properties']: - if setup['remove']: - logger.error("--remove cannot be used with --properties, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - - for fname in setup['args']: - if not os.path.exists(fname): - logger.error("%s does not exist, skipping" % fname) - continue - - # figure out if we need to encrypt this as a Properties file - # or as a Cfg file - props = False - if setup['properties']: - props = True - elif setup['cfg']: - props = False - elif fname.endswith(".xml"): - try: - xroot = lxml.etree.parse(fname).getroot() - if xroot.tag == "Properties": - props = True - else: - props = False - except IOError: - err = sys.exc_info()[1] - logger.error("Error reading %s, skipping: %s" % (fname, err)) - continue - except lxml.etree.XMLSyntaxError: - props = False - else: - props = False - - if props: - if setup['remove']: - logger.info("Cannot use --remove with Properties file %s, " - "ignoring for this file" % fname) - tools = (PropertiesEncryptor, PropertiesDecryptor) - else: - if setup['xpath']: - logger.info("Cannot use xpath with Cfg file %s, ignoring " - "xpath for this file" % fname) - if setup['interactive']: - logger.info("Cannot use interactive mode with Cfg file %s, " - "ignoring -I for this file" % fname) - tools = (CfgEncryptor, CfgDecryptor) - - data = None - mode = None - if setup['encrypt']: - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - elif setup['decrypt']: - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "decrypt" - else: - logger.info("Neither --encrypt nor --decrypt specified, " - "determining mode") - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - - try: - data = tool.decrypt() - mode = "decrypt" - except: # pylint: disable=W0702 - pass - if data is False: - data = None - logger.info("Failed to decrypt %s, trying encryption" % fname) - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - - if data is None: - data = getattr(tool, mode)() - if not data: - logger.error("Failed to %s %s, skipping" % (mode, fname)) - continue - if setup['crypt_stdout']: - if len(setup['args']) > 1: - print("----- %s -----" % fname) - print(data) - if len(setup['args']) > 1: - print("") - else: - tool.write(data) - - if (setup['remove'] and - tool.get_destination_filename(fname) != fname): - try: - os.unlink(fname) - except IOError: - err = sys.exc_info()[1] - logger.error("Error removing %s: %s" % (fname, err)) - continue +from Bcfg2.Server.Encryption import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 6008f8896..adfa96852 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,807 +1,8 @@ #!/usr/bin/env python """This tool loads the Bcfg2 core into an interactive debugger.""" -import os -import re import sys -import cmd -import getopt -import fnmatch -import logging -import lxml.etree -import traceback -from code import InteractiveConsole -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Plugin -import Bcfg2.Client.Tools.POSIX - -try: - try: - import cProfile as profile - except ImportError: - import profile - import pstats - HAS_PROFILE = True -except ImportError: - HAS_PROFILE = False - - -class MockLog(object): - """ Fake logger that just discards all messages in order to mask - errors from builddir being unable to chown files it creates """ - def error(self, *args, **kwargs): - """ discard error messages """ - pass - - def warning(self, *args, **kwargs): - """ discard warning messages """ - pass - - def info(self, *args, **kwargs): - """ discard info messages """ - pass - - def debug(self, *args, **kwargs): - """ discard debug messages """ - pass - - -class FileNotBuilt(Exception): - """Thrown when File entry contains no content.""" - def __init__(self, value): - Exception.__init__(self) - self.value = value - - def __str__(self): - return repr(self.value) - - -def print_tabular(rows): - """Print data in tabular format.""" - cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 - for index in range(len(rows[0]))]) - fstring = (" %%-%ss |" * len(cmax)) % cmax - fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax - print(fstring % rows[0]) - print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') - for row in rows[1:]: - print(fstring % row) - - -def display_trace(trace): - """ display statistics from a profile trace """ - stats = pstats.Stats(trace) - stats.sort_stats('cumulative', 'calls', 'time') - stats.print_stats(200) - - -def load_interpreters(): - """ Load a dict of available Python interpreters """ - interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) - best = "python" - try: - import bpython.cli - interpreters["bpython"] = lambda v: bpython.cli.main(args=[], - locals_=v) - best = "bpython" - except ImportError: - pass - - try: - # whether ipython is actually better than bpython is - # up for debate, but this is the behavior that existed - # before --interpreter was added, so we call IPython - # better - import IPython - # pylint: disable=E1101 - if hasattr(IPython, "Shell"): - interpreters["ipython"] = lambda v: \ - IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() - best = "ipython" - elif hasattr(IPython, "embed"): - interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) - best = "ipython" - else: - print("Unknown IPython API version") - # pylint: enable=E1101 - except ImportError: - pass - - interpreters['best'] = interpreters[best] - return interpreters - - -class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): - """Main class for bcfg2-info.""" - def __init__(self, setup): - cmd.Cmd.__init__(self) - Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) - self.prompt = '> ' - self.cont = True - - def _get_client_list(self, hostglobs): - """ given a host glob, get a list of clients that match it """ - # special cases to speed things up: - if '*' in hostglobs: - return self.metadata.clients - has_wildcards = False - for glob in hostglobs: - # check if any wildcard characters are in the string - if set('*?[]') & set(glob): - has_wildcards = True - break - if not has_wildcards: - return hostglobs - - rv = set() - clist = set(self.metadata.clients) - for glob in hostglobs: - for client in clist: - if fnmatch.fnmatch(client, glob): - rv.update(client) - clist.difference_update(rv) - return list(rv) - - def _get_usage(self, func): - """ get the short usage message for a given function """ - return "Usage: " + re.sub(r'\s+', ' ', func.__doc__).split(" - ", 1)[0] - - def do_loop(self): - """Looping.""" - self.cont = True - while self.cont: - try: - self.cmdloop('Welcome to bcfg2-info\n' - 'Type "help" for more information') - except SystemExit: - raise - except Bcfg2.Server.Plugin.PluginExecutionError: - continue - except KeyboardInterrupt: - print("Ctrl-C pressed exiting...") - self.do_exit([]) - except: - self.logger.error("Command failure", exc_info=1) - - def do_debug(self, args): - """ debug [-n] [-f <command list>] - Shell out to native - python interpreter """ - try: - opts, _ = getopt.getopt(args.split(), 'nf:') - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_debug)) - return - self.cont = False - scriptmode = False - interactive = True - for opt in opts: - if opt[0] == '-f': - scriptmode = True - spath = opt[1] - elif opt[0] == '-n': - interactive = False - if scriptmode: - console = InteractiveConsole(locals()) - for command in [c.strip() for c in open(spath).readlines()]: - if command: - console.push(command) - if interactive: - interpreters = load_interpreters() - if self.setup['interpreter'] in interpreters: - print("Dropping to %s interpreter; press ^D to resume" % - self.setup['interpreter']) - interpreters[self.setup['interpreter']](locals()) - else: - self.logger.error("Invalid interpreter %s" % - self.setup['interpreter']) - self.logger.error("Valid interpreters are: %s" % - ", ".join(interpreters.keys())) - - def do_quit(self, _): - """ quit|exit - Exit program """ - self.shutdown() - os._exit(0) # pylint: disable=W0212 - - do_EOF = do_quit - do_exit = do_quit - - def do_help(self, _): - """ help - Print this list of available commands """ - print(USAGE) - - def do_update(self, _): - """ update - Process pending filesystem events""" - self.fam.handle_events_in_interval(0.1) - - def do_build(self, args): - """ build [-f] <hostname> <filename> - Build config for - hostname, writing to filename""" - alist = args.split() - path_force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, ofile = alist - if not ofile.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - try: - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) - except IOError: - err = sys.exc_info()[1] - print("Failed to write File %s: %s" % (ofile, err)) - else: - print(self._get_usage(self.do_build)) - - def help_builddir(self): - """Display help for builddir command.""" - print("""Usage: builddir [-f] <hostname> <output dir> - -Generates a config for client <hostname> and writes the -individual configuration files out separately in a tree -under <output dir>. The <output dir> directory must be -rooted under /tmp unless the -f argument is provided, in -which case it can be located anywhere. - -NOTE: Currently only handles file entries and writes -all content with the default owner and permissions. These -could be much more permissive than would be created by the -Bcfg2 client itself.""") - - def do_builddir(self, args): - """ builddir [-f] <hostname> <dirname> - Build config for - hostname, writing separate files to dirname""" - alist = args.split() - path_force = False - if '-f' in args: - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, odir = alist - if not odir.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - client_config = self.BuildConfiguration(client) - if client_config.tag == 'error': - print("Building client configuration failed.") - return - - for struct in client_config: - for entry in struct: - if entry.tag == 'Path': - entry.set('name', odir + '/' + entry.get('name')) - - posix = Bcfg2.Client.Tools.POSIX.POSIX(MockLog(), - self.setup, - client_config) - states = dict() - posix.Inventory(states) - posix.Install(list(states.keys()), states) - else: - print('Error: Incorrect number of parameters.') - self.help_builddir() - - def do_buildall(self, args): - """ buildall <directory> [<hostnames*>] - Build configs for - all clients in directory """ - alist = args.split() - if len(alist) < 1: - print(self._get_usage(self.do_buildall)) - return - - destdir = alist[0] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(alist) > 1: - clients = self._get_client_list(alist[1:]) - else: - clients = self.metadata.clients - for client in clients: - self.do_build("%s %s" % (client, os.path.join(destdir, - client + ".xml"))) - - def do_buildallfile(self, args): - """ buildallfile <directory> <filename> [<hostnames*>] - Build - config file for all clients in directory """ - try: - opts, args = getopt.gnu_getopt(args.split(), '', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_buildallfile)) - return - altsrc = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - if len(args) < 2: - print(self._get_usage(self.do_buildallfile)) - return - - destdir = args[0] - filename = args[1] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(args) > 2: - clients = self._get_client_list(args[1:]) - else: - clients = self.metadata.clients - if altsrc: - args = "--altsrc %s -f %%s %%s %%s" % altsrc - else: - args = "-f %s %s %s" - for client in clients: - self.do_buildfile(args % (os.path.join(destdir, client), - filename, client)) - - def do_buildfile(self, args): - """ buildfile [-f <outfile>] [--altsrc=<altsrc>] <filename> - <hostname> - Build config file for hostname (not written to - disk)""" - try: - opts, alist = getopt.gnu_getopt(args.split(), 'f:', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_buildfile.__doc__) - return - altsrc = None - outfile = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - elif opt[0] == '-f': - outfile = opt[1] - if len(alist) != 2: - print(self.do_buildfile.__doc__) - return - - fname, client = alist - entry = lxml.etree.Element('Path', type='file', name=fname) - if altsrc: - entry.set("altsrc", altsrc) - try: - metadata = self.build_metadata(client) - self.Bind(entry, metadata) - data = lxml.etree.tostring(entry, - xml_declaration=False).decode('UTF-8') - except Exception: - print("Failed to build entry %s for host %s: %s" % - (fname, client, traceback.format_exc().splitlines()[-1])) - raise - try: - if outfile: - open(outfile, 'w').write(data) - else: - print(data) - except IOError: - err = sys.exc_info()[1] - print("Could not write to %s: %s" % (outfile, err)) - print(data) - - def do_buildbundle(self, args): - """ buildbundle <bundle> <hostname> - Render a templated - bundle for hostname (not written to disk) """ - if len(args.split()) != 2: - print(self._get_usage(self.do_buildbundle)) - return 1 - - bname, client = args.split() - try: - metadata = self.build_metadata(client) - bundle = self.plugins['Bundler'].entries[bname] - print(lxml.etree.tostring(bundle.get_xml_value(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - except KeyError: - print("No such bundle %s" % bname) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Failed to render bundle %s for host %s: %s" % (bname, - client, - err)) - - def do_automatch(self, args): - """ automatch [-f] <propertyfile> <hostname> - Perform automatch on - a Properties file """ - alist = args.split() - force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - force = True - if len(alist) != 2: - print(self._get_usage(self.do_automatch)) - return - - if 'Properties' not in self.plugins: - print("Properties plugin not enabled") - return - - pname, client = alist - automatch = self.setup.cfp.getboolean("properties", "automatch", - default=False) - pfile = self.plugins['Properties'].entries[pname] - if (not force and - not automatch and - pfile.xdata.get("automatch", "false").lower() != "true"): - print("Automatch not enabled on %s" % pname) - else: - metadata = self.build_metadata(client) - print(lxml.etree.tostring(pfile.XMLMatch(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_bundles(self, _): - """ bundles - Print out group/bundle info """ - data = [('Group', 'Bundles')] - groups = list(self.metadata.groups.keys()) - groups.sort() - for group in groups: - data.append((group, - ','.join(self.metadata.groups[group][0]))) - print_tabular(data) - - def do_clients(self, _): - """ clients - Print out client/profile info """ - data = [('Client', 'Profile')] - for client in sorted(self.metadata.list_clients()): - imd = self.metadata.get_initial_metadata(client) - data.append((client, imd.profile)) - print_tabular(data) - - def do_config(self, _): - """ config - Print out the current configuration of Bcfg2""" - output = [ - ('Description', 'Value'), - ('Path Bcfg2 repository', self.setup['repo']), - ('Plugins', self.setup['plugins']), - ('Password', self.setup['password']), - ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), - ('Path to key', self.setup['key']), - ('Path to SSL certificate', self.setup['cert']), - ('Path to SSL CA certificate', self.setup['ca']), - ('Protocol', self.setup['protocol']), - ('Logging', self.setup['logging'])] - print_tabular(output) - - def do_expirecache(self, args): - """ expirecache [<hostname> [<hostname> ...]]- Expire the - metadata cache """ - alist = args.split() - if len(alist): - for client in self._get_client_list(alist): - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) - else: - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def do_probes(self, args): - """ probes [-p] <hostname> - Get probe list for the given - host, in XML (the default) or human-readable pretty (with -p) - format""" - alist = args.split() - pretty = False - if '-p' in alist: - pretty = True - alist.remove('-p') - if len(alist) != 1: - print(self._get_usage(self.do_probes)) - return - hostname = alist[0] - if pretty: - probes = [] - else: - probes = lxml.etree.Element('probes') - metadata = self.build_metadata(hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): - for probe in plugin.GetProbes(metadata): - probes.append(probe) - if pretty: - for probe in probes: - pname = probe.get("name") - print("=" * (len(pname) + 2)) - print(" %s" % pname) - print("=" * (len(pname) + 2)) - print("") - print(probe.text) - print("") - else: - print(lxml.etree.tostring(probes, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_showentries(self, args): - """ showentries <hostname> <type> - Show abstract - configuration entries for a given host """ - arglen = len(args.split()) - if arglen not in [1, 2]: - print(self._get_usage(self.do_showentries)) - return - client = args.split()[0] - try: - meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to find metadata for host %s" % client) - return - structures = self.GetStructures(meta) - output = [('entrytype', 'name')] - if arglen == 1: - for item in structures: - for child in item.getchildren(): - output.append((child.tag, child.get('name'))) - if arglen == 2: - etype = args.split()[1] - for item in structures: - for child in item.getchildren(): - if child.tag in [etype, "Bound%s" % etype]: - output.append((child.tag, child.get('name'))) - print_tabular(output) - - def do_groups(self, _): - """ groups - Print out group info """ - data = [("Groups", "Profile", "Category")] - grouplist = list(self.metadata.groups.keys()) - grouplist.sort() - for group in grouplist: - if self.metadata.groups[group].is_profile: - prof = 'yes' - else: - prof = 'no' - cat = self.metadata.groups[group].category - data.append((group, prof, cat)) - print_tabular(data) - - def do_showclient(self, args): - """ showclient <client> [<client> ...] - Show metadata for the - given hosts """ - if not len(args): - print(self._get_usage(self.do_showclient)) - return - for client in args.split(): - try: - client_meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Client %s not defined" % client) - continue - fmt = "%-10s %s" - print(fmt % ("Hostname:", client_meta.hostname)) - print(fmt % ("Profile:", client_meta.profile)) - - group_fmt = "%-10s %-30s %s" - header = False - for group in list(client_meta.groups): - category = "" - for cat, grp in client_meta.categories.items(): - if grp == group: - category = "Category: %s" % cat - break - if not header: - print(group_fmt % ("Groups:", group, category)) - header = True - else: - print(group_fmt % ("", group, category)) - - if client_meta.bundles: - print(fmt % ("Bundles:", list(client_meta.bundles)[0])) - for bnd in list(client_meta.bundles)[1:]: - print(fmt % ("", bnd)) - if client_meta.connectors: - print("Connector data") - print("=" * 80) - for conn in client_meta.connectors: - if getattr(client_meta, conn): - print(fmt % (conn + ":", getattr(client_meta, conn))) - print("=" * 80) - - def do_mappings(self, args): - """ mappings <type*> <name*> - Print generator mappings for - optional type and name """ - # Dump all mappings unless type specified - data = [('Plugin', 'Type', 'Name')] - arglen = len(args.split()) - for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): - if arglen == 0: - etypes = list(generator.Entries.keys()) - else: - etypes = [args.split()[0]] - if arglen == 2: - interested = [(etype, [args.split()[1]]) - for etype in etypes] - else: - interested = [(etype, generator.Entries[etype]) - for etype in etypes - if etype in generator.Entries] - for etype, names in interested: - for name in [name for name in names if name in - generator.Entries.get(etype, {})]: - data.append((generator.name, etype, name)) - print_tabular(data) - - def do_event_debug(self, _): - """ event_debug - Display filesystem events as they are - processed """ - self.fam.debug = True - - def do_packageresolve(self, args): - """ packageresolve <hostname> [<package> [<package>...]] - - Resolve packages for the given host, optionally specifying a - set of packages """ - arglist = args.split(" ") - if len(arglist) < 1: - print(self._get_usage(self.do_packageresolve)) - return - - try: - pkgs = self.plugins['Packages'] - except KeyError: - print("Packages plugin not enabled") - return - pkgs.toggle_debug() - - hostname = arglist[0] - metadata = self.build_metadata(hostname) - - indep = lxml.etree.Element("Independent") - if len(arglist) > 1: - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) - else: - structures = self.GetStructures(metadata) - - pkgs._build_packages(metadata, indep, # pylint: disable=W0212 - structures) - print("%d new packages added" % len(indep.getchildren())) - if len(indep.getchildren()): - print(" %s" % "\n ".join(lxml.etree.tostring(p) - for p in indep.getchildren())) - - def do_packagesources(self, args): - """ packagesources <hostname> - Show package sources """ - if not args: - print(self._get_usage(self.do_packagesources)) - return - if 'Packages' not in self.plugins: - print("Packages plugin not enabled") - return - try: - metadata = self.build_metadata(args) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to build metadata for host %s" % args) - return - collection = self.plugins['Packages'].get_collection(metadata) - print(collection.sourcelist()) - - def do_query(self, args): - """ query <-g group|-p profile|-b bundle> - Query clients """ - if len(args) == 0: - print("\n".join(self.metadata.clients)) - return - arglist = args.split(" ") - if len(arglist) != 2: - print(self._get_usage(self.do_query)) - return - - qtype, qparam = arglist - if qtype == '-p': - res = self.metadata.get_client_names_by_profiles(qparam.split(',')) - elif qtype == '-g': - res = self.metadata.get_client_names_by_groups(qparam.split(',')) - elif qtype == '-b': - res = self.metadata.get_client_names_by_bundles(qparam.split(',')) - else: - print(self._get_usage(self.do_query)) - return - print("\n".join(res)) - - def do_profile(self, arg): - """ profile <command> <args> - Profile a single bcfg2-info - command """ - if not HAS_PROFILE: - print("Profiling functionality not available.") - return - if len(arg) == 0: - print(self._get_usage(self.do_profile)) - return - prof = profile.Profile() - prof.runcall(self.onecmd, arg) - display_trace(prof) - - def run(self, args): # pylint: disable=W0221 - try: - self.load_plugins() - self.block_for_fam_events(handle_events=True) - if args: - self.onecmd(" ".join(args)) - else: - self.do_loop() - finally: - self.shutdown() - - def _daemonize(self): - pass - - def _run(self): - pass - - def _block(self): - pass - - -def build_usage(): - """ build usage message """ - cmd_blacklist = ["do_loop", "do_EOF"] - usage = dict() - for attrname in dir(InfoCore): - attr = getattr(InfoCore, attrname) - - # shim for python 2.4, __func__ is im_func - funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) - if (funcattr is not None and - funcattr.func_name not in cmd_blacklist and - funcattr.func_name.startswith("do_") and - funcattr.func_doc): - usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) - return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) - - -USAGE = build_usage() - - -def main(): - optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, - interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER, - command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) - optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", - "Options:", - setup.buildHelpMessage(), - USAGE]) - - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False, level=level) - - if setup['args'] and setup['args'][0] == 'help': - print(setup.hm) - sys.exit(0) - elif setup['profile'] and HAS_PROFILE: - prof = profile.Profile() - loop = prof.runcall(InfoCore, setup) - display_trace(prof) - else: - if setup['profile']: - print("Profiling functionality not available.") - loop = InfoCore(setup) - - loop.run(setup['args']) - +from Bcfg2.Server.Info import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 9ceb1dd04..e818dc3be 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -1,213 +1,8 @@ #!/usr/bin/env python - """This tool examines your Bcfg2 specifications for errors.""" import sys -import time -import inspect -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Lint - -LOGGER = logging.getLogger('bcfg2-lint') - - -def run_serverless_plugins(plugins, setup=None, errorhandler=None, files=None): - """ Run serverless plugins """ - LOGGER.debug("Running serverless plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, errorhandler=errorhandler, - setup=setup, files=files) - - -def run_server_plugins(plugins, setup=None, errorhandler=None, files=None): - """ run plugins that require a running server to run """ - core = load_server(setup) - try: - LOGGER.debug("Running server plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, args=[core], - errorhandler=errorhandler, setup=setup, files=files) - finally: - core.shutdown() - - -def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, - args=None, files=None): - """ run a single plugin, server-ful or serverless. """ - LOGGER.debug(" Running %s" % plugin_name) - if args is None: - args = [] - - if errorhandler is None: - errorhandler = get_errorhandler(setup) - - if setup is not None and setup.cfp.has_section(plugin_name): - arg = setup - for key, val in setup.cfp.items(plugin_name): - arg[key] = val - args.append(arg) - else: - args.append(setup) - - # python 2.5 doesn't support mixing *magic and keyword arguments - start = time.time() - rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() - LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, - time.time() - start)) - return rv - - -def get_errorhandler(setup): - """ get a Bcfg2.Server.Lint.ErrorHandler object """ - if setup.cfp.has_section("errors"): - errors = dict(setup.cfp.items("errors")) - else: - errors = None - return Bcfg2.Server.Lint.ErrorHandler(errors=errors) - - -def load_server(setup): - """ load server """ - core = Bcfg2.Server.Core.BaseCore(setup) - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def load_plugin(module, obj_name=None): - """ load a single plugin """ - parts = module.split(".") - if obj_name is None: - obj_name = parts[-1] - - mod = __import__(module) - for part in parts[1:]: - mod = getattr(mod, part) - return getattr(mod, obj_name) - - -def load_plugins(setup): - """ get list of plugins to run """ - if setup['args']: - plugin_list = setup['args'] - elif "bcfg2-repo-validate" in sys.argv[0]: - plugin_list = 'RequiredAttrs,Validate'.split(',') - elif setup['lint_plugins']: - plugin_list = setup['lint_plugins'] - else: - plugin_list = Bcfg2.Server.Lint.plugins - - allplugins = dict() - for plugin in plugin_list: - try: - allplugins[plugin] = load_plugin("Bcfg2.Server.Lint." + plugin) - except ImportError: - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except (ImportError, AttributeError): - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % - (plugin + "Lint", err)) - except AttributeError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - - for plugin in setup['plugins']: - if plugin in allplugins: - # already loaded - continue - - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except AttributeError: - pass - except ImportError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin + "Lint", - err)) - - serverplugins = dict() - serverlessplugins = dict() - for plugin_name, plugin in allplugins.items(): - if [c for c in inspect.getmro(plugin) - if c == Bcfg2.Server.Lint.ServerPlugin]: - serverplugins[plugin_name] = plugin - else: - serverlessplugins[plugin_name] = plugin - return (serverlessplugins, serverplugins) - - -def main(): - optinfo = dict(lint_config=Bcfg2.Options.LINT_CONFIG, - showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, - stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, - schema=Bcfg2.Options.SCHEMA_PATH, - lint_plugins=Bcfg2.Options.LINT_PLUGINS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-info', **log_args) - - setup.cfp.read(setup['lint_config']) - setup.reparse() - - if setup['stdin']: - files = [s.strip() for s in sys.stdin.readlines()] - else: - files = None - - (serverlessplugins, serverplugins) = load_plugins(setup) - - errorhandler = get_errorhandler(setup) - - if setup['showerrors']: - for plugin in serverplugins.values() + serverlessplugins.values(): - errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) - - print("%-35s %-35s" % ("Error name", "Handler")) - for err, handler in errorhandler.errortypes.items(): - print("%-35s %-35s" % (err, handler.__name__)) - raise SystemExit(0) - - run_serverless_plugins(serverlessplugins, errorhandler=errorhandler, - setup=setup, files=files) - - if serverplugins: - if errorhandler.errors: - # it would be swell if we could try to start the server - # even if there were errors with the serverless plugins, - # but since XML parsing errors occur in the FAM thread - # (not in the core server thread), there's no way we can - # start the server and try to catch exceptions -- - # bcfg2-lint isn't in the same stack as the exceptions. - # so we're forced to assume that a serverless plugin error - # will prevent the server from starting - print("Serverless plugins encountered errors, skipping server " - "plugins") - else: - run_server_plugins(serverplugins, errorhandler=errorhandler, - setup=setup, files=files) - - if errorhandler.errors or errorhandler.warnings or setup['verbose']: - print("%d errors" % errorhandler.errors) - print("%d warnings" % errorhandler.warnings) - - if errorhandler.errors: - raise SystemExit(2) - elif errorhandler.warnings: - raise SystemExit(3) +from Bcfg2.Server.Lint import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-repo-validate b/src/sbin/bcfg2-repo-validate deleted file mode 120000 index cea09cda3..000000000 --- a/src/sbin/bcfg2-repo-validate +++ /dev/null @@ -1 +0,0 @@ -bcfg2-lint
\ No newline at end of file diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector index 594be13bf..00e015100 100755 --- a/src/sbin/bcfg2-report-collector +++ b/src/sbin/bcfg2-report-collector @@ -11,20 +11,14 @@ from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError def main(): + parser = Bcfg2.Options.get_parser(description="Collect Bcfg2 report data", + components=[ReportingCollector]) + parser.parse() logger = logging.getLogger('bcfg2-report-collector') - optinfo = dict(daemon=Bcfg2.Options.DAEMON, - repo=Bcfg2.Options.SERVER_REPOSITORY, - filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, - web_configfile=Bcfg2.Options.WEB_CFILE) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) # run collector try: - collector = ReportingCollector(setup) - collector.run() + ReportingCollector().run() except ReportingError: msg = sys.exc_info()[1] logger.error(msg) diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 4c4a71fa7..274bd3659 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -2,63 +2,47 @@ """The XML-RPC Bcfg2 server.""" -import os import sys import logging -import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Server.Core import CoreInitError -LOGGER = logging.getLogger('bcfg2-server') - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - # check whether the specified bcfg2.conf exists - if not os.path.exists(setup['configfile']): - print("Could not read %s" % setup['configfile']) - sys.exit(1) - - # TODO: normalize case of various core modules so we can add a new - # core without modifying this script - backends = dict(cherrypy='CherryPyCore', - builtin='BuiltinCore', - best='BuiltinCore', - multiprocessing='MultiprocessingCore') - - if setup['backend'] not in backends: - print("Unknown server backend %s, using 'best'" % setup['backend']) - setup['backend'] = 'best' - - coremodule = backends[setup['backend']] - try: - corecls = getattr(__import__("Bcfg2.Server.%s" % coremodule).Server, - coremodule).Core - except ImportError: - err = sys.exc_info()[1] - print("Unable to import %s server core: %s" % (setup['backend'], err)) - raise - except AttributeError: - err = sys.exc_info()[1] - print("Unable to load %s server core: %s" % (setup['backend'], err)) - raise - - try: - core = corecls(setup) - core.run() - except CoreInitError: - msg = sys.exc_info()[1] - LOGGER.error(msg) - sys.exit(1) - except KeyboardInterrupt: - sys.exit(1) - sys.exit(0) +class BackendAction(Bcfg2.Options.ComponentAction): + """ Action to load Bcfg2 backends """ + islist = False + bases = ['Bcfg2.Server'] + + +class CLI(object): + """ bcfg2-server CLI class """ + options = [ + Bcfg2.Options.Option( + cf=('server', 'backend'), help='Server Backend', + default='BuiltinCore', type=lambda b: b.title() + "Core", + action=BackendAction)] + + def __init__(self): + parser = Bcfg2.Options.get_parser("Bcfg2 server", components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run the bcfg2 server """ + try: + core = Bcfg2.Options.setup.backend() + core.run() + except CoreInitError: + self.logger.error(sys.exc_info()[1]) + return 1 + except TypeError: + self.logger.error("Failed to load %s server backend: %s" % + (Bcfg2.Options.setup.backend.__name__, + sys.exc_info()[1])) + raise + except KeyboardInterrupt: + return 1 if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 7c38a65d8..73d9f13a7 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -1,318 +1,9 @@ #!/usr/bin/env python +""" This tool verifies that all clients known to the server build +without failures """ -"""This tool verifies that all clients known to the server build -without failures""" - -import os import sys -import signal -import fnmatch -import logging -import Bcfg2.Logger -import Bcfg2.Server.Core -from math import ceil -from nose.core import TestProgram -from nose.suite import LazySuite -from unittest import TestCase - -try: - from multiprocessing import Process, Queue, active_children - HAS_MULTIPROC = True -except ImportError: - HAS_MULTIPROC = False - active_children = lambda: [] # pylint: disable=C0103 - - -class CapturingLogger(object): - """ Fake logger that captures logging output so that errors are - only displayed for clients that fail tests """ - def __init__(self, *args, **kwargs): # pylint: disable=W0613 - self.output = [] - - def error(self, msg): - """ discard error messages """ - self.output.append(msg) - - def warning(self, msg): - """ discard error messages """ - self.output.append(msg) - - def info(self, msg): - """ discard error messages """ - self.output.append(msg) - - def debug(self, msg): - """ discard error messages """ - self.output.append(msg) - - def reset_output(self): - """ Reset the captured output """ - self.output = [] - - -class ClientTestFromQueue(TestCase): - """ A test case that tests a value that has been enqueued by a - child test process. ``client`` is the name of the client that has - been tested; ``result`` is the result from the :class:`ClientTest` - test. ``None`` indicates a successful test; a string value - indicates a failed test; and an exception indicates an error while - running the test. """ - __test__ = False # Do not collect - - def __init__(self, client, result): - TestCase.__init__(self) - self.client = client - self.result = result - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ parse the result from this test """ - if isinstance(self.result, Exception): - raise self.result - assert self.result is None, self.result - - -class ClientTest(TestCase): - """ A test case representing the build of all of the configuration for - a single host. Checks that none of the build config entities has - had a failure when it is building. Optionally ignores some config - files that we know will cause errors (because they are private - files we don't have access to, for instance) """ - __test__ = False # Do not collect - divider = "-" * 70 - - def __init__(self, core, client, ignore=None): - TestCase.__init__(self) - self.core = core - self.core.logger = CapturingLogger() - self.client = client - if ignore is None: - self.ignore = dict() - else: - self.ignore = ignore - - def ignore_entry(self, tag, name): - """ return True if an error on a given entry should be ignored - """ - if tag in self.ignore: - if name in self.ignore[tag]: - return True - else: - # try wildcard matching - for pattern in self.ignore[tag]: - if fnmatch.fnmatch(name, pattern): - return True - return False - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ run this individual test """ - config = self.core.BuildConfiguration(self.client) - output = self.core.logger.output[:] - if output: - output.append(self.divider) - self.core.logger.reset_output() - - # check for empty client configuration - assert len(config.findall("Bundle")) > 0, \ - "\n".join(output + ["%s has no content" % self.client]) - - # check for missing bundles - metadata = self.core.build_metadata(self.client) - sbundles = [el.get('name') for el in config.findall("Bundle")] - missing = [b for b in metadata.bundles if b not in sbundles] - assert len(missing) == 0, \ - "\n".join(output + ["Configuration is missing bundle(s): %s" % - ':'.join(missing)]) - - # check for unknown packages - unknown_pkgs = [el.get("name") - for el in config.xpath('//Package[@type="unknown"]') - if not self.ignore_entry(el.tag, el.get("name"))] - assert len(unknown_pkgs) == 0, \ - "Configuration contains unknown packages: %s" % \ - ", ".join(unknown_pkgs) - - failures = [] - msg = output + ["Failures:"] - for failure in config.xpath('//*[@failure]'): - if not self.ignore_entry(failure.tag, failure.get('name')): - failures.append(failure) - msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), - failure.get("failure"))) - - assert len(failures) == 0, "\n".join(msg) - - def __str__(self): - return "ClientTest(%s)" % self.client - - id = __str__ - - -def get_core(setup): - """ Get a server core, with events handled """ - core = Bcfg2.Server.Core.BaseCore(setup) - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def get_ignore(setup): - """ Given an options dict, get a dict of entry tags and names to - ignore errors from """ - ignore = dict() - for entry in setup['test_ignore']: - tag, name = entry.split(":") - try: - ignore[tag].append(name) - except KeyError: - ignore[tag] = [name] - return ignore - - -def run_child(setup, clients, queue): - """ Run tests for the given clients in a child process, returning - results via the given Queue """ - core = get_core(setup) - ignore = get_ignore(setup) - for client in clients: - try: - ClientTest(core, client, ignore).runTest() - queue.put((client, None)) - except AssertionError: - queue.put((client, str(sys.exc_info()[1]))) - except: - queue.put((client, sys.exc_info()[1])) - - core.shutdown() - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def parse_args(): - """ Parse command line arguments. """ - optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) - - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = \ - "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ - setup.buildHelpMessage() - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']: - setup['noseopts'].append("-v") - - if setup['children'] and not HAS_MULTIPROC: - logger.warning("Python multiprocessing library not found, running " - "with no children") - setup['children'] = 0 - - if (setup['children'] and ('--with-xunit' in setup['noseopts'] or - '--xunit-file' in setup['noseopts'])): - logger.warning("Use the --xunit option to bcfg2-test instead of the " - "--with-xunit or --xunit-file options to nosetest") - xunitfile = None - if '--with-xunit' in setup['noseopts']: - setup['noseopts'].remove('--with-xunit') - xunitfile = "nosetests.xml" - if '--xunit-file' in setup['noseopts']: - idx = setup['noseopts'].index('--xunit-file') - try: - setup['noseopts'].pop(idx) # remove --xunit-file - # remove the argument to it - xunitfile = setup['noseopts'].pop(idx) - except IndexError: - pass - if xunitfile and not setup['xunit']: - setup['xunit'] = xunitfile - return setup - - -def main(): - setup = parse_args() - logger = logging.getLogger(sys.argv[0]) - core = get_core(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - - if setup['args']: - clients = setup['args'] - else: - clients = core.metadata.clients - - ignore = get_ignore(setup) - - if setup['children']: - if setup['children'] > len(clients): - logger.info("Refusing to spawn more children than clients to test," - " setting children=%s" % len(clients)) - setup['children'] = len(clients) - perchild = int(ceil(len(clients) / float(setup['children'] + 1))) - queue = Queue() - for child in range(setup['children']): - start = child * perchild - end = (child + 1) * perchild - child = Process(target=run_child, - args=(setup, clients[start:end], queue)) - child.start() - - def generate_tests(): - """ Read test results for the clients """ - start = setup['children'] * perchild - for client in clients[start:]: - yield ClientTest(core, client, ignore) - - for i in range(start): # pylint: disable=W0612 - yield ClientTestFromQueue(*queue.get()) - else: - def generate_tests(): - """ Run tests for the clients """ - for client in clients: - yield ClientTest(core, client, ignore) - - result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) - - # block until all children have completed -- should be - # immediate since we've already gotten all the results we - # expect - for child in active_children(): - child.join() - - core.shutdown() - if result.success: - os._exit(0) # pylint: disable=W0212 - else: - os._exit(1) # pylint: disable=W0212 - +from Bcfg2.Server.Test import CLI if __name__ == "__main__": - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 49baeb9c3..95fb9889e 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -5,358 +5,8 @@ the right way to get around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ -import os import sys -import yum -import logging -import Bcfg2.Logger -from Bcfg2.Compat import wraps -from lockfile import FileLock, LockTimeout -from optparse import OptionParser -try: - import json -except ImportError: - import simplejson as json - - -def pkg_to_tuple(package): - """ json doesn't distinguish between tuples and lists, but yum - does, so we convert a package in list format to one in tuple - format """ - if isinstance(package, list): - return tuple(package) - else: - return package - - -def pkgtup_to_string(package): - """ given a package tuple, return a human-readable string - describing the package """ - if package[3] in ['auto', 'any']: - return package[0] - - rv = [package[0], "-"] - if package[2]: - rv.extend([package[2], ':']) - rv.extend([package[3], '-', package[4]]) - if package[1]: - rv.extend(['.', package[1]]) - return ''.join(str(e) for e in rv) - - -class YumHelper(object): - """ Yum helper base object """ - - def __init__(self, cfgfile, verbose=1): - self.cfgfile = cfgfile - self.yumbase = yum.YumBase() - # pylint: disable=E1121,W0212 - try: - self.yumbase.preconf.debuglevel = verbose - self.yumbase.preconf.fn = cfgfile - self.yumbase._getConfig() - except AttributeError: - self.yumbase._getConfig(cfgfile, debuglevel=verbose) - # pylint: enable=E1121,W0212 - self.logger = logging.getLogger(self.__class__.__name__) - - -class DepSolver(YumHelper): - """ Yum dependency solver. This is used for operations that only - read from the yum cache, and thus operates in cacheonly mode. """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - # internally, yum uses an integer, not a boolean, for conf.cache - self.yumbase.conf.cache = 1 - self._groups = None - - def get_groups(self): - """ getter for the groups property """ - if self._groups is not None: - return self._groups - else: - return ["noarch"] - - def set_groups(self, groups): - """ setter for the groups property """ - self._groups = set(groups).union(["noarch"]) - - groups = property(get_groups, set_groups) - - def get_package_object(self, pkgtup, silent=False): - """ given a package tuple, get a yum package object """ - try: - matches = yum.packageSack.packagesNewestByName( - self.yumbase.pkgSack.searchPkgTuple(pkgtup)) - except yum.Errors.PackageSackError: - if not silent: - self.logger.warning("Package '%s' not found" % - self.get_package_name(pkgtup)) - matches = [] - except yum.Errors.RepoError: - err = sys.exc_info()[1] - self.logger.error("Temporary failure loading metadata for %s: %s" % - (self.get_package_name(pkgtup), err)) - matches = [] - - pkgs = self._filter_arch(matches) - if pkgs: - return pkgs[0] - else: - return None - - def get_group(self, group, ptype="default"): - """ Resolve a package group name into a list of packages """ - if group.startswith("@"): - group = group[1:] - - try: - if self.yumbase.comps.has_group(group): - group = self.yumbase.comps.return_group(group) - else: - self.logger.error("%s is not a valid group" % group) - return [] - except yum.Errors.GroupsError: - err = sys.exc_info()[1] - self.logger.warning(err) - return [] - - if ptype == "default": - return [p - for p, d in list(group.default_packages.items()) - if d] - elif ptype == "mandatory": - return [p - for p, m in list(group.mandatory_packages.items()) - if m] - elif ptype == "optional" or ptype == "all": - return group.packages - else: - self.logger.warning("Unknown group package type '%s'" % ptype) - return [] - - def _filter_arch(self, packages): - """ filter packages in the given list that do not have an - architecture in the list of groups for this client """ - matching = [] - for pkg in packages: - if pkg.arch in self.groups: - matching.append(pkg) - else: - self.logger.debug("%s has non-matching architecture (%s)" % - (pkg, pkg.arch)) - if matching: - return matching - else: - # no packages match architecture; we'll assume that the - # user knows what s/he is doing and this is a multiarch - # box. - return packages - - def get_package_name(self, package): - """ get the name of a package or virtual package from the - internal representation used by this Collection class """ - if isinstance(package, tuple): - if len(package) == 3: - return yum.misc.prco_tuple_to_string(package) - else: - return pkgtup_to_string(package) - else: - return str(package) - - def complete(self, packagelist): - """ resolve dependencies and generate a complete package list - from the given list of initial packages """ - packages = set() - unknown = set() - for pkg in packagelist: - if isinstance(pkg, tuple): - pkgtup = pkg - else: - pkgtup = (pkg, None, None, None, None) - pkgobj = self.get_package_object(pkgtup) - if not pkgobj: - self.logger.debug("Unknown package %s" % - self.get_package_name(pkg)) - unknown.add(pkg) - else: - if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): - self.logger.debug("%s added to transaction multiple times" - % pkgobj) - else: - self.logger.debug("Adding %s to transaction" % pkgobj) - self.yumbase.tsInfo.addInstall(pkgobj) - self.yumbase.resolveDeps() - - for txmbr in self.yumbase.tsInfo: - packages.add(txmbr.pkgtup) - return list(packages), list(unknown) - - -def acquire_lock(func): - """ decorator for CacheManager methods that gets and release a - lock while the method runs """ - @wraps(func) - def inner(self, *args, **kwargs): - """ Get and release a lock while running the function this - wraps. """ - self.logger.debug("Acquiring lock at %s" % self.lockfile) - while not self.lock.i_am_locking(): - try: - self.lock.acquire(timeout=60) # wait up to 60 seconds - except LockTimeout: - self.lock.break_lock() - self.lock.acquire() - try: - func(self, *args, **kwargs) - finally: - self.lock.release() - self.logger.debug("Released lock at %s" % self.lockfile) - - return inner - - -class CacheManager(YumHelper): - """ Yum cache manager. Unlike :class:`DepSolver`, this can write - to the yum cache, and so is used for operations that muck with the - cache. (Technically, :func:`CacheManager.clean_cache` could be in - either DepSolver or CacheManager, but for consistency I've put it - here.) """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - self.lockfile = \ - os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), - "lock") - self.lock = FileLock(self.lockfile) - - @acquire_lock - def clean_cache(self): - """ clean the yum cache """ - for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", - "ExpireCache"]: - # for reasons that are entirely obvious, all of the yum - # API clean* methods return a tuple of 0 (zero, always - # zero) and a list containing a single message about how - # many files were deleted. so useful. thanks, yum. - msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] - if not msg.startswith("0 "): - self.logger.info(msg) - - @acquire_lock - def populate_cache(self): - """ populate the yum cache """ - for repo in self.yumbase.repos.findRepos('*'): - repo.metadata_expire = 0 - repo.mdpolicy = "group:all" - self.yumbase.doRepoSetup() - self.yumbase.repos.doSetup() - for repo in self.yumbase.repos.listEnabled(): - # this populates the cache as a side effect - repo.repoXML # pylint: disable=W0104 - try: - repo.getGroups() - except yum.Errors.RepoMDError: - pass # this repo has no groups - self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) - # this does something with the groups cache as a side effect - self.yumbase.comps # pylint: disable=W0104 - - -def main(): - parser = OptionParser() - parser.add_option("-c", "--config", help="Config file") - parser.add_option("-v", "--verbose", help="Verbosity level", - action="count") - (options, args) = parser.parse_args() - - if options.verbose: - level = logging.DEBUG - clevel = logging.DEBUG - else: - level = logging.WARNING - clevel = logging.INFO - Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True, - to_console=clevel, level=level) - logger = logging.getLogger('bcfg2-yum-helper') - - try: - cmd = args[0] - except IndexError: - logger.error("No command given") - return 1 - - if not os.path.exists(options.config): - logger.error("Config file %s not found" % options.config) - return 1 - - # pylint: disable=W0702 - rv = 0 - if cmd == "clean": - cachemgr = CacheManager(options.config, options.verbose) - try: - cachemgr.clean_cache() - print(json.dumps(True)) - except: - logger.error("Unexpected error cleaning cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - rv = 2 - elif cmd == "makecache": - cachemgr = CacheManager(options.config, options.verbose) - try: - # this code copied from yumcommands.py - cachemgr.populate_cache() - print(json.dumps(True)) - except yum.Errors.YumBaseError: - logger.error("Unexpected error creating cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - elif cmd == "complete": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - except: - logger.error("Unexpected error decoding JSON input: %s" % - sys.exc_info()[1]) - rv = 2 - try: - depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete( - [pkg_to_tuple(p) for p in data['packages']]) - print(json.dumps(dict(packages=list(packages), - unknown=list(unknown)))) - except: - logger.error("Unexpected error completing package set: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict(packages=[], unknown=data['packages']))) - rv = 2 - elif cmd == "get_groups": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - rv = dict() - for gdata in data: - if "type" in gdata: - packages = depsolver.get_group(gdata['group'], - ptype=gdata['type']) - else: - packages = depsolver.get_group(gdata['group']) - rv[gdata['group']] = list(packages) - print(json.dumps(rv)) - except: - logger.error("Unexpected error getting groups: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict())) - rv = 2 - else: - logger.error("Unknown command %s" % cmd) - print(json.dumps(None)) - rv = 2 - return rv +from Bcfg2.Server.Plugins.Packages.YumHelper import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) |