From edca0b698637c3fd0a70af7e4752a46afca938d3 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Mon, 23 Jan 2006 22:35:40 +0000 Subject: last step of repo switches git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1716 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Client/Redhat.py | 2 +- src/lib/Client/Solaris.py | 2 +- src/lib/Client/Toolset.py | 22 ++- src/lib/Server/Component.py | 1 - src/lib/Server/Core.py | 18 +- src/lib/Server/Metadata.py | 274 ++++++++++++++++---------- src/lib/Server/Plugin.py | 232 ++++++++++++++-------- src/lib/Server/Plugins/Base.py | 59 ++---- src/lib/Server/Plugins/Bundler.py | 120 ++---------- src/lib/Server/Plugins/Cfg.py | 147 +++++++------- src/lib/Server/Plugins/Hostbase.py | 391 ++++++++++++++++++++++++++----------- src/lib/Server/Plugins/Pkgmgr.py | 128 ++++-------- src/lib/Server/Plugins/Svcmgr.py | 27 ++- src/sbin/Bcfg2Server | 289 --------------------------- src/sbin/Bcfg2debug | 70 ------- src/sbin/GenerateHostInfo | 77 ++++---- src/sbin/StatReports | 33 ++-- src/sbin/ValidateBcfg2Repo | 51 ----- src/sbin/bcfg2 | 0 src/sbin/bcfg2-info | 154 +++++++++++++++ src/sbin/bcfg2-repo-validate | 56 ++++++ src/sbin/bcfg2-server | 291 +++++++++++++++++++++++++++ 22 files changed, 1318 insertions(+), 1126 deletions(-) delete mode 100644 src/sbin/Bcfg2Server delete mode 100644 src/sbin/Bcfg2debug mode change 100644 => 100755 src/sbin/GenerateHostInfo mode change 100644 => 100755 src/sbin/StatReports delete mode 100644 src/sbin/ValidateBcfg2Repo mode change 100644 => 100755 src/sbin/bcfg2 create mode 100755 src/sbin/bcfg2-info create mode 100644 src/sbin/bcfg2-repo-validate create mode 100755 src/sbin/bcfg2-server (limited to 'src') diff --git a/src/lib/Client/Redhat.py b/src/lib/Client/Redhat.py index d68288b79..9ddd0f142 100644 --- a/src/lib/Client/Redhat.py +++ b/src/lib/Client/Redhat.py @@ -117,7 +117,7 @@ class ToolsetImpl(Toolset): if self.setup['remove'] in ['all', 'services']: self.CondDisplayList('verbose', 'Removing services:', self.extra_services) for service in self.extra_services: - if not system("/sbin/chkconfig %s off" % service): + if not system("/sbin/chkconfig --level 123456 %s off" % service): self.extra_services.remove(service) else: self.CondDisplayList('verbose', 'Need to remove services:', self.extra_services) diff --git a/src/lib/Client/Solaris.py b/src/lib/Client/Solaris.py index 69d6ee69b..48f26c1f8 100644 --- a/src/lib/Client/Solaris.py +++ b/src/lib/Client/Solaris.py @@ -83,7 +83,7 @@ class ToolsetImpl(Toolset): def VerifyService(self, entry): '''Verify Service status for entry''' if not entry.attrib.has_key('FMRI'): - (rc, name) = self.saferun("/usr/bin/svcs -H -o FMRI %s 2>/dev/null" % entry.get('name')) + name = self.saferun("/usr/bin/svcs -H -o FMRI %s 2>/dev/null" % entry.get('name'))[1] if name: entry.set('FMRI', name[0]) else: diff --git a/src/lib/Client/Toolset.py b/src/lib/Client/Toolset.py index b358ac92c..9cf9ad4b1 100644 --- a/src/lib/Client/Toolset.py +++ b/src/lib/Client/Toolset.py @@ -4,7 +4,7 @@ __revision__ = '$Revision$' from binascii import a2b_base64 from copy import deepcopy from grp import getgrgid, getgrnam -from os import chown, chmod, lstat, mkdir, stat, system, unlink, rename, readlink, symlink +from os import chown, chmod, lstat, mkdir, stat, unlink, rename, readlink, symlink from pwd import getpwuid, getpwnam from stat import S_ISVTX, S_ISGID, S_ISUID, S_IXUSR, S_IWUSR, S_IRUSR, S_IXGRP from stat import S_IWGRP, S_IRGRP, S_IXOTH, S_IWOTH, S_IROTH, ST_MODE, S_ISDIR @@ -87,6 +87,7 @@ class Toolset(object): pass def get_height_width(self): + '''Get Terminal information''' try: import termios, struct, fcntl height, width = struct.unpack('hhhh', @@ -99,6 +100,7 @@ class Toolset(object): return 25, 80 def FormattedCondPrint(self, state, items): + '''Formatted conditional print''' items.sort() screenWidth = self.width - len("%s[%s]:" % (self.__name__, state)) columnWidth = 1 @@ -121,11 +123,13 @@ class Toolset(object): self.CondPrint(state, lineText.rstrip()) def CondDisplayList(self, state, title, items): + '''Conditionally print a list of data''' self.CondPrint(state, title) self.FormattedCondPrint(state, items) self.CondPrint(state, '') def CondDisplayState(self, state, phase): + '''Conditionally print tracing information''' self.CondPrint(state, 'Phase: %s' % phase) self.CondPrint(state, 'Correct entries:\t%d' % self.states.values().count(True)) @@ -625,17 +629,19 @@ class Toolset(object): self.CondPrint("debug", "Installing packages: :%s:" % pkgargs) self.CondPrint("debug", "Running command ::%s::" % (pkgtool[0] % pkgargs)) - (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % pkgargs) + cmdrc = self.saferun(pkgtool[0] % pkgargs)[0] if cmdrc == 0: self.CondPrint('verbose', "Single Pass Succeded") # set all package states to true and flush workqueues pkgnames = [pkg.get('name') for pkg in pkglist] for entry in [entry for entry in self.states.keys() - if entry.tag == 'Package' and entry.get('type') == pkgtype and entry.get('name') in pkgnames]: + if entry.tag == 'Package' and entry.get('type') == pkgtype + and entry.get('name') in pkgnames]: self.CondPrint('debug', 'Setting state to true for pkg %s' % (entry.get('name'))) self.states[entry] = True - [self.pkgwork[listname].remove(entry) for listname in ['add', 'update'] if self.pkgwork[listname].count(entry)] + [self.pkgwork[listname].remove(entry) for listname in ['add', 'update'] + if self.pkgwork[listname].count(entry)] self.Refresh() else: self.CondPrint("verbose", "Single Pass Failed") @@ -649,10 +655,10 @@ class Toolset(object): else: self.CondPrint("verbose", "Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) - (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % - (pkgtool[1][0] % - tuple([pkg.get(field) for field in pkgtool[1][1]]))) - if cmdrc == 0: + cmdrc = self.saferun(pkgtool[0] % + (pkgtool[1][0] % + tuple([pkg.get(field) for field in pkgtool[1][1]]))) + if cmdrc[0] == 0: self.states[pkg] = True else: self.CondPrint('verbose', "Failed to install package %s" % (pkg.get('name'))) diff --git a/src/lib/Server/Component.py b/src/lib/Server/Component.py index 97444bb10..5c19c3bdd 100644 --- a/src/lib/Server/Component.py +++ b/src/lib/Server/Component.py @@ -5,7 +5,6 @@ from ConfigParser import ConfigParser, NoOptionError from cPickle import loads, dumps from M2Crypto import SSL from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler -from select import select from socket import gethostname from sys import exc_info import sys diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index a2100ea53..91da366b8 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -11,9 +11,11 @@ from ConfigParser import ConfigParser from lxml.etree import Element from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError -from Bcfg2.Server.Metadata import MetadataStore, MetadataConsistencyError + from Bcfg2.Server.Statistics import Statistics +import Bcfg2.Server.Metadata + def log_failure(msg): syslog(LOG_ERR, "Unexpected failure in %s" % (msg)) (trace, val, trb) = exc_info() @@ -210,7 +212,7 @@ class Core(object): mpath = cfile.get('server','repository') try: - self.metadata = MetadataStore("%s/etc/metadata.xml" % mpath, self.fam) + self.metadata = Bcfg2.Server.Metadata.Metadata(self.fam, mpath) except OSError: raise CoreInitError, "metadata path incorrect" @@ -269,21 +271,15 @@ class Core(object): generators = ", ".join([gen.__name__ for gen in glist]) syslog(LOG_ERR, "%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - raise PluginExecutionError, (entry.tag, entry.get('name')) - else: - for gen in self.generators: - if hasattr(gen, "FindHandler"): - return gen.FindHandler(entry)(entry, metadata) - syslog(LOG_ERR, "Failed to find handler for %s:%s" % (entry.tag, entry.get('name'))) - raise PluginExecutionError, (entry.tag, entry.get('name')) + raise PluginExecutionError, (entry.tag, entry.get('name')) def BuildConfiguration(self, client): '''Build Configuration for client''' start = time() config = Element("Configuration", version='2.0') try: - meta = self.metadata.FetchMetadata(client) - except MetadataConsistencyError: + meta = self.metadata.get_metadata(client) + except Bcfg2.Server.Metadata.MetadataConsistencyError: syslog(LOG_ERR, "Metadata consistency error for client %s" % client) return Element("error", type='metadata error') diff --git a/src/lib/Server/Metadata.py b/src/lib/Server/Metadata.py index 47bbb3ecb..ecf636476 100644 --- a/src/lib/Server/Metadata.py +++ b/src/lib/Server/Metadata.py @@ -1,130 +1,190 @@ '''This file stores persistent metadata for the BCFG Configuration Repository''' __revision__ = '$Revision$' -from lxml.etree import XML, SubElement, Element, _Comment, tostring from syslog import syslog, LOG_ERR, LOG_INFO -from Bcfg2.Server.Plugin import SingleXMLFileBacked +import lxml.etree, os, time, threading class MetadataConsistencyError(Exception): '''This error gets raised when metadata is internally inconsistent''' pass -class Metadata(object): - '''The Metadata class is a container for all classes of metadata used by Bcfg2''' - def __init__(self, all, image, classes, bundles, attributes, hostname, toolset): - self.all = all - self.image = image - self.classes = classes +class MetadataRuntimeError(Exception): + '''This error is raised when the metadata engine is called prior to reading enough data''' + pass + +class ClientMetadata(object): + '''This object contains client metadata''' + def __init__(self, client, groups, bundles, toolset): + self.hostname = client self.bundles = bundles - self.attributes = attributes - self.hostname = hostname + self.groups = groups self.toolset = toolset - def Applies(self, other): - '''Check if metadata styled object applies to current metadata''' - if (other.all or (other.image and (self.image == other.image)) or - (other.classes and (other.classes in self.classes)) or - (other.attributes and (other.attributes in self.attributes)) or - (other.bundles and (other.bundles in self.bundles)) or - (other.hostname and (self.hostname == other.hostname)) or - (other.hostname and (self.hostname.split('.')[0] == other.hostname))): - return True - else: - return False - -class Profile(object): - '''Profiles are configuration containers for sets of classes and attributes''' - def __init__(self, xml): - object.__init__(self) - self.classes = [cls.attrib['name'] for cls in xml.findall("Class")] - self.attributes = ["%s.%s" % (attr.attrib['scope'], attr.attrib['name']) for - attr in xml.findall("Attribute")] - -class MetadataStore(SingleXMLFileBacked): - '''The MetadataStore is a filebacked xml repository that contains all setup info for all clients''' +class Metadata: + '''This class contains data for bcfg2 server metadata''' + __name__ = 'Metadata' + __version__ = '$Id$' + __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, filename, fam): - # initialize Index data to avoid race - self.defaults = {} + def __init__(self, fam, datastore): + self.data = "%s/%s" % (datastore, self.__name__) + fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self) + fam.AddMonitor("%s/%s" % (self.data, "clients.xml"), self) + self.states = {'groups.xml':False, 'clients.xml':False} self.clients = {} - self.profiles = {} - self.classes = {} - self.images = {} - self.element = Element("dummy") - SingleXMLFileBacked.__init__(self, filename, fam) - - def Index(self): - '''Build data structures for XML data''' - self.element = XML(self.data) - self.defaults = {} - self.clients = {} - self.profiles = {} - self.classes = {} - self.images = {} - for prof in self.element.findall("Profile"): - self.profiles[prof.attrib['name']] = Profile(prof) - for cli in self.element.findall("Client"): - self.clients[cli.attrib['name']] = (cli.attrib['image'], cli.attrib['profile']) - for cls in self.element.findall("Class"): - self.classes[cls.attrib['name']] = [bundle.attrib['name'] for bundle in cls.findall("Bundle")] - for img in self.element.findall("Image"): - self.images[img.attrib['name']] = img.attrib['toolset'] - for key in [key[8:] for key in self.element.attrib if key[:8] == 'default_']: - self.defaults[key] = self.element.get("default_%s" % key) + self.aliases = {} + self.groups = {} + self.public = [] + self.profiles = [] + self.toolsets = {} + self.categories = {} + self.clientdata = None + self.default = None - def FetchMetadata(self, client, image=None, profile=None): - '''Get metadata for client''' - if ((image != None) and (profile != None)): - # Client asserted profile/image - self.clients[client] = (image, profile) - syslog(LOG_INFO, "Metadata: Asserted metadata for %s: %s, %s" % (client, image, profile)) - [self.element.remove(cli) for cli in self.element.findall("Client") if cli.get('name') == client] - SubElement(self.element, "Client", name=client, image=image, profile=profile) - self.WriteBack() + def HandleEvent(self, event): + '''Handle update events for data files''' + filename = event.filename.split('/')[-1] + if filename not in ['groups.xml', 'clients.xml']: + return + if event.code2str() == 'endExist': + return + try: + xdata = lxml.etree.parse("%s/%s" % (self.data, filename)) + except lxml.etree.XMLSyntaxError: + syslog(LOG_ERR, 'Metadata: Failed to parse %s' % (filename)) + return + if filename == 'clients.xml': + self.clients = {} + self.aliases = {} + self.clientdata = xdata + for client in xdata.findall('./Client'): + self.clients.update({client.get('name'): client.get('profile')}) + [self.aliases.update({alias.get('name'): client.get('name')}) for alias in client.findall('Alias')] else: - # no asserted metadata - if self.clients.has_key(client): - (image, profile) = self.clients[client] - else: - # default profile stuff goes here - (image, profile) = (self.defaults['image'], self.defaults['profile']) - SubElement(self.element, "Client", name=client, profile=profile, image=image) - self.WriteBack() + self.public = [] + self.profiles = [] + self.toolsets = {} + self.groups = {} + grouptmp = {} + self.categories = {} + for group in xdata.findall('./Group'): + grouptmp[group.get('name')] = tuple([[item.get('name') for item in group.findall(spec)] + for spec in ['./Bundle', './Group']]) + grouptmp[group.get('name')][1].append(group.get('name')) + if group.get('default', 'false') == 'true': + self.default = group.get('name') + if group.get('profile', 'false') == 'true': + self.profiles.append(group.get('name')) + if group.get('public', 'false') == 'true': + self.public.append(group.get('name')) + if group.attrib.has_key('toolset'): + self.toolsets[group.get('name')] = group.get('toolset') + if group.attrib.has_key('category'): + self.categories[group.get('name')] = group.get('category') + for group in grouptmp: + self.groups[group] = ([], []) + gcategories = [] + tocheck = [group] + while tocheck: + now = tocheck.pop() + if now not in self.groups[group][1]: + self.groups[group][1].append(now) + if grouptmp.has_key(now): + (bundles, groups) = grouptmp[now] + for ggg in [ggg for ggg in groups if ggg not in self.groups[group][1]]: + if not self.categories.has_key(ggg) or (self.categories[ggg] not in gcategories): + self.groups[group][1].append(ggg) + tocheck.append(ggg) + if self.categories.has_key(ggg): + gcategories.append(self.categories[ggg]) + [self.groups[group][0].append(bund) for bund in bundles + if bund not in self.groups[group][0]] + self.states[filename] = True + if False not in self.states.values(): + # check that all client groups are real and complete + real = self.groups.keys() + for client in self.clients.keys(): + if self.clients[client] not in real or self.clients[client] not in self.profiles: + syslog(LOG_ERR, "Metadata: Client %s set as nonexistant or incomplete group %s" \ + % (client, self.clients[client])) + syslog(LOG_ERR, "Metadata: Removing client mapping for %s" % (client)) + del self.clients[client] - if not self.profiles.has_key(profile): - syslog(LOG_ERR, "Metadata: profile %s not defined" % profile) - raise MetadataConsistencyError - prof = self.profiles[profile] - # should we uniq here? V - bundles = reduce(lambda x, y:x + y, [self.classes.get(cls, []) for cls in prof.classes]) - if not self.images.has_key(image): - syslog(LOG_ERR, "Metadata: Image %s not defined" % image) + def set_group(self, client, group): + '''Set group parameter for provided client''' + if False in self.states.values(): + raise MetadataRuntimeError + if group not in self.public: + syslog(LOG_ERR, "Metadata: Failed to set client %s to private group %s" % (client, + group)) raise MetadataConsistencyError - toolset = self.images[image] - return Metadata(False, image, prof.classes, bundles, prof.attributes, client, toolset) + if self.clients.has_key(client): + syslog(LOG_INFO, "Metadata: Changing %s group from %s to %s" % (client, + self.clients[client], group)) + cli = self.clientdata.xpath('/Clients/Client[@name="%s"]' % (client)) + cli[0].set('group', group) + else: + lxml.etree.SubElement(self.clientdata.getroot(), 'Client', name=client, group=group) + self.clients[client] = group + self.write_back_clients() + + def write_back_clients(self): + '''Write changes to client.xml back to disk''' + try: + datafile = open("%s/%s" % (self.data, 'clients.xml'), 'w') + except IOError: + syslog(LOG_ERR, "Metadata: Failed to write clients.xml") + raise MetadataRuntimeError + datafile.write(lxml.etree.tostring(self.clientdata)) + datafile.close() - def pretty_print(self, element, level=0): - '''Produce a pretty-printed text representation of element''' - if isinstance(element, _Comment): - return (level * " ") + tostring(element) - if element.text: - fmt = "%s<%%s %%s>%%s" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])), - element.text, element.tag) - numchild = len(element.getchildren()) - if numchild: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (numchild * "%s") + "%s\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),) - data += tuple([self.pretty_print(entry, level+2) for entry in element.getchildren()]) + (element.tag, ) + def find_toolset(self, client): + '''Find the toolset for a given client''' + tgroups = [self.toolsets[group] for group in self.groups[client][1] if self.toolsets.has_key(group)] + if len(tgroups) == 1: + return tgroups[0] + elif len(tgroups) == 0: + syslog(LOG_ERR, "Metadata: Couldn't find toolset for client %s" % (client)) + raise MetadataConsistencyError else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])) - return fmt % data + syslog(LOG_ERR, "Metadata: Got goofy toolset result for client %s" % (client)) + raise MetadataConsistencyError - def WriteBack(self): - '''Write metadata changes back to persistent store''' - fout = open(self.name, 'w') - fout.write(self.pretty_print(self.element)) - fout.close() + def get_config_template(self, client): + '''Build the configuration header for a client configuration''' + return lxml.etree.Element("Configuration", version='2.0', toolset=self.find_toolset(client)) + def get_metadata(self, client): + '''Return the metadata for a given client''' + if self.aliases.has_key(client): + client = self.aliases[client] + if self.clients.has_key(client): + [bundles, groups] = self.groups[self.clients[client]] + else: + if self.default == None: + syslog(LOG_ERR, "Cannot set group for client %s; no default group set" % (client)) + raise MetadataConsistencyError + [bundles, groups] = self.groups[self.default] + toolinfo = [self.toolsets[group] for group in groups if self.toolsets.has_key(group)] + if len(toolinfo) > 1: + syslog(LOG_ERR, "Metadata: Found multiple toolsets for client %s; choosing one" % (client)) + elif len(toolinfo) == 0: + syslog(LOG_ERR, "Metadata: Cannot determine toolset for client %s" % (client)) + raise MetadataConsistencyError + toolset = toolinfo[0] + return ClientMetadata(client, groups, bundles, toolset) + + def ping_sweep_clients(self): + '''Find live and dead clients''' + live = {} + dead = {} + work = self.clients.keys() + while work: + client = work.pop() + rc = os.system("/bin/ping -w 5 -c 1 %s > /dev/null 2>&1" % client) + if not rc: + live[client] = time.time() + else: + dead[client] = time.time() + diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 262d6092d..890084c98 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -1,10 +1,12 @@ '''This module provides the baseclass for Bcfg2 Server Plugins''' -__revision__ = '$Revision:$' +__revision__ = '$Revision$' -from lxml.etree import XML, XMLSyntaxError, _Comment, tostring -from os import stat -from stat import ST_MTIME -from syslog import syslog, LOG_ERR, LOG_INFO +import lxml.etree +import os +import stat +import syslog + +from lxml.etree import XML, XMLSyntaxError class PluginInitError(Exception): '''Error raised in cases of Plugin initialization errors''' @@ -39,7 +41,7 @@ class Plugin(object): def LogError(self, msg): '''Log error message tagged with Plugin name''' - syslog(LOG_ERR, "%s: %s" % (self.__name__, msg)) + syslog.syslog(syslog.LOG_ERR, "%s: %s" % (self.__name__, msg)) def BuildStructures(self, metadata): '''Build a set of structures tailored to the client metadata''' @@ -73,15 +75,15 @@ class FileBacked(object): '''Read file upon update''' oldmtime = self.mtime try: - self.mtime = stat(self.name)[ST_MTIME] + self.mtime = os.stat(self.name)[stat.ST_MTIME] except OSError: - syslog(LOG_ERR, "Failed to stat file %s" % (self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to stat file %s" % (self.name)) try: self.data = file(self.name).read() self.Index() except IOError: - syslog(LOG_ERR, "Failed to read file %s" % (self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to read file %s" % (self.name)) def Index(self): '''Update local data structures based on current file state''' @@ -108,9 +110,9 @@ class DirectoryBacked(object): def AddEntry(self, name): '''Add new entry to data structures upon file creation''' if name == '': - syslog(LOG_INFO, "got add for empty name") + syslog.syslog(syslog.LOG_INFO, "got add for empty name") elif self.entries.has_key(name): - syslog(LOG_INFO, "got multiple adds for %s" % name) + syslog.syslog(syslog.LOG_INFO, "got multiple adds for %s" % name) else: if ((name[-1] == '~') or (name[:2] == '.#') or (name[-4:] == '.swp') or (name in ['SCCS', '.svn'])): return @@ -121,7 +123,7 @@ class DirectoryBacked(object): '''Propagate fam events to underlying objects''' action = event.code2str() if event.filename == '': - syslog(LOG_INFO, "Got event for blank filename") + syslog.syslog(syslog.LOG_INFO, "Got event for blank filename") return if action == 'exists': if event.filename != self.name: @@ -153,7 +155,7 @@ class XMLFileBacked(FileBacked): try: xdata = XML(self.data) except XMLSyntaxError: - syslog(LOG_ERR, "Failed to parse %s"%(self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to parse %s"%(self.name)) return self.label = xdata.attrib[self.__identifier__] self.entries = xdata.getchildren() @@ -167,83 +169,141 @@ class SingleXMLFileBacked(XMLFileBacked): XMLFileBacked.__init__(self, filename) fam.AddMonitor(filename, self) -class ScopedXMLFile(SingleXMLFileBacked): - '''Scoped XML files are coherent files with Metadata structured data''' - __containers__ = ['Class', 'Host', 'Image'] - - def __init__(self, filename, fam): - self.store = {} - self.__provides__ = {} - SingleXMLFileBacked.__init__(self, filename, fam) +class StructFile(XMLFileBacked): + '''This file contains a set of structure file formatting logic''' + def __init__(self, name): + XMLFileBacked.__init__(self, name) + self.fragments = {} - def StoreRecord(self, metadata, entry): - '''Store scoped record based on metadata''' - if isinstance(entry, _Comment): - return - elif not entry.attrib.has_key('name'): - syslog(LOG_ERR, "Got malformed record %s" % (tostring(entry))) - if not self.store.has_key(entry.tag): - self.store[entry.tag] = {} - if not self.store[entry.tag].has_key(entry.attrib['name']): - self.store[entry.tag][entry.attrib['name']] = [] - self.store[entry.tag][entry.attrib['name']].append((metadata, entry)) - def Index(self): '''Build internal data structures''' try: - xdata = XML(self.data) - except XMLSyntaxError, msg: - syslog(LOG_ERR, "Failed to parse %s"%(self.name)) - # need to add in lxml error messages, once they are supported + xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + syslog.syslog(syslog.LOG_ERR, "Failed to parse file %s" % self.name) return - self.store = {} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag not in self.__containers__: - self.StoreRecord(('Global','all'), entry) + self.fragments = {} + work = {lambda x:True: xdata.getchildren()} + while work: + (predicate, worklist) = work.popitem() + self.fragments[predicate] = [item for item in worklist if item.tag != 'Group' + and not isinstance(item, lxml.etree._Comment)] + for group in [item for item in worklist if item.tag == 'Group']: + # if only python had forceable early-binding + newpred = eval("lambda x:'%s' in x.groups and predicate(x)" % (group.get('name')), + {'predicate':predicate}) + work[newpred] = group.getchildren() + + def Match(self, metadata): + '''Return matching fragments of independant''' + return reduce(lambda x, y:x+y, [frag for (pred, frag) in self.fragments.iteritems() + if pred(metadata)]) + +class LNode: + '''LNodes provide lists of things available at a particular group intersection''' + raw = {'Client':"lambda x:'%s' == x.hostname and predicate(x)", + 'Group':"lambda x:'%s' in x.groups and predicate(x)"} + __leaf__ = './Child' + + def __init__(self, data, plist, parent=None): + self.data = data + self.contents = {} + if parent == None: + self.predicate = lambda x:True + else: + predicate = parent.predicate + if data.tag in self.raw.keys(): + self.predicate = eval(self.raw[data.tag] % (data.get('name')), {'predicate':predicate}) else: - name = (entry.tag, entry.get('name')) - [self.StoreRecord(name, child) - for child in entry.getchildren() if not isinstance(entry, _Comment)] - # now to build the __provides__ table - for key in self.__provides__.keys(): - del self.__provides__[key] - for key in self.store.keys(): - self.__provides__[key] = {} - for name in self.store[key].keys(): - self.__provides__[key][name] = self.FetchRecord - # also need to sort all leaf node lists - self.store[key][name].sort(self.Sort) - - def Sort(self, meta1, meta2): - '''Sort based on specificity''' - order = ['Global', 'Image', 'Profile', 'Class', 'Host'] - return order.index(meta1[0][0]) - order.index(meta2[0][0]) - - def MatchMetadata(self, mdata, metadata): - '''Match internal metadata representation against metadata''' - (mtype, mvalue) = mdata - if mtype == 'Global': - return True - elif mtype == 'Profile': - if mvalue == metadata.profile: - return True - elif mtype == 'Image': - if mvalue == metadata.image: - return True - elif mtype == 'Class': - if mvalue in metadata.classes: - return True - elif mtype == 'Host': - if mvalue == metadata.hostname: - return True - return False - - def FetchRecord(self, entry, metadata): - '''Build a data for specified metadata''' - dlist = self.store[entry.tag][entry.get('name')] - useful = [ent for ent in dlist if self.MatchMetadata(ent[0], metadata)] - if not useful: - syslog(LOG_ERR, "Failed to FetchRecord %s:%s"%(entry.tag, entry.get('name'))) + print data.tag + raise Exception + mytype = self.__class__ + self.children = [mytype(child, plist, self) for child in data.getchildren() + if child.tag in ['Group', 'Client']] + for leaf in data.findall(self.__leaf__): + self.contents[leaf.get('name')] = leaf.attrib + if leaf.get('name') not in plist: + plist.append(leaf.get('name')) + + def Match(self, metadata, data): + '''Return a dictionary of package mappings''' + if self.predicate(metadata): + data.update(self.contents) + for child in self.children: + child.Match(metadata, data) + +class XMLSrc(XMLFileBacked): + '''XMLSrc files contain a LNode hierarchy that returns matching entries''' + __node__ = LNode + + def __init__(self, filename): + XMLFileBacked.__init__(self, filename) + self.names = [] + self.cache = None + self.pnode = None + self.priority = '1000' + + def Index(self): + self.names = [] + xdata = XML(self.data) + self.pnode = self.__node__(xdata, self.names) + self.cache = None + self.priority = xdata.attrib['priority'] + + def Cache(self, metadata): + '''Build a package dict for a given host''' + if self.cache == None or self.cache[0] != metadata: + cache = (metadata, {}) + if self.pnode == None: + syslog.syslog(syslog.LOG_ERR, + "Cache method called early for %s; forcing data load" % (self.name)) + self.HandleEvent() + return + self.pnode.Match(metadata, cache[1]) + self.cache = cache + +class XMLPrioDir(Plugin, DirectoryBacked): + '''This is a generator that handles package assignments''' + __name__ = 'XMLPrioDir' + __child__ = XMLSrc + __element__ = 'Dummy' + + def __init__(self, core, datastore): + Plugin.__init__(self, core, datastore) + self.Entries[self.__element__] = {} + try: + DirectoryBacked.__init__(self, self.data, self.core.fam) + except OSError: + self.LogError("Failed to load %s indices" % (self.__element__.lower())) + raise PluginInitError + + def HandleEvent(self, event): + '''Handle events and update dispatch table''' + DirectoryBacked.HandleEvent(self, event) + for src in self.entries.values(): + for child in src.names: + self.Entries[self.__element__][child] = self.BindEntry + + def BindEntry(self, entry, metadata): + '''Check package lists of package entries''' + [src.Cache(metadata) for src in self.entries.values()] + name = entry.get('name') + if not src.cache: + self.LogError("Called before data loaded") + raise PluginExecutionError + matching = [src for src in self.entries.values() + if src.cache[1].has_key(name)] + if len(matching) == 0: + raise PluginExecutionError + elif len(matching) == 1: + index = 0 else: - data = useful[-1][-1] - [entry.attrib.__setitem__(x, data.attrib[x]) for x in data.attrib] + prio = [int(src.priority) for src in matching] + if prio.count(max(prio)) > 1: + self.LogError("Found multiple %s sources with same priority for %s, pkg %s" % + (self.__element__.lower(), metadata.hostname, entry.get('name'))) + raise PluginExecutionError + index = prio.index(max(prio)) + + data = matching[index].cache[1][name] + [entry.attrib.__setitem__(key, data[key]) for key in data.keys()] diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 1cdd7599c..3be30bc6a 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -1,62 +1,31 @@ '''This module sets up a base list of configuration entries''' __revision__ = '$Revision$' -from copy import deepcopy -from lxml.etree import Element, XML, XMLSyntaxError, _Comment +import Bcfg2.Server.Plugin +import copy +import lxml.etree -from Bcfg2.Server.Plugin import Plugin, PluginInitError, SingleXMLFileBacked - -class Base(Plugin, SingleXMLFileBacked): +class Base(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): '''This Structure is good for the pile of independent configs needed for most actual systems''' __name__ = 'Base' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = Bcfg2.Server.Plugin.StructFile '''base creates independent clauses based on client metadata''' def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - self.store = {'all':[], 'Class':{'all':[]}, 'Image':{'all':[]}, 'all':[]} + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + self.fragements = {} try: - SingleXMLFileBacked.__init__(self, "%s/etc/base.xml"%(datastore), self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, self.core.fam) except OSError: - self.LogError("Failed to load base.xml") - raise PluginInitError + self.LogError("Failed to load Base repository") + raise Bcfg2.Server.Plugin.PluginInitError - def Index(self): - '''Store XML data in reasonable structures''' - try: - xdata = XML(self.data) - except XMLSyntaxError: - self.LogError("Failed to parse base.xml") - return - self.store = {'all':[], 'Class':{'all':[]}, 'Image':{'all':[]}, 'all':[]} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag in ['Image', 'Class']: - if not self.store[entry.tag].has_key(entry.get('name')): - self.store[entry.tag][entry.get('name')] = {'all':[], 'Class':{}, 'Image':{}} - for child in [ent for ent in entry.getchildren() if not isinstance(ent, _Comment)]: - if child.tag in ['Image', 'Class']: - self.store[entry.tag][entry.get('name')][child.tag][child.get('name')] = \ - [ent for ent in child.getchildren() if \ - not isinstance(ent, _Comment)] - else: - self.store[entry.tag][entry.get('name')]['all'].append(child) - else: - self.store['all'].append(child) - def BuildStructures(self, metadata): '''Build structures for client described by metadata''' - ret = Element("Independant", version='2.0') - [ret.append(deepcopy(entry)) for entry in self.store['all']] - idata = self.store['Image'].get(metadata.image, {'all':[], 'Class':{}}) - for entry in idata['all']: - ret.append(deepcopy(entry)) - for cls in metadata.classes: - for entry in idata['Class'].get(cls, []): - ret.append(deepcopy(entry)) - cdata = self.store['Class'].get(cls, {'all':[], 'Image':{}}) - for entry in cdata['all']: - ret.append(deepcopy(entry)) - for entry in cdata['Image'].get(metadata.image, []): - ret.append(deepcopy(entry)) + ret = lxml.etree.Element("Independant", version='2.0') + fragments = reduce(lambda x, y: x+y, + [base.Match(metadata) for base in self.entries.values()]) + [ret.append(copy.deepcopy(frag)) for frag in fragments] return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 4b357f121..cbbb6c671 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -1,122 +1,36 @@ '''This provides bundle clauses with translation functionality''' __revision__ = '$Revision$' -from copy import deepcopy -from syslog import LOG_ERR, syslog -from lxml.etree import Element, XML, XMLSyntaxError, _Comment +import Bcfg2.Server.Plugin +import copy +import lxml.etree -from Bcfg2.Server.Plugin import Plugin, SingleXMLFileBacked, XMLFileBacked, DirectoryBacked - - -class ImageFile(SingleXMLFileBacked): - '''This file contains image -> system mappings''' - def __init__(self, filename, fam): - self.images = {} - SingleXMLFileBacked.__init__(self, filename, fam) - - def Index(self): - '''Build data structures out of the data''' - try: - xdata = XML(self.data) - except XMLSyntaxError, err: - syslog(LOG_ERR, "Failed to parse file %s" % (self.name)) - syslog(LOG_ERR, err) - del self.data - return - self.images = {} - for child in xdata.getchildren(): - [name, pkg, service] = [child.get(field) for field in ['name', 'package', 'service']] - for grandchild in child.getchildren(): - self.images[grandchild.get('name')] = (name, pkg, service) - -class Bundle(XMLFileBacked): - '''Bundles are configuration specifications (with image/translation abstraction)''' - - def __init__(self, filename): - self.all = [] - self.attributes = {} - self.systems = {} - XMLFileBacked.__init__(self, filename) - - def Index(self): - '''Build data structures from the source data''' - try: - xdata = XML(self.data) - except XMLSyntaxError, err: - syslog(LOG_ERR, "Failed to parse file %s" % (self.name)) - syslog(LOG_ERR, str(err)) - del self.data - return - self.all = [] - self.systems = {} - self.attributes = {} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag == 'System': - self.systems[entry.attrib['name']] = [ent for ent in entry.getchildren() \ - if not isinstance(ent, _Comment)] - elif entry.tag == 'Attribute': - self.attributes[entry.get('name')] = [ent for ent in entry.getchildren() \ - if not isinstance(ent, _Comment)] - else: - self.all.append(entry) - del self.data - - def BuildBundle(self, metadata, system): - '''Build a bundle for a particular client''' - bundlename = self.name.split('/')[-1] - bundle = Element('Bundle', name=bundlename) - for entry in self.all + self.systems.get(system, []): - bundle.append(deepcopy(entry)) - for attribute in [aname for (scope, aname) in [item.split('.') for item in metadata.attributes] - if scope == bundlename[:-4]]: - for entry in self.attributes.get(attribute, []): - bundle.append(deepcopy(entry)) - return bundle - -class BundleSet(DirectoryBacked): - '''The Bundler handles creation of dependent clauses based on bundle definitions''' - __child__ = Bundle - -class Bundler(Plugin): +class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): '''The bundler creates dependent clauses based on the bundle/translation scheme from bcfg1''' __name__ = 'Bundler' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = Bcfg2.Server.Plugin.StructFile def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - self.imageinfo = ImageFile("%s/etc/imageinfo.xml"%(datastore), self.core.fam) - self.bundles = BundleSet(self.data, self.core.fam) + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + try: + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, self.core.fam) + except OSError: + self.LogError("Failed to load Bundle repository") + raise Bcfg2.Server.Plugin.PluginInitError def BuildStructures(self, metadata): '''Build all structures for client (metadata)''' - try: - (system, package, service) = self.GetTransInfo(metadata) - except KeyError: - syslog(LOG_ERR, "Failed to find translation information for image %s" % metadata.image) - return [] bundleset = [] for bundlename in metadata.bundles: - if not self.bundles.entries.has_key("%s.xml"%(bundlename)): - syslog(LOG_ERR, "Client %s requested nonexistent bundle %s"%(metadata.hostname, bundlename)) + if not self.entries.has_key("%s.xml"%(bundlename)): + self.LogError("Client %s requested nonexistent bundle %s" % \ + (metadata.hostname, bundlename)) continue - - bundle = self.bundles.entries["%s.xml" % (bundlename)].BuildBundle(metadata, system) - # now we need to populate service/package types - for entry in bundle.getchildren(): - if entry.tag == 'Package': - entry.attrib['type'] = package - elif entry.tag == 'Service': - entry.attrib['type'] = service + bundle = lxml.etree.Element('Bundle', name=bundlename) + [bundle.append(copy.deepcopy(item)) + for item in self.entries["%s.xml" % (bundlename)].Match(metadata)] bundleset.append(bundle) return bundleset - def GetTransInfo(self, metadata): - '''Get Translation info for metadata.image''' - if self.imageinfo.images.has_key(metadata.image): - return self.imageinfo.images[metadata.image] - else: - raise KeyError, metadata.image - - - diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 953401e7a..b325144e5 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -1,7 +1,6 @@ '''This module implements a config file repository''' __revision__ = '$Revision$' -from binascii import b2a_base64 from os import stat from re import compile as regcompile from stat import S_ISDIR, ST_MODE @@ -9,54 +8,73 @@ from syslog import syslog, LOG_INFO, LOG_ERR from Bcfg2.Server.Plugin import Plugin, PluginExecutionError, FileBacked +import binascii +import exceptions + +specific = regcompile('(.*/)(?P[\S\-.]+)\.((H_(?P\S+))|' + + '(G(?P\d+)_(?P\S+)))$') + +class SpecificityError(Exception): + '''Thrown in case of filename parse failure''' + pass + class FileEntry(FileBacked): '''The File Entry class pertains to the config files contained in a particular directory. This includes :info, all base files and deltas''' - - def __init__(self, name, all, image, classes, bundles, attribs, hostname): + + def __init__(self, myid, name): FileBacked.__init__(self, name) - self.all = all - self.image = image - self.bundles = bundles - self.classes = classes - self.attributes = attribs - self.hostname = hostname + self.name = name + self.identity = myid + self.all = False + self.hostname = False + self.group = False + self.op = False + self.prio = False + if name.split('.')[-1] in ['cat', 'diff']: + self.op = name.split('.')[-1] + name = name[:-(len(self.op) + 1)] + if self.name.split('/')[-1] == myid.split('/')[-1]: + self.all = True + else: + data = specific.match(name) + if not data: + syslog(LOG_ERR, "Cfg: Failed to match %s" % name) + raise SpecificityError + if data.group('hostname') != None: + self.hostname = data.group('hostname') + else: + self.group = data.group('group') + self.prio = int(data.group('prio')) def __cmp__(self, other): - fields = ['all', 'image', 'classes', 'bundles', 'attributes', 'hostname'] - try: - most1 = [index for index in range(len(fields)) if getattr(self, fields[index])][0] - except IndexError: - most1 = 0 - try: - most2 = [index for index in range(len(fields)) if getattr(other, fields[index])][0] - except IndexError: - most2 = 0 - if most1 == most2: - if self.name.split('.')[-1] in ['cat', 'diff']: - meta1 = self.name.split('.')[-2] - else: - meta1 = self.name.split('.')[-1] - if other.name.split('.')[-1] in ['cat', 'diff']: - meta2 = other.name.split('.')[-2] + data = [[getattr(self, field) for field in ['all', 'group', 'hostname']], + [getattr(other, field) for field in ['all', 'group', 'hostname']]] + for index in range(3): + if data[0][index] and not data[1][index]: + return -1 + elif data[1][index] and not data[0][index]: + return 1 + elif data[0][index] and data[1][index]: + if hasattr(self, 'prio') and hasattr(other, 'prio'): + return self.prio - other.prio + else: + return 0 else: - meta2 = other.name.split('.')[-1] + pass + syslog(LOG_ERR, "Cfg: Critical: Ran off of the end of the world sorting %s" % (self.name)) - if meta1[0] not in ['C', 'B']: - return 0 - # need to tiebreak with numeric prio - prio1 = int(meta1[1:3]) - prio2 = int(meta2[1:3]) - return prio1 - prio2 + def applies(self, metadata): + '''Predicate if fragment matches client metadata''' + if self.all or (self.hostname == metadata.hostname) or \ + (self.group in metadata.groups): + return True else: - return most1 - most2 + return False class ConfigFileEntry(object): '''ConfigFileEntry is a repository entry for a single file, containing all data for all clients.''' - specific = regcompile('(.*/)(?P[\S\-.]+)\.((H_(?P\S+))|' + - '(B(?P\d+)_(?P\S+))|(A(?P\d+)_(?P\S+))|' + - '(I_(?P\S+))|(C(?P\d+)_(?P\S+)))$') info = regcompile('^owner:(\s)*(?P\w+)|group:(\s)*(?P\w+)|' + 'perms:(\s)*(?P\w+)|encoding:(\s)*(?P\w+)|' + '(?Pparanoid(\s)*)$') @@ -65,8 +83,7 @@ class ConfigFileEntry(object): object.__init__(self) self.path = path self.repopath = repopath - self.basefiles = [] - self.deltas = [] + self.fragments = [] self.metadata = {'encoding': 'ascii', 'owner':'root', 'group':'root', 'perms':'0644'} self.paranoid = False @@ -94,41 +111,15 @@ class ConfigFileEntry(object): def AddEntry(self, name): '''add new file additions for a single cf file''' - delta = False - oldname = name if name[-5:] == ':info': return self.read_info() - if name.split('/')[-1] == self.path.split('/')[-1]: - self.basefiles.append(FileEntry(name, True, None, [], [], [], None)) - self.basefiles.sort() - return - - if name.split('/')[-1].split('.')[-1] in ['cat']: - delta = True - oldname = name - name = name[:-4] - - specmatch = self.specific.match(name) - if specmatch == None: - syslog(LOG_ERR, "Cfg: Failed to match file %s" % (name)) + try: + self.fragments.append(FileEntry(self.path, name)) + self.fragments.sort() + except SpecificityError: return - data = {} - for item, value in specmatch.groupdict().iteritems(): - if value != None: - data[item] = value - - cfile = FileEntry(oldname, False, data.get('image', None), data.get('class', []), - data.get('bundle', []), data.get('attr', []), data.get('hostname', None)) - - if delta: - self.deltas.append(cfile) - self.deltas.sort() - else: - self.basefiles.append(cfile) - self.basefiles.sort() - def HandleEvent(self, event): '''Handle FAM updates''' action = event.code2str() @@ -136,11 +127,11 @@ class ConfigFileEntry(object): if action in ['changed', 'exists', 'created']: return self.read_info() if event.filename != self.path.split('/')[-1]: - if not self.specific.match('/' + event.filename): + if not specific.match('/' + event.filename): syslog(LOG_INFO, 'Cfg: Suppressing event for bogus file %s' % event.filename) return - entries = [entry for entry in self.basefiles + self.deltas if + entries = [entry for entry in self.fragments if entry.name.split('/')[-1] == event.filename] if len(entries) == 0: @@ -152,10 +143,8 @@ class ConfigFileEntry(object): syslog(LOG_INFO, "Cfg: Removing entry %s" % event.filename) for entry in entries: syslog(LOG_INFO, "Cfg: Removing entry %s" % (entry.name)) - if entry in self.basefiles: - self.basefiles.remove(entry) - if entry in self.deltas: - self.deltas.remove(entry) + self.fragments.remove(entry) + self.fragments.sort() syslog(LOG_INFO, "Cfg: Entry deletion completed") elif action in ['changed', 'exists', 'created']: [entry.HandleEvent(event) for entry in entries] @@ -168,13 +157,13 @@ class ConfigFileEntry(object): filedata = "" # first find basefile try: - basefile = [bfile for bfile in self.basefiles if metadata.Applies(bfile)][-1] + basefile = [bfile for bfile in self.fragments if bfile.applies(metadata) and not bfile.op][-1] except IndexError: syslog(LOG_ERR, "Cfg: Failed to locate basefile for %s" % name) raise PluginExecutionError, ('basefile', name) filedata += basefile.data - for delta in [x for x in self.deltas if metadata.Applies(x)]: + for delta in [delta for delta in self.fragments if delta.applies(metadata) and delta.op]: # find applicable deltas lines = filedata.split('\n') if not lines[-1]: @@ -188,15 +177,15 @@ class ConfigFileEntry(object): lines.append(line[1:]) filedata = "\n".join(lines) + "\n" - [entry.attrib.__setitem__(x,y) for (x,y) in self.metadata.iteritems()] + [entry.attrib.__setitem__(key, value) for (key, value) in self.metadata.iteritems()] if self.paranoid: entry.attrib['paranoid'] = 'true' if entry.attrib['encoding'] == 'base64': - entry.text = b2a_base64(filedata) + entry.text = binascii.b2a_base64(filedata) else: try: entry.text = filedata - except: + except exceptions.AttributeError: syslog(LOG_ERR, "Failed to marshall file %s. Mark it as base64" % (entry.get('name'))) class Cfg(Plugin): diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py index 9a488fc74..e729030fb 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Server/Plugins/Hostbase.py @@ -2,9 +2,12 @@ __revision__ = '$Revision$' from syslog import syslog, LOG_INFO -from lxml.etree import XML +from lxml.etree import XML, SubElement from Cheetah.Template import Template from Bcfg2.Server.Plugin import Plugin, PluginExecutionError, PluginInitError, DirectoryBacked +from time import strftime +from sets import Set +import re class DataNexus(DirectoryBacked): '''DataNexus is an object that watches multiple files and @@ -39,7 +42,7 @@ class Hostbase(Plugin, DataNexus): def __init__(self, core, datastore): self.ready = False - files = ['dnsdata.xml', 'hostbase.xml', 'networks.xml'] + files = ['zones.xml', 'hostbase.xml', 'hostbase-dns.xml', 'hostbase-dhcp.xml'] Plugin.__init__(self, core, datastore) try: DataNexus.__init__(self, datastore + '/Hostbase/data', @@ -49,11 +52,15 @@ class Hostbase(Plugin, DataNexus): raise PluginInitError self.xdata = {} self.filedata = {} + self.dnsservers = ['scotty.mcs.anl.gov'] + self.dhcpservers = ['thwap.mcs.anl.gov', 'squeak.mcs.anl.gov'] self.templates = {'zone':Template(open(self.data + '/templates/' + 'zonetemplate.tmpl').read()), 'reversesoa':Template(open(self.data + '/templates/' + 'reversesoa.tmpl').read()), 'named':Template(open(self.data + '/templates/' + 'namedtemplate.tmpl').read()), 'reverseapp':Template(open(self.data + '/templates/' + 'reverseappend.tmpl').read()), - 'dhcp':Template(open(self.data + '/templates/' + 'dhcpd_template.tmpl').read())} + 'dhcp':Template(open(self.data + '/templates/' + 'dhcpd_template.tmpl').read()), + 'hosts':Template(open(self.data + '/templates/' + 'hosts.tmpl').read()), + 'hostsapp':Template(open(self.data + '/templates/' + 'hostsappend.tmpl').read())} self.Entries['ConfigFile'] = {} def FetchFile(self, entry, metadata): @@ -65,8 +72,38 @@ class Hostbase(Plugin, DataNexus): [entry.attrib.__setitem__(key, value) for (key, value) in perms.iteritems()] entry.text = self.filedata[fname] + def BuildStructures(self, metadata): + '''Build hostbase bundle''' + if metadata.hostname in self.dnsservers or metadata.hostname in self.dhcpservers: + output = [] + if metadata.hostname in self.dnsservers: + dnsbundle = XML(self.entries['hostbase-dns.xml'].data) + for configfile in self.Entries['ConfigFile']: + if re.search('/etc/bind/', configfile): + SubElement(dnsbundle, "ConfigFile", name=configfile) + output.append(dnsbundle) + if metadata.hostname in self.dhcpservers: + dhcpbundle = XML(self.entries['hostbase-dhcp.xml'].data) + output.append(dhcpbundle) + return output + else: + return [] + def rebuildState(self, event): '''Pre-cache all state information for hostbase config files''' + def get_serial(zone): + '''I think this does the zone file serial number hack but whatever''' + todaydate = (strftime('%Y%m%d')) + try: + if todaydate == zone.get('serial')[:8]: + serial = atoi(zone.get('serial')) + 1 + else: + serial = atoi(todaydate) * 100 + return str(serial) + except (KeyError): + serial = atoi(todaydate) * 100 + return str(serial) + if self.entries.has_key(event.filename) and not self.xdata.has_key(event.filename): self.xdata[event.filename] = XML(self.entries[event.filename].data) if [item for item in self.files if not self.entries.has_key(item)]: @@ -74,163 +111,293 @@ class Hostbase(Plugin, DataNexus): # we might be able to rebuild data more sparsely, # but hostbase.xml is the only one that will really change often # rebuild zoneinfo - iplist = [] - for zone in self.xdata['dnsdata.xml']: + hosts = {} + zones = self.xdata['zones.xml'] + hostbase = self.xdata['hostbase.xml'] + ## this now gets all hosts associated with the zone file being initialized + ## all ip addresses and cnames are grabbed from each host and passed to the appropriate template + for zone in zones: + hosts[zone.get('domain')] = [] + for host in hostbase: + if host.get('domain') in hosts: + hosts[host.get('domain')].append(host) + for zone in zones: zonehosts = [] - for host in [host for host in self.xdata['hostbase.xml'] - if host.get('domain') == zone.get('domain')]: - hostname = host.get('hostname') - if zone.get('domain') == 'mcs.anl.gov': - ## special cases for the mcs.anl.gov domain - ## all machines have a "-eth" entry as well as an entry identifying their subnet - ## they also have their mail exchangers after every address - ipnodes = host.findall("interface/ip") - zonehosts.append((hostname, ipnodes[0].attrib['ip'], ipnodes[0].findall("name/mx"), None)) - [zonehosts.append(("-".join([hostname, ipnode.attrib['dnssuffix']]), \ - ipnode.attrib['ip'], ipnode.findall("name/mx"), None)) - for ipnode in ipnodes] - [zonehosts.append(("-".join([hostname, namenode.attrib['name']]), \ - ipnode.attrib['ip'], namenode.findall("mx"), None)) - for ipnode in ipnodes - for namenode in ipnode - if namenode.attrib['name'] != ""] - else: - ipnodes = host.findall("interface/ip") - zonehosts.append((host.attrib['hostname'], ipnodes[0].attrib['ip'], None, None)) - [zonehosts.append(("-".join([host.attrib['hostname'], namenode.attrib['name']]), - ipnode.attrib['ip'], None, None)) - for ipnode in ipnodes - for namenode in ipnode - if namenode.attrib['name'] != ""] - - [zonehosts.append((host.attrib['hostname'], None, None, cnamenode.attrib['cname'])) - for cnamenode in host.findall("interface/ip/name/cname") - if cnamenode.attrib['cname'] != ""] - - [iplist.append(ipnode.attrib['ip']) for ipnode in host.findall("interface/ip")] + for host in hosts[zone.get('domain')]: + hostname = host.attrib['hostname'] + ipnodes = host.findall("interface/ip") + #gets all the forward look up stuff + [zonehosts.append((namenode.get('name').split(".")[0], ipnode.get('ip'), + namenode.findall('mx'))) + for ipnode in ipnodes + for namenode in ipnode] + #gets cname stuff + [zonehosts.append((cnamenode.get('cname') + '.', namenode.get('name').split('.')[0], None)) + for namenode in host.findall("interface/ip/name") + for cnamenode in namenode.findall("cname") + if (cnamenode.get('cname').split(".")[0], namenode.get('name').split('.')[0], None) not in zonehosts + and cnamenode.get('cname') is not None] + zonehosts.sort() self.templates['zone'].zone = zone - self.templates['zone'].root = self.xdata['dnsdata.xml'] + self.templates['zone'].root = zones self.templates['zone'].hosts = zonehosts self.filedata[zone.get('domain')] = str(self.templates['zone']) + self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone.get('domain'))] = self.FetchFile # now all zone forward files are built - iplist.sort() filelist = [] - temp = None - for x in range(len(iplist)-1): - addressparts = iplist[x].split(".") - if addressparts[:3] != iplist[x+1].split(".")[:3] and addressparts[:2] == iplist[x+1].split(".")[:2] \ - and ".".join([addressparts[1], addressparts[0]]) not in filelist: - filelist.append(".".join([addressparts[1], addressparts[0]])) - elif addressparts[:3] != iplist[x+1].split(".")[:3] and \ - addressparts[:2] != iplist[x+1].split(".")[:2] and \ - ".".join([addressparts[1], addressparts[0]]) not in filelist: - filelist.append(".".join([addressparts[2], addressparts[1], addressparts[0]])) - if x+1 == len(iplist) - 1: - temp = iplist[x+1].split(".") - if ".".join([temp[2], temp[1], temp[0]]) not in filelist \ - and ".".join([temp[1], temp[0]]) not in filelist: - filelist.append(".".join([temp[2], temp[1], temp[0]])) - + three_subnet = [ip.get('ip').rstrip('0123456789').rstrip('.') + for ip in hostbase.findall('host/interface/ip')] + three_subnet_set = Set(three_subnet) + two_subnet = [subnet.rstrip('0123456789').rstrip('.') + for subnet in three_subnet_set] + two_subnet_set = Set(two_subnet) + filelist = [each for each in two_subnet_set + if two_subnet.count(each) > 1] + [filelist.append(each) for each in three_subnet_set + if each.rstrip('0123456789').rstrip('.') not in filelist] + + reversenames = [] for filename in filelist: - self.templates['reversesoa'].inaddr = filename + towrite = filename.split('.') + towrite.reverse() + reversename = '.'.join(towrite) + self.templates['reversesoa'].inaddr = reversename self.templates['reversesoa'].zone = zone - self.templates['reversesoa'].root = self.xdata['dnsdata.xml'] - self.filedata["%s.rev" % filename] = str(self.templates['reversesoa']) + self.templates['reversesoa'].root = self.xdata['zones.xml'] + self.filedata['%s.rev' % reversename] = str(self.templates['reversesoa']) + reversenames.append(reversename) - self.templates['named'].zones = self.xdata['dnsdata.xml'] - self.templates['named'].reverses = filelist + self.templates['named'].zones = self.xdata['zones.xml'] + self.templates['named'].reverses = reversenames self.filedata["named.conf"] = str(self.templates['named']) + self.Entries['ConfigFile']["%s/%s" % (self.filepath, 'named.conf')] = self.FetchFile - for filename in filelist: + reversenames.sort() + for filename in reversenames: originlist = [] + reversehosts = [] towrite = filename.split(".") towrite.reverse() if len(towrite) > 2: - self.templates['reverseapp'].hosts = [(ipnode.get('ip').split('.'), host.get('hostname'), - host.get('domain'), ipnode.get('num'), ipnode.get('dnssuffix')) - for host in self.xdata['hostbase.xml'] - for ipnode in host.findall('interface/ip') - if ipnode.get('ip').split('.')[:3] == towrite] - + [reversehosts.append((ipnode.attrib['ip'].split("."), host.attrib['hostname'], + host.attrib['domain'], ipnode.get('num'), None)) + for host in self.xdata['hostbase.xml'] + for ipnode in host.findall("interface/ip") + if ipnode.attrib['ip'].split(".")[:3] == towrite] + self.templates['reverseapp'].hosts = reversehosts self.templates['reverseapp'].inaddr = filename self.templates['reverseapp'].fileorigin = None self.filedata["%s.rev" % filename] += str(self.templates['reverseapp']) else: - revhosts = [(ipnode.get('ip').split('.'), host.get('hostname'), host.get('domain'), - ipnode.get('num'), ipnode.get('dnssuffix')) - for host in self.xdata['hostbase.xml'] - for ipnode in host.findall("interface/ip") - if ipnode.get('ip').split(".")[:2] == towrite] + [reversehosts.append((ipnode.attrib['ip'].split("."), host.attrib['hostname'], + host.attrib['domain'], ipnode.get('num'), None)) + for host in self.xdata['hostbase.xml'] + for ipnode in host.findall("interface/ip") + if ipnode.attrib['ip'].split(".")[:2] == towrite] [originlist.append(".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]])) - for reversehost in revhosts + for reversehost in reversehosts if ".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]]) not in originlist] - revhosts.sort() + reversehosts.sort() originlist.sort() for origin in originlist: - outputlist = [rhost for rhost in revhosts - if ".".join([rhost[0][2], rhost[0][1], rhost[0][0]]) == origin] + outputlist = [] + [outputlist.append(reversehost) + for reversehost in reversehosts + if ".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]]) == origin] self.templates['reverseapp'].fileorigin = filename self.templates['reverseapp'].hosts = outputlist self.templates['reverseapp'].inaddr = origin self.filedata["%s.rev" % filename] += str(self.templates['reverseapp']) + self.Entries['ConfigFile']["%s/%s.rev" % (self.filepath, filename)] = self.FetchFile self.buildDHCP() - for key in self.filedata: - self.Entries['ConfigFile']["%s/%s" % (self.filepath, key)] = self.FetchFile + self.buildHosts() + self.buildHostsLPD() + self.buildPrinters() + self.buildNetgroups() def buildDHCP(self): '''Pre-build dhcpd.conf and stash in the filedata table''' - if 'networks.xml' not in self.xdata.keys(): - print "not running before networks is cached" - return - networkroot = self.xdata['networks.xml'] if 'hostbase.xml' not in self.xdata.keys(): print "not running before hostbase is cached" return hostbase = self.xdata['hostbase.xml'] - vlanandsublist = [] - subnets = networkroot.findall("subnet") - for vlan in networkroot.findall("vlan"): - vlansubs = vlan.findall("subnet") - vlansubs.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - vlanandsublist.append((vlan, vlansubs)) - - subnets140 = [subnet for subnet in subnets if subnet.attrib['address'].split(".")[0] == "140"] - privatesubnets = [subnet for subnet in subnets if subnet.attrib['address'].split(".")[0] != "140"] - subnets140.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - privatesubnets.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - - dhcphosts = [host for host in hostbase if host.get('dhcp') == 'y' \ - and host.find("interface").get('mac') != 'float' \ - and host.find("interface").get('mac') != ""] + dhcphosts = [host for host in hostbase if host.find('dhcp').get('dhcp') == 'y' + and host.find("interface").attrib['mac'] != 'float' + and host.find("interface").attrib['mac'] != "" + and host.find("interface").attrib['mac'] != "unknown"] + numips = 0 hosts = [] for host in dhcphosts: if len(host.findall("interface")) == 1 and len(host.findall("interface/ip")) == 1: - hosts.append([host.get('hostname'), host.get('domain'), \ - host.find("interface").get('mac'), \ - host.find("interface/ip").get('ip')]) - elif len(host.findall("interface")) > 1: + hosts.append([host.attrib['hostname'], host.attrib['domain'], \ + host.find("interface").attrib['mac'], \ + host.find("interface/ip").attrib['ip']]) + else: count = 0 - for interface in host.findall("interface"): + for interface in host.findall('interface'): if count == 0 and interface.find("ip") is not None: - hostdata = [host.get('hostname'), host.get('domain'), \ - interface.get('mac'), interface.find("ip").get('ip')] + hostdata = [host.attrib['hostname'], host.attrib['domain'], + interface.attrib['mac'], interface.find("ip").attrib['ip']] elif count != 0 and interface.find("ip") is not None: - hostdata = [host.get('hostname'), "-".join([host.get('domain'), str(count)]), \ - interface.get('mac'), interface.find("ip").get('ip')] + hostdata = [host.attrib['hostname'], "-".join([host.attrib['domain'], str(count)]), + interface.attrib['mac'], interface.find("ip").attrib['ip']] if len(interface.findall("ip")) > 1: - for ipnode in interface.findall("ip")[1:]: - hostdata[3] = ", ".join([hostdata[3], ipnode.get('ip')]) + for ip in interface.findall("ip")[1:]: + hostdata[3] = ", ".join([hostdata[3], ip.attrib['ip']]) count += 1 hosts.append(hostdata) + + numips += len(host.findall("interface/ip")) hosts.sort(lambda x, y: cmp(x[0], y[0])) self.templates['dhcp'].hosts = hosts - self.templates['dhcp'].privatesubnets = privatesubnets - self.templates['dhcp'].subnets140 = subnets140 - self.templates['dhcp'].vlans = vlanandsublist - self.templates['dhcp'].networkroot = networkroot - self.filedata['/etc/dhcpd.conf'] = str(self.templates['dhcp']) + self.templates['dhcp'].numips = numips + self.templates['dhcp'].timecreated = strftime("%a %b %d %H:%M:%S %Z %Y") + self.filedata['dhcpd.conf'] = str(self.templates['dhcp']) + self.Entries['ConfigFile']['/etc/dhcpd.conf'] = self.FetchFile + + def buildHosts(self): + '''This will rebuild the hosts file to include all important machines''' + hostbase = self.xdata['hostbase.xml'] + domains = [host.get('domain') for host in hostbase] + domains_set = Set(domains) + domain_data = [(domain, domains.count(domain)) for domain in domains_set] + domain_data.sort() + ips = [(ip, host) for host in hostbase.findall('host') + for ip in host.findall("interface/ip")] + three_octets = [ip[0].get('ip').rstrip('0123456789').rstrip('.') + for ip in ips] + three_octets_set = list(Set(three_octets)) + three_sort = [tuple([int(num) for num in each.split('.')]) for each in three_octets_set] + three_sort.sort() + three_octets_set = ['.'.join([str(num) for num in each]) for each in three_sort] + three_octets_data = [(octet, three_octets.count(octet)) + for octet in three_octets_set] + append_data = [(subnet, [ip for ip in ips \ + if ip[0].get('ip').rstrip("0123456789").rstrip('.') + == subnet[0]]) for subnet in three_octets_data] + for each in append_data: + each[1].sort(lambda x, y: cmp(int(x[0].get('ip').split('.')[-1]), int(y[0].get('ip').split('.')[-1]))) + two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] + two_octets_set = list(Set(two_octets)) + two_sort = [tuple([int(num) for num in each.split('.')]) for each in two_octets_set] + two_sort.sort() + two_octets_set = ['.'.join([str(num) for num in each]) for each in two_sort] + two_octets_data = [(octet, two_octets.count(octet)) for octet in two_octets_set] + self.templates['hosts'].domain_data = domain_data + self.templates['hosts'].three_octets_data = three_octets_data + self.templates['hosts'].two_octets_data = two_octets_data + self.templates['hosts'].three_octets = len(three_octets) + self.templates['hosts'].timecreated = strftime("%a %b %d %H:%M:%S %Z %Y") + self.filedata['hosts'] = str(self.templates['hosts']) + for subnet in append_data: + self.templates['hostsapp'].ips = subnet[1] + self.templates['hostsapp'].subnet = subnet[0] + self.filedata['hosts'] += str(self.templates['hostsapp']) + self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile + + + def buildPrinters(self): + '''this will rebuild the printers.data file used in + our local printers script''' + header = """# This file is automatically generated. DO NOT EDIT IT! +# This datafile is for use with /mcs/bin/printers. +# +Name Room User Type Notes +============== ========== ============ ======================== ==================== +""" + + printers = [host for host in self.xdata['hostbase.xml'] + if host.find('whatami').get('whatami') == "printer" + and host.get('domain') == 'mcs.anl.gov'] + self.filedata['printers.data'] = header + output_list = [] + for printer in printers: + if printer.find('printq').get('printq'): + for printq in re.split(',[ ]*', printer.find('printq').get('printq')): + output_list.append((printq, printer.find('room').get('room'), printer.find('user').get('user'), + printer.find('model').get('model'), printer.find('note').get('note'))) + output_list.sort() + for printer in output_list: + self.filedata['printers.data'] += ("%-16s%-12s%-14s%-26s%s\n" % printer) + self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile + + def buildHostsLPD(self): + '''this rebuilds the hosts.lpd file''' + header = """+@machines ++@all-machines +achilles.ctd.anl.gov +raven.ops.anl.gov +seagull.hr.anl.gov +parrot.ops.anl.gov +condor.ops.anl.gov +delphi.esh.anl.gov +anlcv1.ctd.anl.gov +anlvms.ctd.anl.gov +olivia.ctd.anl.gov\n\n""" + + hostbase = self.xdata['hostbase.xml'] + redmachines = [".".join([host.get('hostname'), host.get('domain')]) + for host in hostbase if host.find('netgroup').get('netgroup') == 'red'] + winmachines = [".".join([host.get('hostname'), host.get('domain')]) + for host in hostbase if host.find('netgroup').get('netgroup') == 'win'] + redmachines += [name.get('name') for host in hostbase + for name in host.findall('interface/ip/name') + if host.find('netgroup').get('netgroup') == 'red' and name.get('only') != 'no'] + winmachines += [name.get('name') for host in hostbase + for name in host.findall('interface/ip/name') + if host.find('netgroup').get('netgroup') == 'win' and name.get('only') != 'no'] + redmachines.sort() + winmachines.sort() + self.filedata['hosts.lpd'] = header + for machine in redmachines: + self.filedata['hosts.lpd'] += machine + "\n" + self.filedata['hosts.lpd'] += "\n" + for machine in winmachines: + self.filedata['hosts.lpd'] += machine + "\n" + self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile + + def buildNetgroups(self): + '''this rebuilds the many different files that will eventually + get post processed and converted into a ypmap for netgroups''' + header = """################################################################### +# This file lists hosts in the '%s' machine netgroup, it is +# automatically generated. DO NOT EDIT THIS FILE! To update +# the hosts in this file, edit hostbase and do a 'make nets' +# in /mcs/adm/hostbase. +# +# Number of hosts in '%s' machine netgroup: %i +#\n\n""" + + netgroups = {} + for host in self.xdata['hostbase.xml']: + if host.find('netgroup').get('netgroup') == "" or host.find('netgroup').get('netgroup')== 'none': + continue + if host.find('netgroup').get('netgroup') not in netgroups: + netgroups.update({host.find('netgroup').get('netgroup') : + [".".join([host.get('hostname'), host.get('domain')])]}) + else: + netgroups[host.find('netgroup').get('netgroup')].append(".".join([host.get('hostname'), + host.get('domain')])) + + for name in host.findall('interface/ip/name'): + if name.get('only') != 'no': + netgroups[host.find('netgroup').get('netgroup')].append(name.get('name')) + + for netgroup in netgroups: + self.filedata["%s-machines" % netgroup] = header % (netgroup, netgroup, len(netgroups[netgroup])) + netgroups[netgroup].sort() + for each in netgroups[netgroup]: + self.filedata["%s-machines" % netgroup] += each + "\n" + self.Entries['ConfigFile']["/var/yp/netgroups/%s-machines" % netgroup] = self.FetchFile + + def dumpXML(self): + '''this just dumps the info in the hostbase.xml file to be used + with external programs''' + self.filedata['hostbase.xml'] = self.xdata['hostbase.xml'] + self.Entries['ConfigFile']['/etc/hostbase.xml'] = self.FetchFile + diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py index 8521994e0..e77dd99e5 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Server/Plugins/Pkgmgr.py @@ -1,100 +1,54 @@ '''This module implements a package management scheme for all images''' __revision__ = '$Revision$' -from copy import deepcopy -from re import compile as regcompile +import re from syslog import syslog, LOG_ERR +import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import Plugin, PluginInitError, PluginExecutionError, DirectoryBacked, XMLFileBacked - -class PackageEntry(XMLFileBacked): - '''PackageEntry is a set of packages and locations for a single image''' - __identifier__ = 'image' - splitters = {'rpm':regcompile('^(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ +class PNode(Bcfg2.Server.Plugin.LNode): + '''PNode has a list of packages available at a particular group intersection''' + splitters = {'rpm':re.compile('^(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ '(?P[\w\d\.]+-([\w\d\.]+))\.(?P\w+)\.rpm$'), - 'encap':regcompile('^(?P\w+)-(?P[\w\d\.-]+).encap.*$')} - - def __init__(self, filename): - XMLFileBacked.__init__(self, filename) - self.packages = {} - - def Index(self): - '''Build internal data structures''' - XMLFileBacked.Index(self) - self.packages = {} - for location in self.entries: - for pkg in location.getchildren(): - if location.attrib.has_key('type'): - pkg.set('type', location.get('type')) - if pkg.attrib.has_key("simplefile"): - self.packages[pkg.get('name')] = {} - for key in pkg.attrib: - self.packages[pkg.get('name')][key] = pkg.attrib[key] - # most attribs will be set from pkg - self.packages[pkg.get('name')]['url'] = "%s/%s" % (location.get('uri'), pkg.get('simplefile')) - elif pkg.attrib.has_key("file"): - if self.splitters.has_key(pkg.get('type')): - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) - if not mdata: - syslog(LOG_ERR, "Failed to match pkg %s" % pkg.get('file')) - continue - pkgname = mdata.group('name') - self.packages[pkgname] = mdata.groupdict() - self.packages[pkgname]['url'] = location.get('uri') + '/' + pkg.get('file') - self.packages[pkgname]['type'] = pkg.get('type') - else: - derived = [(ptype, self.splitters[ptype].match(pkg.get('file')).groupdict()) - for ptype in self.splitters if self.splitters[ptype].match(pkg.get('file'))] - if not derived: - syslog("Failed to match pkg %s" % pkg.get('file')) - else: - (ptype, mdata) = derived[0] - pkgname = mdata['name'] - self.packages[pkgname] = mdata - self.packages[pkgname]['url'] = location.get('uri') + '/' + pkg.get('file') - self.packages[pkgname]['type'] = ptype + 'encap':re.compile('^(?P\w+)-(?P[\w\d\.-]+).encap.*$')} + + def __init__(self, data, plist, parent=None): + # copy local attributes to all child nodes if no local attribute exists + for child in data.getchildren(): + for attr in [key for key in data.attrib.keys() if key != 'name' and not child.attrib.has_key(key)]: + child.set(attr, data.get(attr)) + Bcfg2.Server.Plugin.LNode.__init__(self, data, plist, parent) + for pkg in data.findall('./Package'): + if pkg.attrib.has_key('name') and pkg.get('name') not in plist: + plist.append(pkg.get('name')) + if pkg.attrib.has_key('simplefile'): + pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) + self.contents[pkg.get('name')] = pkg.attrib + else: + if pkg.attrib.has_key('file'): + pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) + if self.splitters.has_key(pkg.get('type')): + mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) + if not mdata: + syslog(LOG_ERR, "Pkgmgr: Failed to match pkg %s" % pkg.get('file')) + continue + pkgname = mdata.group('name') + self.contents[pkgname] = mdata.groupdict() + if pkg.attrib.get('file'): + self.contents[pkgname]['url'] = pkg.get('url') + self.contents[pkgname]['type'] = pkg.get('type') + if pkgname not in plist: + plist.append(pkgname) else: - self.packages[pkg.get('name')] = pkg.attrib + self.contents[pkg.get('name')] = pkg.attrib -class PackageDir(DirectoryBacked): - '''A directory of package files''' - __child__ = PackageEntry +class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): + '''PkgSrc files contain a PNode hierarchy that returns matching package entries''' + __node__ = PNode -class Pkgmgr(Plugin): +class Pkgmgr(Bcfg2.Server.Plugin.XMLPrioDir): '''This is a generator that handles package assignments''' __name__ = 'Pkgmgr' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - try: - self.pkgdir = PackageDir(self.data, self.core.fam) - except OSError: - self.LogError("Pkgmgr: Failed to load package indices") - raise PluginInitError - - def FindHandler(self, entry): - '''Non static mechanism of determining entry provisioning''' - if entry.tag != 'Package': - raise PluginExecutionError, (entry.tag, entry.get('name')) - return self.LocatePackage - - def LocatePackage(self, entry, metadata): - '''Locates a package entry for particular metadata''' - pkgname = entry.get('name') - if self.pkgdir.entries.has_key("%s.xml" % metadata.hostname): - pkglist = self.pkgdir["%s.xml" % metadata.hostname] - if pkglist.packages.has_key(pkgname): - pkginfo = pkglist.packages[pkgname] - [entry.attrib.__setitem__(field, pkginfo[field]) for field in pkginfo] - return - elif not self.pkgdir.entries.has_key("%s.xml" % metadata.image): - self.LogError("Pkgmgr: no package index for image %s" % metadata.image) - raise PluginExecutionError, ("Image", metadata.image) - pkglist = self.pkgdir["%s.xml" % (metadata.image)] - if pkglist.packages.has_key(pkgname): - pkginfo = pkglist.packages[pkgname] - [entry.attrib.__setitem__(x, pkginfo[x]) for x in pkginfo] - else: - raise PluginExecutionError, ("Package", pkgname) + __child__ = PkgSrc + __element__ = 'Package' diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Server/Plugins/Svcmgr.py index 2f2c7e5eb..da5ab341c 100644 --- a/src/lib/Server/Plugins/Svcmgr.py +++ b/src/lib/Server/Plugins/Svcmgr.py @@ -1,23 +1,20 @@ '''This generator provides service mappings''' __revision__ = '$Revision$' -from Bcfg2.Server.Plugin import Plugin, ScopedXMLFile, PluginInitError +import Bcfg2.Server.Plugin -class Svcmgr(Plugin): +class SNode(Bcfg2.Server.Plugin.LNode): + '''SNode has a list of services available at a particular group intersection''' + __leaf__ = './Service' + +class SvcSrc(Bcfg2.Server.Plugin.XMLSrc): + '''SvcSrc files contain prioritized service definitions''' + __node__ = SNode + +class Svcmgr(Bcfg2.Server.Plugin.XMLPrioDir): '''This is a generator that handles service assignments''' __name__ = 'Svcmgr' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - try: - self.svc = ScopedXMLFile("%s/etc/services.xml"%(datastore), self.core.fam) - except OSError: - self.LogError("Failed to load service definition file") - raise PluginInitError - self.Entries = self.svc.__provides__ - - - - + __child__ = SvcSrc + __element__ = 'Service' diff --git a/src/sbin/Bcfg2Server b/src/sbin/Bcfg2Server deleted file mode 100644 index c1b93644b..000000000 --- a/src/sbin/Bcfg2Server +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python - -'''The XML-RPC Bcfg2 Server''' -__revision__ = '$Revision:$' - -from getopt import getopt, GetoptError -from sys import argv, exc_info -from syslog import openlog, LOG_LOCAL0, syslog, LOG_INFO, LOG_ERR -from Bcfg2.Server.Core import Core, CoreInitError -from Bcfg2.Server.Metadata import MetadataConsistencyError -from Bcfg2.Server.Component import Component -from threading import Lock -from select import select, error as selecterror -from signal import signal, SIGINT, SIGTERM -from traceback import extract_tb -from xmlrpclib import Fault -from socket import gethostbyaddr, herror -from lxml.etree import XML, Element, tostring -from M2Crypto.SSL import SSLError - -import os, sys - -def daemonize(filename): - '''Do the double fork/setsession dance''' - # Fork once - if os.fork() != 0: - os._exit(0) - os.setsid() # Create new session - pid = os.fork() - if pid != 0: - pidfile = open(filename, "w") - pidfile.write("%i" % pid) - pidfile.close() - os._exit(0) - os.chdir("/") - os.umask(0) - - null = open("/dev/null", "w+") - - os.dup2(null.fileno(), sys.__stdin__.fileno()) - os.dup2(null.fileno(), sys.__stdout__.fileno()) - os.dup2(null.fileno(), sys.__stderr__.fileno()) - - -def critical_error(operation): - '''Print tracebacks in unexpected cases''' - syslog(LOG_ERR, "Traceback information (please include in any bug report):") - (ttype, value, trace) = exc_info() - for line in extract_tb(trace): - syslog(LOG_ERR, "File %s, line %i, in %s\n %s" % (line)) - syslog(LOG_ERR, "%s: %s" % (ttype, value)) - warning_error("An unexpected failure occurred in %s" % (operation) ) - raise Fault, (7, "Critical unexpected failure: %s" % (operation)) - -def fatal_error(message): - '''Signal a fatal error''' - syslog(LOG_ERR, "Fatal error: %s" % (message)) - raise SystemExit, 1 - -def warning_error(message): - '''Warn about a problem but continue''' - syslog(LOG_ERR,"Warning: %s\n" % (message)) - -def usage_error(message, opt, vopt, descs, argDescs): - '''Die because script was called the wrong way''' - print "Usage error: %s" % (message) - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 2 - -verboseMode = False - -def verbose(message): - '''Conditionally output information in verbose mode''' - global verboseMode - - if(verboseMode == True): - syslog(LOG_INFO, "%s" % (message)) - -def print_usage(opt, vopt, descs, argDescs): - print "Bcfg2Server usage:" - for arg in opt.iteritems(): - print " -%s\t\t\t%s" % (arg[0], descs[arg[0]]) - for arg in vopt.iteritems(): - print " -%s %s\t%s" % (arg[0], argDescs[arg[0]], descs[arg[0]]) - -def dgetopt(arglist, opt, vopt, descs, argDescs): - '''parse options into a dictionary''' - global verboseMode - - ret = {} - for optname in opt.values() + vopt.values(): - ret[optname] = False - - gstr = "".join(opt.keys()) + "".join([optionkey + ':' for optionkey in vopt.keys()]) - try: - ginfo = getopt(arglist, gstr) - except GetoptError, gerr: - usage_error(gerr, opt, vopt, descs, argDescs) - - for (gopt, garg) in ginfo[0]: - option = gopt[1:] - if opt.has_key(option): - ret[opt[option]] = True - else: - ret[vopt[option]] = garg - - if ret["help"] == True: - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 0 - - if ret["verbose"] == True: - verboseMode = True - - return ret - -class Bcfg2(Component): - """The Bcfg2 Server component providing XML-RPC access to Bcfg methods""" - __name__ = 'bcfg2' - __implementation__ = 'bcfg2' - - request_queue_size = 15 - - def __init__(self, setup): - Component.__init__(self, setup) - self.shut = False - # set shutdown handlers for sigint and sigterm - signal(SIGINT, self.start_shutdown) - signal(SIGTERM, self.start_shutdown) - try: - self.Core = Core(setup, setup['configfile']) - self.CoreLock = Lock() - except CoreInitError, msg: - fatal_error(msg) - - self.funcs.update({ - "GetConfig": self.Bcfg2GetConfig, - "GetProbes": self.Bcfg2GetProbes, - "RecvProbeData": self.Bcfg2RecvProbeData, - "RecvStats": self.Bcfg2RecvStats - }) - for plugin in self.Core.plugins.values(): - for method in plugin.__rmi__: - self.register_function(getattr(self.Core.plugins[plugin.__name__], method), - "%s.%s" % (plugin.__name__, method)) - - def get_request(self): - '''We need to do work between requests, so select with timeout instead of blocking in accept''' - rsockinfo = [] - famfd = self.Core.fam.fileno() - while self.socket not in rsockinfo: - if self.shut: - raise SSLError - try: - rsockinfo = select([self.socket, famfd], [], [], 15)[0] - except selecterror: - raise SSLError - - if famfd in rsockinfo: - self.Core.fam.Service() - if self.socket in rsockinfo: - # workaround for m2crypto 0.15 bug - self.socket.postConnectionCheck = None - return self.socket.accept() - - def serve_forever(self): - """Handle one request at a time until doomsday.""" - while not self.shut: - self.handle_request() - - def start_shutdown(self, signum, frame): - '''Shutdown on unexpected signals''' - self.shut = True - - def handle_error(self): - '''Catch error path for clean exit''' - return False - - def resolve_client(self, client): - try: - return gethostbyaddr(client)[0] - except herror: - warning = "host resolution error for %s" % (client) - warning_error(warning) - raise Fault, (5, warning) - - def Bcfg2GetProbes(self, address): - '''Fetch probes for a particular client''' - client = self.resolve_client(address[0]) - resp = Element('probes') - - try: - meta = self.Core.metadata.FetchMetadata(client) - - for generator in self.Core.generators: - for probe in generator.GetProbes(meta): - resp.append(probe) - return tostring(resp) - except MetadataConsistencyError: - warning = 'metadata consistency error' - warning_error(warning) - raise Fault, (6, warning) - except: - critical_error("determining client probes") - - - def Bcfg2RecvProbeData(self, address, probedata): - '''Receive probe data from clients''' - client = self.resolve_client(address[0]) - - for data in probedata: - try: - [generator] = [gen for gen in self.Core.generators if gen.__name__ == data.get('source')] - generator.ReceiveData(client, data) - except IndexError: - warning_error("Failed to locate plugin %s" % (data.get('source'))) - except: - critical_error("probe data receipt") - return True - - def Bcfg2GetConfig(self, address, image=False, profile=False): - '''Build config for a client''' - client = self.resolve_client(address[0]) - - if image and profile: - try: - self.Core.metadata.set_group(client, profile) - except MetadataConsistencyError: - warning = 'metadata consistency error' - warning_error(warning) - raise Fault, (6, warning) - return tostring(self.Core.BuildConfiguration(client)) - - def Bcfg2RecvStats(self, address, stats): - '''Act on statistics upload''' - sdata = XML(stats) - state = sdata.find(".//Statistics") - # Versioned stats to prevent tied client/server upgrade - if state.get('version') >= '2.0': - client = self.resolve_client(address[0]) - - # Update statistics - self.Core.stats.updateStats(sdata, client) - - verbose("Client %s reported state %s" % - (client, state.attrib['state'])) - return "" - -if __name__ == '__main__': - openlog("Bcfg2", 0, LOG_LOCAL0) - options = { - 'v':'verbose', - 'd':'debug', - 'h':'help' - } - doptions = { - 'D':'daemon', - 'c':'configfile', - 'C':'client' - } - - descriptions = { - 'v': "enable verbose output", - 'd': "enable debugging output", - 'D': "daemonise the server, storing PID", - 'c': "set the server's config file", - 'C': "always return the given client's config (debug only)", - 'h': "display this usage information" - } - - argDescriptions = { - 'D': " ", - 'c': "", - 'C': "" - } - - ssetup = dgetopt(argv[1:], options, doptions, - descriptions, argDescriptions) - if ssetup['daemon']: - daemonize(ssetup['daemon']) - if not ssetup['configfile']: - ssetup['configfile'] = '/etc/bcfg2.conf' - s = Bcfg2(ssetup) - while not s.shut: - try: - s.serve_forever() - except: - critical_error("service loop") - - syslog(LOG_INFO, "Shutting down") diff --git a/src/sbin/Bcfg2debug b/src/sbin/Bcfg2debug deleted file mode 100644 index 95508a5b4..000000000 --- a/src/sbin/Bcfg2debug +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -'''This tool loads the Bcfg2 core into an interactive debugger''' -__revision__ = '$Revision$' - - -from sys import argv -from time import sleep -from Bcfg2.Server.Core import Core, CoreInitError - -def get_input(): - '''read commands from stdin''' - try: - return raw_input('> ').split(" ") - except: - return [''] - -if __name__ == '__main__': - settings = {} - if '-c' in argv: - cfile = argv[-1] - else: - cfile = '/etc/bcfg2.conf' - try: - core = Core({}, cfile) - except CoreInitError, msg: - print "Core load failed because %s" % msg - raise SystemExit, 1 - for i in range(25): - core.fam.Service() - sleep(0.5) - cmd = get_input() - while cmd != ['']: - if cmd[0] in ['exit', 'quit']: - raise SystemExit, 0 - elif cmd[0] == 'generators': - for generator in core.generators: - print generator.__version__ - elif cmd[0] == 'help': - print 'Commands:' - print 'exit - quit' - print 'generators - list current versions of generators' - print 'help - print this text' - print 'mappings - print generator mappings for optional type' - print 'set - set variable for later use' - print 'settings - display all settings' - print 'shell - shell out to native python interpreter' - print 'update - process pending fam events' - print 'version - print version of this tool' - elif cmd[0] == 'set': - settings[cmd[1]] = cmd[2] - elif cmd[0] == 'settings': - for (key, value) in settings.iteritems(): - print "%s --> %s" % (key, value) - elif cmd[0] == 'shell': - cmd = [''] - continue - elif cmd[0] == 'version': - print 'Bcfg2debug v. %s' % __revision__ - elif cmd[0] == 'mappings': - # dump all mappings unless type specified - for generator in core.generators: - print "Generator -> ", generator.__name__ - for key, value in generator.__provides__.iteritems(): - for instance in generator.__provides__[key].keys(): - print " ", key, instance - elif cmd[0] == 'update': - core.fam.Service() - else: - print "Unknown command %s" % cmd[0] - cmd = get_input() diff --git a/src/sbin/GenerateHostInfo b/src/sbin/GenerateHostInfo old mode 100644 new mode 100755 index 04d257e8a..67521d628 --- a/src/sbin/GenerateHostInfo +++ b/src/sbin/GenerateHostInfo @@ -6,42 +6,30 @@ __revision__ = '$Revision$' from ConfigParser import ConfigParser -from lxml.etree import Element, SubElement, parse -from os import fork, execl, dup2, wait +from lxml.etree import Element, SubElement, parse, tostring +from os import fork, execl, dup2, wait, uname import sys -def pretty_print(element, level=0): - '''Produce a pretty-printed text representation of element''' - if element.text: - fmt = "%s<%%s %%s>%%s" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])), - element.text, element.tag) - children = element.getchildren() - if children: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(children) * "%s") + "%s\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),) - data += tuple([pretty_print(entry, level+2) for entry in children]) + (element.tag, ) - else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])) - return fmt % data - - if __name__ == '__main__': c = ConfigParser() c.read(['/etc/bcfg2.conf']) configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') - hostinfopath = "%s/etc/hostinfo.xml" % c.get('server', 'repository') - metadatapath = "%s/etc/metadata.xml" % c.get('server', 'repository') + clientdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') sendmailpath = c.get('statistics','sendmailpath') - metaElement = parse(metadatapath) - hostlist = [client.get('name') for client in metaElement.findall("Client")] + clientElement = parse(clientdatapath) + hostlist = [client.get('name') for client in clientElement.findall("Client")] - HostInfo = Element("HostInformation") pids = {} fullnames = {} null = open('/dev/null', 'w+') + + + #use uname to detect OS and use -t for darwin and -w for linux + #/bin/ping on linux /sbin/ping on os x + osname = uname()[0] + + while hostlist or pids: if hostlist and len(pids.keys()) < 15: host = hostlist.pop() @@ -51,26 +39,39 @@ if __name__ == '__main__': dup2(null.fileno(), sys.__stdin__.fileno()) dup2(null.fileno(), sys.__stdout__.fileno()) dup2(null.fileno(), sys.__stderr__.fileno()) - execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) + if osname == 'Linux': + execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) + elif osname == 'Darwin': + execl('/sbin/ping', 'ping', '-t', '5', '-c', '1', host) + elif osname == 'SunOS': + execl('/usr/sbin/ping', 'ping', host, '56', '1') + else: #default + execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) else: pids[pid] = host else: try: (cpid, status) = wait() - chost = pids[cpid] - del pids[cpid] - if status == 0: - SubElement(HostInfo, "HostInfo", name=chost, fqdn=chost, pingable='Y') + except OSError: + continue + chost = pids[cpid] + del pids[cpid] + if status == 0: + try: + clientElement.xpath("//Client[@name='%s']"%chost)[0].set("pingable",'Y') + except:#i think this is for a problem with aliases? + clientElement.xpath("//Client[@name='%s']"%fullnames[chost])[0].set("pingable",'Y') + #also set pingtime, if you can get it + + else: + if chost.count('.') > 0: + fullnames[chost.split('.')[0]] = chost + hostlist.append(chost.split('.')[0]) else: - if chost.count('.') > 0: - fullnames[chost.split('.')[0]] = chost - hostlist.append(chost.split('.')[0]) - else: - SubElement(HostInfo, "HostInfo", name=fullnames[chost], fqdn=fullnames[chost], pingable='N') - except: - pass + clientElement.xpath("//Client[@name='%s']"%(fullnames[chost]))[0].set("pingable",'N') + #also set pingtime if you can get it - fout = open(hostinfopath, 'w') - fout.write(pretty_print(HostInfo)) + fout = open(clientdatapath, 'w') + fout.write(tostring(clientElement.getroot())) fout.close() diff --git a/src/sbin/StatReports b/src/sbin/StatReports old mode 100644 new mode 100755 index ef6a8df66..cb74eb4bb --- a/src/sbin/StatReports +++ b/src/sbin/StatReports @@ -30,8 +30,7 @@ def generatereport(rspec, nrpt): pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) for node in nodereprt.findall('Node'): - if not (node.findall("HostInfo") and node.findall("Statistics") and - node.find("HostInfo").get("fqdn") and pattern.match(node.get('name'))): + if not (node.findall("Statistics") and pattern.match(node.get('name'))): # don't know enough about node nodereprt.remove(node) continue @@ -151,8 +150,7 @@ if __name__ == '__main__': c.read(['/etc/bcfg2.conf']) configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') - hostinfopath = "%s/etc/hostinfo.xml" % c.get('server', 'repository') - metadatapath = "%s/etc/metadata.xml" % c.get('server', 'repository') + clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') transformpath = "/usr/share/bcfg2/xsl-transforms/" #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" @@ -173,12 +171,12 @@ if __name__ == '__main__': #See if hostinfo.xml exists, and is less than 23.5 hours old - try: - hostinstat = os.stat(hostinfopath) - if (time() - hostinstat[9])/(60*60) > 23.5: - os.system('GenerateHostInfo')#Generate HostInfo needs to be in path - except OSError: - os.system('GenerateHostInfo')#Generate HostInfo needs to be in path + #try: + #hostinstat = os.stat(hostinfopath) + #if (time() - hostinstat[9])/(60*60) > 23.5: + os.system('GenerateHostInfo')#Generate HostInfo needs to be in path + #except OSError: + # os.system('GenerateHostInfo')#Generate HostInfo needs to be in path '''Reads Data & Config files''' @@ -193,16 +191,10 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s"%(configpath)) raise SystemExit, 1 try: - metadata = XML(open(metadatapath).read()) + clientsdata = XML(open(clientsdatapath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(metadatapath)) + print("StatReports: Failed to parse %s"%(clientsdatapath)) raise SystemExit, 1 - try: - hostinfodata = XML(open(hostinfopath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s. Is GenerateHostInfo in your path?"%(hostinfopath)) - raise SystemExit, 1 - #Merge data from three sources nodereport = Element("Report", attrib={"time" : asctime()}) @@ -210,12 +202,9 @@ if __name__ == '__main__': #should all of the other info in Metadata be appended? #What about all of the package stuff for other types of reports? - for client in metadata.findall("Client"): + for client in clientsdata.findall("Client"): nodel = Element("Node", attrib={"name" : client.get("name")}) nodel.append(client) - for hostinfo in hostinfodata.findall("HostInfo"): - if hostinfo.get("name") == client.get("name"): - nodel.append(hostinfo) for nod in statsdata.findall("Node"): if client.get('name').find(nod.get('name')) == 0: diff --git a/src/sbin/ValidateBcfg2Repo b/src/sbin/ValidateBcfg2Repo deleted file mode 100644 index a1b9552ce..000000000 --- a/src/sbin/ValidateBcfg2Repo +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -'''ValidateBcfg2Repo checks all xml files in Bcfg2 repos against their respective XML schemas''' -__revision__ = '0.7.3' - -from glob import glob -from lxml.etree import parse, XMLSchema -from os import system -from sys import argv -from ConfigParser import ConfigParser, NoSectionError, NoOptionError - -if __name__ == '__main__': - cf = ConfigParser() - schemadir = '/usr/share/bcfg2/schemas' - cf.read(['/etc/bcfg2.conf']) - try: - repo = cf.get('server', 'repository') - except (NoSectionError, NoOptionError): - if len(argv) == 1: - print "Repository location not specified in config file or on command line" - print "Usage: validate_repo " - raise SystemExit, 1 - repo = argv[1] - - # add more validation as more schemas get written - filesets = {'metadata':("%s/etc/metadata.xml", "%s/metadata.xsd"), - 'bundle':("%s/Bundler/*.xml", "%s/bundle.xsd"), - 'pkglist':("%s/Pkgmgr/*.xml", "%s/pkglist.xsd"), - 'base':("%s/etc/base.xml", "%s/base.xsd"), - 'imageinfo':("%s/etc/imageinfo.xml", "%s/translation.xsd"), - 'imageinfo':("%s/etc/reports.xml", "%s/report-configuration.xsd"), - 'services':("%s/etc/services.xml", "%s/services.xsd")} - - for k, (spec, schemaname) in filesets.iteritems(): - schema = XMLSchema(parse(open(schemaname%(schemadir)))) - for filename in glob(spec%(repo)): - try: - datafile = parse(open(filename)) - except SyntaxError: - print "%s ***FAILS*** to parse \t\t<----" % (filename) - system("xmllint %s" % filename) - continue - except IOError: - print "Failed to open file %s \t\t<---" % (filename) - continue - if schema.validate(datafile): - if '-v' in argv: - print "%s checks out" % (filename) - else: - print "%s ***FAILS*** to verify \t\t<----" % (filename) - system("xmllint --schema %s %s" % (schemaname % schemadir, filename)) diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 old mode 100644 new mode 100755 diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info new file mode 100755 index 000000000..e325b2fd0 --- /dev/null +++ b/src/sbin/bcfg2-info @@ -0,0 +1,154 @@ +#!/usr/bin/python -i +'''This tool loads the Bcfg2 core into an interactive debugger''' +__revision__ = '$Revision$' + +from sys import argv +from time import sleep +from Bcfg2.Server.Core import Core, CoreInitError +from lxml.etree import tostring + +def print_tabular(rows): + '''print data in tabular format''' + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 for index in xrange(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print fstring % rows[0] + print (sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=' + for row in rows[1:]: + print fstring % row + +def get_input(): + '''read commands from stdin''' + try: + return raw_input('> ').split(" ") + except: + return [''] + +def do_build(cmd, core): + '''build client configuration''' + if len(cmd) == 3: + output = open(cmd[2], 'w') + output.write(tostring(core.BuildConfiguration(cmd[1]))) + output.close() + else: + print 'Usage: build ' + +def do_bundles(cmd, core): + '''print out group/bundle info''' + data = [('Group', 'Bundles')] + groups = core.metadata.groups.keys() + groups.sort() + for group in groups: + data.append((group, ','.join(core.metadata.groups[group][0]))) + print_tabular(data) + +def do_clients(cmd, core): + '''print out client info''' + data = [('Client', 'Profile')] + clist = core.metadata.clients.keys() + clist.sort() + for client in clist: + data.append((client, core.metadata.clients[client])) + print_tabular(data) + +def do_help(cmd, core): + '''print out usage info''' + print 'Commands:' + print 'build - build config for hostname, writing to filename' + print 'bundles - print out group/bundle information' + print 'clients - print out client/profile information' + print 'debug - shell out to native python interpreter' + print 'generators - list current versions of generators' + print 'groups - list groups' + print 'help - print this text' + print 'mappings - print generator mappings for optional type and name' + print 'quit' + print 'update - process pending file events' + print 'version - print version of this tool' + +def do_generators(cmd, core): + '''print out generator info''' + for generator in core.generators: + print generator.__version__ + +def do_groups(cmd, core): + '''print out group info''' + data = [("Groups", "Profile", "Category", "Contains")] + grouplist = core.metadata.groups.keys() + grouplist.sort() + for group in grouplist: + if group in core.metadata.profiles: + prof = 'yes' + else: + prof = 'no' + if core.metadata.categories.has_key(group): + cat = core.metadata.categories[group] + else: + cat = '' + gdata = [grp for grp in core.metadata.groups[group][1]] + gdata.remove(group) + data.append((group, prof, cat, ','.join(gdata))) + print_tabular(data) + +def do_mappings(cmd, core): + '''print out mapping info''' + # dump all mappings unless type specified + data = [('Plugin', 'Type', 'Name')] + for generator in core.generators: + if len(cmd) == 1: + etypes = generator.Entries.keys() + elif len(cmd) > 1: + etypes = [cmd[1]] + if len(cmd) == 3: + interested = [(etype, [cmd[2]]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype].keys()) for etype in etypes + if generator.Entries.has_key(etype)] + if [etype for (etype, names) in interested + if generator.Entries.has_key(etype) and [name for name in names + if generator.Entries[etype].has_key(name)]]: + for (etype, names) in interested: + for name in names: + if generator.Entries.has_key(etype) and generator.Entries[etype].has_key(name): + data.append((generator.__name__, etype, name)) + print_tabular(data) + +def do_quit(cmd, core): + '''exit program''' + raise SystemExit, 0 + +def do_update(cmd, core): + '''Process pending fs events''' + core.fam.Service() + +def do_version(cmd, core): + '''print out code version''' + print __revision__ + +if __name__ == '__main__': + dispatch = {'build': do_build, 'bundles': do_bundles, 'clients': do_clients, + 'generators': do_generators, 'groups': do_groups, + 'help': do_help, 'mappings': do_mappings, 'quit': do_quit, + 'update': do_update, 'version': do_version} + if '-c' in argv: + cfile = argv[-1] + else: + cfile = '/etc/bcfg2.conf' + try: + bcore = Core({}, cfile) + except CoreInitError, msg: + print "Core load failed because %s" % msg + raise SystemExit, 1 + for i in range(25): + bcore.fam.Service() + sleep(0.5) + ucmd = get_input() + while True: + if ucmd[0] == 'debug': + break + else: + if dispatch.has_key(ucmd[0]): + dispatch[ucmd[0]](ucmd, bcore) + else: + print "Unknown command %s" % ucmd[0] + ucmd = get_input() diff --git a/src/sbin/bcfg2-repo-validate b/src/sbin/bcfg2-repo-validate new file mode 100644 index 000000000..a87b31cc6 --- /dev/null +++ b/src/sbin/bcfg2-repo-validate @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +'''bcfg2-repo-validate checks all xml files in Bcfg2 repos against their respective XML schemas''' +__revision__ = '$Revision$' + +from glob import glob +from lxml.etree import parse, XMLSchema +from os import system +from sys import argv +from ConfigParser import ConfigParser, NoSectionError, NoOptionError + +if __name__ == '__main__': + cf = ConfigParser() + schemadir = '/usr/share/bcfg2/schemas' + if len(argv) > 1: + repo = argv[1] + else: + cf.read(['/etc/bcfg2.conf']) + try: + repo = cf.get('server', 'repository') + except (NoSectionError, NoOptionError): + print "Repository location not specified in config file or on command line" + print "Usage: bcfg2-repo-validate " + raise SystemExit, 1 + + # add more validation as more schemas get written + filesets = {'metadata':("%s/Metadata/groups.xml", "%s/metadata.xsd"), + 'clients':("%s/Metadata/clients.xml", "%s/clients.xsd"), + 'bundle':("%s/Bundler/*.xml", "%s/bundle.xsd"), + 'pkglist':("%s/Pkgmgr/*.xml", "%s/pkglist.xsd"), + 'base':("%s/Base/*.xml", "%s/base.xsd"), + 'imageinfo':("%s/etc/reports.xml", "%s/report-configuration.xsd"), + 'services':("%s/Svcmgr/*.xml", "%s/services.xsd")} + + for k, (spec, schemaname) in filesets.iteritems(): + try: + schema = XMLSchema(parse(open(schemaname%(schemadir)))) + except: + print "Failed to process schema %s" % (schemaname%(schemadir)) + continue + for filename in glob(spec%(repo)): + try: + datafile = parse(open(filename)) + except SyntaxError: + print "%s ***FAILS*** to parse \t\t<----" % (filename) + system("xmllint %s" % filename) + continue + except IOError: + print "Failed to open file %s \t\t<---" % (filename) + continue + if schema.validate(datafile): + if '-v' in argv: + print "%s checks out" % (filename) + else: + print "%s ***FAILS*** to verify \t\t<----" % (filename) + system("xmllint --schema %s %s" % (schemaname % schemadir, filename)) diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server new file mode 100755 index 000000000..2da6ff325 --- /dev/null +++ b/src/sbin/bcfg2-server @@ -0,0 +1,291 @@ +#!/usr/bin/env python + +'''The XML-RPC Bcfg2 Server''' +__revision__ = '$Revision$' + +from getopt import getopt, GetoptError +from sys import argv, exc_info +from syslog import openlog, LOG_LOCAL0, syslog, LOG_INFO, LOG_ERR +from Bcfg2.Server.Core import Core, CoreInitError +from Bcfg2.Server.Metadata import MetadataConsistencyError +from Bcfg2.Server.Component import Component +from threading import Lock +from select import select, error as selecterror +from signal import signal, SIGINT, SIGTERM +from traceback import extract_tb +from xmlrpclib import Fault +from socket import gethostbyaddr, herror +from lxml.etree import XML, Element, tostring +from M2Crypto.SSL import SSLError + +import os, sys + +def daemonize(filename): + '''Do the double fork/setsession dance''' + # Fork once + if os.fork() != 0: + os._exit(0) + os.setsid() # Create new session + pid = os.fork() + if pid != 0: + pidfile = open(filename, "w") + pidfile.write("%i" % pid) + pidfile.close() + os._exit(0) + os.chdir("/") + os.umask(0) + + null = open("/dev/null", "w+") + + os.dup2(null.fileno(), sys.__stdin__.fileno()) + os.dup2(null.fileno(), sys.__stdout__.fileno()) + os.dup2(null.fileno(), sys.__stderr__.fileno()) + + +def critical_error(operation): + '''Print tracebacks in unexpected cases''' + syslog(LOG_ERR, "Traceback information (please include in any bug report):") + (ttype, value, trace) = exc_info() + for line in extract_tb(trace): + syslog(LOG_ERR, "File %s, line %i, in %s\n %s" % (line)) + syslog(LOG_ERR, "%s: %s" % (ttype, value)) + warning_error("An unexpected failure occurred in %s" % (operation) ) + raise Fault, (7, "Critical unexpected failure: %s" % (operation)) + +def fatal_error(message): + '''Signal a fatal error''' + syslog(LOG_ERR, "Fatal error: %s" % (message)) + raise SystemExit, 1 + +def warning_error(message): + '''Warn about a problem but continue''' + syslog(LOG_ERR,"Warning: %s\n" % (message)) + +def usage_error(message, opt, vopt, descs, argDescs): + '''Die because script was called the wrong way''' + print "Usage error: %s" % (message) + print_usage(opt, vopt, descs, argDescs) + raise SystemExit, 2 + +verboseMode = False + +def verbose(message): + '''Conditionally output information in verbose mode''' + global verboseMode + + if(verboseMode == True): + syslog(LOG_INFO, "%s" % (message)) + +def print_usage(opt, vopt, descs, argDescs): + print "bcfg2-server usage:" + for arg in opt.iteritems(): + print " -%s\t\t\t%s" % (arg[0], descs[arg[0]]) + for arg in vopt.iteritems(): + print " -%s %s\t%s" % (arg[0], argDescs[arg[0]], descs[arg[0]]) + +def dgetopt(arglist, opt, vopt, descs, argDescs): + '''parse options into a dictionary''' + global verboseMode + + ret = {} + for optname in opt.values() + vopt.values(): + ret[optname] = False + + gstr = "".join(opt.keys()) + "".join([optionkey + ':' for optionkey in vopt.keys()]) + try: + ginfo = getopt(arglist, gstr) + except GetoptError, gerr: + usage_error(gerr, opt, vopt, descs, argDescs) + + for (gopt, garg) in ginfo[0]: + option = gopt[1:] + if opt.has_key(option): + ret[opt[option]] = True + else: + ret[vopt[option]] = garg + + if ret["help"] == True: + print_usage(opt, vopt, descs, argDescs) + raise SystemExit, 0 + + if ret["verbose"] == True: + verboseMode = True + + return ret + +class Bcfg2(Component): + """The Bcfg2 Server component providing XML-RPC access to Bcfg methods""" + __name__ = 'bcfg2' + __implementation__ = 'bcfg2' + + request_queue_size = 15 + + def __init__(self, setup): + Component.__init__(self, setup) + self.shut = False + # set shutdown handlers for sigint and sigterm + signal(SIGINT, self.start_shutdown) + signal(SIGTERM, self.start_shutdown) + try: + self.Core = Core(setup, setup['configfile']) + self.CoreLock = Lock() + except CoreInitError, msg: + fatal_error(msg) + + self.funcs.update({ + "GetConfig": self.Bcfg2GetConfig, + "GetProbes": self.Bcfg2GetProbes, + "RecvProbeData": self.Bcfg2RecvProbeData, + "RecvStats": self.Bcfg2RecvStats + }) + for plugin in self.Core.plugins.values(): + for method in plugin.__rmi__: + self.register_function(getattr(self.Core.plugins[plugin.__name__], method), + "%s.%s" % (plugin.__name__, method)) + + def get_request(self): + '''We need to do work between requests, so select with timeout instead of blocking in accept''' + rsockinfo = [] + famfd = self.Core.fam.fileno() + while self.socket not in rsockinfo: + if self.shut: + raise SSLError + try: + rsockinfo = select([self.socket, famfd], [], [], 15)[0] + except selecterror: + raise SSLError + + if famfd in rsockinfo: + self.Core.fam.Service() + if self.socket in rsockinfo: + # workaround for m2crypto 0.15 bug + self.socket.postConnectionCheck = None + return self.socket.accept() + + def serve_forever(self): + """Handle one request at a time until doomsday.""" + while not self.shut: + self.handle_request() + + def start_shutdown(self, signum, frame): + '''Shutdown on unexpected signals''' + self.shut = True + + def handle_error(self): + '''Catch error path for clean exit''' + return False + + def resolve_client(self, client): + if self.setup['client']: + return self.setup['client'] + try: + return gethostbyaddr(client)[0] + except herror: + warning = "host resolution error for %s" % (client) + warning_error(warning) + raise Fault, (5, warning) + + def Bcfg2GetProbes(self, address): + '''Fetch probes for a particular client''' + client = self.resolve_client(address[0]) + resp = Element('probes') + + try: + meta = self.Core.metadata.get_metadata(client) + + for generator in self.Core.generators: + for probe in generator.GetProbes(meta): + resp.append(probe) + return tostring(resp) + except MetadataConsistencyError: + warning = 'metadata consistency error' + warning_error(warning) + raise Fault, (6, warning) + except: + critical_error("determining client probes") + + + def Bcfg2RecvProbeData(self, address, probedata): + '''Receive probe data from clients''' + client = self.resolve_client(address[0]) + + for data in probedata: + try: + [generator] = [gen for gen in self.Core.generators if gen.__name__ == data.get('source')] + generator.ReceiveData(client, data) + except IndexError: + warning_error("Failed to locate plugin %s" % (data.get('source'))) + except: + critical_error("probe data receipt") + return True + + def Bcfg2GetConfig(self, address, image=False, profile=False): + '''Build config for a client''' + client = self.resolve_client(address[0]) + + if image and profile: + try: + self.Core.metadata.set_group(client, profile) + except MetadataConsistencyError: + warning = 'metadata consistency error' + warning_error(warning) + raise Fault, (6, warning) + return tostring(self.Core.BuildConfiguration(client)) + + def Bcfg2RecvStats(self, address, stats): + '''Act on statistics upload''' + sdata = XML(stats) + state = sdata.find(".//Statistics") + # Versioned stats to prevent tied client/server upgrade + if state.get('version') >= '2.0': + client = self.resolve_client(address[0]) + + # Update statistics + self.Core.stats.updateStats(sdata, client) + + syslog(LOG_INFO, "Client %s reported state %s" % + (client, state.attrib['state'])) + return "" + +if __name__ == '__main__': + openlog("Bcfg2", 0, LOG_LOCAL0) + options = { + 'v':'verbose', + 'd':'debug', + 'h':'help' + } + doptions = { + 'D':'daemon', + 'c':'configfile', + 'C':'client' + } + + descriptions = { + 'v': "enable verbose output", + 'd': "enable debugging output", + 'D': "daemonise the server, storing PID", + 'c': "set the server's config file", + 'C': "always return the given client's config (debug only)", + 'h': "display this usage information" + } + + argDescriptions = { + 'D': " ", + 'c': "", + 'C': "" + } + + ssetup = dgetopt(argv[1:], options, doptions, + descriptions, argDescriptions) + if ssetup['daemon']: + daemonize(ssetup['daemon']) + if not ssetup['configfile']: + ssetup['configfile'] = '/etc/bcfg2.conf' + s = Bcfg2(ssetup) + while not s.shut: + try: + s.serve_forever() + except: + critical_error("service loop") + + syslog(LOG_INFO, "Shutting down") -- cgit v1.2.3-1-g7c22