diff options
Diffstat (limited to 'src/lib/Server/Plugins')
49 files changed, 0 insertions, 7858 deletions
diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Server/Plugins/Account.py deleted file mode 100644 index f2703dccb..000000000 --- a/src/lib/Server/Plugins/Account.py +++ /dev/null @@ -1,101 +0,0 @@ -"""This handles authentication setup.""" - -import Bcfg2.Server.Plugin - - -class Account(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """This module generates account config files, - based on an internal data repo: - static.(passwd|group|limits.conf) -> static entries - dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch) - useraccess -> users to be granted login access on some hosts - superusers -> users to be granted root privs on all hosts - rootlike -> users to be granted root privs on some hosts - - """ - name = 'Account' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb, - '/etc/group': self.from_yp_cb, - '/etc/security/limits.conf': self.gen_limits_cb, - '/root/.ssh/authorized_keys': self.gen_root_keys_cb, - '/etc/sudoers': self.gen_sudoers}} - try: - self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, - self.core.fam) - except: - self.logger.error("Failed to load repos: %s, %s" % \ - (self.data, "%s/ssh" % (self.data))) - raise Bcfg2.Server.Plugin.PluginInitError - - def from_yp_cb(self, entry, metadata): - """Build password file from cached yp data.""" - fname = entry.attrib['name'].split('/')[-1] - entry.text = self.repository.entries["static.%s" % (fname)].data - entry.text += self.repository.entries["dyn.%s" % (fname)].data - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0644'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - - def gen_limits_cb(self, entry, metadata): - """Build limits entries based on current ACLs.""" - entry.text = self.repository.entries["static.limits.conf"].data - superusers = self.repository.entries["superusers"].data.split() - useraccess = [line.split(':') for line in \ - self.repository.entries["useraccess"].data.split()] - users = [user for (user, host) in \ - useraccess if host == metadata.hostname.split('.')[0]] - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users]) - if "*" not in users: - entry.text += "* hard maxlogins 0\n" - - def gen_root_keys_cb(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - rdata = self.repository.entries - entry.text = "".join([rdata["%s.key" % user].data for user \ - in superusers if \ - ("%s.key" % user) in rdata]) - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] - - def gen_sudoers(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - entry.text = self.repository.entries['static.sudoers'].data - entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \ - for uname in superusers]) - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0440'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] diff --git a/src/lib/Server/Plugins/BB.py b/src/lib/Server/Plugins/BB.py deleted file mode 100644 index c015ec47c..000000000 --- a/src/lib/Server/Plugins/BB.py +++ /dev/null @@ -1,83 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Plugin -import glob -import os -import socket - -#manage boot symlinks - #add statistics check to do build->boot mods - -#map profiles: first array is not empty we replace the -p with a determined profile. -logger = Bcfg2.Server.Plugin.logger - -class BBfile(Bcfg2.Server.Plugin.XMLFileBacked): - """Class for bb files.""" - def Index(self): - """Build data into an xml object.""" - - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % self.name) - return - self.tftppath = self.data.get('tftp', '/tftpboot') - self.macs = {} - self.users = {} - self.actions = {} - self.bootlinks = [] - - for node in self.data.findall('Node'): - iface = node.find('Interface') - if iface != None: - mac = "01-%s" % (iface.get('mac'.replace(':','-').lower())) - self.actions[node.get('name')] = node.get('action') - self.bootlinks.append((mac, node.get('action'))) - try: - ip = socket.gethostbyname(node.get('name')) - except: - logger.error("failed host resolution for %s" % node.get('name')) - - self.macs[node.get('name')] = (iface.get('mac'), ip) - else: - logger.error("%s" % lxml.etree.tostring(node)) - self.users[node.get('name')] = node.get('user',"").split(':') - - def enforce_bootlinks(self): - for mac, target in self.bootlinks: - path = self.tftppath + '/' + mac - if not os.path.islink(path): - logger.error("Boot file %s not a link" % path) - if target != os.readlink(path): - try: - os.unlink(path) - os.symlink(target, path) - except: - logger.error("Failed to modify link %s" % path) - -class BBDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - __child__ = BBfile - - -class BB(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """The BB plugin maps users to machines and metadata to machines.""" - name = 'BB' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.store = BBDirectoryBacked(self.data, core.fam) - - def get_additional_data(self, metadata): - - users = {} - for user in self.store.entries['bb.xml'].users.get(metadata.hostname.split(".")[0], []): - pubkeys = [] - for fname in glob.glob('/home/%s/.ssh/*.pub'%user): - pubkeys.append(open(fname).read()) - - users[user] = pubkeys - - return dict([('users', users), - ('macs', self.store.entries['bb.xml'].macs)]) diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py deleted file mode 100644 index 389ca7a95..000000000 --- a/src/lib/Server/Plugins/Base.py +++ /dev/null @@ -1,43 +0,0 @@ -"""This module sets up a base list of configuration entries.""" - -import copy -import lxml.etree -import sys -# py3k compatibility -if sys.hexversion >= 0x03000000: - from functools import reduce - -import Bcfg2.Server.Plugin - - -class Base(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """This Structure is good for the pile of independent configs - needed for most actual systems. - """ - name = 'Base' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = Bcfg2.Server.Plugin.StructFile - deprecated = True - - """Base creates independent clauses based on client metadata.""" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Base repository") - raise Bcfg2.Server.Plugin.PluginInitError - - def BuildStructures(self, metadata): - """Build structures for client described by metadata.""" - ret = lxml.etree.Element("Independent", version='2.0') - fragments = reduce(lambda x, y: x + y, - [base.Match(metadata) for base - in list(self.entries.values())], []) - [ret.append(copy.copy(frag)) for frag in fragments] - return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py deleted file mode 100644 index ccb99481e..000000000 --- a/src/lib/Server/Plugins/Bundler.py +++ /dev/null @@ -1,99 +0,0 @@ -"""This provides bundle clauses with translation functionality.""" - -import copy -import lxml.etree -import os -import os.path -import re -import sys - -import Bcfg2.Server.Plugin - -try: - import genshi.template - import genshi.template.base - import Bcfg2.Server.Plugins.SGenshi - have_genshi = True -except: - have_genshi = False - - -class BundleFile(Bcfg2.Server.Plugin.StructFile): - - def get_xml_value(self, metadata): - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - [bundle.append(copy.copy(item)) for item in self.Match(metadata)] - return bundle - - -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """The bundler creates dependent clauses based on the - bundle/translation scheme from Bcfg1. - """ - name = 'Bundler' - __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$') - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.encoding - self.__child__ = self.template_dispatch - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Bundle repository") - raise Bcfg2.Server.Plugin.PluginInitError - - def template_dispatch(self, name): - bundle = lxml.etree.parse(name) - nsmap = bundle.getroot().nsmap - if name.endswith('.xml'): - if have_genshi and \ - (nsmap == {'py': 'http://genshi.edgewall.org/'}): - # allow for genshi bundles with .xml extensions - spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) - else: - return BundleFile(name) - elif name.endswith('.genshi'): - if have_genshi: - spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) - - def BuildStructures(self, metadata): - """Build all structures for client (metadata).""" - bundleset = [] - - bundle_entries = {} - for key, item in self.entries.items(): - bundle_entries.setdefault(self.patterns.match(os.path.basename(key)).group('name'), - []).append(item) - - for bundlename in metadata.bundles: - try: - entries = bundle_entries[bundlename] - except KeyError: - self.logger.error("Bundler: Bundle %s does not exist" % - bundlename) - continue - try: - bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: - t = sys.exc_info()[1] - self.logger.error("Bundler: Failed to template genshi bundle %s" - % bundlename) - self.logger.error(t) - except: - self.logger.error("Bundler: Unexpected bundler error for %s" % - bundlename, exc_info=1) - return bundleset diff --git a/src/lib/Server/Plugins/Bzr.py b/src/lib/Server/Plugins/Bzr.py deleted file mode 100644 index a71021cb5..000000000 --- a/src/lib/Server/Plugins/Bzr.py +++ /dev/null @@ -1,35 +0,0 @@ -import Bcfg2.Server.Plugin -from bzrlib.workingtree import WorkingTree -from bzrlib import errors - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Bzr') - -class Bzr(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Bzr is a version plugin for dealing with Bcfg2 repos.""" - name = 'Bzr' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # Read revision from bcfg2 repo - revision = self.get_revision() - - logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision}) - - def get_revision(self): - """Read Bazaar revision information for the Bcfg2 repository.""" - try: - working_tree = WorkingTree.open(self.datastore) - revision = str(working_tree.branch.revno()) - if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns(): - revision += "+" - except errors.NotBranchError: - logger.error("Failed to read Bazaar branch; disabling Bazaar support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py deleted file mode 100644 index c3e807d7a..000000000 --- a/src/lib/Server/Plugins/Cfg.py +++ /dev/null @@ -1,283 +0,0 @@ -"""This module implements a config file repository.""" - -import binascii -import logging -import lxml -import operator -import os -import os.path -import re -import stat -import sys -import tempfile -from subprocess import Popen, PIPE -from Bcfg2.Bcfg2Py3k import u_str - -import Bcfg2.Server.Plugin - -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, NewTextTemplate - have_genshi = True -except: - have_genshi = False - -try: - import Cheetah.Template - import Cheetah.Parser - have_cheetah = True -except: - have_cheetah = False - -# setup logging -logger = logging.getLogger('Bcfg2.Plugins.Cfg') - - -# snipped from TGenshi -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -def process_delta(data, delta): - if not delta.specific.delta: - return data - if delta.specific.delta == 'cat': - datalines = data.strip().split('\n') - for line in delta.data.split('\n'): - if not line: - continue - if line[0] == '+': - datalines.append(line[1:]) - elif line[0] == '-': - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - elif delta.specific.delta == 'diff': - basehandle, basename = tempfile.mkstemp() - basefile = open(basename, 'w') - basefile.write(data) - basefile.close() - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basefile.name] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=delta.data)[1] - ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) - if ret >> 8 != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) - return output - - -class CfgMatcher: - - def __init__(self, fname): - name = re.escape(fname) - self.basefile_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))((?P<genshi>\\.genshi)|(?P<cheetah>\\.cheetah))?$' % name) - self.delta_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % name) - self.cat_count = fname.count(".cat") - self.diff_count = fname.count(".diff") - - def match(self, fname): - if fname.count(".cat") > self.cat_count \ - or fname.count('.diff') > self.diff_count: - return self.delta_reg.match(fname) - return self.basefile_reg.match(fname) - - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - self.specific = CfgMatcher(path.split('/')[-1]) - path = path - - def debug_log(self, message, flag=None): - if (flag is None and self.debug_flag) or flag: - logger.error(message) - - def sort_by_specific(self, one, other): - return cmp(one.specific, other.specific) - - def get_pertinent_entries(self, entry, metadata): - """return a list of all entries pertinent - to a client => [base, delta1, delta2] - """ - matching = [ent for ent in list(self.entries.values()) if \ - ent.specific.matches(metadata)] - matching.sort(key=operator.attrgetter('specific')) - # base entries which apply to a client - # (e.g. foo, foo.G##_groupname, foo.H_hostname) - base_files = [matching.index(m) for m in matching - if not m.specific.delta] - if not base_files: - logger.error("No base file found for %s" % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - base = min(base_files) - used = matching[:base + 1] - used.reverse() - return used - - def bind_entry(self, entry, metadata): - self.bind_info_to_entry(entry, metadata) - used = self.get_pertinent_entries(entry, metadata) - basefile = used.pop(0) - if entry.get('perms').lower() == 'inherit': - # use on-disk permissions - fname = "%s/%s" % (self.path, entry.get('name')) - entry.set('perms', - str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) - if entry.tag == 'Path': - entry.set('type', 'file') - if basefile.name.endswith(".genshi"): - if not have_genshi: - logger.error("Cfg: Genshi is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - template_cls = NewTextTemplate - loader = TemplateLoader() - template = loader.load(basefile.name, cls=template_cls, - encoding=self.encoding) - fname = entry.get('realname', entry.get('name')) - stream = template.generate(name=fname, - metadata=metadata, - path=basefile.name).filter(removecomment) - try: - data = stream.render('text', encoding=self.encoding, - strip_whitespace=False) - except TypeError: - data = stream.render('text', encoding=self.encoding) - if data == '': - entry.set('empty', 'true') - except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: genshi exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError - elif basefile.name.endswith(".cheetah"): - if not have_cheetah: - logger.error("Cfg: Cheetah is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - fname = entry.get('realname', entry.get('name')) - s = {'useStackFrames': False} - template = Cheetah.Template.Template(open(basefile.name).read(), - compilerSettings=s) - template.metadata = metadata - template.path = fname - template.source_path = basefile.name - data = template.respond() - if data == '': - entry.set('empty', 'true') - except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: cheetah exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError - else: - data = basefile.data - for delta in used: - data = process_delta(data, delta) - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(data) - else: - try: - entry.text = u_str(data, self.encoding) - except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError - except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % entry.get('name')) - logger.error("%s" % e) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - if entry.text in ['', None]: - entry.set('empty', 'true') - - def list_accept_choices(self, entry, metadata): - '''return a list of candidate pull locations''' - used = self.get_pertinent_entries(entry, metadata) - ret = [] - if used: - ret.append(used[0].specific) - if not ret[0].hostname: - ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) - return ret - - def build_filename(self, specific): - bfname = self.path + '/' + self.path.split('/')[-1] - if specific.all: - return bfname - elif specific.group: - return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) - elif specific.hostname: - return "%s.H_%s" % (bfname, specific.hostname) - - def write_update(self, specific, new_entry, log): - if 'text' in new_entry: - name = self.build_filename(specific) - if os.path.exists("%s.genshi" % name): - logger.error("Cfg: Unable to pull data for genshi types") - raise Bcfg2.Server.Plugin.PluginExecutionError - elif os.path.exists("%s.cheetah" % name): - logger.error("Cfg: Unable to pull data for cheetah types") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - etext = new_entry['text'].encode(self.encoding) - except: - logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) - raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(etext) - self.debug_log("Wrote file %s" % name, flag=log) - badattr = [attr for attr in ['owner', 'group', 'perms'] - if attr in new_entry] - if badattr: - # check for info files and inform user of their removal - if os.path.exists(self.path + "/:info"): - logger.info("Removing :info file and replacing with " - "info.xml") - os.remove(self.path + "/:info") - if os.path.exists(self.path + "/info"): - logger.info("Removing info file and replacing with " - "info.xml") - os.remove(self.path + "/info") - metadata_updates = {} - metadata_updates.update(self.metadata) - for attr in badattr: - metadata_updates[attr] = new_entry.get(attr) - infoxml = lxml.etree.Element('FileInfo') - infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ - for attr in metadata_updates] - ofile = open(self.path + "/info.xml", "w") - ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) - ofile.close() - self.debug_log("Wrote file %s" % (self.path + "/info.xml"), - flag=log) - - -class Cfg(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.PullTarget): - """This generator in the configuration file repository for Bcfg2.""" - name = 'Cfg' - __author__ = 'bcfg-dev@mcs.anl.gov' - es_cls = CfgEntrySet - es_child_cls = Bcfg2.Server.Plugin.SpecificData - - def AcceptChoices(self, entry, metadata): - return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - - def AcceptPullData(self, specific, new_entry, log): - return self.entries[new_entry.get('name')].write_update(specific, - new_entry, - log) diff --git a/src/lib/Server/Plugins/Cvs.py b/src/lib/Server/Plugins/Cvs.py deleted file mode 100644 index 6ce72acd2..000000000 --- a/src/lib/Server/Plugins/Cvs.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Cvs') - -class Cvs(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """CVS is a version plugin for dealing with Bcfg2 repository.""" - name = 'Cvs' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to cvs directory for Bcfg2 repo - cvs_dir = "%s/CVSROOT" % datastore - - # Read revision from Bcfg2 repo - if os.path.isdir(cvs_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % cvs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir) - - def get_revision(self): - """Read cvs revision information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revision = data[3].strip('\n') - except IndexError: - logger.error("Failed to read cvs log; disabling cvs support") - logger.error('''Ran command "cvs log %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py deleted file mode 100644 index 95395f74e..000000000 --- a/src/lib/Server/Plugins/DBStats.py +++ /dev/null @@ -1,113 +0,0 @@ -import binascii -import difflib -import logging -import lxml.etree -import platform -import time - -try: - from django.core.exceptions import MultipleObjectsReturned -except ImportError: - pass - -import Bcfg2.Server.Plugin -import Bcfg2.Server.Reports.importscript -from Bcfg2.Server.Reports.reports.models import Client -import Bcfg2.Server.Reports.settings -from Bcfg2.Server.Reports.updatefix import update_database -# for debugging output only -logger = logging.getLogger('Bcfg2.Plugins.DBStats') - -class DBStats(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.ThreadedStatistics, - Bcfg2.Server.Plugin.PullSource): - name = 'DBStats' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) - Bcfg2.Server.Plugin.PullSource.__init__(self) - self.cpath = "%s/Metadata/clients.xml" % datastore - self.core = core - logger.debug("Searching for new models to add to the statistics database") - try: - update_database() - except Exception: - inst = sys.exc_info()[1] - logger.debug(str(inst)) - logger.debug(str(type(inst))) - - def handle_statistic(self, metadata, data): - newstats = data.find("Statistics") - newstats.set('time', time.asctime(time.localtime())) - # ick - data = lxml.etree.tostring(newstats) - ndx = lxml.etree.XML(data) - e = lxml.etree.Element('Node', name=metadata.hostname) - e.append(ndx) - container = lxml.etree.Element("ConfigStatistics") - container.append(e) - - # FIXME need to build a metadata interface to expose a list of clients - start = time.time() - for i in [1, 2, 3]: - try: - Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, - container, - self.core.encoding, - 0, - logger, - True, - platform.node()) - logger.info("Imported data for %s in %s seconds" \ - % (metadata.hostname, time.time() - start)) - return - except MultipleObjectsReturned: - e = sys.exc_info()[1] - logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \ - (metadata.hostname, e)) - logger.error("DBStats: Data is inconsistent") - break - except: - logger.error("DBStats: Failed to write to db (lock); retrying", - exc_info=1) - logger.error("DBStats: Retry limit failed for %s; aborting operation" \ - % metadata.hostname) - - def GetExtra(self, client): - c_inst = Client.objects.filter(name=client)[0] - return [(a.entry.kind, a.entry.name) for a in - c_inst.current_interaction.extra()] - - def GetCurrentEntry(self, client, e_type, e_name): - try: - c_inst = Client.objects.filter(name=client)[0] - except IndexError: - self.logger.error("Unknown client: %s" % client) - raise Bcfg2.Server.Plugin.PluginExecutionError - result = c_inst.current_interaction.bad().filter(entry__kind=e_type, - entry__name=e_name) - if not result: - raise Bcfg2.Server.Plugin.PluginExecutionError - entry = result[0] - ret = [] - data = ('owner', 'group', 'perms') - for t in data: - if getattr(entry.reason, "current_%s" % t) == '': - ret.append(getattr(entry.reason, t)) - else: - ret.append(getattr(entry.reason, "current_%s" % t)) - if entry.reason.is_sensitive: - raise Bcfg2.Server.Plugin.PluginExecutionError - elif entry.reason.current_diff != '': - if entry.reason.is_binary: - ret.append(binascii.a2b_base64(entry.reason.current_diff)) - else: - ret.append('\n'.join(difflib.restore(\ - entry.reason.current_diff.split('\n'), 1))) - elif entry.reason.is_binary: - # If len is zero the object was too large to store - raise Bcfg2.Server.Plugin.PluginExecutionError - else: - ret.append(None) - return ret diff --git a/src/lib/Server/Plugins/Darcs.py b/src/lib/Server/Plugins/Darcs.py deleted file mode 100644 index 9fb9ff4f1..000000000 --- a/src/lib/Server/Plugins/Darcs.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Darcs') - -class Darcs(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Darcs is a version plugin for dealing with Bcfg2 repos.""" - name = 'Darcs' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to darcs directory for bcfg2 repo - darcs_dir = "%s/_darcs" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(darcs_dir): - self.get_revision() - else: - logger.error("%s is not present." % darcs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir) - - def get_revision(self): - """Read Darcs changeset information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revision = data[0].strip('\n') - except: - logger.error("Failed to read darcs repository; disabling Darcs support") - logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision - diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Server/Plugins/Decisions.py deleted file mode 100644 index b432474f2..000000000 --- a/src/lib/Server/Plugins/Decisions.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -import lxml.etree -import sys - -import Bcfg2.Server.Plugin -logger = logging.getLogger('Bcfg2.Plugins.Decisions') - -class DecisionFile(Bcfg2.Server.Plugin.SpecificData): - def handle_event(self, event): - Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) - self.contents = lxml.etree.XML(self.data) - - def get_decisions(self): - return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')] - -class DecisionSet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, path, fam, encoding): - """Container for decision specification files. - - Arguments: - - `path`: repository path - - `fam`: reference to the file monitor - - `encoding`: XML character encoding - - """ - pattern = '(white|black)list' - Bcfg2.Server.Plugin.EntrySet.__init__(self, pattern, path, \ - DecisionFile, encoding) - try: - fam.AddMonitor(path, self) - except OSError: - e = sys.exc_info()[1] - logger.error('Adding filemonitor for %s failed. ' - 'Make sure directory exists' % path) - raise Bcfg2.Server.Plugin.PluginInitError(e) - - def HandleEvent(self, event): - if event.filename != self.path: - return self.handle_event(event) - - def GetDecisions(self, metadata, mode): - ret = [] - candidates = [c for c in self.get_matching(metadata) - if c.name.split('/')[-1].startswith(mode)] - for c in candidates: - ret += c.get_decisions() - return ret - -class Decisions(DecisionSet, - Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Decision): - name = 'Decisions' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - """Decisions plugins - - Arguments: - - `core`: Bcfg2.Core instance - - `datastore`: File repository location - - """ - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Decision.__init__(self) - DecisionSet.__init__(self, self.data, core.fam, core.encoding) - diff --git a/src/lib/Server/Plugins/Defaults.py b/src/lib/Server/Plugins/Defaults.py deleted file mode 100644 index 718192e2a..000000000 --- a/src/lib/Server/Plugins/Defaults.py +++ /dev/null @@ -1,53 +0,0 @@ -"""This generator provides rule-based entry mappings.""" - -import re -import Bcfg2.Server.Plugin -import Bcfg2.Server.Plugins.Rules - -class Defaults(Bcfg2.Server.Plugins.Rules.Rules, - Bcfg2.Server.Plugin.StructureValidator): - """Set default attributes on bound entries""" - name = 'Defaults' - __author__ = 'bcfg-dev@mcs.anl.gov' - - # Rules is a Generator that happens to implement all of the - # functionality we want, so we overload it, but Defaults should - # _not_ handle any entries; it does its stuff in the structure - # validation phase. so we overload Handle(s)Entry and HandleEvent - # to ensure that Defaults handles no entries, even though it's a - # Generator. - - def HandlesEntry(self, entry, metadata): - return False - - def HandleEntry(self, entry, metadata): - raise PluginExecutionError - - def HandleEvent(self, event): - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - - def validate_structures(self, metadata, structures): - """ Apply defaults """ - for struct in structures: - for entry in struct.iter(): - if entry.tag.startswith("Bound"): - is_bound = True - entry.tag = entry.tag[5:] - else: - is_bound = False - try: - try: - self.BindEntry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - # either no matching defaults (which is okay), - # or multiple matching defaults (which is not - # okay, but is logged). either way, we don't - # care about the error. - pass - finally: - if is_bound: - entry.tag = "Bound" + entry.tag - - def _regex_enabled(self): - """ Defaults depends on regex matching, so force it enabled """ - return True diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py deleted file mode 100644 index 9b848baae..000000000 --- a/src/lib/Server/Plugins/Deps.py +++ /dev/null @@ -1,134 +0,0 @@ -"""This plugin provides automatic dependency handling.""" - -import lxml.etree - -import Bcfg2.Server.Plugin - - -class DNode(Bcfg2.Server.Plugin.INode): - """DNode provides supports for single predicate types for dependencies.""" - raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"} - containers = ['Group'] - - def __init__(self, data, idict, parent=None): - self.data = data - self.contents = {} - if parent == None: - self.predicate = lambda x, d: True - else: - predicate = parent.predicate - if data.tag in list(self.raw.keys()): - self.predicate = eval(self.raw[data.tag] % - {'name': data.get('name')}, - {'predicate': predicate}) - else: - raise Exception - mytype = self.__class__ - self.children = [] - for item in data.getchildren(): - if item.tag in self.containers: - self.children.append(mytype(item, idict, self)) - else: - data = [(child.tag, child.get('name')) - for child in item.getchildren()] - try: - self.contents[item.tag][item.get('name')] = data - except KeyError: - self.contents[item.tag] = {item.get('name'): data} - - -class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): - __node__ = DNode - - -class Deps(Bcfg2.Server.Plugin.PrioDir, - Bcfg2.Server.Plugin.StructureValidator): - name = 'Deps' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = DepXMLSrc - - # Override the default sort_order (of 500) so that this plugin - # gets handled after others running at the default. In particular, - # we want to run after Packages, so we can see the final set of - # packages that will be installed on the client. - sort_order = 750 - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) - Bcfg2.Server.Plugin.StructureValidator.__init__(self) - self.cache = {} - - def HandleEvent(self, event): - self.cache = {} - Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event) - - def validate_structures(self, metadata, structures): - """Examine the passed structures and append any additional - prerequisite entries as defined by the files in Deps. - """ - entries = [] - for structure in structures: - for entry in structure.getchildren(): - tag = entry.tag - if tag.startswith('Bound'): - tag = tag[5:] - if (tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): - entries.append((tag, entry.get('name'))) - entries.sort() - entries = tuple(entries) - gdata = list(metadata.groups) - gdata.sort() - gdata = tuple(gdata) - - # Check to see if we have cached the prereqs already - if (entries, gdata) in self.cache: - prereqs = self.cache[(entries, gdata)] - else: - prereqs = self.calculate_prereqs(metadata, entries) - self.cache[(entries, gdata)] = prereqs - - newstruct = lxml.etree.Element("Independent") - for tag, name in prereqs: - try: - lxml.etree.SubElement(newstruct, tag, name=name) - except: - self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) - structures.append(newstruct) - - - def calculate_prereqs(self, metadata, entries): - """Calculate the prerequisites defined in Deps for the passed - set of entries. - """ - prereqs = [] - [src.Cache(metadata) for src in self.entries.values()] - - toexamine = list(entries[:]) - while toexamine: - entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] - if len(matching) > 1: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError - index = prio.index(max(prio)) - matching = [matching[index]] - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - # XML comments seem to show up in the cache as a - # tuple with item 0 being callable. The logic - # below filters them out. Would be better to - # exclude them when we load the cache in the first - # place. - if prq not in prereqs and prq not in entries and not callable(prq[0]): - toexamine.append(prq) - prereqs.append(prq) - else: - continue - - return prereqs diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Server/Plugins/Editor.py deleted file mode 100644 index c0d2cfbad..000000000 --- a/src/lib/Server/Plugins/Editor.py +++ /dev/null @@ -1,80 +0,0 @@ -import Bcfg2.Server.Plugin -import re -import lxml.etree - - -def linesub(pattern, repl, filestring): - """Substitutes instances of pattern with repl in filestring.""" - if filestring == None: - filestring = '' - output = list() - fileread = filestring.split('\n') - for line in fileread: - output.append(re.sub(pattern, repl, filestring)) - return '\n'.join(output) - - -class EditDirectives(Bcfg2.Server.Plugin.SpecificData): - """This object handles the editing directives.""" - def ProcessDirectives(self, input): - """Processes a list of edit directives on input.""" - temp = input - for directive in self.data.split('\n'): - directive = directive.split(',') - temp = linesub(directive[0], directive[1], temp) - return temp - - -class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path, entry_type, encoding): - self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1]) - Bcfg2.Server.Plugin.EntrySet.__init__(self, - basename, - path, - entry_type, - encoding) - self.inputs = dict() - - def bind_entry(self, entry, metadata): - client = metadata.hostname - filename = entry.get('name') - permdata = {'owner': 'root', - 'group': 'root', - 'perms': '0644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) - if not entry.text: - entry.set('empty', 'true') - try: - f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w') - f.write(entry.text) - f.close() - except: - pass - - def get_client_data(self, client): - return self.inputs[client] - - -class Editor(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.Probing): - name = 'Editor' - __author__ = 'bcfg2-dev@mcs.anl.gov' - filename_pattern = 'edits' - es_child_cls = EditDirectives - es_cls = EditEntrySet - - def GetProbes(self, _): - '''Return a set of probes for execution on client''' - probelist = list() - for name in list(self.entries.keys()): - probe = lxml.etree.Element('probe') - probe.set('name', name) - probe.set('source', "Editor") - probe.text = "cat %s" % name - probelist.append(probe) - return probelist - - def ReceiveData(self, client, datalist): - for data in datalist: - self.entries[data.get('name')].inputs[client.hostname] = data.text diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py deleted file mode 100644 index a76d7cac4..000000000 --- a/src/lib/Server/Plugins/FileProbes.py +++ /dev/null @@ -1,230 +0,0 @@ -""" This module allows you to probe a client for a file, which is then -added to the specification. On subsequent runs, the file will be -replaced on the client if it is missing; if it has changed on the -client, it can either be updated in the specification or replaced on -the client """ - -import os -import sys -import errno -import binascii -import lxml.etree -import Bcfg2.Options -import Bcfg2.Server.Plugin - -probecode = """#!/usr/bin/env python - -import os -import pwd -import grp -import binascii -import lxml.etree - -path = "%s" - -if not os.path.exists(path): - print "%%s does not exist" %% path - raise SystemExit(1) - -stat = os.stat(path) -data = lxml.etree.Element("ProbedFileData", - name=path, - owner=pwd.getpwuid(stat[4])[0], - group=grp.getgrgid(stat[5])[0], - perms=oct(stat[0] & 07777)) -data.text = binascii.b2a_base64(open(path).read()) -print lxml.etree.tostring(data) -""" - -class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): - """ Config file handler for FileProbes """ - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - - -class FileProbes(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing): - """ This module allows you to probe a client for a file, which is then - added to the specification. On subsequent runs, the file will be - replaced on the client if it is missing; if it has changed on the - client, it can either be updated in the specification or replaced on - the client """ - - name = 'FileProbes' - experimental = True - __author__ = 'chris.a.st.pierre@gmail.com' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Probing.__init__(self) - self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), - core.fam) - self.entries = dict() - self.probes = dict() - - def GetProbes(self, metadata): - """Return a set of probes for execution on client.""" - if metadata.hostname not in self.probes: - cfg = self.core.plugins['Cfg'] - self.entries[metadata.hostname] = dict() - self.probes[metadata.hostname] = [] - for entry in self.config.Match(metadata): - path = entry.get("name") - # do not probe for files that are already in Cfg and - # for which update is false; we can't possibly do - # anything with the data we get from such a probe - try: - if (entry.get('update', 'false').lower() == "false" and - cfg.entries[path].get_pertinent_entries(entry, - metadata)): - continue - except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): - pass - self.entries[metadata.hostname][path] = entry - probe = lxml.etree.Element('probe', name=path, - source=self.name, - interpreter="/usr/bin/env python") - probe.text = probecode % path - self.probes[metadata.hostname].append(probe) - self.debug_log("Adding file probe for %s to %s" % - (path, metadata.hostname)) - return self.probes[metadata.hostname] - - def ReceiveData(self, metadata, datalist): - """Receive data from probe.""" - self.debug_log("Receiving file probe data from %s" % metadata.hostname) - - for data in datalist: - if data.text is None: - self.logger.error("Got null response to %s file probe from %s" % - (data.get('name'), metadata.hostname)) - else: - try: - self.write_data(lxml.etree.XML(data.text), metadata) - except lxml.etree.XMLSyntaxError: - # if we didn't get XML back from the probe, assume - # it's an error message - self.logger.error(data.text) - - def write_data(self, data, metadata): - """Write the probed file data to the bcfg2 specification.""" - filename = data.get("name") - contents = binascii.a2b_base64(data.text) - entry = self.entries[metadata.hostname][filename] - cfg = self.core.plugins['Cfg'] - specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) - # we can't use os.path.join() for this because specific - # already has a leading /, which confuses os.path.join() - fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) - - create = False - try: - cfg.entries[filename].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - create = True - - # get current entry data - if entry.text and entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) - else: - entrydata = entry.text - - if create: - self.logger.info("Writing new probed file %s" % fileloc) - self.write_file(fileloc, contents) - self.verify_file(filename, contents, metadata) - infoxml = os.path.join("%s%s" % (cfg.data, filename), "info.xml") - self.write_infoxml(infoxml, entry, data) - elif entrydata == contents: - self.debug_log("Existing %s contents match probed contents" % - filename) - return - elif (entry.get('update', 'false').lower() == "true"): - self.logger.info("Writing updated probed file %s" % fileloc) - self.write_file(fileloc, contents) - self.verify_file(filename, contents, metadata) - else: - self.logger.info("Skipping updated probed file %s" % fileloc) - return - - def write_file(self, fileloc, contents): - try: - os.makedirs(os.path.dirname(fileloc)) - except OSError: - err = sys.exc_info()[1] - if err.errno == errno.EEXIST: - pass - else: - self.logger.error("Could not create parent directories for %s: " - "%s" % (fileloc, err)) - return - - try: - open(fileloc, 'wb').write(contents) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Could not write %s: %s" % (fileloc, err)) - return - - def verify_file(self, filename, contents, metadata): - # Service the FAM events queued up by the key generation so - # the data structure entries will be available for binding. - # - # NOTE: We wait for up to ten seconds. There is some potential - # for race condition, because if the file monitor doesn't get - # notified about the new key files in time, those entries - # won't be available for binding. In practice, this seems - # "good enough". - entry = self.entries[metadata.hostname][filename] - cfg = self.core.plugins['Cfg'] - tries = 0 - updated = False - while not updated: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - return - self.core.fam.handle_events_in_interval(1) - try: - cfg.entries[filename].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - tries += 1 - continue - - # get current entry data - if entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) - else: - entrydata = entry.text - if entrydata == contents: - updated = True - tries += 1 - - def write_infoxml(self, infoxml, entry, data): - """ write an info.xml for the file """ - if os.path.exists(infoxml): - return - - self.logger.info("Writing info.xml at %s for %s" % - (infoxml, data.get("name"))) - info = \ - lxml.etree.Element("Info", - owner=data.get("owner", - Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", - Bcfg2.Options.MDATA_GROUP.value), - perms=data.get("perms", - Bcfg2.Options.MDATA_PERMS.value), - encoding=entry.get("encoding", - Bcfg2.Options.ENCODING.value)) - - root = lxml.etree.Element("FileInfo") - root.append(info) - try: - open(infoxml, "w").write(lxml.etree.tostring(root, - pretty_print=True)) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Could not write %s: %s" % (fileloc, err)) - return diff --git a/src/lib/Server/Plugins/Fossil.py b/src/lib/Server/Plugins/Fossil.py deleted file mode 100644 index 1b1627688..000000000 --- a/src/lib/Server/Plugins/Fossil.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Fossil') - -class Fossil(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Fossil is a version plugin for dealing with Bcfg2 repos.""" - name = 'Fossil' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to fossil file for bcfg2 repo - fossil_file = "%s/_FOSSIL_" % datastore - - # Read revision from bcfg2 repo - if os.path.isfile(fossil_file): - revision = self.get_revision() - elif not os.path.isdir(datastore): - logger.error("%s is not a directory" % datastore) - raise Bcfg2.Server.Plugin.PluginInitError - else: - logger.error("%s is not a file" % fossil_file) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \ - % {'ffile': fossil_file, 'frev': revision}) - - def get_revision(self): - """Read fossil revision information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data if \ - line.split(': ')[0].strip() == 'checkout'][-1] - revision = revline.split(' ')[0] - except IndexError: - logger.error("Failed to read fossil info; disabling fossil support") - logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/Git.py b/src/lib/Server/Plugins/Git.py deleted file mode 100644 index 8f8ea87f1..000000000 --- a/src/lib/Server/Plugins/Git.py +++ /dev/null @@ -1,44 +0,0 @@ -"""The Git plugin provides a revision interface for Bcfg2 repos using git.""" - -import os -from dulwich.repo import Repo -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Git') - - -class Git(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Git is a version plugin for dealing with Bcfg2 repos.""" - name = 'Git' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to git directory for bcfg2 repo - git_dir = "%s/.git" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(git_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % git_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized git plugin with git directory %s" % git_dir) - - def get_revision(self): - """Read git revision information for the Bcfg2 repository.""" - try: - repo = Repo(self.datastore) - revision = repo.head() - except: - logger.error("Failed to read git repository; disabling git support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Server/Plugins/GroupPatterns.py deleted file mode 100644 index 58b4d4afb..000000000 --- a/src/lib/Server/Plugins/GroupPatterns.py +++ /dev/null @@ -1,124 +0,0 @@ -import re -import logging -import lxml.etree -import Bcfg2.Server.Plugin - -class PackedDigitRange(object): - def __init__(self, digit_range): - self.sparse = list() - self.ranges = list() - for item in digit_range.split(','): - if '-' in item: - self.ranges.append(tuple([int(x) for x in item.split('-')])) - else: - self.sparse.append(int(item)) - - def includes(self, other): - iother = int(other) - if iother in self.sparse: - return True - for (start, end) in self.ranges: - if iother in range(start, end + 1): - return True - return False - - -class PatternMap(object): - range_finder = r'\[\[[\d\-,]+\]\]' - - def __init__(self, pattern, rangestr, groups): - self.pattern = pattern - self.rangestr = rangestr - self.groups = groups - if pattern != None: - self.re = re.compile(pattern) - self.process = self.process_re - elif rangestr != None: - if '\\' in rangestr: - raise Exception("Backslashes are not allowed in NameRanges") - self.process = self.process_range - self.re = re.compile('^' + re.sub(self.range_finder, '(\d+)', - rangestr)) - dmatcher = re.compile(re.sub(self.range_finder, - r'\[\[([\d\-,]+)\]\]', - rangestr)) - self.dranges = [PackedDigitRange(x) - for x in dmatcher.match(rangestr).groups()] - else: - raise Exception("No pattern or range given") - - def process_range(self, name): - match = self.re.match(name) - if not match: - return None - digits = match.groups() - for i in range(len(digits)): - if not self.dranges[i].includes(digits[i]): - return None - return self.groups - - def process_re(self, name): - match = self.re.match(name) - if not match: - return None - ret = list() - sub = match.groups() - for group in self.groups: - newg = group - for idx in range(len(sub)): - newg = newg.replace('$%s' % (idx + 1), sub[idx]) - ret.append(newg) - return ret - - -class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked): - __identifier__ = None - - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - self.patterns = [] - self.logger = logging.getLogger(self.__class__.__name__) - - def Index(self): - Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) - self.patterns = [] - for entry in self.xdata.xpath('//GroupPattern'): - try: - groups = [g.text for g in entry.findall('Group')] - for pat_ent in entry.findall('NamePattern'): - pat = pat_ent.text - self.patterns.append(PatternMap(pat, None, groups)) - for range_ent in entry.findall('NameRange'): - rng = range_ent.text - self.patterns.append(PatternMap(None, rng, groups)) - except: - self.logger.error("GroupPatterns: Failed to initialize pattern " - "%s" % entry.get('pattern')) - - def process_patterns(self, hostname): - ret = [] - for pattern in self.patterns: - try: - gn = pattern.process(hostname) - if gn is not None: - ret.extend(gn) - except: - self.logger.error("GroupPatterns: Failed to process pattern %s " - "for %s" % (pattern.pattern, hostname), - exc_info=1) - return ret - - -class GroupPatterns(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - name = "GroupPatterns" - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = PatternFile(self.data + '/config.xml', - core.fam) - - def get_additional_groups(self, metadata): - return self.config.process_patterns(metadata.hostname) diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Server/Plugins/Guppy.py deleted file mode 100644 index 046aedc0b..000000000 --- a/src/lib/Server/Plugins/Guppy.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -This plugin is used to trace memory leaks within the bcfg2-server -process using Guppy. By default the remote debugger is started -when this plugin is enabled. The debugger can be shutoff in a running -process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using -"bcfg2-admin xcmd Guppy.Enable". - -To attach the console run: - -python -c "from guppy import hpy;hpy().monitor()" - -For example: - -# python -c "from guppy import hpy;hpy().monitor()" -<Monitor> -*** Connection 1 opened *** -<Monitor> lc -CID PID ARGV - 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid'] -<Monitor> sc 1 -Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN> -<Annex> int -Remote interactive console. To return to Annex, type '-'. ->>> hp.heap() -... - - -""" -import re -import Bcfg2.Server.Plugin - -class Guppy(Bcfg2.Server.Plugin.Plugin): - """Guppy is a debugging plugin to help trace memory leaks""" - name = 'Guppy' - __author__ = 'bcfg-dev@mcs.anl.gov' - - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - - self.Enable() - - def Enable(self): - """Enable remote debugging""" - try: - from guppy.heapy import Remote - Remote.on() - except: - self.logger.error("Failed to create Heapy context") - raise Bcfg2.Server.Plugin.PluginInitError - - def Disable(self): - """Disable remote debugging""" - try: - from guppy.heapy import Remote - Remote.off() - except: - self.logger.error("Failed to disable Heapy") - raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Server/Plugins/Hg.py b/src/lib/Server/Plugins/Hg.py deleted file mode 100644 index 0c3537613..000000000 --- a/src/lib/Server/Plugins/Hg.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -from mercurial import ui, hg -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Mercurial') - -class Hg(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Mercurial is a version plugin for dealing with Bcfg2 repository.""" - name = 'Mercurial' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to hg directory for Bcfg2 repo - hg_dir = "%s/.hg" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(hg_dir): - self.get_revision() - else: - logger.error("%s is not present." % hg_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir) - - def get_revision(self): - """Read hg revision information for the Bcfg2 repository.""" - try: - repo_path = "%s/" % self.datastore - repo = hg.repository(ui.ui(), repo_path) - tip = repo.changelog.tip() - revision = repo.changelog.rev(tip) - except: - logger.error("Failed to read hg repository; disabling mercurial support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision - diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py deleted file mode 100644 index e9c1c1cff..000000000 --- a/src/lib/Server/Plugins/Hostbase.py +++ /dev/null @@ -1,593 +0,0 @@ -""" -This file provides the Hostbase plugin. -It manages dns/dhcp/nis host information -""" - -import os -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' -from lxml.etree import Element, SubElement -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError -from time import strftime -from sets import Set -from django.template import Context, loader -from django.db import connection -import re -# Compatibility imports -from Bcfg2.Bcfg2Py3k import StringIO - - -class Hostbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.Generator): - """The Hostbase plugin handles host/network info.""" - name = 'Hostbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - filepath = '/my/adm/hostbase/files/bind' - - def __init__(self, core, datastore): - - self.ready = False - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - files = ['zone.tmpl', - 'reversesoa.tmpl', - 'named.tmpl', - 'reverseappend.tmpl', - 'dhcpd.tmpl', - 'hosts.tmpl', - 'hostsappend.tmpl'] - self.filedata = {} - self.dnsservers = [] - self.dhcpservers = [] - self.templates = {'zone': loader.get_template('zone.tmpl'), - 'reversesoa': loader.get_template('reversesoa.tmpl'), - 'named': loader.get_template('named.tmpl'), - 'namedviews': loader.get_template('namedviews.tmpl'), - 'reverseapp': loader.get_template('reverseappend.tmpl'), - 'dhcp': loader.get_template('dhcpd.tmpl'), - 'hosts': loader.get_template('hosts.tmpl'), - 'hostsapp': loader.get_template('hostsappend.tmpl'), - } - self.Entries['ConfigFile'] = {} - self.__rmi__ = ['rebuildState'] - try: - self.rebuildState(None) - except: - raise PluginInitError - - def FetchFile(self, entry, metadata): - """Return prebuilt file data.""" - fname = entry.get('name').split('/')[-1] - if not fname in self.filedata: - raise PluginExecutionError - perms = {'owner': 'root', - 'group': 'root', - 'perms': '644'} - [entry.attrib.__setitem__(key, value) - for (key, value) in list(perms.items())] - entry.text = self.filedata[fname] - - def BuildStructures(self, metadata): - """Build hostbase bundle.""" - if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: - return [] - output = Element("Bundle", name='hostbase') - if metadata.hostname in self.dnsservers: - for configfile in self.Entries['ConfigFile']: - if re.search('/etc/bind/', configfile): - SubElement(output, "ConfigFile", name=configfile) - if metadata.hostname in self.dhcpservers: - SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") - return [output] - - def rebuildState(self, _): - """Pre-cache all state information for hostbase config files - callable as an XMLRPC function. - - """ - self.buildZones() - self.buildDHCP() - self.buildHosts() - self.buildHostsLPD() - self.buildPrinters() - self.buildNetgroups() - return True - - def buildZones(self): - """Pre-build and stash zone files.""" - cursor = connection.cursor() - - cursor.execute("SELECT id, serial FROM hostbase_zone") - zones = cursor.fetchall() - - for zone in zones: - # update the serial number for all zone files - todaydate = (strftime('%Y%m%d')) - try: - if todaydate == str(zone[1])[:8]: - serial = zone[1] + 1 - else: - serial = int(todaydate) * 100 - except (KeyError): - serial = int(todaydate) * 100 - cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") - zones = cursor.fetchall() - - iplist = [] - hosts = {} - - for zone in zones: - zonefile = StringIO() - externalzonefile = StringIO() - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - nameservers = cursor.fetchall() - cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z - INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - addresses = cursor.fetchall() - cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z - INNER JOIN hostbase_mx m ON z.mx_id = m.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - mxs = cursor.fetchall() - context = Context({ - 'zone': zone, - 'nameservers': nameservers, - 'addresses': addresses, - 'mxs': mxs - }) - zonefile.write(self.templates['zone'].render(context)) - externalzonefile.write(self.templates['zone'].render(context)) - - querystring = """SELECT h.hostname, p.ip_addr, - n.name, c.cname, m.priority, m.mx, n.dns_view - FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE n.name LIKE '%%%%%s' - AND h.status = 'active' - ORDER BY h.hostname, n.name, p.ip_addr - """ % zone[1] - cursor.execute(querystring) - zonehosts = cursor.fetchall() - prevhost = (None, None, None, None) - cnames = StringIO() - cnamesexternal = StringIO() - for host in zonehosts: - if not host[2].split(".", 1)[1] == zone[1]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - continue - if not prevhost[1] == host[1] or not prevhost[2] == host[2]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - zonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - elif not prevhost[5] == host[5]: - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - - if host[3]: - try: - if host[3].split(".", 1)[1] == zone[1]: - cnames.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - else: - cnames.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - - except: - pass - prevhost = host - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - zonefile.write("\n\n%s" % zone[9]) - externalzonefile.write("\n\n%s" % zone[9]) - self.filedata[zone[1]] = zonefile.getvalue() - self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile - self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") - reversezones = cursor.fetchall() - - reversenames = [] - for reversezone in reversezones: - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % reversezone[0]) - reverse_nameservers = cursor.fetchall() - - context = Context({ - 'inaddr': reversezone[1].rstrip('.rev'), - 'zone': reversezone, - 'nameservers': reverse_nameservers, - }) - - self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1]] += reversezone[9] - self.filedata[reversezone[1] + '.external'] += reversezone[9] - - subnet = reversezone[1].split(".") - subnet.reverse() - reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) - - for filename in reversenames: - cursor.execute(""" - SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON n.ip_id = p.id - WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr - """ % filename[1]) - reversehosts = cursor.fetchall() - zonefile = StringIO() - externalzonefile = StringIO() - if len(filename[0].split(".")) == 2: - originlist = [] - [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]]))) - for ip in reversehosts - if (".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] - for origin in originlist: - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if host[1].rstrip('0123456789').rstrip('.') == origin[1]] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].rstrip('0123456789').rstrip('.') == origin[1] - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - else: - originlist = [filename[0]] - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].split("."), host[0]) not in hosts] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if ((host[1].split("."), host[0]) not in hosts_external - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': filename[0], - 'fileorigin': None, - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': filename[0], - 'fileorigin': None, - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() - self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile - self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile - - ## here's where the named.conf file gets written - context = Context({ - 'zones': zones, - 'reverses': reversenames, - }) - self.filedata['named.conf'] = self.templates['named'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile - self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile - - def buildDHCP(self): - """Pre-build dhcpd.conf and stash in the filedata table.""" - - # fetches all the hosts with DHCP == True - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname, mac_addr, ip_addr - FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip ip ON i.id = ip.interface_id - WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' - AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' - ORDER BY h.hostname, i.mac_addr - """) - - dhcphosts = cursor.fetchall() - count = 0 - hosts = [] - hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] - if len(dhcphosts) > 1: - for x in range(1, len(dhcphosts)): - # if an interface has 2 or more ip addresses - # adds the ip to the current interface - if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: - hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) - # if a host has 2 or more interfaces - # writes the current one and grabs the next - elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: - hosts.append(hostdata) - count += 1 - hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] - # new host found, writes current data to the template - else: - hosts.append(hostdata) - count = 0 - hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] - #makes sure the last of the data gets written out - if hostdata not in hosts: - hosts.append(hostdata) - - context = Context({ - 'hosts': hosts, - 'numips': len(hosts), - }) - - self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile - - def buildHosts(self): - """Pre-build and stash /etc/hosts file.""" - - append_data = [] - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host ORDER BY hostname - """) - hostbase = cursor.fetchall() - domains = [host[0].split(".", 1)[1] for host in hostbase] - domains_set = Set(domains) - domain_data = [(domain, domains.count(domain)) for domain in domains_set] - domain_data.sort() - - cursor.execute(""" - SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr - """) - ips = cursor.fetchall() - three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ - for ip in ips] - three_octets_set = Set(three_octets) - three_octets_data = [(octet, three_octets.count(octet)) \ - for octet in three_octets_set] - three_octets_data.sort() - - for three_octet in three_octets_data: - querystring = """SELECT h.hostname, h.primary_user, - p.ip_addr, n.name, c.cname - FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] - cursor.execute(querystring) - tosort = list(cursor.fetchall()) - tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) - append_data.append((three_octet, tuple(tosort))) - - two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] - two_octets_set = Set(two_octets) - two_octets_data = [(octet, two_octets.count(octet)) - for octet in two_octets_set] - two_octets_data.sort() - - context = Context({ - 'domain_data': domain_data, - 'three_octets_data': three_octets_data, - 'two_octets_data': two_octets_data, - 'three_octets': three_octets, - 'num_ips': len(three_octets), - }) - - self.filedata['hosts'] = self.templates['hosts'].render(context) - - for subnet in append_data: - ips = [] - simple = True - namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] - cnamelist = [] - if subnet[1][0][4]: - cnamelist.append(subnet[1][0][4].split('.', 1)[0]) - simple = False - appenddata = subnet[1][0] - for ip in subnet[1][1:]: - if appenddata[2] == ip[2]: - namelist.append(ip[3].split('.', 1)[0]) - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - appenddata = ip - else: - if appenddata[0] == ip[0]: - simple = False - ips.append((appenddata[2], appenddata[0], Set(namelist), - cnamelist, simple, appenddata[1])) - appenddata = ip - simple = True - namelist = [ip[3].split('.', 1)[0]] - cnamelist = [] - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - ips.append((appenddata[2], appenddata[0], Set(namelist), - cnamelist, simple, appenddata[1])) - context = Context({ - 'subnet': subnet[0], - 'ips': ips, - }) - self.filedata['hosts'] += self.templates['hostsapp'].render(context) - self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile - - def buildPrinters(self): - """The /mcs/etc/printers.data file""" - header = """# This file is automatically generated. DO NOT EDIT IT! -# -Name Room User Type Notes -============== ========== ============================== ======================== ==================== -""" - - cursor = connection.cursor() - # fetches all the printers from the database - cursor.execute(""" - SELECT printq, location, primary_user, comments - FROM hostbase_host - WHERE whatami='printer' AND printq <> '' AND status = 'active' - ORDER BY printq - """) - printers = cursor.fetchall() - - printersfile = header - for printer in printers: - # splits up the printq line and gets the - # correct description out of the comments section - temp = printer[3].split('\n') - for printq in re.split(',[ ]*', printer[0]): - if len(temp) > 1: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], temp[1], temp[0])) - else: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], '', printer[3])) - self.filedata['printers.data'] = printersfile - self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile - - def buildHostsLPD(self): - """Creates the /mcs/etc/hosts.lpd file""" - - # this header needs to be changed to be more generic - header = """+@machines -+@all-machines -achilles.ctd.anl.gov -raven.ops.anl.gov -seagull.hr.anl.gov -parrot.ops.anl.gov -condor.ops.anl.gov -delphi.esh.anl.gov -anlcv1.ctd.anl.gov -anlvms.ctd.anl.gov -olivia.ctd.anl.gov\n\n""" - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' - ORDER BY hostname""") - redmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' - """) - redmachines.extend(list(cursor.fetchall())) - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' - ORDER BY hostname""") - winmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' - """) - winmachines.__add__(list(cursor.fetchall())) - hostslpdfile = header - for machine in redmachines: - hostslpdfile += machine[0] + "\n" - hostslpdfile += "\n" - for machine in winmachines: - hostslpdfile += machine[0] + "\n" - self.filedata['hosts.lpd'] = hostslpdfile - self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile - - def buildNetgroups(self): - """Makes the *-machine files""" - header = """################################################################### -# This file lists hosts in the '%s' machine netgroup, it is -# automatically generated. DO NOT EDIT THIS FILE! -# -# Number of hosts in '%s' machine netgroup: %i -#\n\n""" - - cursor = connection.cursor() - # fetches all the hosts that with valid netgroup entries - cursor.execute(""" - SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' - ORDER BY h.netgroup, h.hostname - """) - nameslist = cursor.fetchall() - # gets the first host and initializes the hash - hostdata = nameslist[0] - netgroups = {hostdata[2]: [hostdata[0]]} - for row in nameslist: - # if new netgroup, create it - if row[2] not in netgroups: - netgroups.update({row[2]: []}) - # if it belongs in the netgroup and has multiple interfaces, put them in - if hostdata[0] == row[0] and row[3]: - netgroups[row[2]].append(row[1]) - hostdata = row - # if its a new host, write the old one to the hash - elif hostdata[0] != row[0]: - netgroups[row[2]].append(row[0]) - hostdata = row - - for netgroup in netgroups: - fileoutput = StringIO() - fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) - for each in netgroups[netgroup]: - fileoutput.write(each + "\n") - self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() - fileoutput.close() - self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile - - cursor.execute(""" - UPDATE hostbase_host SET dirty=0 - """) diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py deleted file mode 100644 index 04417339b..000000000 --- a/src/lib/Server/Plugins/Ldap.py +++ /dev/null @@ -1,245 +0,0 @@ -import imp -import logging -import sys -import time -import traceback -import Bcfg2.Options -import Bcfg2.Server.Plugin - -logger = logging.getLogger('Bcfg2.Plugins.Ldap') - -try: - import ldap -except: - logger.error("Unable to load ldap module. Is python-ldap installed?") - raise ImportError - -# time in seconds between retries after failed LDAP connection -RETRY_DELAY = 5 -# how many times to try reaching the LDAP server if a connection is broken -# at the very minimum, one retry is needed to handle a restarted LDAP daemon -RETRY_COUNT = 3 - -SCOPE_MAP = { - "base" : ldap.SCOPE_BASE, - "one" : ldap.SCOPE_ONELEVEL, - "sub" : ldap.SCOPE_SUBTREE, -} - -LDAP_QUERIES = [] - -def register_query(query): - LDAP_QUERIES.append(query) - -class ConfigFile(Bcfg2.Server.Plugin.FileBacked): - """ - Config file for the Ldap plugin - - The config file cannot be 'parsed' in the traditional sense as we would - need some serious type checking ugliness to just get the LdapQuery - subclasses. The alternative would be to have the user create a list with - a predefined name that contains all queries. - The approach implemented here is having the user call a registering - decorator that updates a global variable in this module. - """ - def __init__(self, filename, fam): - self.filename = filename - Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename) - fam.AddMonitor(self.filename, self) - - def Index(self): - """ - Reregisters the queries in the config file - - The config will take care of actually registering the queries, - so we just load it once and don't keep it. - """ - global LDAP_QUERIES - LDAP_QUERIES = [] - imp.load_source("ldap_cfg", self.filename) - -class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ - The Ldap plugin allows adding data from an LDAP server to your metadata. - """ - name = "Ldap" - experimental = True - debug_flag = False - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = ConfigFile(self.data + "/config.py", core.fam) - - def debug_log(self, message, flag = None): - if (flag is None) and self.debug_flag or flag: - self.logger.error(message) - - def get_additional_data(self, metadata): - query = None - try: - data = {} - self.debug_log("LdapPlugin debug: found queries " + - str(LDAP_QUERIES)) - for QueryClass in LDAP_QUERIES: - query = QueryClass() - if query.is_applicable(metadata): - self.debug_log("LdapPlugin debug: processing query '" + - query.name + "'") - data[query.name] = query.get_result(metadata) - else: - self.debug_log("LdapPlugin debug: query '" + query.name + - "' not applicable to host '" + metadata.hostname + "'") - return data - except Exception: - if hasattr(query, "name"): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + - str(query.name) + - "', query results will be empty" + - " and may cause bind failures") - for line in traceback.format_exception(sys.exc_info()[0], - sys.exc_info()[1], - sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - line.replace("\n", "")) - return {} - -class LdapConnection(object): - """ - Connection to an LDAP server. - """ - def __init__(self, host = "localhost", port = 389, - binddn = None, bindpw = None): - self.host = host - self.port = port - self.binddn = binddn - self.bindpw = bindpw - self.conn = None - - def __del__(self): - if self.conn: - self.conn.unbind() - - def init_conn(self): - self.conn = ldap.initialize(self.url) - if self.binddn is not None and self.bindpw is not None: - self.conn.simple_bind_s(self.binddn, self.bindpw) - - def run_query(self, query): - result = None - for attempt in range(RETRY_COUNT + 1): - if attempt >= 1: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "LDAP server down (retry " + str(attempt) + "/" + - str(RETRY_COUNT) + ")") - try: - if not self.conn: - self.init_conn() - result = self.conn.search_s( - query.base, - SCOPE_MAP[query.scope], - query.filter, - query.attrs, - ) - break - except ldap.SERVER_DOWN: - self.conn = None - time.sleep(RETRY_DELAY) - return result - - @property - def url(self): - return "ldap://" + self.host + ":" + str(self.port) - -class LdapQuery(object): - """ - Query referencing an LdapConnection and providing several - methods for query manipulation. - """ - - name = "unknown" - base = "" - scope = "sub" - filter = "(objectClass=*)" - attrs = None - connection = None - result = None - - def __unicode__(self): - return "LdapQuery:" + self.name - - def is_applicable(self, metadata): - """ - Overrideable method to determine if the query is to be executed for - the given metadata object. - Defaults to true. - """ - return True - - def prepare_query(self, metadata): - """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - - In most cases, you will do something like - - self.filter = "(cn=" + metadata.hostname + ")" - - here. - """ - pass - - def process_result(self, metadata): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. - """ - return self.result - - def get_result(self, metadata): - """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): - self.prepare_query(metadata) - self.result = self.connection.run_query(self) - self.result = self.process_result(metadata) - return self.result - else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None - -class LdapSubQuery(LdapQuery): - """ - SubQueries are meant for internal use only and are not added - to the metadata object. They are useful for situations where - you need to run more than one query to obtain some data. - """ - def prepare_query(self, metadata, **kwargs): - """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - """ - pass - - def process_result(self, metadata, **kwargs): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. - """ - return self.result - - def get_result(self, metadata, **kwargs): - """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): - self.prepare_query(metadata, **kwargs) - self.result = self.connection.run_query(self) - return self.process_result(metadata, **kwargs) - else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py deleted file mode 100644 index fa2cf0a78..000000000 --- a/src/lib/Server/Plugins/Metadata.py +++ /dev/null @@ -1,866 +0,0 @@ -""" -This file stores persistent metadata for the Bcfg2 Configuration Repository. -""" - -import copy -import fcntl -import lxml.etree -import os -import os.path -import socket -import sys -import time - -import Bcfg2.Server.FileMonitor -import Bcfg2.Server.Plugin - - -def locked(fd): - """Aquire a lock on a file""" - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - return True - return False - - -class MetadataConsistencyError(Exception): - """This error gets raised when metadata is internally inconsistent.""" - pass - - -class MetadataRuntimeError(Exception): - """This error is raised when the metadata engine - is called prior to reading enough data. - """ - pass - - -class XMLMetadataConfig(object): - """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - self.metadata = metadata - self.basefile = basefile - self.should_monitor = watch_clients - self.extras = [] - self.data = None - self.basedata = None - self.basedir = metadata.data - self.logger = metadata.logger - self.pseudo_monitor = isinstance(metadata.core.fam, - Bcfg2.Server.FileMonitor.Pseudo) - - @property - def xdata(self): - if not self.data: - raise MetadataRuntimeError - return self.data - - @property - def base_xdata(self): - if not self.basedata: - raise MetadataRuntimeError - return self.basedata - - def add_monitor(self, fname): - """Add a fam monitor for an included file""" - if self.should_monitor: - self.metadata.core.fam.AddMonitor(os.path.join(self.basedir, fname), - self.metadata) - self.extras.append(fname) - - def load_xml(self): - """Load changes from XML""" - try: - xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile)) - except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % self.basefile) - return - self.basedata = copy.copy(xdata) - included = [ent.get('href') for ent in \ - xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) - try: - xdata.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % - self.basefile) - self.data = xdata - - def write(self): - """Write changes to xml back to disk.""" - self.write_xml(os.path.join(self.basedir, self.basefile), - self.basedata) - - def write_xml(self, fname, xmltree): - """Write changes to xml back to disk.""" - tmpfile = "%s.new" % fname - try: - datafile = open(tmpfile, 'w') - except IOError: - e = sys.exc_info()[1] - self.logger.error("Failed to write %s: %s" % (tmpfile, e)) - raise MetadataRuntimeError - # prep data - dataroot = xmltree.getroot() - newcontents = lxml.etree.tostring(dataroot, pretty_print=True) - - fd = datafile.fileno() - while locked(fd) == True: - pass - try: - datafile.write(newcontents) - except: - fcntl.lockf(fd, fcntl.LOCK_UN) - self.logger.error("Metadata: Failed to write new xml data to %s" % - tmpfile, exc_info=1) - os.unlink(tmpfile) - raise MetadataRuntimeError - datafile.close() - - # check if clients.xml is a symlink - if os.path.islink(fname): - fname = os.readlink(fname) - - try: - os.rename(tmpfile, fname) - except: - self.logger.error("Metadata: Failed to rename %s" % tmpfile) - raise MetadataRuntimeError - - def find_xml_for_xpath(self, xpath): - """Find and load xml file containing the xpath query""" - if self.pseudo_monitor: - # Reload xml if we don't have a real monitor - self.load_xml() - cli = self.basedata.xpath(xpath) - if len(cli) > 0: - return {'filename': os.path.join(self.basedir, self.basefile), - 'xmltree': self.basedata, - 'xquery': cli} - else: - """Try to find the data in included files""" - for included in self.extras: - try: - xdata = lxml.etree.parse(os.path.join(self.basedir, - included)) - cli = xdata.xpath(xpath) - if len(cli) > 0: - return {'filename': os.path.join(self.basedir, - included), - 'xmltree': xdata, - 'xquery': cli} - except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % (included)) - return {} - - def HandleEvent(self, event): - """Handle fam events""" - filename = event.filename.split('/')[-1] - if filename in self.extras: - if event.code2str() == 'exists': - return False - elif filename != self.basefile: - return False - if event.code2str() == 'endExist': - return False - self.load_xml() - return True - - -class ClientMetadata(object): - """This object contains client metadata.""" - def __init__(self, client, profile, groups, bundles, - aliases, addresses, categories, uuid, password, query): - self.hostname = client - self.profile = profile - self.bundles = bundles - self.aliases = aliases - self.addresses = addresses - self.groups = groups - self.categories = categories - self.uuid = uuid - self.password = password - self.connectors = [] - self.query = query - - def inGroup(self, group): - """Test to see if client is a member of group.""" - return group in self.groups - - def group_in_category(self, category): - for grp in self.query.all_groups_in_category(category): - if grp in self.groups: - return grp - return '' - - -class MetadataQuery(object): - def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): - # resolver is set later - self.by_name = by_name - self.names_by_groups = by_groups - self.names_by_profiles = by_profiles - self.all_clients = get_clients - self.all_groups = all_groups - self.all_groups_in_category = all_groups_in_category - - def by_groups(self, groups): - return [self.by_name(name) for name in self.names_by_groups(groups)] - - def by_profiles(self, profiles): - return [self.by_name(name) for name in self.names_by_profiles(profiles)] - - def all(self): - return [self.by_name(name) for name in self.all_clients()] - - -class Metadata(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Statistics): - """This class contains data for bcfg2 server metadata.""" - __author__ = 'bcfg-dev@mcs.anl.gov' - name = "Metadata" - sort_order = 500 - - def __init__(self, core, datastore, watch_clients=True): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Statistics.__init__(self) - if watch_clients: - try: - core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self) - core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self) - except: - print("Unable to add file monitor for groups.xml or clients.xml") - raise Bcfg2.Server.Plugin.PluginInitError - - self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml') - self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml') - self.states = {} - if watch_clients: - self.states = {"groups.xml": False, - "clients.xml": False} - self.addresses = {} - self.auth = dict() - self.clients = {} - self.aliases = {} - self.groups = {} - self.cgroups = {} - self.public = [] - self.private = [] - self.profiles = [] - self.categories = {} - self.bad_clients = {} - self.uuid = {} - self.secure = [] - self.floating = [] - self.passwords = {} - self.session_cache = {} - self.default = None - self.pdirty = False - self.extra = {'groups.xml': [], - 'clients.xml': []} - self.password = core.password - self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients.keys()), - self.get_client_names_by_groups, - self.get_client_names_by_profiles, - self.get_all_group_names, - self.get_all_groups_in_category) - - @classmethod - def init_repo(cls, repo, groups, os_selection, clients): - path = os.path.join(repo, cls.name) - os.makedirs(path) - open(os.path.join(repo, "Metadata", "groups.xml"), - "w").write(groups % os_selection) - open(os.path.join(repo, "Metadata", "clients.xml"), - "w").write(clients % socket.getfqdn()) - - def get_groups(self): - '''return groups xml tree''' - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) - root = groups_tree.getroot() - return root - - def _search_xdata(self, tag, name, tree, alias=False): - for node in tree.findall("//%s" % tag): - if node.get("name") == name: - return node - elif alias: - for child in node: - if (child.tag == "Alias" and - child.attrib["name"] == name): - return node - return None - - def search_group(self, group_name, tree): - """Find a group.""" - return self._search_xdata("Group", group_name, tree) - - def search_bundle(self, bundle_name, tree): - """Find a bundle.""" - return self._search_xdata("Bundle", bundle_name, tree) - - def search_client(self, client_name, tree): - return self._search_xdata("Client", client_name, tree, alias=True) - - def _add_xdata(self, config, tag, name, attribs=None, alias=False): - node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node != None: - self.logger.error("%s \"%s\" already exists" % (tag, name)) - raise MetadataConsistencyError - element = lxml.etree.SubElement(config.base_xdata.getroot(), - tag, name=name) - if attribs: - for key, val in list(attribs.items()): - element.set(key, val) - config.write() - - def add_group(self, group_name, attribs): - """Add group to groups.xml.""" - return self._add_xdata(self.groups_xml, "Group", group_name, - attribs=attribs) - - def add_bundle(self, bundle_name): - """Add bundle to groups.xml.""" - return self._add_xdata(self.groups_xml, "Bundle", bundle_name) - - def add_client(self, client_name, attribs): - """Add client to clients.xml.""" - return self._add_xdata(self.clients_xml, "Client", client_name, - attribs=attribs, alias=True) - - def _update_xdata(self, config, tag, name, attribs, alias=False): - node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError - xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % - (tag, node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError - for key, val in list(attribs.items()): - xdict['xquery'][0].set(key, val) - config.write_xml(xdict['filename'], xdict['xmltree']) - - def update_group(self, group_name, attribs): - """Update a groups attributes.""" - return self._update_xdata(self.groups_xml, "Group", group_name, attribs) - - def update_client(self, client_name, attribs): - """Update a clients attributes.""" - return self._update_xdata(self.clients_xml, "Client", client_name, - attribs, alias=True) - - def _remove_xdata(self, config, tag, name, alias=False): - node = self._search_xdata(tag, name, config.xdata) - if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError - xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % - (tag, node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError - xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) - self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) - - def remove_group(self, group_name): - """Remove a group.""" - return self._remove_xdata(self.groups_xml, "Group", group_name) - - def remove_bundle(self, bundle_name): - """Remove a bundle.""" - return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) - - def _handle_clients_xml_event(self, event): - xdata = self.clients_xml.xdata - self.clients = {} - self.aliases = {} - self.raliases = {} - self.bad_clients = {} - self.secure = [] - self.floating = [] - self.addresses = {} - self.raddresses = {} - for client in xdata.findall('.//Client'): - clname = client.get('name').lower() - if 'address' in client.attrib: - caddr = client.get('address') - if caddr in self.addresses: - self.addresses[caddr].append(clname) - else: - self.addresses[caddr] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(caddr) - if 'auth' in client.attrib: - self.auth[client.get('name')] = client.get('auth', - 'cert+password') - if 'uuid' in client.attrib: - self.uuid[client.get('uuid')] = clname - if client.get('secure', 'false') == 'true': - self.secure.append(clname) - if client.get('location', 'fixed') == 'floating': - self.floating.append(clname) - if 'password' in client.attrib: - self.passwords[clname] = client.get('password') - - self.raliases[clname] = set() - for alias in client.findall('Alias'): - self.aliases.update({alias.get('name'): clname}) - self.raliases[clname].add(alias.get('name')) - if 'address' not in alias.attrib: - continue - if alias.get('address') in self.addresses: - self.addresses[alias.get('address')].append(clname) - else: - self.addresses[alias.get('address')] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(alias.get('address')) - self.clients.update({clname: client.get('profile')}) - self.states['clients.xml'] = True - - def _handle_groups_xml_event(self, event): - xdata = self.groups_xml.xdata - self.public = [] - self.private = [] - self.profiles = [] - self.groups = {} - grouptmp = {} - self.categories = {} - groupseen = list() - for group in xdata.xpath('//Groups/Group'): - if group.get('name') not in groupseen: - groupseen.append(group.get('name')) - else: - self.logger.error("Metadata: Group %s defined multiply" % - group.get('name')) - grouptmp[group.get('name')] = \ - ([item.get('name') for item in group.findall('./Bundle')], - [item.get('name') for item in group.findall('./Group')]) - grouptmp[group.get('name')][1].append(group.get('name')) - if group.get('default', 'false') == 'true': - self.default = group.get('name') - if group.get('profile', 'false') == 'true': - self.profiles.append(group.get('name')) - if group.get('public', 'false') == 'true': - self.public.append(group.get('name')) - elif group.get('public', 'true') == 'false': - self.private.append(group.get('name')) - if 'category' in group.attrib: - self.categories[group.get('name')] = group.get('category') - - for group in grouptmp: - # self.groups[group] => (bundles, groups, categories) - self.groups[group] = (set(), set(), {}) - tocheck = [group] - group_cat = self.groups[group][2] - while tocheck: - now = tocheck.pop() - self.groups[group][1].add(now) - if now in grouptmp: - (bundles, groups) = grouptmp[now] - for ggg in groups: - if ggg in self.groups[group][1]: - continue - if (ggg not in self.categories or \ - self.categories[ggg] not in self.groups[group][2]): - self.groups[group][1].add(ggg) - tocheck.append(ggg) - if ggg in self.categories: - group_cat[self.categories[ggg]] = ggg - elif ggg in self.categories: - self.logger.info("Group %s: %s cat-suppressed %s" % \ - (group, - group_cat[self.categories[ggg]], - ggg)) - [self.groups[group][0].add(bund) for bund in bundles] - self.states['groups.xml'] = True - - def HandleEvent(self, event): - """Handle update events for data files.""" - if self.clients_xml.HandleEvent(event): - self._handle_clients_xml_event(event) - elif self.groups_xml.HandleEvent(event): - self._handle_groups_xml_event(event) - - if False not in list(self.states.values()): - # check that all client groups are real and complete - real = list(self.groups.keys()) - for client in list(self.clients.keys()): - if self.clients[client] not in self.profiles: - self.logger.error("Client %s set as nonexistent or " - "incomplete group %s" % - (client, self.clients[client])) - self.logger.error("Removing client mapping for %s" % client) - self.bad_clients[client] = self.clients[client] - del self.clients[client] - for bclient in list(self.bad_clients.keys()): - if self.bad_clients[bclient] in self.profiles: - self.logger.info("Restored profile mapping for client %s" % - bclient) - self.clients[bclient] = self.bad_clients[bclient] - del self.bad_clients[bclient] - - def set_profile(self, client, profile, addresspair): - """Set group parameter for provided client.""" - self.logger.info("Asserting client %s profile to %s" % (client, profile)) - if False in list(self.states.values()): - raise MetadataRuntimeError - if profile not in self.public: - self.logger.error("Failed to set client %s to private group %s" % - (client, profile)) - raise MetadataConsistencyError - if client in self.clients: - self.logger.info("Changing %s group from %s to %s" % - (client, self.clients[client], profile)) - self.update_client(client, dict(profile=profile)) - else: - self.logger.info("Creating new client: %s, profile %s" % - (client, profile)) - if addresspair in self.session_cache: - # we are working with a uuid'd client - self.add_client(self.session_cache[addresspair][1], - dict(uuid=client, profile=profile, - address=addresspair[0])) - else: - self.add_client(client, dict(profile=profile)) - self.clients[client] = profile - self.clients_xml.write() - - def resolve_client(self, addresspair, cleanup_cache=False): - """Lookup address locally or in DNS to get a hostname.""" - if addresspair in self.session_cache: - # client _was_ cached, so there can be some expired - # entries. we need to clean them up to avoid potentially - # infinite memory swell - cache_ttl = 90 - if cleanup_cache: - # remove entries for this client's IP address with - # _any_ port numbers - perhaps a priority queue could - # be faster? - curtime = time.time() - for addrpair in self.session_cache.keys(): - if addresspair[0] == addrpair[0]: - (stamp, _) = self.session_cache[addrpair] - if curtime - stamp > cache_ttl: - del self.session_cache[addrpair] - # return the cached data - try: - (stamp, uuid) = self.session_cache[addresspair] - if time.time() - stamp < cache_ttl: - return self.session_cache[addresspair][1] - except KeyError: - # we cleaned all cached data for this client in cleanup_cache - pass - address = addresspair[0] - if address in self.addresses: - if len(self.addresses[address]) != 1: - self.logger.error("Address %s has multiple reverse assignments; " - "a uuid must be used" % (address)) - raise MetadataConsistencyError - return self.addresses[address][0] - try: - cname = socket.gethostbyaddr(address)[0].lower() - if cname in self.aliases: - return self.aliases[cname] - return cname - except socket.herror: - warning = "address resolution error for %s" % (address) - self.logger.warning(warning) - raise MetadataConsistencyError - - def get_initial_metadata(self, client): - """Return the metadata for a given client.""" - if False in list(self.states.values()): - raise MetadataRuntimeError - client = client.lower() - if client in self.aliases: - client = self.aliases[client] - if client in self.clients: - profile = self.clients[client] - (bundles, groups, categories) = self.groups[profile] - else: - if self.default == None: - self.logger.error("Cannot set group for client %s; " - "no default group set" % client) - raise MetadataConsistencyError - self.set_profile(client, self.default, (None, None)) - profile = self.default - [bundles, groups, categories] = self.groups[self.default] - aliases = self.raliases.get(client, set()) - addresses = self.raddresses.get(client, set()) - newgroups = set(groups) - newbundles = set(bundles) - newcategories = {} - newcategories.update(categories) - if client in self.passwords: - password = self.passwords[client] - else: - password = None - uuids = [item for item, value in list(self.uuid.items()) - if value == client] - if uuids: - uuid = uuids[0] - else: - uuid = None - for group in self.cgroups.get(client, []): - if group in self.groups: - nbundles, ngroups, ncategories = self.groups[group] - else: - nbundles, ngroups, ncategories = ([], [group], {}) - [newbundles.add(b) for b in nbundles if b not in newbundles] - [newgroups.add(g) for g in ngroups if g not in newgroups] - newcategories.update(ncategories) - return ClientMetadata(client, profile, newgroups, newbundles, aliases, - addresses, newcategories, uuid, password, - self.query) - - def get_all_group_names(self): - all_groups = set() - [all_groups.update(g[1]) for g in list(self.groups.values())] - return all_groups - - def get_all_groups_in_category(self, category): - all_groups = set() - [all_groups.add(g) for g in self.categories \ - if self.categories[g] == category] - return all_groups - - def get_client_names_by_profiles(self, profiles): - return [client for client, profile in list(self.clients.items()) \ - if profile in profiles] - - def get_client_names_by_groups(self, groups): - mdata = [self.core.build_metadata(client) - for client in list(self.clients.keys())] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] - - def merge_additional_groups(self, imd, groups): - for group in groups: - if (group in self.categories and - self.categories[group] in imd.categories): - continue - newbundles, newgroups, _ = self.groups.get(group, - (list(), - [group], - dict())) - for newbundle in newbundles: - if newbundle not in imd.bundles: - imd.bundles.add(newbundle) - for newgroup in newgroups: - if newgroup not in imd.groups: - if (newgroup in self.categories and - self.categories[newgroup] in imd.categories): - continue - if newgroup in self.private: - self.logger.error("Refusing to add dynamic membership " - "in private group %s for client %s" % - (newgroup, imd.hostname)) - continue - imd.groups.add(newgroup) - - def merge_additional_data(self, imd, source, data): - if not hasattr(imd, source): - setattr(imd, source, data) - imd.connectors.append(source) - - def validate_client_address(self, client, addresspair): - """Check address against client.""" - address = addresspair[0] - if client in self.floating: - self.debug_log("Client %s is floating" % client) - return True - if address in self.addresses: - if client in self.addresses[address]: - self.debug_log("Client %s matches address %s" % - (client, address)) - return True - else: - self.logger.error("Got request for non-float client %s from %s" % - (client, address)) - return False - resolved = self.resolve_client(addresspair) - if resolved.lower() == client.lower(): - return True - else: - self.logger.error("Got request for %s from incorrect address %s" % - (client, address)) - self.logger.error("Resolved to %s" % resolved) - return False - - def AuthenticateConnection(self, cert, user, password, address): - """This function checks auth creds.""" - if cert: - id_method = 'cert' - certinfo = dict([x[0] for x in cert['subject']]) - # look at cert.cN - client = certinfo['commonName'] - self.debug_log("Got cN %s; using as client name" % client) - auth_type = self.auth.get(client, 'cert+password') - elif user == 'root': - id_method = 'address' - try: - client = self.resolve_client(address) - except MetadataConsistencyError: - self.logger.error("Client %s failed to resolve; metadata problem" - % address[0]) - return False - else: - id_method = 'uuid' - # user maps to client - if user not in self.uuid: - client = user - self.uuid[user] = user - else: - client = self.uuid[user] - - # we have the client name - self.debug_log("Authenticating client %s" % client) - - # next we validate the address - if id_method == 'uuid': - addr_is_valid = True - else: - addr_is_valid = self.validate_client_address(client, address) - - if not addr_is_valid: - return False - - if id_method == 'cert' and auth_type != 'cert+password': - # remember the cert-derived client name for this connection - if client in self.floating: - self.session_cache[address] = (time.time(), client) - # we are done if cert+password not required - return True - - if client not in self.passwords: - if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" - % address[0]) - return False - if password != self.password: - self.logger.error("Client %s used incorrect global password" % - address[0]) - return False - if client not in self.secure: - if client in self.passwords: - plist = [self.password, self.passwords[client]] - else: - plist = [self.password] - if password not in plist: - self.logger.error("Client %s failed to use either allowed " - "password" % address[0]) - return False - else: - # client in secure mode and has a client password - if password != self.passwords[client]: - self.logger.error("Client %s failed to use client password in " - "secure mode" % address[0]) - return False - # populate the session cache - if user.decode('utf-8') != 'root': - self.session_cache[address] = (time.time(), client) - return True - - def process_statistics(self, meta, _): - """Hook into statistics interface to toggle clients in bootstrap mode.""" - client = meta.hostname - if client in self.auth and self.auth[client] == 'bootstrap': - self.update_client(client, dict(auth='cert')) - - def viz(self, hosts, bundles, key, only_client, colors): - """Admin mode viz support.""" - if only_client: - clientmeta = self.core.build_metadata(only_client) - - def include_client(client): - return not only_client or client != only_client - - def include_bundle(bundle): - return not only_client or bundle in clientmeta.bundles - - def include_group(group): - return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) - try: - groups_tree.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % dest) - groups = groups_tree.getroot() - categories = {'default': 'grey83'} - viz_str = [] - egroups = groups.findall("Group") + groups.findall('.//Groups/Group') - for group in egroups: - if not group.get('category') in categories: - categories[group.get('category')] = colors.pop() - group.set('color', categories[group.get('category')]) - if None in categories: - del categories[None] - if hosts: - instances = {} - clients = self.clients - for client, profile in list(clients.items()): - if include_client(client): - continue - if profile in instances: - instances[profile].append(client) - else: - instances[profile] = [client] - for profile, clist in list(instances.items()): - clist.sort() - viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % - (profile, '|'.join(clist))) - viz_str.append('"%s-instances" -> "group-%s";' % - (profile, profile)) - if bundles: - bundles = [] - [bundles.append(bund.get('name')) \ - for bund in groups.findall('.//Bundle') \ - if bund.get('name') not in bundles \ - and include_bundle(bund.get('name'))] - bundles.sort() - for bundle in bundles: - viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % - (bundle, bundle)) - gseen = [] - for group in egroups: - if group.get('profile', 'false') == 'true': - style = "filled, bold" - else: - style = "filled" - gseen.append(group.get('name')) - if include_group(group.get('name')): - viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % - (group.get('name'), group.get('name'), style, - group.get('color'))) - if bundles: - for bundle in group.findall('Bundle'): - viz_str.append('"group-%s" -> "bundle-%s";' % - (group.get('name'), bundle.get('name'))) - gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' - for group in egroups: - for parent in group.findall('Group'): - if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str.append(gfmt % (parent.get('name'), - parent.get('name'))) - gseen.append(parent.get("name")) - if include_group(group.get('name')): - viz_str.append('"group-%s" -> "group-%s";' % - (group.get('name'), parent.get('name'))) - if key: - for category in categories: - viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % - (category, category, categories[category])) - return "\n".join("\t" + s for s in viz_str) diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py deleted file mode 100644 index d67dac6a6..000000000 --- a/src/lib/Server/Plugins/NagiosGen.py +++ /dev/null @@ -1,151 +0,0 @@ -'''This module implements a Nagios configuration generator''' - -import os -import re -import sys -import glob -import socket -import logging -import lxml.etree - -import Bcfg2.Server.Plugin - -LOGGER = logging.getLogger('Bcfg2.Plugins.NagiosGen') - -line_fmt = '\t%-32s %s' - -class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - - -class NagiosGen(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """NagiosGen is a Bcfg2 plugin that dynamically generates - Nagios configuration file based on Bcfg2 data. - """ - name = 'NagiosGen' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'), - core.fam) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} - - self.client_attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0400'} - self.server_attrib = {'encoding': 'ascii', - 'owner': 'nagios', - 'group': 'nagios', - 'type': 'file', - 'perms': '0440'} - - def createhostconfig(self, entry, metadata): - """Build host specific configuration file.""" - host_address = socket.gethostbyname(metadata.hostname) - host_groups = [grp for grp in metadata.groups - if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] - host_config = ['define host {', - line_fmt % ('host_name', metadata.hostname), - line_fmt % ('alias', metadata.hostname), - line_fmt % ('address', host_address)] - - if host_groups: - host_config.append(line_fmt % ("hostgroups", - ",".join(host_groups))) - - # read the old-style Properties config, but emit a warning. - xtra = dict() - props = None - if (hasattr(metadata, 'Properties') and - 'NagiosGen.xml' in metadata.Properties): - props = metadata.Properties['NagiosGen.xml'].data - if props is not None: - LOGGER.warn("Parsing deprecated Properties/NagiosGen.xml. " - "Update to the new-style config with " - "nagiosgen-convert.py.") - xtra = dict((el.tag, el.text) - for el in props.find(metadata.hostname)) - # hold off on parsing the defaults until we've checked for - # a new-style config - - # read the old-style parents.xml, but emit a warning - pfile = os.path.join(self.data, "parents.xml") - if os.path.exists(pfile): - LOGGER.warn("Parsing deprecated NagiosGen/parents.xml. " - "Update to the new-style config with " - "nagiosgen-convert.py.") - parents = lxml.etree.parse(pfile) - for el in parents.xpath("//Depend[@name='%s']" % metadata.hostname): - if 'parent' in xtra: - xtra['parent'] += "," + el.get("on") - else: - xtra['parent'] = el.get("on") - - # read the new-style config and overwrite the old-style config - for el in self.config.Match(metadata): - if el.tag == 'Option': - xtra[el.get("name")] = el.text - - # if we haven't found anything in the new- or old-style - # configs, finally read defaults from old-style config - if not xtra and props is not None: - xtra = dict((el.tag, el.text) for el in props.find('default')) - - if xtra: - host_config.extend([line_fmt % (opt, val) - for opt, val in list(xtra.items())]) - else: - host_config.append(line_fmt % ('use', 'default')) - - host_config.append('}') - entry.text = "%s\n" % "\n".join(host_config) - [entry.attrib.__setitem__(key, value) - for (key, value) in list(self.client_attrib.items())] - try: - fileh = open("%s/%s-host.cfg" % - (self.data, metadata.hostname), 'w') - fileh.write(entry.text) - fileh.close() - except OSError: - ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/%s-host.cfg" % - (self.data, metadata.hostname)) - LOGGER.error(ioerr) - - def createserverconfig(self, entry, _): - """Build monolithic server configuration file.""" - host_configs = glob.glob('%s/*-host.cfg' % self.data) - group_configs = glob.glob('%s/*-group.cfg' % self.data) - host_data = [] - group_data = [] - for host in host_configs: - host_data.append(open(host, 'r').read()) - - for group in group_configs: - group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) - if "\n".join(host_data).find(group_name) != -1: - groupfile = open(group, 'r') - group_data.append(groupfile.read()) - groupfile.close() - - entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data)) - [entry.attrib.__setitem__(key, value) - for (key, value) in list(self.server_attrib.items())] - try: - fileh = open("%s/nagiosgen.cfg" % self.data, 'w') - fileh.write(entry.text) - fileh.close() - except OSError: - ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/nagiosgen.cfg" % self.data) - LOGGER.error(ioerr) diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Server/Plugins/Ohai.py deleted file mode 100644 index 5fff20d98..000000000 --- a/src/lib/Server/Plugins/Ohai.py +++ /dev/null @@ -1,92 +0,0 @@ -import lxml.etree -import os - -import logging -logger = logging.getLogger('Bcfg2.Plugins.Ohai') - -import Bcfg2.Server.Plugin - -try: - import json -except: - # FIXME: can be removed when server prereq is >= python 2.6 - # necessary for clients without the in-tree json module - try: - import simplejson as json - except: - logger.error("Unable to load any json modules. Make sure " - "python-simplejson is installed.") - raise ImportError - - -probecode = """#!/bin/sh - -export PATH=$PATH:/sbin:/usr/sbin - -if type ohai >& /dev/null; then - ohai -else - # an empty dict, so "'foo' in metadata.Ohai" tests succeed - echo '{}' -fi -""" - -class OhaiCache(object): - def __init__(self, dirname): - self.dirname = dirname - self.cache = dict() - - def __setitem__(self, item, value): - if value == None: - # simply return if the client returned nothing - return - self.cache[item] = json.loads(value) - file("%s/%s.json" % (self.dirname, item), 'w').write(value) - - def __getitem__(self, item): - if item not in self.cache: - try: - data = open("%s/%s.json" % (self.dirname, item)).read() - except: - raise KeyError(item) - self.cache[item] = json.loads(data) - return self.cache[item] - - def __iter__(self): - data = list(self.cache.keys()) - data.extend([x[:-5] for x in os.listdir(self.dirname)]) - return data.__iter__() - - -class Ohai(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Connector): - """The Ohai plugin is used to detect information - about the client operating system. - """ - name = 'Ohai' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', - interpreter='/bin/sh') - self.probe.text = probecode - try: - os.stat(self.data) - except: - os.makedirs(self.data) - self.cache = OhaiCache(self.data) - - def GetProbes(self, meta, force=False): - return [self.probe] - - def ReceiveData(self, meta, datalist): - self.cache[meta.hostname] = datalist[0].text - - def get_additional_data(self, meta): - if meta.hostname in self.cache: - return self.cache[meta.hostname] - return dict() diff --git a/src/lib/Server/Plugins/Packages/Apt.py b/src/lib/Server/Plugins/Packages/Apt.py deleted file mode 100644 index f76bf7fa1..000000000 --- a/src/lib/Server/Plugins/Packages/Apt.py +++ /dev/null @@ -1,139 +0,0 @@ -import re -import gzip -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import Source -from Bcfg2.Bcfg2Py3k import cPickle, file - -class AptCollection(Collection): - def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") - return [] - -class AptSource(Source): - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - ptype = 'deb' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = file(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = file(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) - return rv - else: - return ["%sPackages.gz" % self.rawurl] - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except: - self.logger.error("Packages: Failed to read file %s" % fname) - raise - for line in reader.readlines(): - words = str(line.strip()).split(':', 1) - if words[0] == 'Package': - pkgname = words[1].strip().rstrip() - self.pkgnames.add(pkgname) - bdeps[barch][pkgname] = [] - elif words[0] in depfnames: - vindex = 0 - for dep in words[1].split(','): - if '|' in dep: - cdeps = [re.sub('\s+', '', - re.sub('\(.*\)', '', cdep)) - for cdep in dep.split('|')] - dyn_dname = "choice-%s-%s-%s" % (pkgname, - barch, - vindex) - vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) - bprov[barch][dyn_dname] = set(cdeps) - else: - raw_dep = re.sub('\(.*\)', '', dep) - raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) - elif words[0] == 'Provides': - for pkg in words[1].split(','): - dname = pkg.rstrip().strip() - if dname not in bprov[barch]: - bprov[barch][dname] = set() - bprov[barch][dname].add(pkgname) - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Server/Plugins/Packages/Collection.py deleted file mode 100644 index 32eeda1ec..000000000 --- a/src/lib/Server/Plugins/Packages/Collection.py +++ /dev/null @@ -1,343 +0,0 @@ -import copy -import logging -import Bcfg2.Server.Plugin - -logger = logging.getLogger(__name__) - -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - -# we have to cache Collection objects so that calling Packages.Refresh -# or .Reload can tell the collection objects to clean up their cache, -# but we don't actually use the cache to return a Collection object -# when one is requested, because that prevents new machines from -# working, since a Collection object gets created by -# get_additional_data(), which is called for all clients at server -# startup. (It would also prevent machines that change groups from -# working properly; e.g., if you reinstall a machine with a new OS, -# then returning a cached Collection object would give the wrong -# sources to that client.) -collections = dict() - -class Collection(Bcfg2.Server.Plugin.Debuggable): - def __init__(self, metadata, sources, basepath, debug=False): - """ don't call this directly; use the factory function """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.debug_flag = debug - self.metadata = metadata - self.sources = sources - self.basepath = basepath - self.virt_pkgs = dict() - - try: - self.config = sources[0].config - self.cachepath = sources[0].basepath - self.ptype = sources[0].ptype - except IndexError: - self.config = None - self.cachepath = None - self.ptype = "unknown" - - self.cachefile = None - - @property - def cachekey(self): - return md5(self.get_config()).hexdigest() - - def get_config(self): - self.logger.error("Packages: Cannot generate config for host with " - "multiple source types (%s)" % self.metadata.hostname) - return "" - - def get_relevant_groups(self): - groups = [] - for source in self.sources: - groups.extend(source.get_relevant_groups(self.metadata)) - return sorted(list(set(groups))) - - @property - def basegroups(self): - groups = set() - for source in self.sources: - groups.update(source.basegroups) - return list(groups) - - @property - def cachefiles(self): - cachefiles = set([self.cachefile]) - for source in self.sources: - cachefiles.add(source.cachefile) - return list(cachefiles) - - def get_group(self, group, ptype=None): - for source in self.sources: - pkgs = source.get_group(self.metadata, group, ptype=ptype) - if pkgs: - return pkgs - self.logger.warning("Packages: '%s' is not a valid group" % group) - return [] - - def is_package(self, package): - for source in self.sources: - if source.is_package(self.metadata, package): - return True - return False - - def is_virtual_package(self, package): - for source in self.sources: - if source.is_virtual_package(self.metadata, package): - return True - return False - - def get_deps(self, package): - for source in self.sources: - if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) - return [] - - def get_provides(self, package): - for source in self.sources: - providers = source.get_provides(self.metadata, package) - if providers: - return providers - return [] - - def get_vpkgs(self): - """ get virtual packages """ - vpkgs = dict() - for source in self.sources: - s_vpkgs = source.get_vpkgs(self.metadata) - for name, prov_set in list(s_vpkgs.items()): - if name not in vpkgs: - vpkgs[name] = set(prov_set) - else: - vpkgs[name].update(prov_set) - return vpkgs - - def filter_unknown(self, unknown): - for source in self.sources: - source.filter_unknown(unknown) - - def magic_groups_match(self): - for source in self.sources: - if source.magic_groups_match(self.metadata): - return True - - def build_extra_structures(self, independent): - pass - - def get_additional_data(self): - sdata = [] - for source in self.sources: - sdata.extend(copy.deepcopy(source.url_map)) - return sdata - - def setup_data(self, force_update=False): - """ do any collection-level data setup tasks """ - pass - - def complete(self, packagelist): - '''Build the transitive closure of all package dependencies - - Arguments: - packageslist - set of package names - returns => (set(packages), set(unsatisfied requirements)) - ''' - - # setup vpkg cache - pgrps = tuple(self.get_relevant_groups()) - if pgrps not in self.virt_pkgs: - self.virt_pkgs[pgrps] = self.get_vpkgs() - vpkg_cache = self.virt_pkgs[pgrps] - - # unclassified is set of unsatisfied requirements (may be pkg - # for vpkg) - unclassified = set(packagelist) - vpkgs = set() - both = set() - pkgs = set(packagelist) - - packages = set() - examined = set() - unknown = set() - - final_pass = False - really_done = False - # do while unclassified or vpkgs or both or pkgs - while unclassified or pkgs or both or final_pass: - if really_done: - break - if len(unclassified) + len(pkgs) + len(both) == 0: - # one more pass then exit - really_done = True - - while unclassified: - current = unclassified.pop() - examined.add(current) - is_pkg = False - if self.is_package(current): - is_pkg = True - - is_vpkg = current in vpkg_cache - - if is_pkg and is_vpkg: - both.add(current) - elif is_pkg and not is_vpkg: - pkgs.add(current) - elif is_vpkg and not is_pkg: - vpkgs.add(current) - elif not is_vpkg and not is_pkg: - unknown.add(current) - - while pkgs: - # direct packages; current can be added, and all deps - # should be resolved - current = pkgs.pop() - self.debug_log("Packages: handling package requirement %s" % - current) - packages.add(current) - deps = self.get_deps(current) - newdeps = set(deps).difference(examined) - if newdeps: - self.debug_log("Packages: Package %s added requirements %s" - % (current, newdeps)) - unclassified.update(newdeps) - - satisfied_vpkgs = set() - for current in vpkgs: - # virtual dependencies, satisfied if one of N in the - # config, or can be forced if only one provider - if len(vpkg_cache[current]) == 1: - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, vpkg_cache[current])) - unclassified.update(vpkg_cache[current].difference(examined)) - satisfied_vpkgs.add(current) - else: - satisfiers = [item for item in vpkg_cache[current] - if item in packages] - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, satisfiers)) - satisfied_vpkgs.add(current) - vpkgs.difference_update(satisfied_vpkgs) - - satisfied_both = set() - for current in both: - # packages that are both have virtual providers as - # well as a package with that name. allow use of virt - # through explicit specification, then fall back to - # forcing current on last pass - satisfiers = [item for item in vpkg_cache[current] - if item in packages] - if satisfiers: - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, satisfiers)) - satisfied_both.add(current) - elif current in packagelist or final_pass: - pkgs.add(current) - satisfied_both.add(current) - both.difference_update(satisfied_both) - - if len(unclassified) + len(pkgs) == 0: - final_pass = True - else: - final_pass = False - - self.filter_unknown(unknown) - - return packages, unknown - - def __len__(self): - return len(self.sources) - - def __getitem__(self, item): - return self.sources[item] - - def __setitem__(self, item, value): - self.sources[item] = value - - def __delitem__(self, item): - del self.sources[item] - - def append(self, item): - self.sources.append(item) - - def count(self): - return self.sources.count() - - def index(self, item): - return self.sources.index(item) - - def extend(self, items): - self.sources.extend(items) - - def insert(self, index, item): - self.sources.insert(index, item) - - def pop(self, index=None): - self.sources.pop(index) - - def remove(self, item): - self.sources.remove(item) - - def sort(self, cmp=None, key=None, reverse=False): - self.sources.sort(cmp, key, reverse) - -def clear_cache(): - global collections - collections = dict() - -def factory(metadata, sources, basepath, debug=False): - global collections - - if not sources.loaded: - # if sources.xml has not received a FAM event yet, defer; - # instantiate a dummy Collection object - return Collection(metadata, [], basepath) - - sclasses = set() - relevant = list() - - for source in sources: - if source.applies(metadata): - relevant.append(source) - sclasses.update([source.__class__]) - - if len(sclasses) > 1: - logger.warning("Packages: Multiple source types found for %s: %s" % - ",".join([s.__name__ for s in sclasses])) - cclass = Collection - elif len(sclasses) == 0: - # you'd think this should be a warning, but it happens all the - # freaking time if you have a) machines in your clients.xml - # that do not have the proper groups set up yet (e.g., if you - # have multiple Bcfg2 servers and Packages-relevant groups set - # by probes); and b) templates that query all or multiple - # machines (e.g., with metadata.query.all_clients()) - if debug: - logger.error("Packages: No sources found for %s" % - metadata.hostname) - cclass = Collection - else: - stype = sclasses.pop().__name__.replace("Source", "") - try: - module = \ - getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cclass = getattr(module, "%sCollection" % stype.title()) - except ImportError: - logger.error("Packages: Unknown source type %s" % stype) - except AttributeError: - logger.warning("Packages: No collection class found for %s sources" - % stype) - - if debug: - logger.error("Packages: Using %s for Collection of sources for %s" % - (cclass.__name__, metadata.hostname)) - - collection = cclass(metadata, relevant, basepath, debug=debug) - collections[metadata.hostname] = collection - return collection - diff --git a/src/lib/Server/Plugins/Packages/Pac.py b/src/lib/Server/Plugins/Packages/Pac.py deleted file mode 100644 index 9db6b0535..000000000 --- a/src/lib/Server/Plugins/Packages/Pac.py +++ /dev/null @@ -1,120 +0,0 @@ -import gzip -import tarfile -from Bcfg2.Bcfg2Py3k import cPickle, file -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import Source - -class PacCollection(Collection): - def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") - return [] - -class PacSource(Source): - basegroups = ['arch', 'parabola'] - ptype = 'pacman' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = file(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = file(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) - return rv - else: - raise Exception("PacSource : RAWUrl not supported (yet)") - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - - for fname in self.files: - if not self.rawurl: - barch = [x for x in fname.split('@') if x in self.arches][0] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] - - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - self.debug_log("Packages: try to read %s" % fname) - tar = tarfile.open(fname, "r") - reader = gzip.GzipFile(fname) - except: - self.logger.error("Packages: Failed to read file %s" % fname) - raise - - for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - self.debug_log("Packages: added %s" % - tarinfo.name.rsplit("-", 2)[0]) - tar.close() - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py deleted file mode 100644 index 7950f15e6..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesConfig.py +++ /dev/null @@ -1,15 +0,0 @@ -import Bcfg2.Server.Plugin - -class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - - def Index(self): - """ Build local data structures """ - Bcfg2.Server.Plugin.SimpleConfig.Index(self) - - if hasattr(self.plugin, "sources") and self.plugin.sources.loaded: - # only reload Packages plugin if sources have been loaded. - # otherwise, this is getting called on server startup, and - # we have to wait until all sources have been indexed - # before we can call Packages.Reload() - self.plugin.Reload() diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Server/Plugins/Packages/PackagesSources.py deleted file mode 100644 index da79c00e9..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import sys -import lxml.etree -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Packages.Source import SourceInitError - -class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): - __identifier__ = None - - def __init__(self, filename, cachepath, fam, packages, config): - Bcfg2.Server.Plugin.Debuggable.__init__(self) - try: - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, - filename, - fam) - except OSError: - err = sys.exc_info()[1] - msg = "Packages: Failed to read configuration file: %s" % err - if not os.path.exists(self.name): - msg += " Have you created it?" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - self.cachepath = cachepath - self.config = config - if not os.path.exists(self.cachepath): - # create cache directory if needed - try: - os.makedirs(self.cachepath) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Could not create Packages cache at %s: %s" % - (self.cachepath, err)) - self.pkg_obj = packages - self.parsed = set() - self.loaded = False - - def toggle_debug(self): - Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) - for source in self.entries: - source.toggle_debug() - - def HandleEvent(self, event=None): - Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) - if event.filename != self.name: - self.parsed.add(os.path.basename(event.filename)) - - if sorted(list(self.parsed)) == sorted(self.extras): - self.logger.info("Reloading Packages plugin") - self.pkg_obj.Reload() - self.loaded = True - - def Index(self): - Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) - self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) - - def source_from_xml(self, xsource): - """ create a *Source object from its XML representation in - sources.xml """ - stype = xsource.get("type") - if stype is None: - self.logger.error("Packages: No type specified for source, " - "skipping") - return None - - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - self.logger.error("Packages: Unknown source type %s" % stype) - return None - - try: - source = cls(self.cachepath, xsource, self.config) - except SourceInitError: - err = sys.exc_info()[1] - self.logger.error("Packages: %s" % err) - source = None - - return source - - def __getitem__(self, key): - return self.entries[key] - - def __repr__(self): - return "PackagesSources: %s" % repr(self.entries) - - def __str__(self): - return "PackagesSources: %s" % str(self.entries) diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Server/Plugins/Packages/Source.py deleted file mode 100644 index 1dfeecc40..000000000 --- a/src/lib/Server/Plugins/Packages/Source.py +++ /dev/null @@ -1,278 +0,0 @@ -import os -import re -import sys -import base64 -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ - urlopen, file, cPickle - -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - -def fetch_url(url): - if '@' in url: - mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) - if not mobj: - raise ValueError - user = mobj.group(2) - passwd = mobj.group(3) - url = mobj.group(1) + mobj.group(4) - auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) - auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() - - -class SourceInitError(Exception): - pass - - -class Source(Bcfg2.Server.Plugin.Debuggable): - mrepo_re = re.compile(r'/RPMS\.([^/]+)') - pulprepo_re = re.compile(r'pulp/repos/([^/]+)') - genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$') - basegroups = [] - - def __init__(self, basepath, xsource, config): - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.basepath = basepath - self.xsource = xsource - self.config = config - - try: - self.version = xsource.find('Version').text - except AttributeError: - pass - - for key, tag in [('components', 'Component'), ('arches', 'Arch'), - ('blacklist', 'Blacklist'), - ('whitelist', 'Whitelist')]: - self.__dict__[key] = [item.text for item in xsource.findall(tag)] - - self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] - - self.recommended = xsource.get('recommended', 'false').lower() == 'true' - - self.rawurl = xsource.get('rawurl', '') - if self.rawurl and not self.rawurl.endswith("/"): - self.rawurl += "/" - self.url = xsource.get('url', '') - if self.url and not self.url.endswith("/"): - self.url += "/" - self.version = xsource.get('version', '') - - # build the set of conditions to see if this source applies to - # a given set of metadata - self.conditions = [] - self.groups = [] # provided for some limited backwards compat - for el in xsource.iterancestors(): - if el.tag == "Group": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") not in m.groups) - else: - self.groups.append(el.get("name")) - self.conditions.append(lambda m, el=el: - el.get("name") in m.groups) - elif el.tag == "Client": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") != m.hostname) - else: - self.conditions.append(lambda m, el=el: - el.get("name") == m.hostname) - - self.deps = dict() - self.provides = dict() - - self.cachefile = os.path.join(self.basepath, - "cache-%s" % self.cachekey) - self.url_map = [] - - @property - def cachekey(self): - return md5(cPickle.dumps([self.version, self.components, self.url, - self.rawurl, self.arches])).hexdigest() - - def get_relevant_groups(self, metadata): - return sorted(list(set([g for g in metadata.groups - if (g in self.basegroups or - g in self.groups or - g in self.arches)]))) - - def load_state(self): - pass - - def setup_data(self, force_update=False): - should_read = True - should_download = False - if os.path.exists(self.cachefile): - try: - self.load_state() - should_read = False - except: - self.logger.error("Packages: Cachefile %s load failed; " - "falling back to file read" % self.cachefile) - if should_read: - try: - self.read_files() - except: - self.logger.error("Packages: File read failed; " - "falling back to file download") - should_download = True - - if should_download or force_update: - try: - self.update() - self.read_files() - except: - self.logger.error("Packages: Failed to load data for Source " - "of %s. Some Packages will be missing." % - self.urls) - - def get_repo_name(self, url_map): - # try to find a sensible name for a repo - if url_map['component']: - rname = url_map['component'] - else: - name = None - for repo_re in (self.mrepo_re, - self.pulprepo_re, - self.genericrepo_re): - match = repo_re.search(url_map['url']) - if match: - break - if name is None: - # couldn't figure out the name from the URL or URL map - # (which probably means its a screwy URL), so we just - # generate a random one - name = base64.b64encode(os.urandom(16))[:-2] - rname = "%s-%s" % (self.groups[0], name) - # see yum/__init__.py in the yum source, lines 441-449, for - # the source of this regex. yum doesn't like anything but - # string.ascii_letters, string.digits, and [-_.:]. There - # doesn't seem to be a reason for this, because yum. - return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname) - - def __str__(self): - if self.rawurl: - return "%s at %s" % (self.__class__.__name__, self.rawurl) - elif self.url: - return "%s at %s" % (self.__class__.__name__, self.url) - else: - return self.__class__.__name__ - - def get_urls(self): - return [] - urls = property(get_urls) - - def get_files(self): - return [self.escape_url(url) for url in self.urls] - files = property(get_files) - - def get_vpkgs(self, metadata): - agroups = ['global'] + [a for a in self.arches - if a in metadata.groups] - vdict = dict() - for agrp in agroups: - for key, value in list(self.provides[agrp].items()): - if key not in vdict: - vdict[key] = set(value) - else: - vdict[key].update(value) - return vdict - - def is_virtual_package(self, metadata, package): - """ called to determine if a package is a virtual package. - this is only invoked if the package is not listed in the dict - returned by get_vpkgs """ - return False - - def escape_url(self, url): - return os.path.join(self.basepath, url.replace('/', '@')) - - def file_init(self): - pass - - def read_files(self): - pass - - def filter_unknown(self, unknown): - pass - - def update(self): - for url in self.urls: - self.logger.info("Packages: Updating %s" % url) - fname = self.escape_url(url) - try: - data = fetch_url(url) - file(fname, 'w').write(data) - except ValueError: - self.logger.error("Packages: Bad url string %s" % url) - raise - except HTTPError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to fetch url %s. HTTP " - "response code=%s" % (url, err.code)) - raise - - def applies(self, metadata): - # check base groups - if not self.magic_groups_match(metadata): - return False - - # check Group/Client tags from sources.xml - for condition in self.conditions: - if not condition(metadata): - return False - - return True - - def get_arches(self, metadata): - return ['global'] + [a for a in self.arches if a in metadata.groups] - - def get_deps(self, metadata, pkgname): - for arch in self.get_arches(metadata): - if pkgname in self.deps[arch]: - return self.deps[arch][pkgname] - return [] - - def get_provides(self, metadata, required): - for arch in self.get_arches(metadata): - if required in self.provides[arch]: - return self.provides[arch][required] - return [] - - def is_package(self, metadata, _): - return False - - def get_package(self, metadata, package): - return package - - def get_group(self, metadata, group, ptype=None): - return [] - - def magic_groups_match(self, metadata): - """ check to see if this source applies to the given host - metadata by checking 'magic' (base) groups only, or if magic - groups are off """ - # we always check that arch matches - found_arch = False - for arch in self.arches: - if arch in metadata.groups: - found_arch = True - break - if not found_arch: - return False - - if self.config.getboolean("global", "magic_groups", - default=True) == False: - return True - else: - for group in self.basegroups: - if group in metadata.groups: - return True - return False diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py deleted file mode 100644 index e13b28251..000000000 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ /dev/null @@ -1,688 +0,0 @@ -import os -import sys -import time -import copy -import glob -import socket -import random -import logging -import threading -import lxml.etree -from UserDict import DictMixin -from subprocess import Popen, PIPE, STDOUT -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ - fetch_url - -logger = logging.getLogger(__name__) - -try: - from pulp.client.consumer.config import ConsumerConfig - from pulp.client.api.repository import RepositoryAPI - from pulp.client.api.consumer import ConsumerAPI - from pulp.client.api import server - has_pulp = True -except ImportError: - has_pulp = False - -try: - import yum - has_yum = True -except ImportError: - has_yum = False - logger.info("Packages: No yum libraries found; forcing use of internal " - "dependency resolver") - -try: - import json -except ImportError: - import simplejson as json - -XP = '{http://linux.duke.edu/metadata/common}' -RP = '{http://linux.duke.edu/metadata/rpm}' -RPO = '{http://linux.duke.edu/metadata/repo}' -FL = '{http://linux.duke.edu/metadata/filelists}' - -PULPSERVER = None -PULPCONFIG = None - - -def _setup_pulp(config): - global PULPSERVER, PULPCONFIG - if not has_pulp: - logger.error("Packages: Cannot create Pulp collection: Pulp libraries not " - "found") - raise Bcfg2.Server.Plugin.PluginInitError - - if PULPSERVER is None: - try: - username = config.get("pulp", "username") - password = config.get("pulp", "password") - except ConfigParser.NoSectionError: - logger.error("Packages: No [pulp] section found in Packages/packages.conf") - raise Bcfg2.Server.Plugin.PluginInitError - except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - logger.error("Packages: Required option not found in " - "Packages/packages.conf: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - - PULPCONFIG = ConsumerConfig() - serveropts = PULPCONFIG.server - - PULPSERVER = server.PulpServer(serveropts['host'], - int(serveropts['port']), - serveropts['scheme'], - serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) - server.set_active_server(PULPSERVER) - return PULPSERVER - - -class YumCollection(Collection): - # options that are included in the [yum] section but that should - # not be included in the temporary yum.conf we write out - option_blacklist = ["use_yum_libraries", "helper"] - - def __init__(self, metadata, sources, basepath, debug=False): - Collection.__init__(self, metadata, sources, basepath, debug=debug) - self.keypath = os.path.join(self.basepath, "keys") - - if len(sources): - config = sources[0].config - self.use_yum = has_yum and config.getboolean("yum", - "use_yum_libraries", - default=False) - else: - self.use_yum = False - - if self.use_yum: - self.cachefile = os.path.join(self.cachepath, - "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) - - self.configdir = os.path.join(self.basepath, "yum") - if not os.path.exists(self.configdir): - os.mkdir(self.configdir) - self.cfgfile = os.path.join(self.configdir, - "%s-yum.conf" % self.cachekey) - self.write_config() - - self.helper = self.config.get("yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") - if has_pulp: - _setup_pulp(self.config) - - def write_config(self): - if not os.path.exists(self.cfgfile): - yumconf = self.get_config(raw=True) - yumconf.add_section("main") - - mainopts = dict(cachedir=self.cachefile, - keepcache="0", - sslverify="0", - debuglevel="0", - reposdir="/dev/null") - try: - for opt in self.config.options("yum"): - if opt not in self.option_blacklist: - mainopts[opt] = self.config.get("yum", opt) - except ConfigParser.NoSectionError: - pass - - for opt, val in list(mainopts.items()): - yumconf.set("main", opt, val) - - yumconf.write(open(self.cfgfile, 'w')) - - def get_config(self, raw=False): - config = ConfigParser.SafeConfigParser() - for source in self.sources: - # get_urls() loads url_map as a side-effect - source.get_urls() - for url_map in source.url_map: - if url_map['arch'] in self.metadata.groups: - basereponame = source.get_repo_name(url_map) - reponame = basereponame - - added = False - while not added: - try: - config.add_section(reponame) - added = True - except ConfigParser.DuplicateSectionError: - match = re.match("-(\d)", reponame) - if match: - rid = int(match.group(1)) + 1 - else: - rid = 1 - reponame = "%s-%d" % (basereponame, rid) - - config.set(reponame, "name", reponame) - config.set(reponame, "baseurl", url_map['url']) - config.set(reponame, "enabled", "1") - if len(source.gpgkeys): - config.set(reponame, "gpgcheck", "1") - config.set(reponame, "gpgkey", - " ".join(source.gpgkeys)) - else: - config.set(reponame, "gpgcheck", "0") - - if len(source.blacklist): - config.set(reponame, "exclude", - " ".join(source.blacklist)) - if len(source.whitelist): - config.set(reponame, "includepkgs", - " ".join(source.whitelist)) - - if raw: - return config - else: - # configparser only writes to file, so we have to use a - # StringIO object to get the data out as a string - buf = StringIO() - config.write(buf) - return "# This config was generated automatically by the Bcfg2 " \ - "Packages plugin\n\n" + buf.getvalue() - - def build_extra_structures(self, independent): - """ build list of gpg keys to be added to the specification by - validate_structures() """ - needkeys = set() - for source in self.sources: - for key in source.gpgkeys: - needkeys.add(key) - - if len(needkeys): - if has_yum: - # this must be be has_yum, not use_yum, because - # regardless of whether the user wants to use the yum - # resolver we want to include gpg key data - keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", - type=self.ptype, origin='Packages') - else: - self.logger.warning("GPGKeys were specified for yum sources in " - "sources.xml, but no yum libraries were " - "found") - self.logger.warning("GPG key version/release data cannot be " - "determined automatically") - self.logger.warning("Install yum libraries, or manage GPG keys " - "manually") - keypkg = None - - for key in needkeys: - # figure out the path of the key on the client - keydir = self.config.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") - remotekey = os.path.join(keydir, os.path.basename(key)) - localkey = os.path.join(self.keypath, os.path.basename(key)) - kdata = open(localkey).read() - - # copy the key to the client - keypath = lxml.etree.Element("BoundPath", name=remotekey, - encoding='ascii', - owner='root', group='root', - type='file', perms='0644', - important='true') - keypath.text = kdata - - # hook to add version/release info if possible - self._add_gpg_instances(keypkg, kdata, localkey, remotekey) - independent.append(keypath) - if keypkg is not None: - independent.append(keypkg) - - # see if there are any pulp sources to handle - has_pulp_sources = False - for source in self.sources: - if source.pulp_id: - has_pulp_sources = True - break - - if has_pulp_sources: - consumerapi = ConsumerAPI() - consumer = self._get_pulp_consumer(consumerapi=consumerapi) - if consumer is None: - consumer = consumerapi.create(self.metadata.hostname, - self.metadata.hostname) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-update", timing="pre", - when="always", status="check", - command="pulp-consumer consumer update") - - for source in self.sources: - # each pulp source can only have one arch, so we don't - # have to check the arch in url_map - if (source.pulp_id and - source.pulp_id not in consumer['repoids']): - consumerapi.bind(self.metadata.hostname, source.pulp_id) - - crt = lxml.etree.SubElement(independent, "BoundPath", - name="/etc/pki/consumer/cert.pem", - type="file", owner="root", - group="root", perms="0644") - crt.text = consumerapi.certificate(self.metadata.hostname) - - def _get_pulp_consumer(self, consumerapi=None): - if consumerapi is None: - consumerapi = ConsumerAPI() - consumer = None - try: - consumer = consumerapi.consumer(self.metadata.hostname) - except server.ServerRequestError: - # consumer does not exist - pass - except socket.error: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not contact Pulp server: %s" % - err) - except: - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown error querying Pulp server: %s" - % err) - return consumer - - def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey): - """ add gpg keys to the specification to ensure they get - installed """ - # this must be be has_yum, not use_yum, because regardless of - # whether the user wants to use the yum resolver we want to - # include gpg key data - if not has_yum: - return - - try: - kinfo = yum.misc.getgpgkeyinfo(keydata) - version = yum.misc.keyIdToRPMVer(kinfo['keyid']) - release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - - lxml.etree.SubElement(keyentry, 'Instance', - version=version, - release=release, - simplefile=remotekey) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not read GPG key %s: %s" % - (localkey, err)) - - def is_package(self, package): - if not self.use_yum: - return Collection.is_package(self, package) - elif isinstance(package, tuple): - if package[1] is None and package[2] == (None, None, None): - package = package[0] - else: - return None - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_package", package) - - def is_virtual_package(self, package): - if not self.use_yum: - return Collection.is_virtual_package(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_virtual_package", package) - - def get_deps(self, package): - if not self.use_yum: - return Collection.get_deps(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_deps", package) - - def get_provides(self, required, all=False, silent=False): - if not self.use_yum: - return Collection.get_provides(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_provides", package) - - def get_group(self, group, ptype="default"): - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported by " - "Bcfg2's internal Yum dependency generator") - return [] - - if group.startswith("@"): - group = group[1:] - - pkgs = self.call_helper("get_group", dict(group=group, type=ptype)) - return pkgs - - def complete(self, packagelist): - if not self.use_yum: - return Collection.complete(self, packagelist) - - packages = set() - unknown = set(packagelist) - - if unknown: - result = \ - self.call_helper("complete", - dict(packages=list(unknown), - groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - - self.filter_unknown(unknown) - - return packages, unknown - - def call_helper(self, command, input=None): - """ Make a call to bcfg2-yum-helper. The yum libs have - horrific memory leaks, so apparently the right way to get - around that in long-running processes it to have a short-lived - helper. No, seriously -- check out the yum-updatesd code. - It's pure madness. """ - # it'd be nice if we could change this to be more verbose if - # -v was given to bcfg2-server, but Collection objects don't - # get the 'setup' variable, so we don't know how verbose - # bcfg2-server is. It'd also be nice if we could tell yum to - # log to syslog. So would a unicorn. - cmd = [self.helper, "-c", self.cfgfile] - if self.debug_flag: - cmd.append("-v") - cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd)) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None - - if input: - idata = json.dumps(input) - (stdout, stderr) = helper.communicate(idata) - else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - if rv: - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) - elif self.debug_flag: - self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr) - try: - return json.loads(stdout) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) - return None - - def setup_data(self, force_update=False): - if not self.use_yum: - return Collection.setup_data(self, force_update) - - if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config - self.call_helper("clean") - - os.unlink(self.cfgfile) - self.write_config() - - if force_update: - self.call_helper("clean") - - -class YumSource(Source): - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - ptype = 'yum' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pulp_id = None - if has_pulp and xsource.get("pulp_id"): - self.pulp_id = xsource.get("pulp_id") - - _setup_pulp(self.config) - repoapi = RepositoryAPI() - try: - self.repo = repoapi.repository(self.pulp_id) - self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key) - for key in repoapi.listkeys(self.pulp_id)] - except server.ServerRequestError: - err = sys.exc_info()[1] - if err[0] == 401: - msg = "Packages: Error authenticating to Pulp: %s" % err[1] - elif err[0] == 404: - msg = "Packages: Pulp repo id %s not found: %s" % \ - (self.pulp_id, err[1]) - else: - msg = "Packages: Error %d fetching pulp repo %s: %s" % \ - (err[0], self.pulp_id, err[1]) - raise SourceInitError(msg) - except socket.error: - err = sys.exc_info()[1] - raise SourceInitError("Could not contact Pulp server: %s" % err) - except: - err = sys.exc_info()[1] - raise SourceInitError("Unknown error querying Pulp server: %s" % - err) - self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], - self.repo['relative_path']) - self.arches = [self.repo['arch']] - - if not self.rawurl: - self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" - else: - self.baseurl = self.rawurl - self.packages = dict() - self.deps = dict([('global', dict())]) - self.provides = dict([('global', dict())]) - self.filemap = dict([(x, dict()) - for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - - self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries", - default=False) - - def save_state(self): - if not self.use_yum: - cache = file(self.cachefile, 'wb') - cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) - cache.close() - - - def load_state(self): - if not self.use_yum: - data = file(self.cachefile) - (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) - - def get_urls(self): - surls = list() - self.url_map = [] - for arch in self.arches: - if self.url: - usettings = [{'version':self.version, 'component':comp, - 'arch':arch} - for comp in self.components] - else: # rawurl given - usettings = [{'version':self.version, 'component':None, - 'arch':arch}] - - for setting in usettings: - setting['url'] = self.baseurl % setting - self.url_map.append(copy.deepcopy(setting)) - surls.append((arch, [setting['url'] for setting in usettings])) - urls = [] - for (sarch, surl_list) in surls: - for surl in surl_list: - urls.extend(self._get_urls_from_repodata(surl, sarch)) - return urls - urls = property(get_urls) - - def _get_urls_from_repodata(self, url, arch): - if self.use_yum: - return [url] - - rmdurl = '%srepodata/repomd.xml' % url - try: - repomd = fetch_url(rmdurl) - xdata = lxml.etree.XML(repomd) - except ValueError: - self.logger.error("Packages: Bad url string %s" % rmdurl) - return [] - except HTTPError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to fetch url %s. code=%s" % - (rmdurl, err.code)) - return [] - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to process metadata at %s: %s" % - (rmdurl, err)) - return [] - - urls = [] - for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: - floc = elt.find(RPO + 'location') - fullurl = url + floc.get('href') - urls.append(fullurl) - self.file_to_arch[self.escape_url(fullurl)] = arch - return urls - - def read_files(self): - # we have to read primary.xml first, and filelists.xml afterwards; - primaries = list() - filelists = list() - for fname in self.files: - if fname.endswith('primary.xml.gz'): - primaries.append(fname) - elif fname.endswith('filelists.xml.gz'): - filelists.append(fname) - - for fname in primaries: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_primary(fdata, farch) - for fname in filelists: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_filelist(fdata, farch) - - # merge data - sdata = list(self.packages.values()) - try: - self.packages['global'] = copy.deepcopy(sdata.pop()) - except IndexError: - logger.error("Packages: No packages in repo") - while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) - - for key in self.packages: - if key == 'global': - continue - self.packages[key] = \ - self.packages[key].difference(self.packages['global']) - self.save_state() - - def parse_filelist(self, data, arch): - if arch not in self.filemap: - self.filemap[arch] = dict() - for pkg in data.findall(FL + 'package'): - for fentry in pkg.findall(FL + 'file'): - if fentry.text in self.needed_paths: - if fentry.text in self.filemap[arch]: - self.filemap[arch][fentry.text].add(pkg.get('name')) - else: - self.filemap[arch][fentry.text] = \ - set([pkg.get('name')]) - - def parse_primary(self, data, arch): - if arch not in self.packages: - self.packages[arch] = set() - if arch not in self.deps: - self.deps[arch] = dict() - if arch not in self.provides: - self.provides[arch] = dict() - for pkg in data.getchildren(): - if not pkg.tag.endswith('package'): - continue - pkgname = pkg.find(XP + 'name').text - self.packages[arch].add(pkgname) - - pdata = pkg.find(XP + 'format') - self.deps[arch][pkgname] = set() - pre = pdata.find(RP + 'requires') - if pre is not None: - for entry in pre.getchildren(): - self.deps[arch][pkgname].add(entry.get('name')) - if entry.get('name').startswith('/'): - self.needed_paths.add(entry.get('name')) - pro = pdata.find(RP + 'provides') - if pro != None: - for entry in pro.getchildren(): - prov = entry.get('name') - if prov not in self.provides[arch]: - self.provides[arch][prov] = list() - self.provides[arch][prov].append(pkgname) - - def is_package(self, metadata, item): - arch = [a for a in self.arches if a in metadata.groups] - if not arch: - return False - return ((item in self.packages['global'] or - item in self.packages[arch[0]]) and - item not in self.blacklist and - (len(self.whitelist) == 0 or item in self.whitelist)) - - def get_vpkgs(self, metadata): - if self.use_yum: - return dict() - - rv = Source.get_vpkgs(self, metadata) - for arch, fmdata in list(self.filemap.items()): - if arch not in metadata.groups and arch != 'global': - continue - for filename, pkgs in list(fmdata.items()): - rv[filename] = pkgs - return rv - - def filter_unknown(self, unknown): - if self.use_yum: - filtered = set() - for unk in unknown: - try: - if unk.startswith('rpmlib'): - filtered.update(unk) - except AttributeError: - try: - if unk[0].startswith('rpmlib'): - filtered.update(unk) - except (IndexError, AttributeError): - pass - else: - filtered = set([u for u in unknown if u.startswith('rpmlib')]) - unknown.difference_update(filtered) - - def setup_data(self, force_update=False): - if not self.use_yum: - Source.setup_data(self, force_update=force_update) - - def get_repo_name(self, url_map): - if self.pulp_id: - return self.pulp_id - else: - return Source.get_repo_name(self, url_map) diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Server/Plugins/Packages/__init__.py deleted file mode 100644 index da5832e90..000000000 --- a/src/lib/Server/Plugins/Packages/__init__.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import sys -import time -import copy -import glob -import shutil -import lxml.etree -import Bcfg2.Logger -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen -from Bcfg2.Server.Plugins.Packages import Collection -from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources -from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig - -class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.StructureValidator, - Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.Connector): - name = 'Packages' - conflicts = ['Pkgmgr'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.StructureValidator.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) - - self.sentinels = set() - self.cachepath = os.path.join(self.data, 'cache') - self.keypath = os.path.join(self.data, 'keys') - if not os.path.exists(self.keypath): - # create key directory if needed - os.makedirs(self.keypath) - - # set up config files - self.config = PackagesConfig(self) - self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self, - self.config) - - def toggle_debug(self): - Bcfg2.Server.Plugin.Plugin.toggle_debug(self) - self.sources.toggle_debug() - - @property - def disableResolver(self): - try: - return not self.config.getboolean("global", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.config.get("global", "resolver", - default="enabled").lower() == "disabled" - - @property - def disableMetaData(self): - try: - return not self.config.getboolean("global", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.config.get("global", "metadata", - default="enabled").lower() == "disabled" - - def create_config(self, entry, metadata): - """ create yum/apt config for the specified host """ - attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} - - collection = self._get_collection(metadata) - entry.text = collection.get_config() - for (key, value) in list(attrib.items()): - entry.attrib.__setitem__(key, value) - - def HandleEntry(self, entry, metadata): - if entry.tag == 'Package': - collection = self._get_collection(metadata) - entry.set('version', 'auto') - entry.set('type', collection.ptype) - elif entry.tag == 'Path': - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): - self.create_config(entry, metadata) - - def HandlesEntry(self, entry, metadata): - if entry.tag == 'Package': - if self.config.getboolean("global", "magic_groups", - default=True) == True: - collection = self._get_collection(metadata) - if collection.magic_groups_match(): - return True - else: - return True - elif entry.tag == 'Path': - # managed entries for yum/apt configs - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): - return True - return False - - def validate_structures(self, metadata, structures): - '''Ensure client configurations include all needed prerequisites - - Arguments: - metadata - client metadata instance - structures - a list of structure-stage entry combinations - ''' - collection = self._get_collection(metadata) - indep = lxml.etree.Element('Independent') - self._build_packages(metadata, indep, structures, - collection=collection) - collection.build_extra_structures(indep) - structures.append(indep) - - def _build_packages(self, metadata, independent, structures, - collection=None): - """ build list of packages that need to be included in the - specification by validate_structures() """ - if self.disableResolver: - # Config requests no resolver - return - - if collection is None: - collection = self._get_collection(metadata) - # initial is the set of packages that are explicitly specified - # in the configuration - initial = set() - # base is the set of initial packages with groups expanded - base = set() - to_remove = [] - for struct in structures: - for pkg in struct.xpath('//Package | //BoundPackage'): - if pkg.get("name"): - initial.add(pkg.get("name")) - elif pkg.get("group"): - try: - if pkg.get("type"): - gpkgs = collection.get_group(pkg.get("group"), - ptype=pkg.get("type")) - else: - gpkgs = collection.get_group(pkg.get("group")) - base.update(gpkgs) - except TypeError: - raise - self.logger.error("Could not resolve group %s" % - pkg.get("group")) - to_remove.append(pkg) - else: - self.logger.error("Packages: Malformed Package: %s" % - lxml.etree.tostring(pkg)) - base.update(initial) - for el in to_remove: - el.getparent().remove(el) - - packages, unknown = collection.complete(base) - if unknown: - self.logger.info("Packages: Got %d unknown entries" % len(unknown)) - self.logger.info("Packages: %s" % list(unknown)) - newpkgs = list(packages.difference(initial)) - self.debug_log("Packages: %d initial, %d complete, %d new" % - (len(initial), len(packages), len(newpkgs))) - newpkgs.sort() - for pkg in newpkgs: - lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version='auto', type=collection.ptype, - origin='Packages') - - def Refresh(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and download sources\n''' - self._load_config(force_update=True) - return True - - def Reload(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and sources\n''' - self._load_config() - return True - - def _load_config(self, force_update=False): - ''' - Load the configuration data and setup sources - - Keyword args: - force_update Force downloading repo data - ''' - self._load_sources(force_update) - self._load_gpg_keys(force_update) - - def _load_sources(self, force_update): - """ Load sources from the config """ - self.sentinels = set() - cachefiles = set() - - for collection in list(Collection.collections.values()): - cachefiles.update(collection.cachefiles) - if not self.disableMetaData: - collection.setup_data(force_update) - self.sentinels.update(collection.basegroups) - - Collection.clear_cache() - - for source in self.sources: - cachefiles.add(source.cachefile) - if not self.disableMetaData: - source.setup_data(force_update) - - for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): - if cfile not in cachefiles: - try: - if os.path.isdir(cfile): - shutil.rmtree(cfile) - else: - os.unlink(cfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not remove cache file " - "%s: %s" % (cfile, err)) - - def _load_gpg_keys(self, force_update): - """ Load gpg keys from the config """ - keyfiles = [] - keys = [] - for source in self.sources: - for key in source.gpgkeys: - localfile = os.path.join(self.keypath, - os.path.basename(key.rstrip("/"))) - if localfile not in keyfiles: - keyfiles.append(localfile) - if ((force_update and key not in keys) or - not os.path.exists(localfile)): - self.logger.info("Packages: Downloading and parsing %s" % key) - response = urlopen(key) - open(localfile, 'w').write(response.read()) - keys.append(key) - - for kfile in glob.glob(os.path.join(self.keypath, "*")): - if kfile not in keyfiles: - os.unlink(kfile) - - def _get_collection(self, metadata): - return Collection.factory(metadata, self.sources, self.data, - debug=self.debug_flag) - - def get_additional_data(self, metadata): - collection = self._get_collection(metadata) - return dict(sources=collection.get_additional_data()) diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py deleted file mode 100644 index e9254cdcc..000000000 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ /dev/null @@ -1,169 +0,0 @@ -'''This module implements a package management scheme for all images''' - -import logging -import re -import Bcfg2.Server.Plugin -import lxml -try: - set -except NameError: - from sets import Set as set - -logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') - - -class FuzzyDict(dict): - fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)') - - def __getitem__(self, key): - if isinstance(key, str): - mdata = self.fuzzy.match(key) - if mdata: - return dict.__getitem__(self, mdata.groupdict()['name']) - else: - print("got non-string key %s" % str(key)) - return dict.__getitem__(self, key) - - def has_key(self, key): - if isinstance(key, str): - mdata = self.fuzzy.match(key) - if self.fuzzy.match(key): - return dict.has_key(self, mdata.groupdict()['name']) - return dict.has_key(self, key) - - def get(self, key, default=None): - try: - return self.__getitem__(key) - except: - if default: - return default - raise - - -class PNode(Bcfg2.Server.Plugin.INode): - """PNode has a list of packages available at a - particular group intersection. - """ - splitters = {'rpm': re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ - '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'), - 'encap': re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')} - ignore = ['Package'] - - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = FuzzyDict() - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data) - - def __init__(self, data, pdict, parent=None): - # copy local attributes to all child nodes if no local attribute exists - if 'Package' not in pdict: - pdict['Package'] = set() - for child in data.getchildren(): - attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) - for attr in attrs: - try: - child.set(attr, data.get(attr)) - except: - # don't fail on things like comments and other immutable elements - pass - Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) - if 'Package' not in self.contents: - self.contents['Package'] = FuzzyDict() - for pkg in data.findall('./Package'): - if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: - pdict['Package'].add(pkg.get('name')) - if pkg.get('name') != None: - self.contents['Package'][pkg.get('name')] = {} - if pkg.getchildren(): - self.contents['Package'][pkg.get('name')]['__children__'] \ - = pkg.getchildren() - if 'simplefile' in pkg.attrib: - pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) - self.contents['Package'][pkg.get('name')].update(pkg.attrib) - else: - if 'file' in pkg.attrib: - if 'multiarch' in pkg.attrib: - archs = pkg.get('multiarch').split() - srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), - pkg.get('file') % {'src':srcs[idx], - 'arch':archs[idx]}) - for idx in range(len(archs))]) - pkg.set('url', url) - else: - pkg.set('url', '%s/%s' % (pkg.get('uri'), - pkg.get('file'))) - if pkg.get('type') in self.splitters and pkg.get('file') != None: - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) - if not mdata: - logger.error("Failed to match pkg %s" % pkg.get('file')) - continue - pkgname = mdata.group('name') - self.contents['Package'][pkgname] = mdata.groupdict() - self.contents['Package'][pkgname].update(pkg.attrib) - if pkg.attrib.get('file'): - self.contents['Package'][pkgname]['url'] = pkg.get('url') - self.contents['Package'][pkgname]['type'] = pkg.get('type') - if pkg.get('verify'): - self.contents['Package'][pkgname]['verify'] = pkg.get('verify') - if pkg.get('multiarch'): - self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch') - if pkgname not in pdict['Package']: - pdict['Package'].add(pkgname) - if pkg.getchildren(): - self.contents['Package'][pkgname]['__children__'] = pkg.getchildren() - else: - self.contents['Package'][pkg.get('name')].update(pkg.attrib) - - -class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that - returns matching package entries. - """ - __node__ = PNode - __cacheobj__ = FuzzyDict - - -class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): - """This is a generator that handles package assignments.""" - name = 'Pkgmgr' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = PkgSrc - __element__ = 'Package' - - def HandleEvent(self, event): - '''Handle events and update dispatch table''' - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = FuzzyDict([(child, - self.BindEntry)]) - - def BindEntry(self, entry, metadata): - """Bind data for entry, and remove instances that are not requested.""" - pname = entry.get('name') - Bcfg2.Server.Plugin.PrioDir.BindEntry(self, entry, metadata) - if entry.findall('Instance'): - mdata = FuzzyDict.fuzzy.match(pname) - if mdata: - arches = mdata.group('alist').split(',') - [entry.remove(inst) for inst in \ - entry.findall('Instance') \ - if inst.get('arch') not in arches] - - def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) - - def HandleEntry(self, entry, metadata): - self.BindEntry(entry, metadata) diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py deleted file mode 100644 index 2af3635e9..000000000 --- a/src/lib/Server/Plugins/Probes.py +++ /dev/null @@ -1,285 +0,0 @@ -import time -import lxml.etree -import operator -import re - -try: - import json - has_json = True -except ImportError: - try: - import simplejson as json - has_json = True - except ImportError: - has_json = False - -try: - import syck - has_syck = True -except ImportError: - has_syck = False - try: - import yaml - has_yaml = True - except ImportError: - has_yaml = False - -import Bcfg2.Server.Plugin - -specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)") -probe_matcher = re.compile("(.*/)?(?P<basename>\S+)") - -class ClientProbeDataSet(dict): - """ dict of probe => [probe data] that records a for each host """ - def __init__(self, *args, **kwargs): - if "timestamp" in kwargs and kwargs['timestamp'] is not None: - self.timestamp = kwargs.pop("timestamp") - else: - self.timestamp = time.time() - dict.__init__(self, *args, **kwargs) - - -class ProbeData(object): - """ a ProbeData object emulates a str object, but also has .xdata - and .json properties to provide convenient ways to use ProbeData - objects as XML or JSON data """ - def __init__(self, data): - self.data = data - self._xdata = None - self._json = None - self._yaml = None - - def __str__(self): - return str(self.data) - - def __repr__(self): - return repr(self.data) - - def __getattr__(self, name): - """ make ProbeData act like a str object """ - return getattr(self.data, name) - - def __complex__(self): - return complex(self.data) - - def __int__(self): - return int(self.data) - - def __long__(self): - return long(self.data) - - def __float__(self): - return float(self.data) - - def __eq__(self, other): - return str(self) == str(other) - - def __ne__(self, other): - return str(self) != str(other) - - def __gt__(self, other): - return str(self) > str(other) - - def __lt__(self, other): - return str(self) < str(other) - - def __ge__(self, other): - return self > other or self == other - - def __le__(self, other): - return self < other or self == other - - @property - def xdata(self): - if self._xdata is None: - try: - self._xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - pass - return self._xdata - - @property - def json(self): - if self._json is None and has_json: - try: - self._json = json.loads(self.data) - except ValueError: - pass - return self._json - - @property - def yaml(self): - if self._yaml is None: - if has_yaml: - try: - self._yaml = yaml.load(self.data) - except yaml.YAMLError: - pass - elif has_syck: - try: - self._yaml = syck.load(self.data) - except syck.error: - pass - return self._yaml - - -class ProbeSet(Bcfg2.Server.Plugin.EntrySet): - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") - - def __init__(self, path, fam, encoding, plugin_name): - fpattern = '[0-9A-Za-z_\-]+' - self.plugin_name = plugin_name - Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, - Bcfg2.Server.Plugin.SpecificData, - encoding) - fam.AddMonitor(path, self) - self.bangline = re.compile('^#!(?P<interpreter>.*)$') - - def HandleEvent(self, event): - if event.filename != self.path: - if (event.code2str == 'changed' and - event.filename.endswith("probed.xml") and - event.filename not in self.entries): - # for some reason, probed.xml is particularly prone to - # getting changed events before created events, - # because gamin is the worst ever. anyhow, we - # specifically handle it here to avoid a warning on - # every single server startup. - self.entry_init(event) - return - return self.handle_event(event) - - def get_probe_data(self, metadata): - ret = [] - build = dict() - candidates = self.get_matching(metadata) - candidates.sort(key=operator.attrgetter('specific')) - for entry in candidates: - rem = specific_probe_matcher.match(entry.name) - if not rem: - rem = probe_matcher.match(entry.name) - pname = rem.group('basename') - if pname not in build: - build[pname] = entry - - for (name, entry) in list(build.items()): - probe = lxml.etree.Element('probe') - probe.set('name', name.split('/')[-1]) - probe.set('source', self.plugin_name) - probe.text = entry.data - match = self.bangline.match(entry.data.split('\n')[0]) - if match: - probe.set('interpreter', match.group('interpreter')) - else: - probe.set('interpreter', '/bin/sh') - ret.append(probe) - return ret - - -class Probes(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Connector): - """A plugin to gather information from a client machine.""" - name = 'Probes' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) - - try: - self.probes = ProbeSet(self.data, core.fam, core.encoding, - self.name) - except: - raise Bcfg2.Server.Plugin.PluginInitError - - self.probedata = dict() - self.cgroups = dict() - self.load_data() - - def write_data(self): - """Write probe data out for use with bcfg2-info.""" - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - cx = lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probed.timestamp))) - for probe in sorted(probed): - lxml.etree.SubElement(cx, 'Probe', name=probe, - value=str(self.probedata[client][probe])) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(cx, "Group", name=group) - data = lxml.etree.tostring(top, encoding='UTF-8', - xml_declaration=True, - pretty_print='true') - try: - datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') - except IOError: - self.logger.error("Failed to write probed.xml") - datafile.write(data.decode('utf-8')) - - def load_data(self): - try: - data = lxml.etree.parse(self.data + '/probed.xml').getroot() - except: - self.logger.error("Failed to read file probed.xml") - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get('value')) - elif (pdata.tag == 'Group'): - self.cgroups[client.get('name')].append(pdata.get('name')) - - def GetProbes(self, meta, force=False): - """Return a set of probes for execution on client.""" - return self.probes.get_probe_data(meta) - - def ReceiveData(self, client, datalist): - self.cgroups[client.hostname] = [] - self.probedata[client.hostname] = ClientProbeDataSet() - for data in datalist: - self.ReceiveDataItem(client, data) - self.write_data() - - def ReceiveDataItem(self, client, data): - """Receive probe results pertaining to client.""" - if client.hostname not in self.cgroups: - self.cgroups[client.hostname] = [] - if data.text == None: - self.logger.error("Got null response to probe %s from %s" % \ - (data.get('name'), client.hostname)) - try: - self.probedata[client.hostname].update({data.get('name'): - ProbeData('')}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), ProbeData(''))]) - return - dlines = data.text.split('\n') - self.logger.debug("%s:probe:%s:%s" % (client.hostname, - data.get('name'), [line.strip() for line in dlines])) - for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in self.cgroups[client.hostname]: - self.cgroups[client.hostname].append(newgroup) - dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - try: - self.probedata[client.hostname].update({data.get('name'): dobj}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), dobj)]) - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py deleted file mode 100644 index 680881858..000000000 --- a/src/lib/Server/Plugins/Properties.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import re -import sys -import copy -import logging -import lxml.etree -import Bcfg2.Server.Plugin - -logger = logging.getLogger('Bcfg2.Plugins.Properties') - -class PropertyFile(Bcfg2.Server.Plugin.StructFile): - """Class for properties files.""" - def write(self): - """ Write the data in this data structure back to the property - file """ - if self.validate_data(): - try: - open(self.name, - "wb").write(lxml.etree.tostring(self.xdata, - pretty_print=True)) - return True - except IOError: - err = sys.exc_info()[1] - logger.error("Failed to write %s: %s" % (self.name, err)) - return False - else: - return False - - def validate_data(self): - """ ensure that the data in this object validates against the - XML schema for this property file (if a schema exists) """ - schemafile = self.name.replace(".xml", ".xsd") - if os.path.exists(schemafile): - try: - schema = lxml.etree.XMLSchema(file=schemafile) - except: - logger.error("Failed to process schema for %s" % self.name) - return False - else: - # no schema exists - return True - - if not schema.validate(self.xdata): - logger.error("Data for %s fails to validate; run bcfg2-lint for " - "more details" % self.name) - return False - else: - return True - - -class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - __child__ = PropertyFile - patterns = re.compile(r'.*\.xml$') - - -class Properties(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """ - The properties plugin maps property - files into client metadata instances. - """ - name = 'Properties' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - try: - self.store = PropDirectoryBacked(self.data, core.fam) - except OSError: - e = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s %s" % - (e.strerror, e.filename)) - raise Bcfg2.Server.Plugin.PluginInitError - - def get_additional_data(self, _): - return copy.copy(self.store.entries) diff --git a/src/lib/Server/Plugins/Rules.py b/src/lib/Server/Plugins/Rules.py deleted file mode 100644 index 684f0c290..000000000 --- a/src/lib/Server/Plugins/Rules.py +++ /dev/null @@ -1,55 +0,0 @@ -"""This generator provides rule-based entry mappings.""" - -import re -import Bcfg2.Server.Plugin - -class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - -class Rules(Bcfg2.Server.Plugin.PrioDir): - """This is a generator that handles service assignments.""" - name = 'Rules' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) - self.config = RulesConfig(self) - self._regex_cache = dict() - - def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) - return False - - def HandleEntry(self, entry, metadata): - return self.BindEntry(entry, metadata) - - def BindEntry(self, entry, metadata): - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - if key not in entry.attrib: - entry.attrib[key] = val - - def _matches(self, entry, metadata, rules): - if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules): - return True - elif (entry.tag == "Path" and - ((entry.get('name').endswith("/") and - entry.get('name').rstrip("/") in rules) or - (not entry.get('name').endswith("/") and - entry.get('name') + '/' in rules))): - # special case for Path tags: - # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967 - return True - elif self._regex_enabled: - # attempt regular expression matching - for rule in rules: - if rule not in self._regex_cache: - self._regex_cache[rule] = re.compile("%s$" % rule) - if self._regex_cache[rule].match(entry.get('name')): - return True - return False - - def _regex_enabled(self): - return self.config.getboolean("rules", "regex", default=False) diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py deleted file mode 100644 index 0ba08125e..000000000 --- a/src/lib/Server/Plugins/SGenshi.py +++ /dev/null @@ -1,97 +0,0 @@ -'''This module implements a templating generator based on Genshi''' - -import genshi.input -import genshi.template -import lxml.etree -import logging -import copy -import sys -import os.path - -import Bcfg2.Server.Plugin -import Bcfg2.Server.Plugins.TGenshi - -logger = logging.getLogger('Bcfg2.Plugins.SGenshi') - - -class SGenshiTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile, - Bcfg2.Server.Plugin.StructFile): - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name, - specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name) - - def get_xml_value(self, metadata): - if not hasattr(self, 'template'): - logger.error("No parsed template information for %s" % (self.name)) - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - stream = self.template.generate(metadata=metadata).filter( \ - Bcfg2.Server.Plugins.TGenshi.removecomment) - data = lxml.etree.XML(stream.render('xml', strip_whitespace=False)) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - except LookupError: - lerror = sys.exc_info()[1] - logger.error('Genshi lookup error: %s' % lerror) - except genshi.template.TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - raise - except genshi.input.ParseError: - perror = sys.exc_info()[1] - logger.error('Genshi parse error: %s' % perror) - raise - - def Match(self, metadata, xdata): - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - logger.debug("File %s got %d match(es)" % (self.name, len(rv))) - return rv - -class SGenshiEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, path, fam, encoding): - fpattern = '\S+\.xml' - Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, - SGenshiTemplateFile, encoding) - fam.AddMonitor(path, self) - - def HandleEvent(self, event): - '''passthrough event handler for old calling convention''' - if event.filename != self.path: - return self.handle_event(event) - - def BuildStructures(self, metadata): - """Build SGenshi structures.""" - ret = [] - for entry in self.get_matching(metadata): - try: - ret.append(entry.get_xml_value(metadata)) - except: - logger.error("SGenshi: Failed to template file %s" % entry.name) - return ret - - -class SGenshi(SGenshiEntrySet, - Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure): - """The SGenshi plugin provides templated structures.""" - name = 'SGenshi' - __author__ = 'bcfg-dev@mcs.anl.gov' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - try: - SGenshiEntrySet.__init__(self, self.data, self.core.fam, core.encoding) - except: - logger.error("Failed to load %s repository; disabling %s" \ - % (self.name, self.name)) - raise Bcfg2.Server.Plugin.PluginInitError diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py deleted file mode 100644 index 2e247caa7..000000000 --- a/src/lib/Server/Plugins/SSHbase.py +++ /dev/null @@ -1,413 +0,0 @@ -'''This module manages ssh key files for bcfg2''' - -import binascii -import re -import os -import socket -import shutil -import sys -import tempfile -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import u_str - -if sys.hexversion >= 0x03000000: - from functools import reduce - -import logging -logger = logging.getLogger(__name__) - -class KeyData(Bcfg2.Server.Plugin.SpecificData): - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - self.encoding = encoding - - def bind_entry(self, entry, metadata): - entry.set('type', 'file') - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(self.data) - else: - try: - entry.text = u_str(self.data, self.encoding) - except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError - except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % - entry.get('name')) - logger.error(str(e)) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - if entry.text in ['', None]: - entry.set('empty', 'true') - -class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - encoding = "base64" - else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) - self.metadata = {'owner': 'root', - 'group': 'root', - 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding - if basename.endswith('.pub'): - self.metadata['perms'] = '0644' - else: - self.metadata['perms'] = '0600' - - -class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, path): - Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) - self.metadata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} - - -class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.PullTarget): - """ - The sshbase generator manages ssh host keys (both v1 and v2) - for hosts. It also manages the ssh_known_hosts file. It can - integrate host keys from other management domains and similarly - export its keys. The repository contains files in the following - formats: - - ssh_host_key.H_(hostname) -> the v1 host private key for - (hostname) - ssh_host_key.pub.H_(hostname) -> the v1 host public key - for (hostname) - ssh_host_(ec)(dr)sa_key.H_(hostname) -> the v2 ssh host - private key for (hostname) - ssh_host_(ec)(dr)sa_key.pub.H_(hostname) -> the v2 ssh host - public key for (hostname) - ssh_known_hosts -> the current known hosts file. this - is regenerated each time a new key is generated. - - """ - name = 'SSHbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - - keypatterns = ["ssh_host_dsa_key", - "ssh_host_ecdsa_key", - "ssh_host_rsa_key", - "ssh_host_key", - "ssh_host_dsa_key.pub", - "ssh_host_ecdsa_key.pub", - "ssh_host_rsa_key.pub", - "ssh_host_key.pub"] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - Bcfg2.Server.Plugin.PullTarget.__init__(self) - self.ipcache = {} - self.namecache = {} - self.__skn = False - - # keep track of which bogus keys we've warned about, and only - # do so once - self.badnames = dict() - - core.fam.AddMonitor(self.data, self) - - self.static = dict() - self.entries = dict() - self.Entries['Path'] = dict() - - self.entries['/etc/ssh/ssh_known_hosts'] = KnownHostsEntrySet(self.data) - self.Entries['Path']['/etc/ssh/ssh_known_hosts'] = self.build_skn - for keypattern in self.keypatterns: - self.entries["/etc/ssh/" + keypattern] = HostKeyEntrySet(keypattern, - self.data) - self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk - - def get_skn(self): - """Build memory cache of the ssh known hosts file.""" - if not self.__skn: - # if no metadata is registered yet, defer - if len(self.core.metadata.query.all()) == 0: - self.__skn = False - return self.__skn - - skn = [s.data.decode().rstrip() - for s in list(self.static.values())] - - mquery = self.core.metadata.query - - # build hostname cache - names = dict() - for cmeta in mquery.all(): - names[cmeta.hostname] = set([cmeta.hostname]) - names[cmeta.hostname].update(cmeta.aliases) - newnames = set() - newips = set() - for name in names[cmeta.hostname]: - newnames.add(name.split('.')[0]) - try: - newips.add(self.get_ipcache_entry(name)[0]) - except: - continue - names[cmeta.hostname].update(newnames) - names[cmeta.hostname].update(cmeta.addresses) - names[cmeta.hostname].update(newips) - # TODO: Only perform reverse lookups on IPs if an option is set. - if True: - for ip in newips: - try: - names[cmeta.hostname].update(self.get_namecache_entry(ip)) - except: - continue - names[cmeta.hostname] = sorted(names[cmeta.hostname]) - - pubkeys = [pubk for pubk in list(self.entries.keys()) - if pubk.endswith('.pub')] - pubkeys.sort() - for pubkey in pubkeys: - for entry in sorted(self.entries[pubkey].entries.values(), - key=lambda e: e.specific.hostname or e.specific.group): - specific = entry.specific - hostnames = [] - if specific.hostname and specific.hostname in names: - hostnames = names[specific.hostname] - elif specific.group: - hostnames = \ - reduce(lambda x, y: x + y, - [names[cmeta.hostname] - for cmeta in \ - mquery.by_groups([specific.group])], []) - elif specific.all: - # a generic key for all hosts? really? - hostnames = reduce(lambda x, y: x + y, - list(names.values()), []) - if not hostnames: - if specific.hostname: - key = specific.hostname - ktype = "host" - elif specific.group: - key = specific.group - ktype = "group" - else: - # user has added a global SSH key, but - # have no clients yet. don't warn about - # this. - continue - - if key not in self.badnames: - self.badnames[key] = True - self.logger.info("Ignoring key for unknown %s %s" % - (ktype, key)) - continue - - skn.append("%s %s" % (','.join(hostnames), - entry.data.decode().rstrip())) - - self.__skn = "\n".join(skn) + "\n" - return self.__skn - - def set_skn(self, value): - """Set backing data for skn.""" - self.__skn = value - skn = property(get_skn, set_skn) - - def HandleEvent(self, event=None): - """Local event handler that does skn regen on pubkey change.""" - # skip events we don't care about - action = event.code2str() - if action == "endExist" or event.filename == self.data: - return - - for entry in list(self.entries.values()): - if entry.specific.match(event.filename): - entry.handle_event(event) - if event.filename.endswith(".pub"): - self.logger.info("New public key %s; invalidating " - "ssh_known_hosts cache" % event.filename) - self.skn = False - return - - if event.filename in ['info', 'info.xml', ':info']: - for entry in list(self.entries.values()): - entry.handle_event(event) - return - - if event.filename.endswith('.static'): - self.logger.info("Static key %s %s; invalidating ssh_known_hosts " - "cache" % (event.filename, action)) - if action == "deleted" and event.filename in self.static: - del self.static[event.filename] - self.skn = False - else: - self.static[event.filename] = \ - Bcfg2.Server.Plugin.FileBacked(os.path.join(self.data, - event.filename)) - self.static[event.filename].HandleEvent(event) - self.skn = False - return - - self.logger.warn("SSHbase: Got unknown event %s %s" % - (event.filename, action)) - - def get_ipcache_entry(self, client): - """Build a cache of dns results.""" - if client in self.ipcache: - if self.ipcache[client]: - return self.ipcache[client] - else: - raise socket.gaierror - else: - # need to add entry - try: - ipaddr = socket.gethostbyname(client) - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) - except socket.gaierror: - ipaddr = Popen(["getent", "hosts", client], - stdout=PIPE).stdout.read().strip().split() - if ipaddr: - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) - self.ipcache[client] = False - self.logger.error("Failed to find IP address for %s" % client) - raise socket.gaierror - - def get_namecache_entry(self, cip): - """Build a cache of name lookups from client IP addresses.""" - if cip in self.namecache: - # lookup cached name from IP - if self.namecache[cip]: - return self.namecache[cip] - else: - raise socket.gaierror - else: - # add an entry that has not been cached - try: - rvlookup = socket.gethostbyaddr(cip) - if rvlookup[0]: - self.namecache[cip] = [rvlookup[0]] - else: - self.namecache[cip] = [] - self.namecache[cip].extend(rvlookup[1]) - return self.namecache[cip] - except socket.gaierror: - self.namecache[cip] = False - self.logger.error("Failed to find any names associated with IP address %s" % cip) - raise - - def build_skn(self, entry, metadata): - """This function builds builds a host specific known_hosts file.""" - try: - rv = self.entries[entry.get('name')].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - client = metadata.hostname - entry.text = self.skn - hostkeys = [] - for k in self.keypatterns: - if k.endswith(".pub"): - try: - hostkeys.append(self.entries["/etc/ssh/" + - k].best_matching(metadata)) - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - hostkeys.sort() - for hostkey in hostkeys: - entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - hostkey.data.decode()) - self.entries[entry.get('name')].bind_info_to_entry(entry, metadata) - - def build_hk(self, entry, metadata): - """This binds host key data into entries.""" - try: - self.entries[entry.get('name')].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - filename = entry.get('name').split('/')[-1] - self.GenerateHostKeyPair(metadata.hostname, filename) - # Service the FAM events queued up by the key generation - # so the data structure entries will be available for - # binding. - # - # NOTE: We wait for up to ten seconds. There is some - # potential for race condition, because if the file - # monitor doesn't get notified about the new key files in - # time, those entries won't be available for binding. In - # practice, this seems "good enough". - tries = 0 - is_bound = False - while not is_bound: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError - self.core.fam.handle_events_in_interval(1) - tries += 1 - try: - self.entries[entry.get('name')].bind_entry(entry, metadata) - is_bound = True - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - - def GenerateHostKeyPair(self, client, filename): - """Generate new host key pair for client.""" - match = re.search(r'(ssh_host_(?:((?:ecd|d|r)sa)_)?key)', filename) - if match: - hostkey = "%s.H_%s" % (match.group(1), client) - if match.group(2): - keytype = match.group(2) - else: - keytype = 'rsa1' - else: - self.logger.error("Unknown key filename: %s" % filename) - return - - fileloc = "%s/%s" % (self.data, hostkey) - publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', - "H_%s" % client]) - tempdir = tempfile.mkdtemp() - temploc = "%s/%s" % (tempdir, hostkey) - cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", - "-t", keytype, "-C", "root@%s" % client] - proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - proc.communicate() - proc.wait() - - try: - shutil.copy(temploc, fileloc) - shutil.copy("%s.pub" % temploc, publoc) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Temporary SSH keys not found: %s" % err) - - try: - os.unlink(temploc) - os.unlink("%s.pub" % temploc) - os.rmdir(tempdir) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Failed to unlink temporary ssh keys: %s" % err) - - def AcceptChoices(self, _, metadata): - return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)] - - def AcceptPullData(self, specific, entry, log): - """Per-plugin bcfg2-admin pull support.""" - # specific will always be host specific - filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1], - specific.hostname) - try: - open(filename, 'w').write(entry['text']) - if log: - print("Wrote file %s" % filename) - except KeyError: - self.logger.error("Failed to pull %s. This file does not currently " - "exist on the client" % entry.get('name')) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Server/Plugins/SSLCA.py deleted file mode 100644 index 0072dc62d..000000000 --- a/src/lib/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,274 +0,0 @@ -import Bcfg2.Server.Plugin -import Bcfg2.Options -import lxml.etree -import posixpath -import tempfile -import pipes -import os -from subprocess import Popen, PIPE, STDOUT -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ - The SSLCA generator handles the creation and - management of ssl certificates and their keys. - """ - name = 'SSLCA' - __author__ = 'g.hagger@gmail.com' - __child__ = Bcfg2.Server.Plugin.FileBacked - key_specs = {} - cert_specs = {} - CAs = {} - - def HandleEvent(self, event=None): - """ - Updates which files this plugin handles based upon filesystem events. - Allows configuration items to be added/removed without server restarts. - """ - action = event.code2str() - if event.filename[0] == '/': - return - epath = "".join([self.data, self.handles[event.requestID], - event.filename]) - if posixpath.isdir(epath): - ident = self.handles[event.requestID] + event.filename - else: - ident = self.handles[event.requestID][:-1] - - fname = "".join([ident, '/', event.filename]) - - if event.filename.endswith('.xml'): - if action in ['exists', 'created', 'changed']: - if event.filename.endswith('key.xml'): - key_spec = dict(list(lxml.etree.parse(epath).find('Key').items())) - self.key_specs[ident] = { - 'bits': key_spec.get('bits', 2048), - 'type': key_spec.get('type', 'rsa') - } - self.Entries['Path'][ident] = self.get_key - elif event.filename.endswith('cert.xml'): - cert_spec = dict(list(lxml.etree.parse(epath).find('Cert').items())) - ca = cert_spec.get('ca', 'default') - self.cert_specs[ident] = { - 'ca': ca, - 'format': cert_spec.get('format', 'pem'), - 'key': cert_spec.get('key'), - 'days': cert_spec.get('days', 365), - 'C': cert_spec.get('c'), - 'L': cert_spec.get('l'), - 'ST': cert_spec.get('st'), - 'OU': cert_spec.get('ou'), - 'O': cert_spec.get('o'), - 'emailAddress': cert_spec.get('emailaddress') - } - cp = ConfigParser.ConfigParser() - cp.read(self.core.cfile) - self.CAs[ca] = dict(cp.items('sslca_' + ca)) - self.Entries['Path'][ident] = self.get_cert - if action == 'deleted': - if ident in self.Entries['Path']: - del self.Entries['Path'][ident] - else: - if action in ['exists', 'created']: - if posixpath.isdir(epath): - self.AddDirectoryMonitor(epath[len(self.data):]) - if ident not in self.entries and posixpath.isfile(epath): - self.entries[fname] = self.__child__(epath) - self.entries[fname].HandleEvent(event) - if action == 'changed': - self.entries[fname].HandleEvent(event) - elif action == 'deleted': - if fname in self.entries: - del self.entries[fname] - else: - self.entries[fname].HandleEvent(event) - - def get_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # set path type and permissions, otherwise bcfg2 won't bind the file - permdata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - - # check if we already have a hostfile, or need to generate a new key - # TODO: verify key fits the specs - path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - if filename not in list(self.entries.keys()): - key = self.build_key(filename, entry, metadata) - open(self.data + filename, 'w').write(key) - entry.text = key - self.entries[filename] = self.__child__("%s%s" % (self.data, - filename)) - self.entries[filename].HandleEvent() - else: - entry.text = self.entries[filename].data - - def build_key(self, filename, entry, metadata): - """ - generates a new key according the the specification - """ - type = self.key_specs[entry.get('name')]['type'] - bits = self.key_specs[entry.get('name')]['bits'] - if type == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif type == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - key = Popen(cmd, stdout=PIPE).stdout.read() - return key - - def get_cert(self, entry, metadata): - """ - either grabs a prexisting cert hostfile, or triggers the generation - of a new cert if one doesn't exist. - """ - # set path type and permissions, otherwise bcfg2 won't bind the file - permdata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - - path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - - # first - ensure we have a key to work with - key = self.cert_specs[entry.get('name')].get('key') - key_filename = "".join([key, '/', key.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - if key_filename not in self.entries: - e = lxml.etree.Element('Path') - e.attrib['name'] = key - self.core.Bind(e, metadata) - - # check if we have a valid hostfile - if filename in list(self.entries.keys()) and self.verify_cert(filename, - key_filename, - entry): - entry.text = self.entries[filename].data - else: - cert = self.build_cert(key_filename, entry, metadata) - open(self.data + filename, 'w').write(cert) - self.entries[filename] = self.__child__("%s%s" % (self.data, - filename)) - self.entries[filename].HandleEvent() - entry.text = cert - - def verify_cert(self, filename, key_filename, entry): - if self.verify_cert_against_ca(filename, entry): - if self.verify_cert_against_key(filename, key_filename): - return True - return False - - def verify_cert_against_ca(self, filename, entry): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - chaincert = self.CAs[self.cert_specs[entry.get('name')]['ca']].get('chaincert') - cert = self.data + filename - res = Popen(["openssl", "verify", "-CAfile", chaincert, cert], - stdout=PIPE, stderr=STDOUT).stdout.read() - if res == cert + ": OK\n": - return True - return False - - def verify_cert_against_key(self, filename, key_filename): - """ - check that a certificate validates against its private key. - """ - cert = self.data + filename - key = self.data + key_filename - cmd = ("openssl x509 -noout -modulus -in %s | openssl md5" % - pipes.quote(cert)) - cert_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read() - cmd = ("openssl rsa -noout -modulus -in %s | openssl md5" % - pipes.quote(key)) - key_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read() - if cert_md5 == key_md5: - return True - return False - - def build_cert(self, key_filename, entry, metadata): - """ - creates a new certificate according to the specification - """ - req_config = self.build_req_config(entry, metadata) - req = self.build_request(key_filename, req_config, entry) - ca = self.cert_specs[entry.get('name')]['ca'] - ca_config = self.CAs[ca]['config'] - days = self.cert_specs[entry.get('name')]['days'] - passphrase = self.CAs[ca].get('passphrase') - cmd = ["openssl", "ca", "-config", ca_config, "-in", req, - "-days", days, "-batch"] - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - cert = Popen(cmd, stdout=PIPE).stdout.read() - try: - os.unlink(req_config) - os.unlink(req) - except OSError: - self.logger.error("Failed to unlink temporary files") - return cert - - def build_req_config(self, entry, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - conffile = open(tempfile.mkstemp()[1], 'w') - cp = ConfigParser.ConfigParser({}) - cp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cp.add_section(section) - for key in defaults[section]: - cp.set(section, key, defaults[section][key]) - x = 1 - altnames = list(metadata.aliases) - altnames.append(metadata.hostname) - for altname in altnames: - cp.set('alt_names', 'DNS.' + str(x), altname) - x += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if self.cert_specs[entry.get('name')][item]: - cp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item]) - cp.set('req_distinguished_name', 'CN', metadata.hostname) - cp.write(conffile) - conffile.close() - return conffile.name - - def build_request(self, key_filename, req_config, entry): - """ - creates the certificate request - """ - req = tempfile.mkstemp()[1] - days = self.cert_specs[entry.get('name')]['days'] - key = self.data + key_filename - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", key, "-text", "-out", req] - res = Popen(cmd, stdout=PIPE).stdout.read() - return req diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py deleted file mode 100644 index aeb3b9f74..000000000 --- a/src/lib/Server/Plugins/Snapshots.py +++ /dev/null @@ -1,142 +0,0 @@ -#import lxml.etree -import logging -import binascii -import difflib -#import sqlalchemy -#import sqlalchemy.orm -import Bcfg2.Server.Plugin -import Bcfg2.Server.Snapshots -import Bcfg2.Logger -from Bcfg2.Server.Snapshots.model import Snapshot -import sys -import time -import threading - -# Compatibility import -from Bcfg2.Bcfg2Py3k import Queue - -logger = logging.getLogger('Snapshots') - -ftypes = ['ConfigFile', 'SymLink', 'Directory'] -datafields = { - 'Package': ['version'], - 'Path': ['type'], - 'Service': ['status'], - 'ConfigFile': ['owner', 'group', 'perms'], - 'Directory': ['owner', 'group', 'perms'], - 'SymLink': ['to'], - } - - -# py3k compatibility -def u_str(string): - if sys.hexversion >= 0x03000000: - return string - else: - return unicode(string) - - -def build_snap_ent(entry): - basefields = [] - if entry.tag in ['Package', 'Service']: - basefields += ['type'] - desired = dict([(key, u_str(entry.get(key))) for key in basefields]) - state = dict([(key, u_str(entry.get(key))) for key in basefields]) - desired.update([(key, u_str(entry.get(key))) for key in \ - datafields[entry.tag]]) - if entry.tag == 'ConfigFile' or \ - ((entry.tag == 'Path') and (entry.get('type') == 'file')): - if entry.text == None: - desired['contents'] = None - else: - if entry.get('encoding', 'ascii') == 'ascii': - desired['contents'] = u_str(entry.text) - else: - desired['contents'] = u_str(binascii.a2b_base64(entry.text)) - - if 'current_bfile' in entry.attrib: - state['contents'] = u_str(binascii.a2b_base64( \ - entry.get('current_bfile'))) - elif 'current_bdiff' in entry.attrib: - diff = binascii.a2b_base64(entry.get('current_bdiff')) - state['contents'] = u_str( \ - '\n'.join(difflib.restore(diff.split('\n'), 1))) - - state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \ - for key in datafields[entry.tag]]) - if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false': - state = None - return [desired, state] - - -class Snapshots(Bcfg2.Server.Plugin.Statistics, - Bcfg2.Server.Plugin.Plugin): - name = 'Snapshots' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Statistics.__init__(self) - self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) - self.work_queue = Queue() - self.loader = threading.Thread(target=self.load_snapshot) - self.loader.start() - - def load_snapshot(self): - while self.running: - try: - (metadata, data) = self.work_queue.get(block=True, timeout=5) - except: - continue - self.statistics_from_old_stats(metadata, data) - - def process_statistics(self, metadata, data): - return self.work_queue.put((metadata, data)) - - def statistics_from_old_stats(self, metadata, xdata): - # entries are name -> (modified, correct, start, desired, end) - # not sure we can get all of this from old format stats - t1 = time.time() - entries = dict([('Package', dict()), - ('Service', dict()), ('Path', dict())]) - extra = dict([('Package', dict()), ('Service', dict()), - ('Path', dict())]) - bad = [] - state = xdata.find('.//Statistics') - correct = state.get('state') == 'clean' - revision = u_str(state.get('revision', '-1')) - for entry in state.find('.//Bad'): - data = [False, False, u_str(entry.get('name'))] \ - + build_snap_ent(entry) - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - entries[etag][entry.get('name')] = data - for entry in state.find('.//Modified'): - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - if entry.get('name') in entries[etag]: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - else: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - for entry in state.find('.//Extra'): - if entry.tag in datafields: - data = build_snap_ent(entry)[1] - ename = u_str(entry.get('name')) - data['name'] = ename - extra[entry.tag][ename] = data - else: - print("extra", entry.tag, entry.get('name')) - t2 = time.time() - snap = Snapshot.from_data(self.session, correct, revision, - metadata, entries, extra) - self.session.add(snap) - self.session.commit() - t3 = time.time() - logger.info("Snapshot storage took %fs" % (t3 - t2)) - return True diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Server/Plugins/Statistics.py deleted file mode 100644 index 265ef95a8..000000000 --- a/src/lib/Server/Plugins/Statistics.py +++ /dev/null @@ -1,161 +0,0 @@ -'''This file manages the statistics collected by the BCFG2 Server''' - -import binascii -import copy -import difflib -import logging -from lxml.etree import XML, SubElement, Element, XMLSyntaxError -import lxml.etree -import os -from time import asctime, localtime, time, strptime, mktime -import threading - -import Bcfg2.Server.Plugin - - -class StatisticsStore(object): - """Manages the memory and file copy of statistics collected about client runs.""" - __min_write_delay__ = 0 - - def __init__(self, filename): - self.filename = filename - self.element = Element('Dummy') - self.dirty = 0 - self.lastwrite = 0 - self.logger = logging.getLogger('Bcfg2.Server.Statistics') - self.ReadFromFile() - - def WriteBack(self, force=0): - """Write statistics changes back to persistent store.""" - if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \ - or force: - try: - fout = open(self.filename + '.new', 'w') - except IOError: - ioerr = sys.exc_info()[1] - self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) - else: - fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True)) - fout.close() - os.rename(self.filename + '.new', self.filename) - self.dirty = 0 - self.lastwrite = time() - - def ReadFromFile(self): - """Reads current state regarding statistics.""" - try: - fin = open(self.filename, 'r') - data = fin.read() - fin.close() - self.element = XML(data) - self.dirty = 0 - except (IOError, XMLSyntaxError): - self.logger.error("Creating new statistics file %s"%(self.filename)) - self.element = Element('ConfigStatistics') - self.WriteBack() - self.dirty = 0 - - def updateStats(self, xml, client): - """Updates the statistics of a current node with new data.""" - - # Current policy: - # - Keep anything less than 24 hours old - # - Keep latest clean run for clean nodes - # - Keep latest clean and dirty run for dirty nodes - newstat = xml.find('Statistics') - - if newstat.get('state') == 'clean': - node_dirty = 0 - else: - node_dirty = 1 - - # Find correct node entry in stats data - # The following list comprehension should be guarenteed to return at - # most one result - nodes = [elem for elem in self.element.findall('Node') \ - if elem.get('name') == client] - nummatch = len(nodes) - if nummatch == 0: - # Create an entry for this node - node = SubElement(self.element, 'Node', name=client) - elif nummatch == 1 and not node_dirty: - # Delete old instance - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if self.isOlderThan24h(elem.get('time'))] - elif nummatch == 1 and node_dirty: - # Delete old dirty statistics entry - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if (elem.get('state') == 'dirty' \ - and self.isOlderThan24h(elem.get('time')))] - else: - # Shouldn't be reached - self.logger.error("Duplicate node entry for %s"%(client)) - - # Set current time for stats - newstat.set('time', asctime(localtime())) - - # Add statistic - node.append(copy.copy(newstat)) - - # Set dirty - self.dirty = 1 - self.WriteBack(force=1) - - def isOlderThan24h(self, testTime): - """Helper function to determine if <time> string is older than 24 hours.""" - now = time() - utime = mktime(strptime(testTime)) - secondsPerDay = 60*60*24 - - return (now-utime) > secondsPerDay - - -class Statistics(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.ThreadedStatistics, - Bcfg2.Server.Plugin.PullSource): - name = 'Statistics' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) - Bcfg2.Server.Plugin.PullSource.__init__(self) - fpath = "%s/etc/statistics.xml" % datastore - self.data_file = StatisticsStore(fpath) - - def handle_statistic(self, metadata, data): - self.data_file.updateStats(data, metadata.hostname) - - def FindCurrent(self, client): - rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0] - maxtime = max([strptime(stat.get('time')) for stat \ - in rt.findall('Statistics')]) - return [stat for stat in rt.findall('Statistics') \ - if strptime(stat.get('time')) == maxtime][0] - - def GetExtra(self, client): - return [(entry.tag, entry.get('name')) for entry \ - in self.FindCurrent(client).xpath('.//Extra/*')] - - def GetCurrentEntry(self, client, e_type, e_name): - curr = self.FindCurrent(client) - entry = curr.xpath('.//Bad/%s[@name="%s"]' % (e_type, e_name)) - if not entry: - raise Bcfg2.Server.Plugin.PluginExecutionError - cfentry = entry[-1] - - owner = cfentry.get('current_owner', cfentry.get('owner')) - group = cfentry.get('current_group', cfentry.get('group')) - perms = cfentry.get('current_perms', cfentry.get('perms')) - if cfentry.get('sensitive') in ['true', 'True']: - raise Bcfg2.Server.Plugin.PluginExecutionError - elif 'current_bfile' in cfentry.attrib: - contents = binascii.a2b_base64(cfentry.get('current_bfile')) - elif 'current_bdiff' in cfentry.attrib: - diff = binascii.a2b_base64(cfentry.get('current_bdiff')) - contents = '\n'.join(difflib.restore(diff.split('\n'), 1)) - else: - contents = None - - return (owner, group, perms, contents) diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Server/Plugins/Svcmgr.py deleted file mode 100644 index f4232ad5c..000000000 --- a/src/lib/Server/Plugins/Svcmgr.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This generator provides service mappings.""" - -import Bcfg2.Server.Plugin - - -class Svcmgr(Bcfg2.Server.Plugin.PrioDir): - """This is a generator that handles service assignments.""" - name = 'Svcmgr' - __author__ = 'bcfg-dev@mcs.anl.gov' - deprecated = True diff --git a/src/lib/Server/Plugins/Svn.py b/src/lib/Server/Plugins/Svn.py deleted file mode 100644 index ae43388ea..000000000 --- a/src/lib/Server/Plugins/Svn.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import pipes -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Svn') - - -class Svn(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to svn directory for bcfg2 repo - svn_dir = "%s/.svn" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(svn_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % svn_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized svn plugin with svn directory = %s" % svn_dir) - - def get_revision(self): - """Read svn revision information for the Bcfg2 repository.""" - try: - data = Popen(("env LC_ALL=C svn info %s" % - pipes.quote(self.datastore)), shell=True, - stdout=PIPE).communicate()[0].split('\n') - return [line.split(': ')[1] for line in data \ - if line[:9] == 'Revision:'][-1] - except IndexError: - logger.error("Failed to read svn info; disabling svn support") - logger.error('''Ran command "svn info %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py deleted file mode 100644 index 8d79348f8..000000000 --- a/src/lib/Server/Plugins/Svn2.py +++ /dev/null @@ -1,125 +0,0 @@ -try: - import pysvn - missing = False -except: - missing = True -import Bcfg2.Server.Plugin - -class Svn2(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn2' - __author__ = 'bcfg-dev@mcs.anl.gov' - - conflicts = ['Svn'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update','Commit'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - - if missing: - self.logger.error("Svn2: Missing PySvn") - raise Bcfg2.Server.Plugin.PluginInitError - - self.client = pysvn.Client() - - self.core = core - self.datastore = datastore - self.svn_root = None - self.revision = None - - # Read revision from bcfg2 repo - revision = self.get_revision() - if not self.revision: - raise Bcfg2.Server.Plugin.PluginInitError - - self.logger.debug("Initialized svn plugin with svn root %s at revision %s" - % (self.svn_root, revision)) - - def get_revision(self): - """Read svn revision information for the Bcfg2 repository.""" - try: - info = self.client.info(self.datastore) - self.revision = info.revision - self.svn_root = info.url - return str(self.revision.number) - except: - self.logger.error("Svn2: Failed to get revision", exc_info=1) - self.revision = None - return str(-1) - - def commit_data(self, file_list, comment=None): - """Commit changes into the repository""" - if not comment: - comment = 'Svn2: autocommit' - - # First try to update - if not self.Update(): - self.logger.error("Failed to update svn repository, refusing to commit changes") - return - - #FIXME - look for conflicts? - - for fname in file_list: - stat = self.client.status(fname) - self.client.add([f.path for f in stat \ - if f.text_status == pysvn.wc_status_kind.unversioned]) - try: - self.revision = self.client.checkin([self.datastore], comment, - recurse=True) - self.revision = self.client.update(self.datastore, recurse=True)[0] - self.logger.info("Svn2: Commited changes. At %s" % - self.revision.number) - except Exception, err: - # try to be smart about the error we got back - details = None - if "callback_ssl_server_trust_prompt" in str(err): - details = "SVN server certificate is not trusted" - elif "callback_get_login" in str(err): - details = "SVN credentials not cached" - - if details is None: - self.logger.error("Svn2: Failed to commit changes", - exc_info=1) - else: - self.logger.error("Svn2: Failed to commit changes: %s" % - details) - - def Update(self): - '''Svn2.Update() => True|False\nUpdate svn working copy\n''' - try: - old_revision = self.revision.number - self.revision = self.client.update(self.datastore, recurse=True)[0] - except Exception, err: - # try to be smart about the error we got back - details = None - if "callback_ssl_server_trust_prompt" in str(err): - details = "SVN server certificate is not trusted" - elif "callback_get_login" in str(err): - details = "SVN credentials not cached" - - if details is None: - self.logger.error("Svn2: Failed to update server repository", - exc_info=1) - else: - self.logger.error("Svn2: Failed to update server repository: %s" % - details) - return False - - if old_revision == self.revision.number: - self.logger.debug("repository is current") - else: - self.logger.info("Updated %s from revision %s to %s" % \ - (self.datastore, old_revision, self.revision.number)) - return True - - def Commit(self): - """Svn2.Commit() => True|False\nCommit svn repository\n""" - try: - self.commit_changes([]) - return True - except: - return False - - diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py deleted file mode 100644 index 8879fdef1..000000000 --- a/src/lib/Server/Plugins/TCheetah.py +++ /dev/null @@ -1,80 +0,0 @@ -'''This module implements a templating generator based on Cheetah''' - -import binascii -import logging -import sys -import traceback -import Bcfg2.Server.Plugin -# py3k compatibility -if sys.hexversion >= 0x03000000: - unicode = str - -logger = logging.getLogger('Bcfg2.Plugins.TCheetah') - -try: - import Cheetah.Template - import Cheetah.Parser -except: - logger.error("TCheetah: Failed to import Cheetah. Is it installed?") - raise - - -class TemplateFile: - """Template file creates Cheetah template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - self.template = None - self.searchlist = dict() - - def handle_event(self, event): - """Handle all fs events for this template.""" - if event.code2str() == 'deleted': - return - try: - s = {'useStackFrames': False} - self.template = Cheetah.Template.Template(open(self.name).read(), - compilerSettings=s, - searchList=self.searchlist) - except Cheetah.Parser.ParseError: - perror = sys.exc_info()[1] - logger.error("Cheetah parse error for file %s" % (self.name)) - logger.error(perror.report()) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - self.template.metadata = metadata - self.searchlist['metadata'] = metadata - self.template.path = entry.get('realname', entry.get('name')) - self.searchlist['path'] = entry.get('realname', entry.get('name')) - self.template.source_path = self.name - self.searchlist['source_path'] = self.name - - if entry.tag == 'Path': - entry.set('type', 'file') - try: - if type(self.template) == unicode: - entry.text = self.template - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = binascii.b2a_base64(self.template) - else: - entry.text = unicode(str(self.template), self.encoding) - except: - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - logger.error(msg) - logger.error("TCheetah template error for %s" % self.searchlist['path']) - del a, b, c - raise Bcfg2.Server.Plugin.PluginExecutionError - - -class TCheetah(Bcfg2.Server.Plugin.GroupSpool): - """The TCheetah generator implements a templating mechanism for configuration files.""" - name = 'TCheetah' - __author__ = 'bcfg-dev@mcs.anl.gov' - filename_pattern = 'template' - es_child_cls = TemplateFile diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py deleted file mode 100644 index 3ba0f4272..000000000 --- a/src/lib/Server/Plugins/TGenshi.py +++ /dev/null @@ -1,135 +0,0 @@ -"""This module implements a templating generator based on Genshi.""" - -import binascii -import logging -import sys -import Bcfg2.Server.Plugin -# py3k compatibility -if sys.hexversion >= 0x03000000: - unicode = str - -logger = logging.getLogger('Bcfg2.Plugins.TGenshi') - -# try to import genshi stuff -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, \ - TextTemplate, MarkupTemplate, TemplateError -except ImportError: - logger.error("TGenshi: Failed to import Genshi. Is it installed?") - raise Bcfg2.Server.Plugin.PluginInitError -try: - from genshi.template import NewTextTemplate - have_ntt = True -except: - have_ntt = False - -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -class TemplateFile: - """Template file creates Genshi template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - if self.specific.all: - matchname = self.name - elif self.specific.group: - matchname = self.name[:self.name.find('.G')] - else: - matchname = self.name[:self.name.find('.H')] - if matchname.endswith('.txt'): - self.template_cls = TextTemplate - elif matchname.endswith('.newtxt'): - if not have_ntt: - logger.error("Genshi NewTextTemplates not supported by this version of Genshi") - else: - self.template_cls = NewTextTemplate - else: - self.template_cls = MarkupTemplate - self.HandleEvent = self.handle_event - - def handle_event(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - try: - self.template = loader.load(self.name, cls=self.template_cls, - encoding=self.encoding) - except LookupError: - lerror = sys.exc_info()[1] - logger.error('Genshi lookup error: %s' % lerror) - except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - except genshi.input.ParseError: - perror = sys.exc_info()[1] - logger.error('Genshi parse error: %s' % perror) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - fname = entry.get('realname', entry.get('name')) - if entry.tag == 'Path': - entry.set('type', 'file') - try: - stream = self.template.generate( \ - name=fname, metadata=metadata, - path=self.name).filter(removecomment) - if have_ntt: - ttypes = [TextTemplate, NewTextTemplate] - else: - ttypes = [TextTemplate] - if True in [isinstance(self.template, t) for t in ttypes]: - try: - textdata = stream.render('text', strip_whitespace=False) - except TypeError: - textdata = stream.render('text') - if type(textdata) == unicode: - entry.text = textdata - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = binascii.b2a_base64(textdata) - else: - entry.text = unicode(textdata, self.encoding) - else: - try: - xmldata = stream.render('xml', strip_whitespace=False) - except TypeError: - xmldata = stream.render('xml') - if type(xmldata) == unicode: - entry.text = xmldata - else: - entry.text = unicode(xmldata, self.encoding) - if entry.text == '': - entry.set('empty', 'true') - except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - raise Bcfg2.Server.Plugin.PluginExecutionError - except AttributeError: - err = sys.exc_info()[1] - logger.error('Genshi template loading error: %s' % err) - raise Bcfg2.Server.Plugin.PluginExecutionError - - -class TGenshi(Bcfg2.Server.Plugin.GroupSpool): - """ - The TGenshi generator implements a templating - mechanism for configuration files. - - """ - name = 'TGenshi' - __author__ = 'jeff@ocjtech.us' - filename_pattern = 'template\.(txt|newtxt|xml)' - es_child_cls = TemplateFile diff --git a/src/lib/Server/Plugins/TemplateHelper.py b/src/lib/Server/Plugins/TemplateHelper.py deleted file mode 100644 index 42eafed56..000000000 --- a/src/lib/Server/Plugins/TemplateHelper.py +++ /dev/null @@ -1,83 +0,0 @@ -import re -import imp -import sys -import logging -import Bcfg2.Server.Plugin - -logger = logging.getLogger(__name__) - -class HelperModule(Bcfg2.Server.Plugin.SpecificData): - _module_name_re = re.compile(r'([^/]+?)\.py') - - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - match = self._module_name_re.search(self.name) - if match: - self._module_name = match.group(1) - else: - self._module_name = name - self._attrs = [] - - def handle_event(self, event): - Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) - try: - module = imp.load_source(self._module_name, self.name) - except: - err = sys.exc_info()[1] - logger.error("TemplateHelper: Failed to import %s: %s" % - (self.name, err)) - return - - if not hasattr(module, "__export__"): - logger.error("TemplateHelper: %s has no __export__ list" % - self.name) - return - - for sym in module.__export__: - if sym not in self._attrs and hasattr(self, sym): - logger.warning("TemplateHelper: %s: %s is a reserved keyword, " - "skipping export" % (self.name, sym)) - setattr(self, sym, getattr(module, sym)) - # remove old exports - for sym in set(self._attrs) - set(module.__export__): - delattr(self, sym) - - self._attrs = module.__export__ - - -class HelperSet(Bcfg2.Server.Plugin.EntrySet): - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") - - def __init__(self, path, fam, encoding, plugin_name): - fpattern = '[0-9A-Za-z_\-]+\.py' - self.plugin_name = plugin_name - Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, - HelperModule, encoding) - fam.AddMonitor(path, self) - - def HandleEvent(self, event): - if (event.filename != self.path and - not self.ignore.match(event.filename)): - return self.handle_event(event) - - -class TemplateHelper(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """ A plugin to provide helper classes and functions to templates """ - name = 'TemplateHelper' - __author__ = 'chris.a.st.pierre@gmail.com' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - - try: - self.helpers = HelperSet(self.data, core.fam, core.encoding, - self.name) - except: - raise Bcfg2.Server.Plugin.PluginInitError - - def get_additional_data(self, metadata): - return dict([(h._module_name, h) - for h in list(self.helpers.entries.values())]) diff --git a/src/lib/Server/Plugins/Trigger.py b/src/lib/Server/Plugins/Trigger.py deleted file mode 100644 index b0d21545c..000000000 --- a/src/lib/Server/Plugins/Trigger.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import Bcfg2.Server.Plugin - - -def async_run(prog, args): - pid = os.fork() - if pid: - os.waitpid(pid, 0) - else: - dpid = os.fork() - if not dpid: - os.system(" ".join([prog] + args)) - os._exit(0) - - -class Trigger(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Statistics): - """Trigger is a plugin that calls external scripts (on the server).""" - name = 'Trigger' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Statistics.__init__(self) - try: - os.stat(self.data) - except: - self.logger.error("Trigger: spool directory %s does not exist; " - "unloading" % self.data) - raise Bcfg2.Server.Plugin.PluginInitError - - def process_statistics(self, metadata, _): - args = [metadata.hostname, '-p', metadata.profile, '-g', - ':'.join([g for g in metadata.groups])] - for notifier in os.listdir(self.data): - if ((notifier[-1] == '~') or - (notifier[:2] == '.#') or - (notifier[-4:] == '.swp') or - (notifier in ['SCCS', '.svn', '4913'])): - continue - npath = self.data + '/' + notifier - self.logger.debug("Running %s %s" % (npath, " ".join(args))) - async_run(npath, args) diff --git a/src/lib/Server/Plugins/__init__.py b/src/lib/Server/Plugins/__init__.py deleted file mode 100644 index a36a9b766..000000000 --- a/src/lib/Server/Plugins/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Imports for Bcfg2.Server.Plugins.""" - -__all__ = [ - 'Account', - 'Base', - 'Bundler', - 'Bzr', - 'Cfg', - 'Cvs', - 'Darcs', - 'Decisions', - 'Fossil', - 'Git', - 'GroupPatterns', - 'Hg', - 'Hostbase', - 'Metadata', - 'NagiosGen', - 'Ohai', - 'Packages', - 'Properties', - 'Probes', - 'Pkgmgr', - 'Rules', - 'SSHbase', - 'Snapshots', - 'Statistics', - 'Svcmgr', - 'Svn', - 'TCheetah', - 'Trigger', - 'SGenshi', - 'TGenshi', - ] |