summaryrefslogtreecommitdiffstats
path: root/src/lib/Server/Plugins
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Server/Plugins')
-rw-r--r--src/lib/Server/Plugins/Account.py101
-rw-r--r--src/lib/Server/Plugins/BB.py83
-rw-r--r--src/lib/Server/Plugins/Base.py43
-rw-r--r--src/lib/Server/Plugins/Bundler.py99
-rw-r--r--src/lib/Server/Plugins/Bzr.py35
-rw-r--r--src/lib/Server/Plugins/Cfg.py283
-rw-r--r--src/lib/Server/Plugins/Cvs.py46
-rw-r--r--src/lib/Server/Plugins/DBStats.py113
-rw-r--r--src/lib/Server/Plugins/Darcs.py48
-rw-r--r--src/lib/Server/Plugins/Decisions.py66
-rw-r--r--src/lib/Server/Plugins/Defaults.py53
-rw-r--r--src/lib/Server/Plugins/Deps.py134
-rw-r--r--src/lib/Server/Plugins/Editor.py80
-rw-r--r--src/lib/Server/Plugins/FileProbes.py230
-rw-r--r--src/lib/Server/Plugins/Fossil.py51
-rw-r--r--src/lib/Server/Plugins/Git.py44
-rw-r--r--src/lib/Server/Plugins/GroupPatterns.py124
-rw-r--r--src/lib/Server/Plugins/Guppy.py62
-rw-r--r--src/lib/Server/Plugins/Hg.py45
-rw-r--r--src/lib/Server/Plugins/Hostbase.py593
-rw-r--r--src/lib/Server/Plugins/Ldap.py245
-rw-r--r--src/lib/Server/Plugins/Metadata.py866
-rw-r--r--src/lib/Server/Plugins/NagiosGen.py151
-rw-r--r--src/lib/Server/Plugins/Ohai.py92
-rw-r--r--src/lib/Server/Plugins/Packages/Apt.py139
-rw-r--r--src/lib/Server/Plugins/Packages/Collection.py343
-rw-r--r--src/lib/Server/Plugins/Packages/Pac.py120
-rw-r--r--src/lib/Server/Plugins/Packages/PackagesConfig.py15
-rw-r--r--src/lib/Server/Plugins/Packages/PackagesSources.py97
-rw-r--r--src/lib/Server/Plugins/Packages/Source.py278
-rw-r--r--src/lib/Server/Plugins/Packages/Yum.py688
-rw-r--r--src/lib/Server/Plugins/Packages/__init__.py263
-rw-r--r--src/lib/Server/Plugins/Pkgmgr.py169
-rw-r--r--src/lib/Server/Plugins/Probes.py285
-rw-r--r--src/lib/Server/Plugins/Properties.py76
-rw-r--r--src/lib/Server/Plugins/Rules.py55
-rw-r--r--src/lib/Server/Plugins/SGenshi.py97
-rw-r--r--src/lib/Server/Plugins/SSHbase.py413
-rw-r--r--src/lib/Server/Plugins/SSLCA.py274
-rw-r--r--src/lib/Server/Plugins/Snapshots.py142
-rw-r--r--src/lib/Server/Plugins/Statistics.py161
-rw-r--r--src/lib/Server/Plugins/Svcmgr.py10
-rw-r--r--src/lib/Server/Plugins/Svn.py46
-rw-r--r--src/lib/Server/Plugins/Svn2.py125
-rw-r--r--src/lib/Server/Plugins/TCheetah.py80
-rw-r--r--src/lib/Server/Plugins/TGenshi.py135
-rw-r--r--src/lib/Server/Plugins/TemplateHelper.py83
-rw-r--r--src/lib/Server/Plugins/Trigger.py43
-rw-r--r--src/lib/Server/Plugins/__init__.py34
49 files changed, 0 insertions, 7858 deletions
diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Server/Plugins/Account.py
deleted file mode 100644
index f2703dccb..000000000
--- a/src/lib/Server/Plugins/Account.py
+++ /dev/null
@@ -1,101 +0,0 @@
-"""This handles authentication setup."""
-
-import Bcfg2.Server.Plugin
-
-
-class Account(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Generator):
- """This module generates account config files,
- based on an internal data repo:
- static.(passwd|group|limits.conf) -> static entries
- dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch)
- useraccess -> users to be granted login access on some hosts
- superusers -> users to be granted root privs on all hosts
- rootlike -> users to be granted root privs on some hosts
-
- """
- name = 'Account'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Generator.__init__(self)
- self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb,
- '/etc/group': self.from_yp_cb,
- '/etc/security/limits.conf': self.gen_limits_cb,
- '/root/.ssh/authorized_keys': self.gen_root_keys_cb,
- '/etc/sudoers': self.gen_sudoers}}
- try:
- self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data,
- self.core.fam)
- except:
- self.logger.error("Failed to load repos: %s, %s" % \
- (self.data, "%s/ssh" % (self.data)))
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def from_yp_cb(self, entry, metadata):
- """Build password file from cached yp data."""
- fname = entry.attrib['name'].split('/')[-1]
- entry.text = self.repository.entries["static.%s" % (fname)].data
- entry.text += self.repository.entries["dyn.%s" % (fname)].data
- perms = {'owner': 'root',
- 'group': 'root',
- 'perms': '0644'}
- [entry.attrib.__setitem__(key, value) for (key, value) in \
- list(perms.items())]
-
- def gen_limits_cb(self, entry, metadata):
- """Build limits entries based on current ACLs."""
- entry.text = self.repository.entries["static.limits.conf"].data
- superusers = self.repository.entries["superusers"].data.split()
- useraccess = [line.split(':') for line in \
- self.repository.entries["useraccess"].data.split()]
- users = [user for (user, host) in \
- useraccess if host == metadata.hostname.split('.')[0]]
- perms = {'owner': 'root',
- 'group': 'root',
- 'perms': '0600'}
- [entry.attrib.__setitem__(key, value) for (key, value) in \
- list(perms.items())]
- entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users])
- if "*" not in users:
- entry.text += "* hard maxlogins 0\n"
-
- def gen_root_keys_cb(self, entry, metadata):
- """Build root authorized keys file based on current ACLs."""
- superusers = self.repository.entries['superusers'].data.split()
- try:
- rootlike = [line.split(':', 1) for line in \
- self.repository.entries['rootlike'].data.split()]
- superusers += [user for (user, host) in rootlike \
- if host == metadata.hostname.split('.')[0]]
- except:
- pass
- rdata = self.repository.entries
- entry.text = "".join([rdata["%s.key" % user].data for user \
- in superusers if \
- ("%s.key" % user) in rdata])
- perms = {'owner': 'root',
- 'group': 'root',
- 'perms': '0600'}
- [entry.attrib.__setitem__(key, value) for (key, value) \
- in list(perms.items())]
-
- def gen_sudoers(self, entry, metadata):
- """Build root authorized keys file based on current ACLs."""
- superusers = self.repository.entries['superusers'].data.split()
- try:
- rootlike = [line.split(':', 1) for line in \
- self.repository.entries['rootlike'].data.split()]
- superusers += [user for (user, host) in rootlike \
- if host == metadata.hostname.split('.')[0]]
- except:
- pass
- entry.text = self.repository.entries['static.sudoers'].data
- entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \
- for uname in superusers])
- perms = {'owner': 'root',
- 'group': 'root',
- 'perms': '0440'}
- [entry.attrib.__setitem__(key, value) for (key, value) \
- in list(perms.items())]
diff --git a/src/lib/Server/Plugins/BB.py b/src/lib/Server/Plugins/BB.py
deleted file mode 100644
index c015ec47c..000000000
--- a/src/lib/Server/Plugins/BB.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import lxml.etree
-import Bcfg2.Server.Plugin
-import glob
-import os
-import socket
-
-#manage boot symlinks
- #add statistics check to do build->boot mods
-
-#map profiles: first array is not empty we replace the -p with a determined profile.
-logger = Bcfg2.Server.Plugin.logger
-
-class BBfile(Bcfg2.Server.Plugin.XMLFileBacked):
- """Class for bb files."""
- def Index(self):
- """Build data into an xml object."""
-
- try:
- self.data = lxml.etree.XML(self.data)
- except lxml.etree.XMLSyntaxError:
- Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % self.name)
- return
- self.tftppath = self.data.get('tftp', '/tftpboot')
- self.macs = {}
- self.users = {}
- self.actions = {}
- self.bootlinks = []
-
- for node in self.data.findall('Node'):
- iface = node.find('Interface')
- if iface != None:
- mac = "01-%s" % (iface.get('mac'.replace(':','-').lower()))
- self.actions[node.get('name')] = node.get('action')
- self.bootlinks.append((mac, node.get('action')))
- try:
- ip = socket.gethostbyname(node.get('name'))
- except:
- logger.error("failed host resolution for %s" % node.get('name'))
-
- self.macs[node.get('name')] = (iface.get('mac'), ip)
- else:
- logger.error("%s" % lxml.etree.tostring(node))
- self.users[node.get('name')] = node.get('user',"").split(':')
-
- def enforce_bootlinks(self):
- for mac, target in self.bootlinks:
- path = self.tftppath + '/' + mac
- if not os.path.islink(path):
- logger.error("Boot file %s not a link" % path)
- if target != os.readlink(path):
- try:
- os.unlink(path)
- os.symlink(target, path)
- except:
- logger.error("Failed to modify link %s" % path)
-
-class BBDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
- __child__ = BBfile
-
-
-class BB(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Connector):
- """The BB plugin maps users to machines and metadata to machines."""
- name = 'BB'
- deprecated = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- self.store = BBDirectoryBacked(self.data, core.fam)
-
- def get_additional_data(self, metadata):
-
- users = {}
- for user in self.store.entries['bb.xml'].users.get(metadata.hostname.split(".")[0], []):
- pubkeys = []
- for fname in glob.glob('/home/%s/.ssh/*.pub'%user):
- pubkeys.append(open(fname).read())
-
- users[user] = pubkeys
-
- return dict([('users', users),
- ('macs', self.store.entries['bb.xml'].macs)])
diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py
deleted file mode 100644
index 389ca7a95..000000000
--- a/src/lib/Server/Plugins/Base.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""This module sets up a base list of configuration entries."""
-
-import copy
-import lxml.etree
-import sys
-# py3k compatibility
-if sys.hexversion >= 0x03000000:
- from functools import reduce
-
-import Bcfg2.Server.Plugin
-
-
-class Base(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Structure,
- Bcfg2.Server.Plugin.XMLDirectoryBacked):
- """This Structure is good for the pile of independent configs
- needed for most actual systems.
- """
- name = 'Base'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- __child__ = Bcfg2.Server.Plugin.StructFile
- deprecated = True
-
- """Base creates independent clauses based on client metadata."""
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Structure.__init__(self)
- try:
- Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
- self.data,
- self.core.fam)
- except OSError:
- self.logger.error("Failed to load Base repository")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def BuildStructures(self, metadata):
- """Build structures for client described by metadata."""
- ret = lxml.etree.Element("Independent", version='2.0')
- fragments = reduce(lambda x, y: x + y,
- [base.Match(metadata) for base
- in list(self.entries.values())], [])
- [ret.append(copy.copy(frag)) for frag in fragments]
- return [ret]
diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py
deleted file mode 100644
index ccb99481e..000000000
--- a/src/lib/Server/Plugins/Bundler.py
+++ /dev/null
@@ -1,99 +0,0 @@
-"""This provides bundle clauses with translation functionality."""
-
-import copy
-import lxml.etree
-import os
-import os.path
-import re
-import sys
-
-import Bcfg2.Server.Plugin
-
-try:
- import genshi.template
- import genshi.template.base
- import Bcfg2.Server.Plugins.SGenshi
- have_genshi = True
-except:
- have_genshi = False
-
-
-class BundleFile(Bcfg2.Server.Plugin.StructFile):
-
- def get_xml_value(self, metadata):
- bundlename = os.path.splitext(os.path.basename(self.name))[0]
- bundle = lxml.etree.Element('Bundle', name=bundlename)
- [bundle.append(copy.copy(item)) for item in self.Match(metadata)]
- return bundle
-
-
-class Bundler(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Structure,
- Bcfg2.Server.Plugin.XMLDirectoryBacked):
- """The bundler creates dependent clauses based on the
- bundle/translation scheme from Bcfg1.
- """
- name = 'Bundler'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$')
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Structure.__init__(self)
- self.encoding = core.encoding
- self.__child__ = self.template_dispatch
- try:
- Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
- self.data,
- self.core.fam)
- except OSError:
- self.logger.error("Failed to load Bundle repository")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def template_dispatch(self, name):
- bundle = lxml.etree.parse(name)
- nsmap = bundle.getroot().nsmap
- if name.endswith('.xml'):
- if have_genshi and \
- (nsmap == {'py': 'http://genshi.edgewall.org/'}):
- # allow for genshi bundles with .xml extensions
- spec = Bcfg2.Server.Plugin.Specificity()
- return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name,
- spec,
- self.encoding)
- else:
- return BundleFile(name)
- elif name.endswith('.genshi'):
- if have_genshi:
- spec = Bcfg2.Server.Plugin.Specificity()
- return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name,
- spec,
- self.encoding)
-
- def BuildStructures(self, metadata):
- """Build all structures for client (metadata)."""
- bundleset = []
-
- bundle_entries = {}
- for key, item in self.entries.items():
- bundle_entries.setdefault(self.patterns.match(os.path.basename(key)).group('name'),
- []).append(item)
-
- for bundlename in metadata.bundles:
- try:
- entries = bundle_entries[bundlename]
- except KeyError:
- self.logger.error("Bundler: Bundle %s does not exist" %
- bundlename)
- continue
- try:
- bundleset.append(entries[0].get_xml_value(metadata))
- except genshi.template.base.TemplateError:
- t = sys.exc_info()[1]
- self.logger.error("Bundler: Failed to template genshi bundle %s"
- % bundlename)
- self.logger.error(t)
- except:
- self.logger.error("Bundler: Unexpected bundler error for %s" %
- bundlename, exc_info=1)
- return bundleset
diff --git a/src/lib/Server/Plugins/Bzr.py b/src/lib/Server/Plugins/Bzr.py
deleted file mode 100644
index a71021cb5..000000000
--- a/src/lib/Server/Plugins/Bzr.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import Bcfg2.Server.Plugin
-from bzrlib.workingtree import WorkingTree
-from bzrlib import errors
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Bzr')
-
-class Bzr(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Bzr is a version plugin for dealing with Bcfg2 repos."""
- name = 'Bzr'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- self.core = core
- self.datastore = datastore
-
- # Read revision from bcfg2 repo
- revision = self.get_revision()
-
- logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision})
-
- def get_revision(self):
- """Read Bazaar revision information for the Bcfg2 repository."""
- try:
- working_tree = WorkingTree.open(self.datastore)
- revision = str(working_tree.branch.revno())
- if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns():
- revision += "+"
- except errors.NotBranchError:
- logger.error("Failed to read Bazaar branch; disabling Bazaar support")
- raise Bcfg2.Server.Plugin.PluginInitError
- return revision
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
deleted file mode 100644
index c3e807d7a..000000000
--- a/src/lib/Server/Plugins/Cfg.py
+++ /dev/null
@@ -1,283 +0,0 @@
-"""This module implements a config file repository."""
-
-import binascii
-import logging
-import lxml
-import operator
-import os
-import os.path
-import re
-import stat
-import sys
-import tempfile
-from subprocess import Popen, PIPE
-from Bcfg2.Bcfg2Py3k import u_str
-
-import Bcfg2.Server.Plugin
-
-try:
- import genshi.core
- import genshi.input
- from genshi.template import TemplateLoader, NewTextTemplate
- have_genshi = True
-except:
- have_genshi = False
-
-try:
- import Cheetah.Template
- import Cheetah.Parser
- have_cheetah = True
-except:
- have_cheetah = False
-
-# setup logging
-logger = logging.getLogger('Bcfg2.Plugins.Cfg')
-
-
-# snipped from TGenshi
-def removecomment(stream):
- """A genshi filter that removes comments from the stream."""
- for kind, data, pos in stream:
- if kind is genshi.core.COMMENT:
- continue
- yield kind, data, pos
-
-
-def process_delta(data, delta):
- if not delta.specific.delta:
- return data
- if delta.specific.delta == 'cat':
- datalines = data.strip().split('\n')
- for line in delta.data.split('\n'):
- if not line:
- continue
- if line[0] == '+':
- datalines.append(line[1:])
- elif line[0] == '-':
- if line[1:] in datalines:
- datalines.remove(line[1:])
- return "\n".join(datalines) + "\n"
- elif delta.specific.delta == 'diff':
- basehandle, basename = tempfile.mkstemp()
- basefile = open(basename, 'w')
- basefile.write(data)
- basefile.close()
- os.close(basehandle)
-
- cmd = ["patch", "-u", "-f", basefile.name]
- patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
- stderr = patch.communicate(input=delta.data)[1]
- ret = patch.wait()
- output = open(basefile.name, 'r').read()
- os.unlink(basefile.name)
- if ret >> 8 != 0:
- logger.error("Error applying diff %s: %s" % (delta.name, stderr))
- raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta)
- return output
-
-
-class CfgMatcher:
-
- def __init__(self, fname):
- name = re.escape(fname)
- self.basefile_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))((?P<genshi>\\.genshi)|(?P<cheetah>\\.cheetah))?$' % name)
- self.delta_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % name)
- self.cat_count = fname.count(".cat")
- self.diff_count = fname.count(".diff")
-
- def match(self, fname):
- if fname.count(".cat") > self.cat_count \
- or fname.count('.diff') > self.diff_count:
- return self.delta_reg.match(fname)
- return self.basefile_reg.match(fname)
-
-
-class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
-
- def __init__(self, basename, path, entry_type, encoding):
- Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
- entry_type, encoding)
- self.specific = CfgMatcher(path.split('/')[-1])
- path = path
-
- def debug_log(self, message, flag=None):
- if (flag is None and self.debug_flag) or flag:
- logger.error(message)
-
- def sort_by_specific(self, one, other):
- return cmp(one.specific, other.specific)
-
- def get_pertinent_entries(self, entry, metadata):
- """return a list of all entries pertinent
- to a client => [base, delta1, delta2]
- """
- matching = [ent for ent in list(self.entries.values()) if \
- ent.specific.matches(metadata)]
- matching.sort(key=operator.attrgetter('specific'))
- # base entries which apply to a client
- # (e.g. foo, foo.G##_groupname, foo.H_hostname)
- base_files = [matching.index(m) for m in matching
- if not m.specific.delta]
- if not base_files:
- logger.error("No base file found for %s" % entry.get('name'))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- base = min(base_files)
- used = matching[:base + 1]
- used.reverse()
- return used
-
- def bind_entry(self, entry, metadata):
- self.bind_info_to_entry(entry, metadata)
- used = self.get_pertinent_entries(entry, metadata)
- basefile = used.pop(0)
- if entry.get('perms').lower() == 'inherit':
- # use on-disk permissions
- fname = "%s/%s" % (self.path, entry.get('name'))
- entry.set('perms',
- str(oct(stat.S_IMODE(os.stat(fname).st_mode))))
- if entry.tag == 'Path':
- entry.set('type', 'file')
- if basefile.name.endswith(".genshi"):
- if not have_genshi:
- logger.error("Cfg: Genshi is not available")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- try:
- template_cls = NewTextTemplate
- loader = TemplateLoader()
- template = loader.load(basefile.name, cls=template_cls,
- encoding=self.encoding)
- fname = entry.get('realname', entry.get('name'))
- stream = template.generate(name=fname,
- metadata=metadata,
- path=basefile.name).filter(removecomment)
- try:
- data = stream.render('text', encoding=self.encoding,
- strip_whitespace=False)
- except TypeError:
- data = stream.render('text', encoding=self.encoding)
- if data == '':
- entry.set('empty', 'true')
- except Exception:
- e = sys.exc_info()[1]
- logger.error("Cfg: genshi exception: %s" % e)
- raise Bcfg2.Server.Plugin.PluginExecutionError
- elif basefile.name.endswith(".cheetah"):
- if not have_cheetah:
- logger.error("Cfg: Cheetah is not available")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- try:
- fname = entry.get('realname', entry.get('name'))
- s = {'useStackFrames': False}
- template = Cheetah.Template.Template(open(basefile.name).read(),
- compilerSettings=s)
- template.metadata = metadata
- template.path = fname
- template.source_path = basefile.name
- data = template.respond()
- if data == '':
- entry.set('empty', 'true')
- except Exception:
- e = sys.exc_info()[1]
- logger.error("Cfg: cheetah exception: %s" % e)
- raise Bcfg2.Server.Plugin.PluginExecutionError
- else:
- data = basefile.data
- for delta in used:
- data = process_delta(data, delta)
- if entry.get('encoding') == 'base64':
- entry.text = binascii.b2a_base64(data)
- else:
- try:
- entry.text = u_str(data, self.encoding)
- except UnicodeDecodeError:
- e = sys.exc_info()[1]
- logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
- logger.error("Please verify you are using the proper encoding.")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- except ValueError:
- e = sys.exc_info()[1]
- logger.error("Error in specification for %s" % entry.get('name'))
- logger.error("%s" % e)
- logger.error("You need to specify base64 encoding for %s." %
- entry.get('name'))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- if entry.text in ['', None]:
- entry.set('empty', 'true')
-
- def list_accept_choices(self, entry, metadata):
- '''return a list of candidate pull locations'''
- used = self.get_pertinent_entries(entry, metadata)
- ret = []
- if used:
- ret.append(used[0].specific)
- if not ret[0].hostname:
- ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname))
- return ret
-
- def build_filename(self, specific):
- bfname = self.path + '/' + self.path.split('/')[-1]
- if specific.all:
- return bfname
- elif specific.group:
- return "%s.G%02d_%s" % (bfname, specific.prio, specific.group)
- elif specific.hostname:
- return "%s.H_%s" % (bfname, specific.hostname)
-
- def write_update(self, specific, new_entry, log):
- if 'text' in new_entry:
- name = self.build_filename(specific)
- if os.path.exists("%s.genshi" % name):
- logger.error("Cfg: Unable to pull data for genshi types")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- elif os.path.exists("%s.cheetah" % name):
- logger.error("Cfg: Unable to pull data for cheetah types")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- try:
- etext = new_entry['text'].encode(self.encoding)
- except:
- logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- open(name, 'w').write(etext)
- self.debug_log("Wrote file %s" % name, flag=log)
- badattr = [attr for attr in ['owner', 'group', 'perms']
- if attr in new_entry]
- if badattr:
- # check for info files and inform user of their removal
- if os.path.exists(self.path + "/:info"):
- logger.info("Removing :info file and replacing with "
- "info.xml")
- os.remove(self.path + "/:info")
- if os.path.exists(self.path + "/info"):
- logger.info("Removing info file and replacing with "
- "info.xml")
- os.remove(self.path + "/info")
- metadata_updates = {}
- metadata_updates.update(self.metadata)
- for attr in badattr:
- metadata_updates[attr] = new_entry.get(attr)
- infoxml = lxml.etree.Element('FileInfo')
- infotag = lxml.etree.SubElement(infoxml, 'Info')
- [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \
- for attr in metadata_updates]
- ofile = open(self.path + "/info.xml", "w")
- ofile.write(lxml.etree.tostring(infoxml, pretty_print=True))
- ofile.close()
- self.debug_log("Wrote file %s" % (self.path + "/info.xml"),
- flag=log)
-
-
-class Cfg(Bcfg2.Server.Plugin.GroupSpool,
- Bcfg2.Server.Plugin.PullTarget):
- """This generator in the configuration file repository for Bcfg2."""
- name = 'Cfg'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- es_cls = CfgEntrySet
- es_child_cls = Bcfg2.Server.Plugin.SpecificData
-
- def AcceptChoices(self, entry, metadata):
- return self.entries[entry.get('name')].list_accept_choices(entry, metadata)
-
- def AcceptPullData(self, specific, new_entry, log):
- return self.entries[new_entry.get('name')].write_update(specific,
- new_entry,
- log)
diff --git a/src/lib/Server/Plugins/Cvs.py b/src/lib/Server/Plugins/Cvs.py
deleted file mode 100644
index 6ce72acd2..000000000
--- a/src/lib/Server/Plugins/Cvs.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-from subprocess import Popen, PIPE
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Cvs')
-
-class Cvs(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """CVS is a version plugin for dealing with Bcfg2 repository."""
- name = 'Cvs'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- self.core = core
- self.datastore = datastore
-
- # path to cvs directory for Bcfg2 repo
- cvs_dir = "%s/CVSROOT" % datastore
-
- # Read revision from Bcfg2 repo
- if os.path.isdir(cvs_dir):
- self.get_revision()
- else:
- logger.error("%s is not a directory" % cvs_dir)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir)
-
- def get_revision(self):
- """Read cvs revision information for the Bcfg2 repository."""
- try:
- data = Popen("env LC_ALL=C cvs log",
- shell=True,
- cwd=self.datastore,
- stdout=PIPE).stdout.readlines()
- revision = data[3].strip('\n')
- except IndexError:
- logger.error("Failed to read cvs log; disabling cvs support")
- logger.error('''Ran command "cvs log %s"''' % (self.datastore))
- logger.error("Got output: %s" % data)
- raise Bcfg2.Server.Plugin.PluginInitError
-
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
deleted file mode 100644
index 95395f74e..000000000
--- a/src/lib/Server/Plugins/DBStats.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import binascii
-import difflib
-import logging
-import lxml.etree
-import platform
-import time
-
-try:
- from django.core.exceptions import MultipleObjectsReturned
-except ImportError:
- pass
-
-import Bcfg2.Server.Plugin
-import Bcfg2.Server.Reports.importscript
-from Bcfg2.Server.Reports.reports.models import Client
-import Bcfg2.Server.Reports.settings
-from Bcfg2.Server.Reports.updatefix import update_database
-# for debugging output only
-logger = logging.getLogger('Bcfg2.Plugins.DBStats')
-
-class DBStats(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.ThreadedStatistics,
- Bcfg2.Server.Plugin.PullSource):
- name = 'DBStats'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.PullSource.__init__(self)
- self.cpath = "%s/Metadata/clients.xml" % datastore
- self.core = core
- logger.debug("Searching for new models to add to the statistics database")
- try:
- update_database()
- except Exception:
- inst = sys.exc_info()[1]
- logger.debug(str(inst))
- logger.debug(str(type(inst)))
-
- def handle_statistic(self, metadata, data):
- newstats = data.find("Statistics")
- newstats.set('time', time.asctime(time.localtime()))
- # ick
- data = lxml.etree.tostring(newstats)
- ndx = lxml.etree.XML(data)
- e = lxml.etree.Element('Node', name=metadata.hostname)
- e.append(ndx)
- container = lxml.etree.Element("ConfigStatistics")
- container.append(e)
-
- # FIXME need to build a metadata interface to expose a list of clients
- start = time.time()
- for i in [1, 2, 3]:
- try:
- Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
- container,
- self.core.encoding,
- 0,
- logger,
- True,
- platform.node())
- logger.info("Imported data for %s in %s seconds" \
- % (metadata.hostname, time.time() - start))
- return
- except MultipleObjectsReturned:
- e = sys.exc_info()[1]
- logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \
- (metadata.hostname, e))
- logger.error("DBStats: Data is inconsistent")
- break
- except:
- logger.error("DBStats: Failed to write to db (lock); retrying",
- exc_info=1)
- logger.error("DBStats: Retry limit failed for %s; aborting operation" \
- % metadata.hostname)
-
- def GetExtra(self, client):
- c_inst = Client.objects.filter(name=client)[0]
- return [(a.entry.kind, a.entry.name) for a in
- c_inst.current_interaction.extra()]
-
- def GetCurrentEntry(self, client, e_type, e_name):
- try:
- c_inst = Client.objects.filter(name=client)[0]
- except IndexError:
- self.logger.error("Unknown client: %s" % client)
- raise Bcfg2.Server.Plugin.PluginExecutionError
- result = c_inst.current_interaction.bad().filter(entry__kind=e_type,
- entry__name=e_name)
- if not result:
- raise Bcfg2.Server.Plugin.PluginExecutionError
- entry = result[0]
- ret = []
- data = ('owner', 'group', 'perms')
- for t in data:
- if getattr(entry.reason, "current_%s" % t) == '':
- ret.append(getattr(entry.reason, t))
- else:
- ret.append(getattr(entry.reason, "current_%s" % t))
- if entry.reason.is_sensitive:
- raise Bcfg2.Server.Plugin.PluginExecutionError
- elif entry.reason.current_diff != '':
- if entry.reason.is_binary:
- ret.append(binascii.a2b_base64(entry.reason.current_diff))
- else:
- ret.append('\n'.join(difflib.restore(\
- entry.reason.current_diff.split('\n'), 1)))
- elif entry.reason.is_binary:
- # If len is zero the object was too large to store
- raise Bcfg2.Server.Plugin.PluginExecutionError
- else:
- ret.append(None)
- return ret
diff --git a/src/lib/Server/Plugins/Darcs.py b/src/lib/Server/Plugins/Darcs.py
deleted file mode 100644
index 9fb9ff4f1..000000000
--- a/src/lib/Server/Plugins/Darcs.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-from subprocess import Popen, PIPE
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Darcs')
-
-class Darcs(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Darcs is a version plugin for dealing with Bcfg2 repos."""
- name = 'Darcs'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Version.__init__(self)
- self.core = core
- self.datastore = datastore
-
- # path to darcs directory for bcfg2 repo
- darcs_dir = "%s/_darcs" % datastore
-
- # Read changeset from bcfg2 repo
- if os.path.isdir(darcs_dir):
- self.get_revision()
- else:
- logger.error("%s is not present." % darcs_dir)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir)
-
- def get_revision(self):
- """Read Darcs changeset information for the Bcfg2 repository."""
- try:
- data = Popen("env LC_ALL=C darcs changes",
- shell=True,
- cwd=self.datastore,
- stdout=PIPE).stdout.readlines()
- revision = data[0].strip('\n')
- except:
- logger.error("Failed to read darcs repository; disabling Darcs support")
- logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore))
- logger.error("Got output: %s" % data)
- raise Bcfg2.Server.Plugin.PluginInitError
- return revision
-
diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Server/Plugins/Decisions.py
deleted file mode 100644
index b432474f2..000000000
--- a/src/lib/Server/Plugins/Decisions.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import logging
-import lxml.etree
-import sys
-
-import Bcfg2.Server.Plugin
-logger = logging.getLogger('Bcfg2.Plugins.Decisions')
-
-class DecisionFile(Bcfg2.Server.Plugin.SpecificData):
- def handle_event(self, event):
- Bcfg2.Server.Plugin.SpecificData.handle_event(self, event)
- self.contents = lxml.etree.XML(self.data)
-
- def get_decisions(self):
- return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')]
-
-class DecisionSet(Bcfg2.Server.Plugin.EntrySet):
- def __init__(self, path, fam, encoding):
- """Container for decision specification files.
-
- Arguments:
- - `path`: repository path
- - `fam`: reference to the file monitor
- - `encoding`: XML character encoding
-
- """
- pattern = '(white|black)list'
- Bcfg2.Server.Plugin.EntrySet.__init__(self, pattern, path, \
- DecisionFile, encoding)
- try:
- fam.AddMonitor(path, self)
- except OSError:
- e = sys.exc_info()[1]
- logger.error('Adding filemonitor for %s failed. '
- 'Make sure directory exists' % path)
- raise Bcfg2.Server.Plugin.PluginInitError(e)
-
- def HandleEvent(self, event):
- if event.filename != self.path:
- return self.handle_event(event)
-
- def GetDecisions(self, metadata, mode):
- ret = []
- candidates = [c for c in self.get_matching(metadata)
- if c.name.split('/')[-1].startswith(mode)]
- for c in candidates:
- ret += c.get_decisions()
- return ret
-
-class Decisions(DecisionSet,
- Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Decision):
- name = 'Decisions'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- """Decisions plugins
-
- Arguments:
- - `core`: Bcfg2.Core instance
- - `datastore`: File repository location
-
- """
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Decision.__init__(self)
- DecisionSet.__init__(self, self.data, core.fam, core.encoding)
-
diff --git a/src/lib/Server/Plugins/Defaults.py b/src/lib/Server/Plugins/Defaults.py
deleted file mode 100644
index 718192e2a..000000000
--- a/src/lib/Server/Plugins/Defaults.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""This generator provides rule-based entry mappings."""
-
-import re
-import Bcfg2.Server.Plugin
-import Bcfg2.Server.Plugins.Rules
-
-class Defaults(Bcfg2.Server.Plugins.Rules.Rules,
- Bcfg2.Server.Plugin.StructureValidator):
- """Set default attributes on bound entries"""
- name = 'Defaults'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- # Rules is a Generator that happens to implement all of the
- # functionality we want, so we overload it, but Defaults should
- # _not_ handle any entries; it does its stuff in the structure
- # validation phase. so we overload Handle(s)Entry and HandleEvent
- # to ensure that Defaults handles no entries, even though it's a
- # Generator.
-
- def HandlesEntry(self, entry, metadata):
- return False
-
- def HandleEntry(self, entry, metadata):
- raise PluginExecutionError
-
- def HandleEvent(self, event):
- Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event)
-
- def validate_structures(self, metadata, structures):
- """ Apply defaults """
- for struct in structures:
- for entry in struct.iter():
- if entry.tag.startswith("Bound"):
- is_bound = True
- entry.tag = entry.tag[5:]
- else:
- is_bound = False
- try:
- try:
- self.BindEntry(entry, metadata)
- except Bcfg2.Server.Plugin.PluginExecutionError:
- # either no matching defaults (which is okay),
- # or multiple matching defaults (which is not
- # okay, but is logged). either way, we don't
- # care about the error.
- pass
- finally:
- if is_bound:
- entry.tag = "Bound" + entry.tag
-
- def _regex_enabled(self):
- """ Defaults depends on regex matching, so force it enabled """
- return True
diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py
deleted file mode 100644
index 9b848baae..000000000
--- a/src/lib/Server/Plugins/Deps.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""This plugin provides automatic dependency handling."""
-
-import lxml.etree
-
-import Bcfg2.Server.Plugin
-
-
-class DNode(Bcfg2.Server.Plugin.INode):
- """DNode provides supports for single predicate types for dependencies."""
- raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"}
- containers = ['Group']
-
- def __init__(self, data, idict, parent=None):
- self.data = data
- self.contents = {}
- if parent == None:
- self.predicate = lambda x, d: True
- else:
- predicate = parent.predicate
- if data.tag in list(self.raw.keys()):
- self.predicate = eval(self.raw[data.tag] %
- {'name': data.get('name')},
- {'predicate': predicate})
- else:
- raise Exception
- mytype = self.__class__
- self.children = []
- for item in data.getchildren():
- if item.tag in self.containers:
- self.children.append(mytype(item, idict, self))
- else:
- data = [(child.tag, child.get('name'))
- for child in item.getchildren()]
- try:
- self.contents[item.tag][item.get('name')] = data
- except KeyError:
- self.contents[item.tag] = {item.get('name'): data}
-
-
-class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc):
- __node__ = DNode
-
-
-class Deps(Bcfg2.Server.Plugin.PrioDir,
- Bcfg2.Server.Plugin.StructureValidator):
- name = 'Deps'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- __child__ = DepXMLSrc
-
- # Override the default sort_order (of 500) so that this plugin
- # gets handled after others running at the default. In particular,
- # we want to run after Packages, so we can see the final set of
- # packages that will be installed on the client.
- sort_order = 750
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.StructureValidator.__init__(self)
- self.cache = {}
-
- def HandleEvent(self, event):
- self.cache = {}
- Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event)
-
- def validate_structures(self, metadata, structures):
- """Examine the passed structures and append any additional
- prerequisite entries as defined by the files in Deps.
- """
- entries = []
- for structure in structures:
- for entry in structure.getchildren():
- tag = entry.tag
- if tag.startswith('Bound'):
- tag = tag[5:]
- if (tag, entry.get('name')) not in entries \
- and not isinstance(entry, lxml.etree._Comment):
- entries.append((tag, entry.get('name')))
- entries.sort()
- entries = tuple(entries)
- gdata = list(metadata.groups)
- gdata.sort()
- gdata = tuple(gdata)
-
- # Check to see if we have cached the prereqs already
- if (entries, gdata) in self.cache:
- prereqs = self.cache[(entries, gdata)]
- else:
- prereqs = self.calculate_prereqs(metadata, entries)
- self.cache[(entries, gdata)] = prereqs
-
- newstruct = lxml.etree.Element("Independent")
- for tag, name in prereqs:
- try:
- lxml.etree.SubElement(newstruct, tag, name=name)
- except:
- self.logger.error("Failed to add dep entry for %s:%s" % (tag, name))
- structures.append(newstruct)
-
-
- def calculate_prereqs(self, metadata, entries):
- """Calculate the prerequisites defined in Deps for the passed
- set of entries.
- """
- prereqs = []
- [src.Cache(metadata) for src in self.entries.values()]
-
- toexamine = list(entries[:])
- while toexamine:
- entry = toexamine.pop()
- matching = [src for src in list(self.entries.values())
- if src.cache and entry[0] in src.cache[1]
- and entry[1] in src.cache[1][entry[0]]]
- if len(matching) > 1:
- prio = [int(src.priority) for src in matching]
- if prio.count(max(prio)) > 1:
- self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" %
- (entry[0].lower(), metadata.hostname, entry[1]))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- index = prio.index(max(prio))
- matching = [matching[index]]
- elif len(matching) == 1:
- for prq in matching[0].cache[1][entry[0]][entry[1]]:
- # XML comments seem to show up in the cache as a
- # tuple with item 0 being callable. The logic
- # below filters them out. Would be better to
- # exclude them when we load the cache in the first
- # place.
- if prq not in prereqs and prq not in entries and not callable(prq[0]):
- toexamine.append(prq)
- prereqs.append(prq)
- else:
- continue
-
- return prereqs
diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Server/Plugins/Editor.py
deleted file mode 100644
index c0d2cfbad..000000000
--- a/src/lib/Server/Plugins/Editor.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import Bcfg2.Server.Plugin
-import re
-import lxml.etree
-
-
-def linesub(pattern, repl, filestring):
- """Substitutes instances of pattern with repl in filestring."""
- if filestring == None:
- filestring = ''
- output = list()
- fileread = filestring.split('\n')
- for line in fileread:
- output.append(re.sub(pattern, repl, filestring))
- return '\n'.join(output)
-
-
-class EditDirectives(Bcfg2.Server.Plugin.SpecificData):
- """This object handles the editing directives."""
- def ProcessDirectives(self, input):
- """Processes a list of edit directives on input."""
- temp = input
- for directive in self.data.split('\n'):
- directive = directive.split(',')
- temp = linesub(directive[0], directive[1], temp)
- return temp
-
-
-class EditEntrySet(Bcfg2.Server.Plugin.EntrySet):
- def __init__(self, basename, path, entry_type, encoding):
- self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1])
- Bcfg2.Server.Plugin.EntrySet.__init__(self,
- basename,
- path,
- entry_type,
- encoding)
- self.inputs = dict()
-
- def bind_entry(self, entry, metadata):
- client = metadata.hostname
- filename = entry.get('name')
- permdata = {'owner': 'root',
- 'group': 'root',
- 'perms': '0644'}
- [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
- entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client))
- if not entry.text:
- entry.set('empty', 'true')
- try:
- f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w')
- f.write(entry.text)
- f.close()
- except:
- pass
-
- def get_client_data(self, client):
- return self.inputs[client]
-
-
-class Editor(Bcfg2.Server.Plugin.GroupSpool,
- Bcfg2.Server.Plugin.Probing):
- name = 'Editor'
- __author__ = 'bcfg2-dev@mcs.anl.gov'
- filename_pattern = 'edits'
- es_child_cls = EditDirectives
- es_cls = EditEntrySet
-
- def GetProbes(self, _):
- '''Return a set of probes for execution on client'''
- probelist = list()
- for name in list(self.entries.keys()):
- probe = lxml.etree.Element('probe')
- probe.set('name', name)
- probe.set('source', "Editor")
- probe.text = "cat %s" % name
- probelist.append(probe)
- return probelist
-
- def ReceiveData(self, client, datalist):
- for data in datalist:
- self.entries[data.get('name')].inputs[client.hostname] = data.text
diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py
deleted file mode 100644
index a76d7cac4..000000000
--- a/src/lib/Server/Plugins/FileProbes.py
+++ /dev/null
@@ -1,230 +0,0 @@
-""" This module allows you to probe a client for a file, which is then
-added to the specification. On subsequent runs, the file will be
-replaced on the client if it is missing; if it has changed on the
-client, it can either be updated in the specification or replaced on
-the client """
-
-import os
-import sys
-import errno
-import binascii
-import lxml.etree
-import Bcfg2.Options
-import Bcfg2.Server.Plugin
-
-probecode = """#!/usr/bin/env python
-
-import os
-import pwd
-import grp
-import binascii
-import lxml.etree
-
-path = "%s"
-
-if not os.path.exists(path):
- print "%%s does not exist" %% path
- raise SystemExit(1)
-
-stat = os.stat(path)
-data = lxml.etree.Element("ProbedFileData",
- name=path,
- owner=pwd.getpwuid(stat[4])[0],
- group=grp.getgrgid(stat[5])[0],
- perms=oct(stat[0] & 07777))
-data.text = binascii.b2a_base64(open(path).read())
-print lxml.etree.tostring(data)
-"""
-
-class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked,
- Bcfg2.Server.Plugin.StructFile):
- """ Config file handler for FileProbes """
- def __init__(self, filename, fam):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
- Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
-
-
-class FileProbes(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Probing):
- """ This module allows you to probe a client for a file, which is then
- added to the specification. On subsequent runs, the file will be
- replaced on the client if it is missing; if it has changed on the
- client, it can either be updated in the specification or replaced on
- the client """
-
- name = 'FileProbes'
- experimental = True
- __author__ = 'chris.a.st.pierre@gmail.com'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Probing.__init__(self)
- self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'),
- core.fam)
- self.entries = dict()
- self.probes = dict()
-
- def GetProbes(self, metadata):
- """Return a set of probes for execution on client."""
- if metadata.hostname not in self.probes:
- cfg = self.core.plugins['Cfg']
- self.entries[metadata.hostname] = dict()
- self.probes[metadata.hostname] = []
- for entry in self.config.Match(metadata):
- path = entry.get("name")
- # do not probe for files that are already in Cfg and
- # for which update is false; we can't possibly do
- # anything with the data we get from such a probe
- try:
- if (entry.get('update', 'false').lower() == "false" and
- cfg.entries[path].get_pertinent_entries(entry,
- metadata)):
- continue
- except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError):
- pass
- self.entries[metadata.hostname][path] = entry
- probe = lxml.etree.Element('probe', name=path,
- source=self.name,
- interpreter="/usr/bin/env python")
- probe.text = probecode % path
- self.probes[metadata.hostname].append(probe)
- self.debug_log("Adding file probe for %s to %s" %
- (path, metadata.hostname))
- return self.probes[metadata.hostname]
-
- def ReceiveData(self, metadata, datalist):
- """Receive data from probe."""
- self.debug_log("Receiving file probe data from %s" % metadata.hostname)
-
- for data in datalist:
- if data.text is None:
- self.logger.error("Got null response to %s file probe from %s" %
- (data.get('name'), metadata.hostname))
- else:
- try:
- self.write_data(lxml.etree.XML(data.text), metadata)
- except lxml.etree.XMLSyntaxError:
- # if we didn't get XML back from the probe, assume
- # it's an error message
- self.logger.error(data.text)
-
- def write_data(self, data, metadata):
- """Write the probed file data to the bcfg2 specification."""
- filename = data.get("name")
- contents = binascii.a2b_base64(data.text)
- entry = self.entries[metadata.hostname][filename]
- cfg = self.core.plugins['Cfg']
- specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname)
- # we can't use os.path.join() for this because specific
- # already has a leading /, which confuses os.path.join()
- fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific))
-
- create = False
- try:
- cfg.entries[filename].bind_entry(entry, metadata)
- except Bcfg2.Server.Plugin.PluginExecutionError:
- create = True
-
- # get current entry data
- if entry.text and entry.get("encoding") == "base64":
- entrydata = binascii.a2b_base64(entry.text)
- else:
- entrydata = entry.text
-
- if create:
- self.logger.info("Writing new probed file %s" % fileloc)
- self.write_file(fileloc, contents)
- self.verify_file(filename, contents, metadata)
- infoxml = os.path.join("%s%s" % (cfg.data, filename), "info.xml")
- self.write_infoxml(infoxml, entry, data)
- elif entrydata == contents:
- self.debug_log("Existing %s contents match probed contents" %
- filename)
- return
- elif (entry.get('update', 'false').lower() == "true"):
- self.logger.info("Writing updated probed file %s" % fileloc)
- self.write_file(fileloc, contents)
- self.verify_file(filename, contents, metadata)
- else:
- self.logger.info("Skipping updated probed file %s" % fileloc)
- return
-
- def write_file(self, fileloc, contents):
- try:
- os.makedirs(os.path.dirname(fileloc))
- except OSError:
- err = sys.exc_info()[1]
- if err.errno == errno.EEXIST:
- pass
- else:
- self.logger.error("Could not create parent directories for %s: "
- "%s" % (fileloc, err))
- return
-
- try:
- open(fileloc, 'wb').write(contents)
- except IOError:
- err = sys.exc_info()[1]
- self.logger.error("Could not write %s: %s" % (fileloc, err))
- return
-
- def verify_file(self, filename, contents, metadata):
- # Service the FAM events queued up by the key generation so
- # the data structure entries will be available for binding.
- #
- # NOTE: We wait for up to ten seconds. There is some potential
- # for race condition, because if the file monitor doesn't get
- # notified about the new key files in time, those entries
- # won't be available for binding. In practice, this seems
- # "good enough".
- entry = self.entries[metadata.hostname][filename]
- cfg = self.core.plugins['Cfg']
- tries = 0
- updated = False
- while not updated:
- if tries >= 10:
- self.logger.error("%s still not registered" % filename)
- return
- self.core.fam.handle_events_in_interval(1)
- try:
- cfg.entries[filename].bind_entry(entry, metadata)
- except Bcfg2.Server.Plugin.PluginExecutionError:
- tries += 1
- continue
-
- # get current entry data
- if entry.get("encoding") == "base64":
- entrydata = binascii.a2b_base64(entry.text)
- else:
- entrydata = entry.text
- if entrydata == contents:
- updated = True
- tries += 1
-
- def write_infoxml(self, infoxml, entry, data):
- """ write an info.xml for the file """
- if os.path.exists(infoxml):
- return
-
- self.logger.info("Writing info.xml at %s for %s" %
- (infoxml, data.get("name")))
- info = \
- lxml.etree.Element("Info",
- owner=data.get("owner",
- Bcfg2.Options.MDATA_OWNER.value),
- group=data.get("group",
- Bcfg2.Options.MDATA_GROUP.value),
- perms=data.get("perms",
- Bcfg2.Options.MDATA_PERMS.value),
- encoding=entry.get("encoding",
- Bcfg2.Options.ENCODING.value))
-
- root = lxml.etree.Element("FileInfo")
- root.append(info)
- try:
- open(infoxml, "w").write(lxml.etree.tostring(root,
- pretty_print=True))
- except IOError:
- err = sys.exc_info()[1]
- self.logger.error("Could not write %s: %s" % (fileloc, err))
- return
diff --git a/src/lib/Server/Plugins/Fossil.py b/src/lib/Server/Plugins/Fossil.py
deleted file mode 100644
index 1b1627688..000000000
--- a/src/lib/Server/Plugins/Fossil.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import os
-from subprocess import Popen, PIPE
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Fossil')
-
-class Fossil(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Fossil is a version plugin for dealing with Bcfg2 repos."""
- name = 'Fossil'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- self.core = core
- self.datastore = datastore
-
- # path to fossil file for bcfg2 repo
- fossil_file = "%s/_FOSSIL_" % datastore
-
- # Read revision from bcfg2 repo
- if os.path.isfile(fossil_file):
- revision = self.get_revision()
- elif not os.path.isdir(datastore):
- logger.error("%s is not a directory" % datastore)
- raise Bcfg2.Server.Plugin.PluginInitError
- else:
- logger.error("%s is not a file" % fossil_file)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \
- % {'ffile': fossil_file, 'frev': revision})
-
- def get_revision(self):
- """Read fossil revision information for the Bcfg2 repository."""
- try:
- data = Popen("env LC_ALL=C fossil info",
- shell=True,
- cwd=self.datastore,
- stdout=PIPE).stdout.readlines()
- revline = [line.split(': ')[1].strip() for line in data if \
- line.split(': ')[0].strip() == 'checkout'][-1]
- revision = revline.split(' ')[0]
- except IndexError:
- logger.error("Failed to read fossil info; disabling fossil support")
- logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore))
- logger.error("Got output: %s" % data)
- raise Bcfg2.Server.Plugin.PluginInitError
- return revision
diff --git a/src/lib/Server/Plugins/Git.py b/src/lib/Server/Plugins/Git.py
deleted file mode 100644
index 8f8ea87f1..000000000
--- a/src/lib/Server/Plugins/Git.py
+++ /dev/null
@@ -1,44 +0,0 @@
-"""The Git plugin provides a revision interface for Bcfg2 repos using git."""
-
-import os
-from dulwich.repo import Repo
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Git')
-
-
-class Git(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Git is a version plugin for dealing with Bcfg2 repos."""
- name = 'Git'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Version.__init__(self)
- self.core = core
- self.datastore = datastore
-
- # path to git directory for bcfg2 repo
- git_dir = "%s/.git" % datastore
-
- # Read revision from bcfg2 repo
- if os.path.isdir(git_dir):
- self.get_revision()
- else:
- logger.error("%s is not a directory" % git_dir)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized git plugin with git directory %s" % git_dir)
-
- def get_revision(self):
- """Read git revision information for the Bcfg2 repository."""
- try:
- repo = Repo(self.datastore)
- revision = repo.head()
- except:
- logger.error("Failed to read git repository; disabling git support")
- raise Bcfg2.Server.Plugin.PluginInitError
- return revision
diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Server/Plugins/GroupPatterns.py
deleted file mode 100644
index 58b4d4afb..000000000
--- a/src/lib/Server/Plugins/GroupPatterns.py
+++ /dev/null
@@ -1,124 +0,0 @@
-import re
-import logging
-import lxml.etree
-import Bcfg2.Server.Plugin
-
-class PackedDigitRange(object):
- def __init__(self, digit_range):
- self.sparse = list()
- self.ranges = list()
- for item in digit_range.split(','):
- if '-' in item:
- self.ranges.append(tuple([int(x) for x in item.split('-')]))
- else:
- self.sparse.append(int(item))
-
- def includes(self, other):
- iother = int(other)
- if iother in self.sparse:
- return True
- for (start, end) in self.ranges:
- if iother in range(start, end + 1):
- return True
- return False
-
-
-class PatternMap(object):
- range_finder = r'\[\[[\d\-,]+\]\]'
-
- def __init__(self, pattern, rangestr, groups):
- self.pattern = pattern
- self.rangestr = rangestr
- self.groups = groups
- if pattern != None:
- self.re = re.compile(pattern)
- self.process = self.process_re
- elif rangestr != None:
- if '\\' in rangestr:
- raise Exception("Backslashes are not allowed in NameRanges")
- self.process = self.process_range
- self.re = re.compile('^' + re.sub(self.range_finder, '(\d+)',
- rangestr))
- dmatcher = re.compile(re.sub(self.range_finder,
- r'\[\[([\d\-,]+)\]\]',
- rangestr))
- self.dranges = [PackedDigitRange(x)
- for x in dmatcher.match(rangestr).groups()]
- else:
- raise Exception("No pattern or range given")
-
- def process_range(self, name):
- match = self.re.match(name)
- if not match:
- return None
- digits = match.groups()
- for i in range(len(digits)):
- if not self.dranges[i].includes(digits[i]):
- return None
- return self.groups
-
- def process_re(self, name):
- match = self.re.match(name)
- if not match:
- return None
- ret = list()
- sub = match.groups()
- for group in self.groups:
- newg = group
- for idx in range(len(sub)):
- newg = newg.replace('$%s' % (idx + 1), sub[idx])
- ret.append(newg)
- return ret
-
-
-class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked):
- __identifier__ = None
-
- def __init__(self, filename, fam):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
- self.patterns = []
- self.logger = logging.getLogger(self.__class__.__name__)
-
- def Index(self):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self)
- self.patterns = []
- for entry in self.xdata.xpath('//GroupPattern'):
- try:
- groups = [g.text for g in entry.findall('Group')]
- for pat_ent in entry.findall('NamePattern'):
- pat = pat_ent.text
- self.patterns.append(PatternMap(pat, None, groups))
- for range_ent in entry.findall('NameRange'):
- rng = range_ent.text
- self.patterns.append(PatternMap(None, rng, groups))
- except:
- self.logger.error("GroupPatterns: Failed to initialize pattern "
- "%s" % entry.get('pattern'))
-
- def process_patterns(self, hostname):
- ret = []
- for pattern in self.patterns:
- try:
- gn = pattern.process(hostname)
- if gn is not None:
- ret.extend(gn)
- except:
- self.logger.error("GroupPatterns: Failed to process pattern %s "
- "for %s" % (pattern.pattern, hostname),
- exc_info=1)
- return ret
-
-
-class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Connector):
- name = "GroupPatterns"
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- self.config = PatternFile(self.data + '/config.xml',
- core.fam)
-
- def get_additional_groups(self, metadata):
- return self.config.process_patterns(metadata.hostname)
diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Server/Plugins/Guppy.py
deleted file mode 100644
index 046aedc0b..000000000
--- a/src/lib/Server/Plugins/Guppy.py
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-This plugin is used to trace memory leaks within the bcfg2-server
-process using Guppy. By default the remote debugger is started
-when this plugin is enabled. The debugger can be shutoff in a running
-process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using
-"bcfg2-admin xcmd Guppy.Enable".
-
-To attach the console run:
-
-python -c "from guppy import hpy;hpy().monitor()"
-
-For example:
-
-# python -c "from guppy import hpy;hpy().monitor()"
-<Monitor>
-*** Connection 1 opened ***
-<Monitor> lc
-CID PID ARGV
- 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid']
-<Monitor> sc 1
-Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN>
-<Annex> int
-Remote interactive console. To return to Annex, type '-'.
->>> hp.heap()
-...
-
-
-"""
-import re
-import Bcfg2.Server.Plugin
-
-class Guppy(Bcfg2.Server.Plugin.Plugin):
- """Guppy is a debugging plugin to help trace memory leaks"""
- name = 'Guppy'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- experimental = True
- __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable']
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
-
- self.Enable()
-
- def Enable(self):
- """Enable remote debugging"""
- try:
- from guppy.heapy import Remote
- Remote.on()
- except:
- self.logger.error("Failed to create Heapy context")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def Disable(self):
- """Disable remote debugging"""
- try:
- from guppy.heapy import Remote
- Remote.off()
- except:
- self.logger.error("Failed to disable Heapy")
- raise Bcfg2.Server.Plugin.PluginInitError
-
diff --git a/src/lib/Server/Plugins/Hg.py b/src/lib/Server/Plugins/Hg.py
deleted file mode 100644
index 0c3537613..000000000
--- a/src/lib/Server/Plugins/Hg.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import os
-from mercurial import ui, hg
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Mercurial')
-
-class Hg(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Mercurial is a version plugin for dealing with Bcfg2 repository."""
- name = 'Mercurial'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Version.__init__(self)
- self.core = core
- self.datastore = datastore
-
- # path to hg directory for Bcfg2 repo
- hg_dir = "%s/.hg" % datastore
-
- # Read changeset from bcfg2 repo
- if os.path.isdir(hg_dir):
- self.get_revision()
- else:
- logger.error("%s is not present." % hg_dir)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir)
-
- def get_revision(self):
- """Read hg revision information for the Bcfg2 repository."""
- try:
- repo_path = "%s/" % self.datastore
- repo = hg.repository(ui.ui(), repo_path)
- tip = repo.changelog.tip()
- revision = repo.changelog.rev(tip)
- except:
- logger.error("Failed to read hg repository; disabling mercurial support")
- raise Bcfg2.Server.Plugin.PluginInitError
- return revision
-
diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py
deleted file mode 100644
index e9c1c1cff..000000000
--- a/src/lib/Server/Plugins/Hostbase.py
+++ /dev/null
@@ -1,593 +0,0 @@
-"""
-This file provides the Hostbase plugin.
-It manages dns/dhcp/nis host information
-"""
-
-import os
-os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings'
-from lxml.etree import Element, SubElement
-import Bcfg2.Server.Plugin
-from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError
-from time import strftime
-from sets import Set
-from django.template import Context, loader
-from django.db import connection
-import re
-# Compatibility imports
-from Bcfg2.Bcfg2Py3k import StringIO
-
-
-class Hostbase(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Structure,
- Bcfg2.Server.Plugin.Generator):
- """The Hostbase plugin handles host/network info."""
- name = 'Hostbase'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- filepath = '/my/adm/hostbase/files/bind'
-
- def __init__(self, core, datastore):
-
- self.ready = False
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Structure.__init__(self)
- Bcfg2.Server.Plugin.Generator.__init__(self)
- files = ['zone.tmpl',
- 'reversesoa.tmpl',
- 'named.tmpl',
- 'reverseappend.tmpl',
- 'dhcpd.tmpl',
- 'hosts.tmpl',
- 'hostsappend.tmpl']
- self.filedata = {}
- self.dnsservers = []
- self.dhcpservers = []
- self.templates = {'zone': loader.get_template('zone.tmpl'),
- 'reversesoa': loader.get_template('reversesoa.tmpl'),
- 'named': loader.get_template('named.tmpl'),
- 'namedviews': loader.get_template('namedviews.tmpl'),
- 'reverseapp': loader.get_template('reverseappend.tmpl'),
- 'dhcp': loader.get_template('dhcpd.tmpl'),
- 'hosts': loader.get_template('hosts.tmpl'),
- 'hostsapp': loader.get_template('hostsappend.tmpl'),
- }
- self.Entries['ConfigFile'] = {}
- self.__rmi__ = ['rebuildState']
- try:
- self.rebuildState(None)
- except:
- raise PluginInitError
-
- def FetchFile(self, entry, metadata):
- """Return prebuilt file data."""
- fname = entry.get('name').split('/')[-1]
- if not fname in self.filedata:
- raise PluginExecutionError
- perms = {'owner': 'root',
- 'group': 'root',
- 'perms': '644'}
- [entry.attrib.__setitem__(key, value)
- for (key, value) in list(perms.items())]
- entry.text = self.filedata[fname]
-
- def BuildStructures(self, metadata):
- """Build hostbase bundle."""
- if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers:
- return []
- output = Element("Bundle", name='hostbase')
- if metadata.hostname in self.dnsservers:
- for configfile in self.Entries['ConfigFile']:
- if re.search('/etc/bind/', configfile):
- SubElement(output, "ConfigFile", name=configfile)
- if metadata.hostname in self.dhcpservers:
- SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf")
- return [output]
-
- def rebuildState(self, _):
- """Pre-cache all state information for hostbase config files
- callable as an XMLRPC function.
-
- """
- self.buildZones()
- self.buildDHCP()
- self.buildHosts()
- self.buildHostsLPD()
- self.buildPrinters()
- self.buildNetgroups()
- return True
-
- def buildZones(self):
- """Pre-build and stash zone files."""
- cursor = connection.cursor()
-
- cursor.execute("SELECT id, serial FROM hostbase_zone")
- zones = cursor.fetchall()
-
- for zone in zones:
- # update the serial number for all zone files
- todaydate = (strftime('%Y%m%d'))
- try:
- if todaydate == str(zone[1])[:8]:
- serial = zone[1] + 1
- else:
- serial = int(todaydate) * 100
- except (KeyError):
- serial = int(todaydate) * 100
- cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0]))
-
- cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'")
- zones = cursor.fetchall()
-
- iplist = []
- hosts = {}
-
- for zone in zones:
- zonefile = StringIO()
- externalzonefile = StringIO()
- cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z
- INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id
- WHERE z.zone_id = \'%s\'""" % zone[0])
- nameservers = cursor.fetchall()
- cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z
- INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id
- WHERE z.zone_id = \'%s\'""" % zone[0])
- addresses = cursor.fetchall()
- cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z
- INNER JOIN hostbase_mx m ON z.mx_id = m.id
- WHERE z.zone_id = \'%s\'""" % zone[0])
- mxs = cursor.fetchall()
- context = Context({
- 'zone': zone,
- 'nameservers': nameservers,
- 'addresses': addresses,
- 'mxs': mxs
- })
- zonefile.write(self.templates['zone'].render(context))
- externalzonefile.write(self.templates['zone'].render(context))
-
- querystring = """SELECT h.hostname, p.ip_addr,
- n.name, c.cname, m.priority, m.mx, n.dns_view
- FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id)
- INNER JOIN hostbase_name n ON p.id = n.ip_id)
- INNER JOIN hostbase_name_mxs x ON n.id = x.name_id)
- INNER JOIN hostbase_mx m ON m.id = x.mx_id)
- LEFT JOIN hostbase_cname c ON n.id = c.name_id
- WHERE n.name LIKE '%%%%%s'
- AND h.status = 'active'
- ORDER BY h.hostname, n.name, p.ip_addr
- """ % zone[1]
- cursor.execute(querystring)
- zonehosts = cursor.fetchall()
- prevhost = (None, None, None, None)
- cnames = StringIO()
- cnamesexternal = StringIO()
- for host in zonehosts:
- if not host[2].split(".", 1)[1] == zone[1]:
- zonefile.write(cnames.getvalue())
- externalzonefile.write(cnamesexternal.getvalue())
- cnames = StringIO()
- cnamesexternal = StringIO()
- continue
- if not prevhost[1] == host[1] or not prevhost[2] == host[2]:
- zonefile.write(cnames.getvalue())
- externalzonefile.write(cnamesexternal.getvalue())
- cnames = StringIO()
- cnamesexternal = StringIO()
- zonefile.write("%-32s%-10s%-32s\n" %
- (host[2].split(".", 1)[0], 'A', host[1]))
- zonefile.write("%-32s%-10s%-3s%s.\n" %
- ('', 'MX', host[4], host[5]))
- if host[6] == 'global':
- externalzonefile.write("%-32s%-10s%-32s\n" %
- (host[2].split(".", 1)[0], 'A', host[1]))
- externalzonefile.write("%-32s%-10s%-3s%s.\n" %
- ('', 'MX', host[4], host[5]))
- elif not prevhost[5] == host[5]:
- zonefile.write("%-32s%-10s%-3s%s.\n" %
- ('', 'MX', host[4], host[5]))
- if host[6] == 'global':
- externalzonefile.write("%-32s%-10s%-3s%s.\n" %
- ('', 'MX', host[4], host[5]))
-
- if host[3]:
- try:
- if host[3].split(".", 1)[1] == zone[1]:
- cnames.write("%-32s%-10s%-32s\n" %
- (host[3].split(".", 1)[0],
- 'CNAME', host[2].split(".", 1)[0]))
- if host[6] == 'global':
- cnamesexternal.write("%-32s%-10s%-32s\n" %
- (host[3].split(".", 1)[0],
- 'CNAME', host[2].split(".", 1)[0]))
- else:
- cnames.write("%-32s%-10s%-32s\n" %
- (host[3] + ".",
- 'CNAME',
- host[2].split(".", 1)[0]))
- if host[6] == 'global':
- cnamesexternal.write("%-32s%-10s%-32s\n" %
- (host[3] + ".",
- 'CNAME',
- host[2].split(".", 1)[0]))
-
- except:
- pass
- prevhost = host
- zonefile.write(cnames.getvalue())
- externalzonefile.write(cnamesexternal.getvalue())
- zonefile.write("\n\n%s" % zone[9])
- externalzonefile.write("\n\n%s" % zone[9])
- self.filedata[zone[1]] = zonefile.getvalue()
- self.filedata[zone[1] + ".external"] = externalzonefile.getvalue()
- zonefile.close()
- externalzonefile.close()
- self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile
- self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile
-
- cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'")
- reversezones = cursor.fetchall()
-
- reversenames = []
- for reversezone in reversezones:
- cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z
- INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id
- WHERE z.zone_id = \'%s\'""" % reversezone[0])
- reverse_nameservers = cursor.fetchall()
-
- context = Context({
- 'inaddr': reversezone[1].rstrip('.rev'),
- 'zone': reversezone,
- 'nameservers': reverse_nameservers,
- })
-
- self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context)
- self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context)
- self.filedata[reversezone[1]] += reversezone[9]
- self.filedata[reversezone[1] + '.external'] += reversezone[9]
-
- subnet = reversezone[1].split(".")
- subnet.reverse()
- reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:])))
-
- for filename in reversenames:
- cursor.execute("""
- SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h
- INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id)
- INNER JOIN hostbase_name n ON n.ip_id = p.id
- WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr
- """ % filename[1])
- reversehosts = cursor.fetchall()
- zonefile = StringIO()
- externalzonefile = StringIO()
- if len(filename[0].split(".")) == 2:
- originlist = []
- [originlist.append((".".join([ip[1].split(".")[2], filename[0]]),
- ".".join([filename[1], ip[1].split(".")[2]])))
- for ip in reversehosts
- if (".".join([ip[1].split(".")[2], filename[0]]),
- ".".join([filename[1], ip[1].split(".")[2]])) not in originlist]
- for origin in originlist:
- hosts = [(host[1].split("."), host[0])
- for host in reversehosts
- if host[1].rstrip('0123456789').rstrip('.') == origin[1]]
- hosts_external = [(host[1].split("."), host[0])
- for host in reversehosts
- if (host[1].rstrip('0123456789').rstrip('.') == origin[1]
- and host[2] == 'global')]
- context = Context({
- 'hosts': hosts,
- 'inaddr': origin[0],
- 'fileorigin': filename[0],
- })
- zonefile.write(self.templates['reverseapp'].render(context))
- context = Context({
- 'hosts': hosts_external,
- 'inaddr': origin[0],
- 'fileorigin': filename[0],
- })
- externalzonefile.write(self.templates['reverseapp'].render(context))
- else:
- originlist = [filename[0]]
- hosts = [(host[1].split("."), host[0])
- for host in reversehosts
- if (host[1].split("."), host[0]) not in hosts]
- hosts_external = [(host[1].split("."), host[0])
- for host in reversehosts
- if ((host[1].split("."), host[0]) not in hosts_external
- and host[2] == 'global')]
- context = Context({
- 'hosts': hosts,
- 'inaddr': filename[0],
- 'fileorigin': None,
- })
- zonefile.write(self.templates['reverseapp'].render(context))
- context = Context({
- 'hosts': hosts_external,
- 'inaddr': filename[0],
- 'fileorigin': None,
- })
- externalzonefile.write(self.templates['reverseapp'].render(context))
- self.filedata['%s.rev' % filename[0]] += zonefile.getvalue()
- self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue()
- zonefile.close()
- externalzonefile.close()
- self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile
- self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile
-
- ## here's where the named.conf file gets written
- context = Context({
- 'zones': zones,
- 'reverses': reversenames,
- })
- self.filedata['named.conf'] = self.templates['named'].render(context)
- self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile
- self.filedata['named.conf.views'] = self.templates['namedviews'].render(context)
- self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile
-
- def buildDHCP(self):
- """Pre-build dhcpd.conf and stash in the filedata table."""
-
- # fetches all the hosts with DHCP == True
- cursor = connection.cursor()
- cursor.execute("""
- SELECT hostname, mac_addr, ip_addr
- FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip ip ON i.id = ip.interface_id
- WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> ''
- AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown'
- ORDER BY h.hostname, i.mac_addr
- """)
-
- dhcphosts = cursor.fetchall()
- count = 0
- hosts = []
- hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]]
- if len(dhcphosts) > 1:
- for x in range(1, len(dhcphosts)):
- # if an interface has 2 or more ip addresses
- # adds the ip to the current interface
- if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]:
- hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]])
- # if a host has 2 or more interfaces
- # writes the current one and grabs the next
- elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]:
- hosts.append(hostdata)
- count += 1
- hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]]
- # new host found, writes current data to the template
- else:
- hosts.append(hostdata)
- count = 0
- hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]]
- #makes sure the last of the data gets written out
- if hostdata not in hosts:
- hosts.append(hostdata)
-
- context = Context({
- 'hosts': hosts,
- 'numips': len(hosts),
- })
-
- self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context)
- self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile
-
- def buildHosts(self):
- """Pre-build and stash /etc/hosts file."""
-
- append_data = []
-
- cursor = connection.cursor()
- cursor.execute("""
- SELECT hostname FROM hostbase_host ORDER BY hostname
- """)
- hostbase = cursor.fetchall()
- domains = [host[0].split(".", 1)[1] for host in hostbase]
- domains_set = Set(domains)
- domain_data = [(domain, domains.count(domain)) for domain in domains_set]
- domain_data.sort()
-
- cursor.execute("""
- SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr
- """)
- ips = cursor.fetchall()
- three_octets = [ip[0].rstrip('0123456789').rstrip('.') \
- for ip in ips]
- three_octets_set = Set(three_octets)
- three_octets_data = [(octet, three_octets.count(octet)) \
- for octet in three_octets_set]
- three_octets_data.sort()
-
- for three_octet in three_octets_data:
- querystring = """SELECT h.hostname, h.primary_user,
- p.ip_addr, n.name, c.cname
- FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id)
- INNER JOIN hostbase_name n ON p.id = n.ip_id)
- LEFT JOIN hostbase_cname c ON n.id = c.name_id
- WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0]
- cursor.execute(querystring)
- tosort = list(cursor.fetchall())
- tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1])))
- append_data.append((three_octet, tuple(tosort)))
-
- two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets]
- two_octets_set = Set(two_octets)
- two_octets_data = [(octet, two_octets.count(octet))
- for octet in two_octets_set]
- two_octets_data.sort()
-
- context = Context({
- 'domain_data': domain_data,
- 'three_octets_data': three_octets_data,
- 'two_octets_data': two_octets_data,
- 'three_octets': three_octets,
- 'num_ips': len(three_octets),
- })
-
- self.filedata['hosts'] = self.templates['hosts'].render(context)
-
- for subnet in append_data:
- ips = []
- simple = True
- namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]]
- cnamelist = []
- if subnet[1][0][4]:
- cnamelist.append(subnet[1][0][4].split('.', 1)[0])
- simple = False
- appenddata = subnet[1][0]
- for ip in subnet[1][1:]:
- if appenddata[2] == ip[2]:
- namelist.append(ip[3].split('.', 1)[0])
- if ip[4]:
- cnamelist.append(ip[4].split('.', 1)[0])
- simple = False
- appenddata = ip
- else:
- if appenddata[0] == ip[0]:
- simple = False
- ips.append((appenddata[2], appenddata[0], Set(namelist),
- cnamelist, simple, appenddata[1]))
- appenddata = ip
- simple = True
- namelist = [ip[3].split('.', 1)[0]]
- cnamelist = []
- if ip[4]:
- cnamelist.append(ip[4].split('.', 1)[0])
- simple = False
- ips.append((appenddata[2], appenddata[0], Set(namelist),
- cnamelist, simple, appenddata[1]))
- context = Context({
- 'subnet': subnet[0],
- 'ips': ips,
- })
- self.filedata['hosts'] += self.templates['hostsapp'].render(context)
- self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile
-
- def buildPrinters(self):
- """The /mcs/etc/printers.data file"""
- header = """# This file is automatically generated. DO NOT EDIT IT!
-#
-Name Room User Type Notes
-============== ========== ============================== ======================== ====================
-"""
-
- cursor = connection.cursor()
- # fetches all the printers from the database
- cursor.execute("""
- SELECT printq, location, primary_user, comments
- FROM hostbase_host
- WHERE whatami='printer' AND printq <> '' AND status = 'active'
- ORDER BY printq
- """)
- printers = cursor.fetchall()
-
- printersfile = header
- for printer in printers:
- # splits up the printq line and gets the
- # correct description out of the comments section
- temp = printer[3].split('\n')
- for printq in re.split(',[ ]*', printer[0]):
- if len(temp) > 1:
- printersfile += ("%-16s%-12s%-32s%-26s%s\n" %
- (printq, printer[1], printer[2], temp[1], temp[0]))
- else:
- printersfile += ("%-16s%-12s%-32s%-26s%s\n" %
- (printq, printer[1], printer[2], '', printer[3]))
- self.filedata['printers.data'] = printersfile
- self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile
-
- def buildHostsLPD(self):
- """Creates the /mcs/etc/hosts.lpd file"""
-
- # this header needs to be changed to be more generic
- header = """+@machines
-+@all-machines
-achilles.ctd.anl.gov
-raven.ops.anl.gov
-seagull.hr.anl.gov
-parrot.ops.anl.gov
-condor.ops.anl.gov
-delphi.esh.anl.gov
-anlcv1.ctd.anl.gov
-anlvms.ctd.anl.gov
-olivia.ctd.anl.gov\n\n"""
-
- cursor = connection.cursor()
- cursor.execute("""
- SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active'
- ORDER BY hostname""")
- redmachines = list(cursor.fetchall())
- cursor.execute("""
- SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id
- WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active'
- """)
- redmachines.extend(list(cursor.fetchall()))
- cursor.execute("""
- SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active'
- ORDER BY hostname""")
- winmachines = list(cursor.fetchall())
- cursor.execute("""
- SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id
- WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active'
- """)
- winmachines.__add__(list(cursor.fetchall()))
- hostslpdfile = header
- for machine in redmachines:
- hostslpdfile += machine[0] + "\n"
- hostslpdfile += "\n"
- for machine in winmachines:
- hostslpdfile += machine[0] + "\n"
- self.filedata['hosts.lpd'] = hostslpdfile
- self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile
-
- def buildNetgroups(self):
- """Makes the *-machine files"""
- header = """###################################################################
-# This file lists hosts in the '%s' machine netgroup, it is
-# automatically generated. DO NOT EDIT THIS FILE!
-#
-# Number of hosts in '%s' machine netgroup: %i
-#\n\n"""
-
- cursor = connection.cursor()
- # fetches all the hosts that with valid netgroup entries
- cursor.execute("""
- SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h
- INNER JOIN hostbase_interface i ON h.id = i.host_id)
- INNER JOIN hostbase_ip p ON i.id = p.interface_id)
- INNER JOIN hostbase_name n ON p.id = n.ip_id
- WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active'
- ORDER BY h.netgroup, h.hostname
- """)
- nameslist = cursor.fetchall()
- # gets the first host and initializes the hash
- hostdata = nameslist[0]
- netgroups = {hostdata[2]: [hostdata[0]]}
- for row in nameslist:
- # if new netgroup, create it
- if row[2] not in netgroups:
- netgroups.update({row[2]: []})
- # if it belongs in the netgroup and has multiple interfaces, put them in
- if hostdata[0] == row[0] and row[3]:
- netgroups[row[2]].append(row[1])
- hostdata = row
- # if its a new host, write the old one to the hash
- elif hostdata[0] != row[0]:
- netgroups[row[2]].append(row[0])
- hostdata = row
-
- for netgroup in netgroups:
- fileoutput = StringIO()
- fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup])))
- for each in netgroups[netgroup]:
- fileoutput.write(each + "\n")
- self.filedata['%s-machines' % netgroup] = fileoutput.getvalue()
- fileoutput.close()
- self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile
-
- cursor.execute("""
- UPDATE hostbase_host SET dirty=0
- """)
diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py
deleted file mode 100644
index 04417339b..000000000
--- a/src/lib/Server/Plugins/Ldap.py
+++ /dev/null
@@ -1,245 +0,0 @@
-import imp
-import logging
-import sys
-import time
-import traceback
-import Bcfg2.Options
-import Bcfg2.Server.Plugin
-
-logger = logging.getLogger('Bcfg2.Plugins.Ldap')
-
-try:
- import ldap
-except:
- logger.error("Unable to load ldap module. Is python-ldap installed?")
- raise ImportError
-
-# time in seconds between retries after failed LDAP connection
-RETRY_DELAY = 5
-# how many times to try reaching the LDAP server if a connection is broken
-# at the very minimum, one retry is needed to handle a restarted LDAP daemon
-RETRY_COUNT = 3
-
-SCOPE_MAP = {
- "base" : ldap.SCOPE_BASE,
- "one" : ldap.SCOPE_ONELEVEL,
- "sub" : ldap.SCOPE_SUBTREE,
-}
-
-LDAP_QUERIES = []
-
-def register_query(query):
- LDAP_QUERIES.append(query)
-
-class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
- """
- Config file for the Ldap plugin
-
- The config file cannot be 'parsed' in the traditional sense as we would
- need some serious type checking ugliness to just get the LdapQuery
- subclasses. The alternative would be to have the user create a list with
- a predefined name that contains all queries.
- The approach implemented here is having the user call a registering
- decorator that updates a global variable in this module.
- """
- def __init__(self, filename, fam):
- self.filename = filename
- Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename)
- fam.AddMonitor(self.filename, self)
-
- def Index(self):
- """
- Reregisters the queries in the config file
-
- The config will take care of actually registering the queries,
- so we just load it once and don't keep it.
- """
- global LDAP_QUERIES
- LDAP_QUERIES = []
- imp.load_source("ldap_cfg", self.filename)
-
-class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector):
- """
- The Ldap plugin allows adding data from an LDAP server to your metadata.
- """
- name = "Ldap"
- experimental = True
- debug_flag = False
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- self.config = ConfigFile(self.data + "/config.py", core.fam)
-
- def debug_log(self, message, flag = None):
- if (flag is None) and self.debug_flag or flag:
- self.logger.error(message)
-
- def get_additional_data(self, metadata):
- query = None
- try:
- data = {}
- self.debug_log("LdapPlugin debug: found queries " +
- str(LDAP_QUERIES))
- for QueryClass in LDAP_QUERIES:
- query = QueryClass()
- if query.is_applicable(metadata):
- self.debug_log("LdapPlugin debug: processing query '" +
- query.name + "'")
- data[query.name] = query.get_result(metadata)
- else:
- self.debug_log("LdapPlugin debug: query '" + query.name +
- "' not applicable to host '" + metadata.hostname + "'")
- return data
- except Exception:
- if hasattr(query, "name"):
- Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
- "Exception during processing of query named '" +
- str(query.name) +
- "', query results will be empty" +
- " and may cause bind failures")
- for line in traceback.format_exception(sys.exc_info()[0],
- sys.exc_info()[1],
- sys.exc_info()[2]):
- Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
- line.replace("\n", ""))
- return {}
-
-class LdapConnection(object):
- """
- Connection to an LDAP server.
- """
- def __init__(self, host = "localhost", port = 389,
- binddn = None, bindpw = None):
- self.host = host
- self.port = port
- self.binddn = binddn
- self.bindpw = bindpw
- self.conn = None
-
- def __del__(self):
- if self.conn:
- self.conn.unbind()
-
- def init_conn(self):
- self.conn = ldap.initialize(self.url)
- if self.binddn is not None and self.bindpw is not None:
- self.conn.simple_bind_s(self.binddn, self.bindpw)
-
- def run_query(self, query):
- result = None
- for attempt in range(RETRY_COUNT + 1):
- if attempt >= 1:
- Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
- "LDAP server down (retry " + str(attempt) + "/" +
- str(RETRY_COUNT) + ")")
- try:
- if not self.conn:
- self.init_conn()
- result = self.conn.search_s(
- query.base,
- SCOPE_MAP[query.scope],
- query.filter,
- query.attrs,
- )
- break
- except ldap.SERVER_DOWN:
- self.conn = None
- time.sleep(RETRY_DELAY)
- return result
-
- @property
- def url(self):
- return "ldap://" + self.host + ":" + str(self.port)
-
-class LdapQuery(object):
- """
- Query referencing an LdapConnection and providing several
- methods for query manipulation.
- """
-
- name = "unknown"
- base = ""
- scope = "sub"
- filter = "(objectClass=*)"
- attrs = None
- connection = None
- result = None
-
- def __unicode__(self):
- return "LdapQuery:" + self.name
-
- def is_applicable(self, metadata):
- """
- Overrideable method to determine if the query is to be executed for
- the given metadata object.
- Defaults to true.
- """
- return True
-
- def prepare_query(self, metadata):
- """
- Overrideable method to alter the query based on metadata.
- Defaults to doing nothing.
-
- In most cases, you will do something like
-
- self.filter = "(cn=" + metadata.hostname + ")"
-
- here.
- """
- pass
-
- def process_result(self, metadata):
- """
- Overrideable method to post-process the query result.
- Defaults to returning the unaltered result.
- """
- return self.result
-
- def get_result(self, metadata):
- """
- Method to handle preparing, executing and processing the query.
- """
- if isinstance(self.connection, LdapConnection):
- self.prepare_query(metadata)
- self.result = self.connection.run_query(self)
- self.result = self.process_result(metadata)
- return self.result
- else:
- Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
- "No valid connection defined for query " + str(self))
- return None
-
-class LdapSubQuery(LdapQuery):
- """
- SubQueries are meant for internal use only and are not added
- to the metadata object. They are useful for situations where
- you need to run more than one query to obtain some data.
- """
- def prepare_query(self, metadata, **kwargs):
- """
- Overrideable method to alter the query based on metadata.
- Defaults to doing nothing.
- """
- pass
-
- def process_result(self, metadata, **kwargs):
- """
- Overrideable method to post-process the query result.
- Defaults to returning the unaltered result.
- """
- return self.result
-
- def get_result(self, metadata, **kwargs):
- """
- Method to handle preparing, executing and processing the query.
- """
- if isinstance(self.connection, LdapConnection):
- self.prepare_query(metadata, **kwargs)
- self.result = self.connection.run_query(self)
- return self.process_result(metadata, **kwargs)
- else:
- Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
- "No valid connection defined for query " + str(self))
- return None
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
deleted file mode 100644
index fa2cf0a78..000000000
--- a/src/lib/Server/Plugins/Metadata.py
+++ /dev/null
@@ -1,866 +0,0 @@
-"""
-This file stores persistent metadata for the Bcfg2 Configuration Repository.
-"""
-
-import copy
-import fcntl
-import lxml.etree
-import os
-import os.path
-import socket
-import sys
-import time
-
-import Bcfg2.Server.FileMonitor
-import Bcfg2.Server.Plugin
-
-
-def locked(fd):
- """Aquire a lock on a file"""
- try:
- fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError:
- return True
- return False
-
-
-class MetadataConsistencyError(Exception):
- """This error gets raised when metadata is internally inconsistent."""
- pass
-
-
-class MetadataRuntimeError(Exception):
- """This error is raised when the metadata engine
- is called prior to reading enough data.
- """
- pass
-
-
-class XMLMetadataConfig(object):
- """Handles xml config files and all XInclude statements"""
- def __init__(self, metadata, watch_clients, basefile):
- self.metadata = metadata
- self.basefile = basefile
- self.should_monitor = watch_clients
- self.extras = []
- self.data = None
- self.basedata = None
- self.basedir = metadata.data
- self.logger = metadata.logger
- self.pseudo_monitor = isinstance(metadata.core.fam,
- Bcfg2.Server.FileMonitor.Pseudo)
-
- @property
- def xdata(self):
- if not self.data:
- raise MetadataRuntimeError
- return self.data
-
- @property
- def base_xdata(self):
- if not self.basedata:
- raise MetadataRuntimeError
- return self.basedata
-
- def add_monitor(self, fname):
- """Add a fam monitor for an included file"""
- if self.should_monitor:
- self.metadata.core.fam.AddMonitor(os.path.join(self.basedir, fname),
- self.metadata)
- self.extras.append(fname)
-
- def load_xml(self):
- """Load changes from XML"""
- try:
- xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile))
- except lxml.etree.XMLSyntaxError:
- self.logger.error('Failed to parse %s' % self.basefile)
- return
- self.basedata = copy.copy(xdata)
- included = [ent.get('href') for ent in \
- xdata.findall('./{http://www.w3.org/2001/XInclude}include')]
- if included:
- for name in included:
- if name not in self.extras:
- self.add_monitor(name)
- try:
- xdata.xinclude()
- except lxml.etree.XIncludeError:
- self.logger.error("Failed to process XInclude for file %s" %
- self.basefile)
- self.data = xdata
-
- def write(self):
- """Write changes to xml back to disk."""
- self.write_xml(os.path.join(self.basedir, self.basefile),
- self.basedata)
-
- def write_xml(self, fname, xmltree):
- """Write changes to xml back to disk."""
- tmpfile = "%s.new" % fname
- try:
- datafile = open(tmpfile, 'w')
- except IOError:
- e = sys.exc_info()[1]
- self.logger.error("Failed to write %s: %s" % (tmpfile, e))
- raise MetadataRuntimeError
- # prep data
- dataroot = xmltree.getroot()
- newcontents = lxml.etree.tostring(dataroot, pretty_print=True)
-
- fd = datafile.fileno()
- while locked(fd) == True:
- pass
- try:
- datafile.write(newcontents)
- except:
- fcntl.lockf(fd, fcntl.LOCK_UN)
- self.logger.error("Metadata: Failed to write new xml data to %s" %
- tmpfile, exc_info=1)
- os.unlink(tmpfile)
- raise MetadataRuntimeError
- datafile.close()
-
- # check if clients.xml is a symlink
- if os.path.islink(fname):
- fname = os.readlink(fname)
-
- try:
- os.rename(tmpfile, fname)
- except:
- self.logger.error("Metadata: Failed to rename %s" % tmpfile)
- raise MetadataRuntimeError
-
- def find_xml_for_xpath(self, xpath):
- """Find and load xml file containing the xpath query"""
- if self.pseudo_monitor:
- # Reload xml if we don't have a real monitor
- self.load_xml()
- cli = self.basedata.xpath(xpath)
- if len(cli) > 0:
- return {'filename': os.path.join(self.basedir, self.basefile),
- 'xmltree': self.basedata,
- 'xquery': cli}
- else:
- """Try to find the data in included files"""
- for included in self.extras:
- try:
- xdata = lxml.etree.parse(os.path.join(self.basedir,
- included))
- cli = xdata.xpath(xpath)
- if len(cli) > 0:
- return {'filename': os.path.join(self.basedir,
- included),
- 'xmltree': xdata,
- 'xquery': cli}
- except lxml.etree.XMLSyntaxError:
- self.logger.error('Failed to parse %s' % (included))
- return {}
-
- def HandleEvent(self, event):
- """Handle fam events"""
- filename = event.filename.split('/')[-1]
- if filename in self.extras:
- if event.code2str() == 'exists':
- return False
- elif filename != self.basefile:
- return False
- if event.code2str() == 'endExist':
- return False
- self.load_xml()
- return True
-
-
-class ClientMetadata(object):
- """This object contains client metadata."""
- def __init__(self, client, profile, groups, bundles,
- aliases, addresses, categories, uuid, password, query):
- self.hostname = client
- self.profile = profile
- self.bundles = bundles
- self.aliases = aliases
- self.addresses = addresses
- self.groups = groups
- self.categories = categories
- self.uuid = uuid
- self.password = password
- self.connectors = []
- self.query = query
-
- def inGroup(self, group):
- """Test to see if client is a member of group."""
- return group in self.groups
-
- def group_in_category(self, category):
- for grp in self.query.all_groups_in_category(category):
- if grp in self.groups:
- return grp
- return ''
-
-
-class MetadataQuery(object):
- def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category):
- # resolver is set later
- self.by_name = by_name
- self.names_by_groups = by_groups
- self.names_by_profiles = by_profiles
- self.all_clients = get_clients
- self.all_groups = all_groups
- self.all_groups_in_category = all_groups_in_category
-
- def by_groups(self, groups):
- return [self.by_name(name) for name in self.names_by_groups(groups)]
-
- def by_profiles(self, profiles):
- return [self.by_name(name) for name in self.names_by_profiles(profiles)]
-
- def all(self):
- return [self.by_name(name) for name in self.all_clients()]
-
-
-class Metadata(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Metadata,
- Bcfg2.Server.Plugin.Statistics):
- """This class contains data for bcfg2 server metadata."""
- __author__ = 'bcfg-dev@mcs.anl.gov'
- name = "Metadata"
- sort_order = 500
-
- def __init__(self, core, datastore, watch_clients=True):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Metadata.__init__(self)
- Bcfg2.Server.Plugin.Statistics.__init__(self)
- if watch_clients:
- try:
- core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self)
- core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self)
- except:
- print("Unable to add file monitor for groups.xml or clients.xml")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml')
- self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml')
- self.states = {}
- if watch_clients:
- self.states = {"groups.xml": False,
- "clients.xml": False}
- self.addresses = {}
- self.auth = dict()
- self.clients = {}
- self.aliases = {}
- self.groups = {}
- self.cgroups = {}
- self.public = []
- self.private = []
- self.profiles = []
- self.categories = {}
- self.bad_clients = {}
- self.uuid = {}
- self.secure = []
- self.floating = []
- self.passwords = {}
- self.session_cache = {}
- self.default = None
- self.pdirty = False
- self.extra = {'groups.xml': [],
- 'clients.xml': []}
- self.password = core.password
- self.query = MetadataQuery(core.build_metadata,
- lambda: list(self.clients.keys()),
- self.get_client_names_by_groups,
- self.get_client_names_by_profiles,
- self.get_all_group_names,
- self.get_all_groups_in_category)
-
- @classmethod
- def init_repo(cls, repo, groups, os_selection, clients):
- path = os.path.join(repo, cls.name)
- os.makedirs(path)
- open(os.path.join(repo, "Metadata", "groups.xml"),
- "w").write(groups % os_selection)
- open(os.path.join(repo, "Metadata", "clients.xml"),
- "w").write(clients % socket.getfqdn())
-
- def get_groups(self):
- '''return groups xml tree'''
- groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"))
- root = groups_tree.getroot()
- return root
-
- def _search_xdata(self, tag, name, tree, alias=False):
- for node in tree.findall("//%s" % tag):
- if node.get("name") == name:
- return node
- elif alias:
- for child in node:
- if (child.tag == "Alias" and
- child.attrib["name"] == name):
- return node
- return None
-
- def search_group(self, group_name, tree):
- """Find a group."""
- return self._search_xdata("Group", group_name, tree)
-
- def search_bundle(self, bundle_name, tree):
- """Find a bundle."""
- return self._search_xdata("Bundle", bundle_name, tree)
-
- def search_client(self, client_name, tree):
- return self._search_xdata("Client", client_name, tree, alias=True)
-
- def _add_xdata(self, config, tag, name, attribs=None, alias=False):
- node = self._search_xdata(tag, name, config.xdata, alias=alias)
- if node != None:
- self.logger.error("%s \"%s\" already exists" % (tag, name))
- raise MetadataConsistencyError
- element = lxml.etree.SubElement(config.base_xdata.getroot(),
- tag, name=name)
- if attribs:
- for key, val in list(attribs.items()):
- element.set(key, val)
- config.write()
-
- def add_group(self, group_name, attribs):
- """Add group to groups.xml."""
- return self._add_xdata(self.groups_xml, "Group", group_name,
- attribs=attribs)
-
- def add_bundle(self, bundle_name):
- """Add bundle to groups.xml."""
- return self._add_xdata(self.groups_xml, "Bundle", bundle_name)
-
- def add_client(self, client_name, attribs):
- """Add client to clients.xml."""
- return self._add_xdata(self.clients_xml, "Client", client_name,
- attribs=attribs, alias=True)
-
- def _update_xdata(self, config, tag, name, attribs, alias=False):
- node = self._search_xdata(tag, name, config.xdata, alias=alias)
- if node == None:
- self.logger.error("%s \"%s\" does not exist" % (tag, name))
- raise MetadataConsistencyError
- xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
- (tag, node.get('name')))
- if not xdict:
- self.logger.error("Unexpected error finding %s \"%s\"" %
- (tag, name))
- raise MetadataConsistencyError
- for key, val in list(attribs.items()):
- xdict['xquery'][0].set(key, val)
- config.write_xml(xdict['filename'], xdict['xmltree'])
-
- def update_group(self, group_name, attribs):
- """Update a groups attributes."""
- return self._update_xdata(self.groups_xml, "Group", group_name, attribs)
-
- def update_client(self, client_name, attribs):
- """Update a clients attributes."""
- return self._update_xdata(self.clients_xml, "Client", client_name,
- attribs, alias=True)
-
- def _remove_xdata(self, config, tag, name, alias=False):
- node = self._search_xdata(tag, name, config.xdata)
- if node == None:
- self.logger.error("%s \"%s\" does not exist" % (tag, name))
- raise MetadataConsistencyError
- xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
- (tag, node.get('name')))
- if not xdict:
- self.logger.error("Unexpected error finding %s \"%s\"" %
- (tag, name))
- raise MetadataConsistencyError
- xdict['xquery'][0].getparent().remove(xdict['xquery'][0])
- self.groups_xml.write_xml(xdict['filename'], xdict['xmltree'])
-
- def remove_group(self, group_name):
- """Remove a group."""
- return self._remove_xdata(self.groups_xml, "Group", group_name)
-
- def remove_bundle(self, bundle_name):
- """Remove a bundle."""
- return self._remove_xdata(self.groups_xml, "Bundle", bundle_name)
-
- def _handle_clients_xml_event(self, event):
- xdata = self.clients_xml.xdata
- self.clients = {}
- self.aliases = {}
- self.raliases = {}
- self.bad_clients = {}
- self.secure = []
- self.floating = []
- self.addresses = {}
- self.raddresses = {}
- for client in xdata.findall('.//Client'):
- clname = client.get('name').lower()
- if 'address' in client.attrib:
- caddr = client.get('address')
- if caddr in self.addresses:
- self.addresses[caddr].append(clname)
- else:
- self.addresses[caddr] = [clname]
- if clname not in self.raddresses:
- self.raddresses[clname] = set()
- self.raddresses[clname].add(caddr)
- if 'auth' in client.attrib:
- self.auth[client.get('name')] = client.get('auth',
- 'cert+password')
- if 'uuid' in client.attrib:
- self.uuid[client.get('uuid')] = clname
- if client.get('secure', 'false') == 'true':
- self.secure.append(clname)
- if client.get('location', 'fixed') == 'floating':
- self.floating.append(clname)
- if 'password' in client.attrib:
- self.passwords[clname] = client.get('password')
-
- self.raliases[clname] = set()
- for alias in client.findall('Alias'):
- self.aliases.update({alias.get('name'): clname})
- self.raliases[clname].add(alias.get('name'))
- if 'address' not in alias.attrib:
- continue
- if alias.get('address') in self.addresses:
- self.addresses[alias.get('address')].append(clname)
- else:
- self.addresses[alias.get('address')] = [clname]
- if clname not in self.raddresses:
- self.raddresses[clname] = set()
- self.raddresses[clname].add(alias.get('address'))
- self.clients.update({clname: client.get('profile')})
- self.states['clients.xml'] = True
-
- def _handle_groups_xml_event(self, event):
- xdata = self.groups_xml.xdata
- self.public = []
- self.private = []
- self.profiles = []
- self.groups = {}
- grouptmp = {}
- self.categories = {}
- groupseen = list()
- for group in xdata.xpath('//Groups/Group'):
- if group.get('name') not in groupseen:
- groupseen.append(group.get('name'))
- else:
- self.logger.error("Metadata: Group %s defined multiply" %
- group.get('name'))
- grouptmp[group.get('name')] = \
- ([item.get('name') for item in group.findall('./Bundle')],
- [item.get('name') for item in group.findall('./Group')])
- grouptmp[group.get('name')][1].append(group.get('name'))
- if group.get('default', 'false') == 'true':
- self.default = group.get('name')
- if group.get('profile', 'false') == 'true':
- self.profiles.append(group.get('name'))
- if group.get('public', 'false') == 'true':
- self.public.append(group.get('name'))
- elif group.get('public', 'true') == 'false':
- self.private.append(group.get('name'))
- if 'category' in group.attrib:
- self.categories[group.get('name')] = group.get('category')
-
- for group in grouptmp:
- # self.groups[group] => (bundles, groups, categories)
- self.groups[group] = (set(), set(), {})
- tocheck = [group]
- group_cat = self.groups[group][2]
- while tocheck:
- now = tocheck.pop()
- self.groups[group][1].add(now)
- if now in grouptmp:
- (bundles, groups) = grouptmp[now]
- for ggg in groups:
- if ggg in self.groups[group][1]:
- continue
- if (ggg not in self.categories or \
- self.categories[ggg] not in self.groups[group][2]):
- self.groups[group][1].add(ggg)
- tocheck.append(ggg)
- if ggg in self.categories:
- group_cat[self.categories[ggg]] = ggg
- elif ggg in self.categories:
- self.logger.info("Group %s: %s cat-suppressed %s" % \
- (group,
- group_cat[self.categories[ggg]],
- ggg))
- [self.groups[group][0].add(bund) for bund in bundles]
- self.states['groups.xml'] = True
-
- def HandleEvent(self, event):
- """Handle update events for data files."""
- if self.clients_xml.HandleEvent(event):
- self._handle_clients_xml_event(event)
- elif self.groups_xml.HandleEvent(event):
- self._handle_groups_xml_event(event)
-
- if False not in list(self.states.values()):
- # check that all client groups are real and complete
- real = list(self.groups.keys())
- for client in list(self.clients.keys()):
- if self.clients[client] not in self.profiles:
- self.logger.error("Client %s set as nonexistent or "
- "incomplete group %s" %
- (client, self.clients[client]))
- self.logger.error("Removing client mapping for %s" % client)
- self.bad_clients[client] = self.clients[client]
- del self.clients[client]
- for bclient in list(self.bad_clients.keys()):
- if self.bad_clients[bclient] in self.profiles:
- self.logger.info("Restored profile mapping for client %s" %
- bclient)
- self.clients[bclient] = self.bad_clients[bclient]
- del self.bad_clients[bclient]
-
- def set_profile(self, client, profile, addresspair):
- """Set group parameter for provided client."""
- self.logger.info("Asserting client %s profile to %s" % (client, profile))
- if False in list(self.states.values()):
- raise MetadataRuntimeError
- if profile not in self.public:
- self.logger.error("Failed to set client %s to private group %s" %
- (client, profile))
- raise MetadataConsistencyError
- if client in self.clients:
- self.logger.info("Changing %s group from %s to %s" %
- (client, self.clients[client], profile))
- self.update_client(client, dict(profile=profile))
- else:
- self.logger.info("Creating new client: %s, profile %s" %
- (client, profile))
- if addresspair in self.session_cache:
- # we are working with a uuid'd client
- self.add_client(self.session_cache[addresspair][1],
- dict(uuid=client, profile=profile,
- address=addresspair[0]))
- else:
- self.add_client(client, dict(profile=profile))
- self.clients[client] = profile
- self.clients_xml.write()
-
- def resolve_client(self, addresspair, cleanup_cache=False):
- """Lookup address locally or in DNS to get a hostname."""
- if addresspair in self.session_cache:
- # client _was_ cached, so there can be some expired
- # entries. we need to clean them up to avoid potentially
- # infinite memory swell
- cache_ttl = 90
- if cleanup_cache:
- # remove entries for this client's IP address with
- # _any_ port numbers - perhaps a priority queue could
- # be faster?
- curtime = time.time()
- for addrpair in self.session_cache.keys():
- if addresspair[0] == addrpair[0]:
- (stamp, _) = self.session_cache[addrpair]
- if curtime - stamp > cache_ttl:
- del self.session_cache[addrpair]
- # return the cached data
- try:
- (stamp, uuid) = self.session_cache[addresspair]
- if time.time() - stamp < cache_ttl:
- return self.session_cache[addresspair][1]
- except KeyError:
- # we cleaned all cached data for this client in cleanup_cache
- pass
- address = addresspair[0]
- if address in self.addresses:
- if len(self.addresses[address]) != 1:
- self.logger.error("Address %s has multiple reverse assignments; "
- "a uuid must be used" % (address))
- raise MetadataConsistencyError
- return self.addresses[address][0]
- try:
- cname = socket.gethostbyaddr(address)[0].lower()
- if cname in self.aliases:
- return self.aliases[cname]
- return cname
- except socket.herror:
- warning = "address resolution error for %s" % (address)
- self.logger.warning(warning)
- raise MetadataConsistencyError
-
- def get_initial_metadata(self, client):
- """Return the metadata for a given client."""
- if False in list(self.states.values()):
- raise MetadataRuntimeError
- client = client.lower()
- if client in self.aliases:
- client = self.aliases[client]
- if client in self.clients:
- profile = self.clients[client]
- (bundles, groups, categories) = self.groups[profile]
- else:
- if self.default == None:
- self.logger.error("Cannot set group for client %s; "
- "no default group set" % client)
- raise MetadataConsistencyError
- self.set_profile(client, self.default, (None, None))
- profile = self.default
- [bundles, groups, categories] = self.groups[self.default]
- aliases = self.raliases.get(client, set())
- addresses = self.raddresses.get(client, set())
- newgroups = set(groups)
- newbundles = set(bundles)
- newcategories = {}
- newcategories.update(categories)
- if client in self.passwords:
- password = self.passwords[client]
- else:
- password = None
- uuids = [item for item, value in list(self.uuid.items())
- if value == client]
- if uuids:
- uuid = uuids[0]
- else:
- uuid = None
- for group in self.cgroups.get(client, []):
- if group in self.groups:
- nbundles, ngroups, ncategories = self.groups[group]
- else:
- nbundles, ngroups, ncategories = ([], [group], {})
- [newbundles.add(b) for b in nbundles if b not in newbundles]
- [newgroups.add(g) for g in ngroups if g not in newgroups]
- newcategories.update(ncategories)
- return ClientMetadata(client, profile, newgroups, newbundles, aliases,
- addresses, newcategories, uuid, password,
- self.query)
-
- def get_all_group_names(self):
- all_groups = set()
- [all_groups.update(g[1]) for g in list(self.groups.values())]
- return all_groups
-
- def get_all_groups_in_category(self, category):
- all_groups = set()
- [all_groups.add(g) for g in self.categories \
- if self.categories[g] == category]
- return all_groups
-
- def get_client_names_by_profiles(self, profiles):
- return [client for client, profile in list(self.clients.items()) \
- if profile in profiles]
-
- def get_client_names_by_groups(self, groups):
- mdata = [self.core.build_metadata(client)
- for client in list(self.clients.keys())]
- return [md.hostname for md in mdata if md.groups.issuperset(groups)]
-
- def merge_additional_groups(self, imd, groups):
- for group in groups:
- if (group in self.categories and
- self.categories[group] in imd.categories):
- continue
- newbundles, newgroups, _ = self.groups.get(group,
- (list(),
- [group],
- dict()))
- for newbundle in newbundles:
- if newbundle not in imd.bundles:
- imd.bundles.add(newbundle)
- for newgroup in newgroups:
- if newgroup not in imd.groups:
- if (newgroup in self.categories and
- self.categories[newgroup] in imd.categories):
- continue
- if newgroup in self.private:
- self.logger.error("Refusing to add dynamic membership "
- "in private group %s for client %s" %
- (newgroup, imd.hostname))
- continue
- imd.groups.add(newgroup)
-
- def merge_additional_data(self, imd, source, data):
- if not hasattr(imd, source):
- setattr(imd, source, data)
- imd.connectors.append(source)
-
- def validate_client_address(self, client, addresspair):
- """Check address against client."""
- address = addresspair[0]
- if client in self.floating:
- self.debug_log("Client %s is floating" % client)
- return True
- if address in self.addresses:
- if client in self.addresses[address]:
- self.debug_log("Client %s matches address %s" %
- (client, address))
- return True
- else:
- self.logger.error("Got request for non-float client %s from %s" %
- (client, address))
- return False
- resolved = self.resolve_client(addresspair)
- if resolved.lower() == client.lower():
- return True
- else:
- self.logger.error("Got request for %s from incorrect address %s" %
- (client, address))
- self.logger.error("Resolved to %s" % resolved)
- return False
-
- def AuthenticateConnection(self, cert, user, password, address):
- """This function checks auth creds."""
- if cert:
- id_method = 'cert'
- certinfo = dict([x[0] for x in cert['subject']])
- # look at cert.cN
- client = certinfo['commonName']
- self.debug_log("Got cN %s; using as client name" % client)
- auth_type = self.auth.get(client, 'cert+password')
- elif user == 'root':
- id_method = 'address'
- try:
- client = self.resolve_client(address)
- except MetadataConsistencyError:
- self.logger.error("Client %s failed to resolve; metadata problem"
- % address[0])
- return False
- else:
- id_method = 'uuid'
- # user maps to client
- if user not in self.uuid:
- client = user
- self.uuid[user] = user
- else:
- client = self.uuid[user]
-
- # we have the client name
- self.debug_log("Authenticating client %s" % client)
-
- # next we validate the address
- if id_method == 'uuid':
- addr_is_valid = True
- else:
- addr_is_valid = self.validate_client_address(client, address)
-
- if not addr_is_valid:
- return False
-
- if id_method == 'cert' and auth_type != 'cert+password':
- # remember the cert-derived client name for this connection
- if client in self.floating:
- self.session_cache[address] = (time.time(), client)
- # we are done if cert+password not required
- return True
-
- if client not in self.passwords:
- if client in self.secure:
- self.logger.error("Client %s in secure mode but has no password"
- % address[0])
- return False
- if password != self.password:
- self.logger.error("Client %s used incorrect global password" %
- address[0])
- return False
- if client not in self.secure:
- if client in self.passwords:
- plist = [self.password, self.passwords[client]]
- else:
- plist = [self.password]
- if password not in plist:
- self.logger.error("Client %s failed to use either allowed "
- "password" % address[0])
- return False
- else:
- # client in secure mode and has a client password
- if password != self.passwords[client]:
- self.logger.error("Client %s failed to use client password in "
- "secure mode" % address[0])
- return False
- # populate the session cache
- if user.decode('utf-8') != 'root':
- self.session_cache[address] = (time.time(), client)
- return True
-
- def process_statistics(self, meta, _):
- """Hook into statistics interface to toggle clients in bootstrap mode."""
- client = meta.hostname
- if client in self.auth and self.auth[client] == 'bootstrap':
- self.update_client(client, dict(auth='cert'))
-
- def viz(self, hosts, bundles, key, only_client, colors):
- """Admin mode viz support."""
- if only_client:
- clientmeta = self.core.build_metadata(only_client)
-
- def include_client(client):
- return not only_client or client != only_client
-
- def include_bundle(bundle):
- return not only_client or bundle in clientmeta.bundles
-
- def include_group(group):
- return not only_client or group in clientmeta.groups
-
- groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"))
- try:
- groups_tree.xinclude()
- except lxml.etree.XIncludeError:
- self.logger.error("Failed to process XInclude for file %s" % dest)
- groups = groups_tree.getroot()
- categories = {'default': 'grey83'}
- viz_str = []
- egroups = groups.findall("Group") + groups.findall('.//Groups/Group')
- for group in egroups:
- if not group.get('category') in categories:
- categories[group.get('category')] = colors.pop()
- group.set('color', categories[group.get('category')])
- if None in categories:
- del categories[None]
- if hosts:
- instances = {}
- clients = self.clients
- for client, profile in list(clients.items()):
- if include_client(client):
- continue
- if profile in instances:
- instances[profile].append(client)
- else:
- instances[profile] = [client]
- for profile, clist in list(instances.items()):
- clist.sort()
- viz_str.append('"%s-instances" [ label="%s", shape="record" ];' %
- (profile, '|'.join(clist)))
- viz_str.append('"%s-instances" -> "group-%s";' %
- (profile, profile))
- if bundles:
- bundles = []
- [bundles.append(bund.get('name')) \
- for bund in groups.findall('.//Bundle') \
- if bund.get('name') not in bundles \
- and include_bundle(bund.get('name'))]
- bundles.sort()
- for bundle in bundles:
- viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' %
- (bundle, bundle))
- gseen = []
- for group in egroups:
- if group.get('profile', 'false') == 'true':
- style = "filled, bold"
- else:
- style = "filled"
- gseen.append(group.get('name'))
- if include_group(group.get('name')):
- viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' %
- (group.get('name'), group.get('name'), style,
- group.get('color')))
- if bundles:
- for bundle in group.findall('Bundle'):
- viz_str.append('"group-%s" -> "bundle-%s";' %
- (group.get('name'), bundle.get('name')))
- gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];'
- for group in egroups:
- for parent in group.findall('Group'):
- if parent.get('name') not in gseen and include_group(parent.get('name')):
- viz_str.append(gfmt % (parent.get('name'),
- parent.get('name')))
- gseen.append(parent.get("name"))
- if include_group(group.get('name')):
- viz_str.append('"group-%s" -> "group-%s";' %
- (group.get('name'), parent.get('name')))
- if key:
- for category in categories:
- viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' %
- (category, category, categories[category]))
- return "\n".join("\t" + s for s in viz_str)
diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py
deleted file mode 100644
index d67dac6a6..000000000
--- a/src/lib/Server/Plugins/NagiosGen.py
+++ /dev/null
@@ -1,151 +0,0 @@
-'''This module implements a Nagios configuration generator'''
-
-import os
-import re
-import sys
-import glob
-import socket
-import logging
-import lxml.etree
-
-import Bcfg2.Server.Plugin
-
-LOGGER = logging.getLogger('Bcfg2.Plugins.NagiosGen')
-
-line_fmt = '\t%-32s %s'
-
-class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked,
- Bcfg2.Server.Plugin.StructFile):
- def __init__(self, filename, fam):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
- Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
-
-
-class NagiosGen(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Generator):
- """NagiosGen is a Bcfg2 plugin that dynamically generates
- Nagios configuration file based on Bcfg2 data.
- """
- name = 'NagiosGen'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Generator.__init__(self)
- self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'),
- core.fam)
- self.Entries = {'Path':
- {'/etc/nagiosgen.status': self.createhostconfig,
- '/etc/nagios/nagiosgen.cfg': self.createserverconfig}}
-
- self.client_attrib = {'encoding': 'ascii',
- 'owner': 'root',
- 'group': 'root',
- 'type': 'file',
- 'perms': '0400'}
- self.server_attrib = {'encoding': 'ascii',
- 'owner': 'nagios',
- 'group': 'nagios',
- 'type': 'file',
- 'perms': '0440'}
-
- def createhostconfig(self, entry, metadata):
- """Build host specific configuration file."""
- host_address = socket.gethostbyname(metadata.hostname)
- host_groups = [grp for grp in metadata.groups
- if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))]
- host_config = ['define host {',
- line_fmt % ('host_name', metadata.hostname),
- line_fmt % ('alias', metadata.hostname),
- line_fmt % ('address', host_address)]
-
- if host_groups:
- host_config.append(line_fmt % ("hostgroups",
- ",".join(host_groups)))
-
- # read the old-style Properties config, but emit a warning.
- xtra = dict()
- props = None
- if (hasattr(metadata, 'Properties') and
- 'NagiosGen.xml' in metadata.Properties):
- props = metadata.Properties['NagiosGen.xml'].data
- if props is not None:
- LOGGER.warn("Parsing deprecated Properties/NagiosGen.xml. "
- "Update to the new-style config with "
- "nagiosgen-convert.py.")
- xtra = dict((el.tag, el.text)
- for el in props.find(metadata.hostname))
- # hold off on parsing the defaults until we've checked for
- # a new-style config
-
- # read the old-style parents.xml, but emit a warning
- pfile = os.path.join(self.data, "parents.xml")
- if os.path.exists(pfile):
- LOGGER.warn("Parsing deprecated NagiosGen/parents.xml. "
- "Update to the new-style config with "
- "nagiosgen-convert.py.")
- parents = lxml.etree.parse(pfile)
- for el in parents.xpath("//Depend[@name='%s']" % metadata.hostname):
- if 'parent' in xtra:
- xtra['parent'] += "," + el.get("on")
- else:
- xtra['parent'] = el.get("on")
-
- # read the new-style config and overwrite the old-style config
- for el in self.config.Match(metadata):
- if el.tag == 'Option':
- xtra[el.get("name")] = el.text
-
- # if we haven't found anything in the new- or old-style
- # configs, finally read defaults from old-style config
- if not xtra and props is not None:
- xtra = dict((el.tag, el.text) for el in props.find('default'))
-
- if xtra:
- host_config.extend([line_fmt % (opt, val)
- for opt, val in list(xtra.items())])
- else:
- host_config.append(line_fmt % ('use', 'default'))
-
- host_config.append('}')
- entry.text = "%s\n" % "\n".join(host_config)
- [entry.attrib.__setitem__(key, value)
- for (key, value) in list(self.client_attrib.items())]
- try:
- fileh = open("%s/%s-host.cfg" %
- (self.data, metadata.hostname), 'w')
- fileh.write(entry.text)
- fileh.close()
- except OSError:
- ioerr = sys.exc_info()[1]
- LOGGER.error("Failed to write %s/%s-host.cfg" %
- (self.data, metadata.hostname))
- LOGGER.error(ioerr)
-
- def createserverconfig(self, entry, _):
- """Build monolithic server configuration file."""
- host_configs = glob.glob('%s/*-host.cfg' % self.data)
- group_configs = glob.glob('%s/*-group.cfg' % self.data)
- host_data = []
- group_data = []
- for host in host_configs:
- host_data.append(open(host, 'r').read())
-
- for group in group_configs:
- group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group)
- if "\n".join(host_data).find(group_name) != -1:
- groupfile = open(group, 'r')
- group_data.append(groupfile.read())
- groupfile.close()
-
- entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data))
- [entry.attrib.__setitem__(key, value)
- for (key, value) in list(self.server_attrib.items())]
- try:
- fileh = open("%s/nagiosgen.cfg" % self.data, 'w')
- fileh.write(entry.text)
- fileh.close()
- except OSError:
- ioerr = sys.exc_info()[1]
- LOGGER.error("Failed to write %s/nagiosgen.cfg" % self.data)
- LOGGER.error(ioerr)
diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Server/Plugins/Ohai.py
deleted file mode 100644
index 5fff20d98..000000000
--- a/src/lib/Server/Plugins/Ohai.py
+++ /dev/null
@@ -1,92 +0,0 @@
-import lxml.etree
-import os
-
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Ohai')
-
-import Bcfg2.Server.Plugin
-
-try:
- import json
-except:
- # FIXME: can be removed when server prereq is >= python 2.6
- # necessary for clients without the in-tree json module
- try:
- import simplejson as json
- except:
- logger.error("Unable to load any json modules. Make sure "
- "python-simplejson is installed.")
- raise ImportError
-
-
-probecode = """#!/bin/sh
-
-export PATH=$PATH:/sbin:/usr/sbin
-
-if type ohai >& /dev/null; then
- ohai
-else
- # an empty dict, so "'foo' in metadata.Ohai" tests succeed
- echo '{}'
-fi
-"""
-
-class OhaiCache(object):
- def __init__(self, dirname):
- self.dirname = dirname
- self.cache = dict()
-
- def __setitem__(self, item, value):
- if value == None:
- # simply return if the client returned nothing
- return
- self.cache[item] = json.loads(value)
- file("%s/%s.json" % (self.dirname, item), 'w').write(value)
-
- def __getitem__(self, item):
- if item not in self.cache:
- try:
- data = open("%s/%s.json" % (self.dirname, item)).read()
- except:
- raise KeyError(item)
- self.cache[item] = json.loads(data)
- return self.cache[item]
-
- def __iter__(self):
- data = list(self.cache.keys())
- data.extend([x[:-5] for x in os.listdir(self.dirname)])
- return data.__iter__()
-
-
-class Ohai(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Probing,
- Bcfg2.Server.Plugin.Connector):
- """The Ohai plugin is used to detect information
- about the client operating system.
- """
- name = 'Ohai'
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Probing.__init__(self)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai',
- interpreter='/bin/sh')
- self.probe.text = probecode
- try:
- os.stat(self.data)
- except:
- os.makedirs(self.data)
- self.cache = OhaiCache(self.data)
-
- def GetProbes(self, meta, force=False):
- return [self.probe]
-
- def ReceiveData(self, meta, datalist):
- self.cache[meta.hostname] = datalist[0].text
-
- def get_additional_data(self, meta):
- if meta.hostname in self.cache:
- return self.cache[meta.hostname]
- return dict()
diff --git a/src/lib/Server/Plugins/Packages/Apt.py b/src/lib/Server/Plugins/Packages/Apt.py
deleted file mode 100644
index f76bf7fa1..000000000
--- a/src/lib/Server/Plugins/Packages/Apt.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import re
-import gzip
-from Bcfg2.Server.Plugins.Packages.Collection import Collection
-from Bcfg2.Server.Plugins.Packages.Source import Source
-from Bcfg2.Bcfg2Py3k import cPickle, file
-
-class AptCollection(Collection):
- def get_group(self, group):
- self.logger.warning("Packages: Package groups are not supported by APT")
- return []
-
-class AptSource(Source):
- basegroups = ['apt', 'debian', 'ubuntu', 'nexenta']
- ptype = 'deb'
-
- def __init__(self, basepath, xsource, config):
- Source.__init__(self, basepath, xsource, config)
- self.pkgnames = set()
-
- self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
- 'version': self.version,
- 'components': self.components, 'arches': self.arches}]
-
- def save_state(self):
- cache = file(self.cachefile, 'wb')
- cPickle.dump((self.pkgnames, self.deps, self.provides),
- cache, 2)
- cache.close()
-
- def load_state(self):
- data = file(self.cachefile)
- self.pkgnames, self.deps, self.provides = cPickle.load(data)
-
- def filter_unknown(self, unknown):
- filtered = set([u for u in unknown if u.startswith('choice')])
- unknown.difference_update(filtered)
-
- def get_urls(self):
- if not self.rawurl:
- rv = []
- for part in self.components:
- for arch in self.arches:
- rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
- (self.url, self.version, part, arch))
- return rv
- else:
- return ["%sPackages.gz" % self.rawurl]
- urls = property(get_urls)
-
- def read_files(self):
- bdeps = dict()
- bprov = dict()
- if self.recommended:
- depfnames = ['Depends', 'Pre-Depends', 'Recommends']
- else:
- depfnames = ['Depends', 'Pre-Depends']
- for fname in self.files:
- if not self.rawurl:
- barch = [x
- for x in fname.split('@')
- if x.startswith('binary-')][0][7:]
- else:
- # RawURL entries assume that they only have one <Arch></Arch>
- # element and that it is the architecture of the source.
- barch = self.arches[0]
- if barch not in bdeps:
- bdeps[barch] = dict()
- bprov[barch] = dict()
- try:
- reader = gzip.GzipFile(fname)
- except:
- self.logger.error("Packages: Failed to read file %s" % fname)
- raise
- for line in reader.readlines():
- words = str(line.strip()).split(':', 1)
- if words[0] == 'Package':
- pkgname = words[1].strip().rstrip()
- self.pkgnames.add(pkgname)
- bdeps[barch][pkgname] = []
- elif words[0] in depfnames:
- vindex = 0
- for dep in words[1].split(','):
- if '|' in dep:
- cdeps = [re.sub('\s+', '',
- re.sub('\(.*\)', '', cdep))
- for cdep in dep.split('|')]
- dyn_dname = "choice-%s-%s-%s" % (pkgname,
- barch,
- vindex)
- vindex += 1
- bdeps[barch][pkgname].append(dyn_dname)
- bprov[barch][dyn_dname] = set(cdeps)
- else:
- raw_dep = re.sub('\(.*\)', '', dep)
- raw_dep = raw_dep.rstrip().strip()
- bdeps[barch][pkgname].append(raw_dep)
- elif words[0] == 'Provides':
- for pkg in words[1].split(','):
- dname = pkg.rstrip().strip()
- if dname not in bprov[barch]:
- bprov[barch][dname] = set()
- bprov[barch][dname].add(pkgname)
-
- self.deps['global'] = dict()
- self.provides['global'] = dict()
- for barch in bdeps:
- self.deps[barch] = dict()
- self.provides[barch] = dict()
- for pkgname in self.pkgnames:
- pset = set()
- for barch in bdeps:
- if pkgname not in bdeps[barch]:
- bdeps[barch][pkgname] = []
- pset.add(tuple(bdeps[barch][pkgname]))
- if len(pset) == 1:
- self.deps['global'][pkgname] = pset.pop()
- else:
- for barch in bdeps:
- self.deps[barch][pkgname] = bdeps[barch][pkgname]
- provided = set()
- for bprovided in list(bprov.values()):
- provided.update(set(bprovided))
- for prov in provided:
- prset = set()
- for barch in bprov:
- if prov not in bprov[barch]:
- continue
- prset.add(tuple(bprov[barch].get(prov, ())))
- if len(prset) == 1:
- self.provides['global'][prov] = prset.pop()
- else:
- for barch in bprov:
- self.provides[barch][prov] = bprov[barch].get(prov, ())
- self.save_state()
-
- def is_package(self, _, pkg):
- return (pkg in self.pkgnames and
- pkg not in self.blacklist and
- (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Server/Plugins/Packages/Collection.py
deleted file mode 100644
index 32eeda1ec..000000000
--- a/src/lib/Server/Plugins/Packages/Collection.py
+++ /dev/null
@@ -1,343 +0,0 @@
-import copy
-import logging
-import Bcfg2.Server.Plugin
-
-logger = logging.getLogger(__name__)
-
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
-
-# we have to cache Collection objects so that calling Packages.Refresh
-# or .Reload can tell the collection objects to clean up their cache,
-# but we don't actually use the cache to return a Collection object
-# when one is requested, because that prevents new machines from
-# working, since a Collection object gets created by
-# get_additional_data(), which is called for all clients at server
-# startup. (It would also prevent machines that change groups from
-# working properly; e.g., if you reinstall a machine with a new OS,
-# then returning a cached Collection object would give the wrong
-# sources to that client.)
-collections = dict()
-
-class Collection(Bcfg2.Server.Plugin.Debuggable):
- def __init__(self, metadata, sources, basepath, debug=False):
- """ don't call this directly; use the factory function """
- Bcfg2.Server.Plugin.Debuggable.__init__(self)
- self.debug_flag = debug
- self.metadata = metadata
- self.sources = sources
- self.basepath = basepath
- self.virt_pkgs = dict()
-
- try:
- self.config = sources[0].config
- self.cachepath = sources[0].basepath
- self.ptype = sources[0].ptype
- except IndexError:
- self.config = None
- self.cachepath = None
- self.ptype = "unknown"
-
- self.cachefile = None
-
- @property
- def cachekey(self):
- return md5(self.get_config()).hexdigest()
-
- def get_config(self):
- self.logger.error("Packages: Cannot generate config for host with "
- "multiple source types (%s)" % self.metadata.hostname)
- return ""
-
- def get_relevant_groups(self):
- groups = []
- for source in self.sources:
- groups.extend(source.get_relevant_groups(self.metadata))
- return sorted(list(set(groups)))
-
- @property
- def basegroups(self):
- groups = set()
- for source in self.sources:
- groups.update(source.basegroups)
- return list(groups)
-
- @property
- def cachefiles(self):
- cachefiles = set([self.cachefile])
- for source in self.sources:
- cachefiles.add(source.cachefile)
- return list(cachefiles)
-
- def get_group(self, group, ptype=None):
- for source in self.sources:
- pkgs = source.get_group(self.metadata, group, ptype=ptype)
- if pkgs:
- return pkgs
- self.logger.warning("Packages: '%s' is not a valid group" % group)
- return []
-
- def is_package(self, package):
- for source in self.sources:
- if source.is_package(self.metadata, package):
- return True
- return False
-
- def is_virtual_package(self, package):
- for source in self.sources:
- if source.is_virtual_package(self.metadata, package):
- return True
- return False
-
- def get_deps(self, package):
- for source in self.sources:
- if source.is_package(self.metadata, package):
- return source.get_deps(self.metadata, package)
- return []
-
- def get_provides(self, package):
- for source in self.sources:
- providers = source.get_provides(self.metadata, package)
- if providers:
- return providers
- return []
-
- def get_vpkgs(self):
- """ get virtual packages """
- vpkgs = dict()
- for source in self.sources:
- s_vpkgs = source.get_vpkgs(self.metadata)
- for name, prov_set in list(s_vpkgs.items()):
- if name not in vpkgs:
- vpkgs[name] = set(prov_set)
- else:
- vpkgs[name].update(prov_set)
- return vpkgs
-
- def filter_unknown(self, unknown):
- for source in self.sources:
- source.filter_unknown(unknown)
-
- def magic_groups_match(self):
- for source in self.sources:
- if source.magic_groups_match(self.metadata):
- return True
-
- def build_extra_structures(self, independent):
- pass
-
- def get_additional_data(self):
- sdata = []
- for source in self.sources:
- sdata.extend(copy.deepcopy(source.url_map))
- return sdata
-
- def setup_data(self, force_update=False):
- """ do any collection-level data setup tasks """
- pass
-
- def complete(self, packagelist):
- '''Build the transitive closure of all package dependencies
-
- Arguments:
- packageslist - set of package names
- returns => (set(packages), set(unsatisfied requirements))
- '''
-
- # setup vpkg cache
- pgrps = tuple(self.get_relevant_groups())
- if pgrps not in self.virt_pkgs:
- self.virt_pkgs[pgrps] = self.get_vpkgs()
- vpkg_cache = self.virt_pkgs[pgrps]
-
- # unclassified is set of unsatisfied requirements (may be pkg
- # for vpkg)
- unclassified = set(packagelist)
- vpkgs = set()
- both = set()
- pkgs = set(packagelist)
-
- packages = set()
- examined = set()
- unknown = set()
-
- final_pass = False
- really_done = False
- # do while unclassified or vpkgs or both or pkgs
- while unclassified or pkgs or both or final_pass:
- if really_done:
- break
- if len(unclassified) + len(pkgs) + len(both) == 0:
- # one more pass then exit
- really_done = True
-
- while unclassified:
- current = unclassified.pop()
- examined.add(current)
- is_pkg = False
- if self.is_package(current):
- is_pkg = True
-
- is_vpkg = current in vpkg_cache
-
- if is_pkg and is_vpkg:
- both.add(current)
- elif is_pkg and not is_vpkg:
- pkgs.add(current)
- elif is_vpkg and not is_pkg:
- vpkgs.add(current)
- elif not is_vpkg and not is_pkg:
- unknown.add(current)
-
- while pkgs:
- # direct packages; current can be added, and all deps
- # should be resolved
- current = pkgs.pop()
- self.debug_log("Packages: handling package requirement %s" %
- current)
- packages.add(current)
- deps = self.get_deps(current)
- newdeps = set(deps).difference(examined)
- if newdeps:
- self.debug_log("Packages: Package %s added requirements %s"
- % (current, newdeps))
- unclassified.update(newdeps)
-
- satisfied_vpkgs = set()
- for current in vpkgs:
- # virtual dependencies, satisfied if one of N in the
- # config, or can be forced if only one provider
- if len(vpkg_cache[current]) == 1:
- self.debug_log("Packages: requirement %s satisfied by %s" %
- (current, vpkg_cache[current]))
- unclassified.update(vpkg_cache[current].difference(examined))
- satisfied_vpkgs.add(current)
- else:
- satisfiers = [item for item in vpkg_cache[current]
- if item in packages]
- self.debug_log("Packages: requirement %s satisfied by %s" %
- (current, satisfiers))
- satisfied_vpkgs.add(current)
- vpkgs.difference_update(satisfied_vpkgs)
-
- satisfied_both = set()
- for current in both:
- # packages that are both have virtual providers as
- # well as a package with that name. allow use of virt
- # through explicit specification, then fall back to
- # forcing current on last pass
- satisfiers = [item for item in vpkg_cache[current]
- if item in packages]
- if satisfiers:
- self.debug_log("Packages: requirement %s satisfied by %s" %
- (current, satisfiers))
- satisfied_both.add(current)
- elif current in packagelist or final_pass:
- pkgs.add(current)
- satisfied_both.add(current)
- both.difference_update(satisfied_both)
-
- if len(unclassified) + len(pkgs) == 0:
- final_pass = True
- else:
- final_pass = False
-
- self.filter_unknown(unknown)
-
- return packages, unknown
-
- def __len__(self):
- return len(self.sources)
-
- def __getitem__(self, item):
- return self.sources[item]
-
- def __setitem__(self, item, value):
- self.sources[item] = value
-
- def __delitem__(self, item):
- del self.sources[item]
-
- def append(self, item):
- self.sources.append(item)
-
- def count(self):
- return self.sources.count()
-
- def index(self, item):
- return self.sources.index(item)
-
- def extend(self, items):
- self.sources.extend(items)
-
- def insert(self, index, item):
- self.sources.insert(index, item)
-
- def pop(self, index=None):
- self.sources.pop(index)
-
- def remove(self, item):
- self.sources.remove(item)
-
- def sort(self, cmp=None, key=None, reverse=False):
- self.sources.sort(cmp, key, reverse)
-
-def clear_cache():
- global collections
- collections = dict()
-
-def factory(metadata, sources, basepath, debug=False):
- global collections
-
- if not sources.loaded:
- # if sources.xml has not received a FAM event yet, defer;
- # instantiate a dummy Collection object
- return Collection(metadata, [], basepath)
-
- sclasses = set()
- relevant = list()
-
- for source in sources:
- if source.applies(metadata):
- relevant.append(source)
- sclasses.update([source.__class__])
-
- if len(sclasses) > 1:
- logger.warning("Packages: Multiple source types found for %s: %s" %
- ",".join([s.__name__ for s in sclasses]))
- cclass = Collection
- elif len(sclasses) == 0:
- # you'd think this should be a warning, but it happens all the
- # freaking time if you have a) machines in your clients.xml
- # that do not have the proper groups set up yet (e.g., if you
- # have multiple Bcfg2 servers and Packages-relevant groups set
- # by probes); and b) templates that query all or multiple
- # machines (e.g., with metadata.query.all_clients())
- if debug:
- logger.error("Packages: No sources found for %s" %
- metadata.hostname)
- cclass = Collection
- else:
- stype = sclasses.pop().__name__.replace("Source", "")
- try:
- module = \
- getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
- stype.title()).Server.Plugins.Packages,
- stype.title())
- cclass = getattr(module, "%sCollection" % stype.title())
- except ImportError:
- logger.error("Packages: Unknown source type %s" % stype)
- except AttributeError:
- logger.warning("Packages: No collection class found for %s sources"
- % stype)
-
- if debug:
- logger.error("Packages: Using %s for Collection of sources for %s" %
- (cclass.__name__, metadata.hostname))
-
- collection = cclass(metadata, relevant, basepath, debug=debug)
- collections[metadata.hostname] = collection
- return collection
-
diff --git a/src/lib/Server/Plugins/Packages/Pac.py b/src/lib/Server/Plugins/Packages/Pac.py
deleted file mode 100644
index 9db6b0535..000000000
--- a/src/lib/Server/Plugins/Packages/Pac.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import gzip
-import tarfile
-from Bcfg2.Bcfg2Py3k import cPickle, file
-from Bcfg2.Server.Plugins.Packages.Collection import Collection
-from Bcfg2.Server.Plugins.Packages.Source import Source
-
-class PacCollection(Collection):
- def get_group(self, group):
- self.logger.warning("Packages: Package groups are not supported by APT")
- return []
-
-class PacSource(Source):
- basegroups = ['arch', 'parabola']
- ptype = 'pacman'
-
- def __init__(self, basepath, xsource, config):
- Source.__init__(self, basepath, xsource, config)
- self.pkgnames = set()
-
- self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
- 'version': self.version,
- 'components': self.components, 'arches': self.arches}]
-
- def save_state(self):
- cache = file(self.cachefile, 'wb')
- cPickle.dump((self.pkgnames, self.deps, self.provides),
- cache, 2)
- cache.close()
-
- def load_state(self):
- data = file(self.cachefile)
- self.pkgnames, self.deps, self.provides = cPickle.load(data)
-
- def filter_unknown(self, unknown):
- filtered = set([u for u in unknown if u.startswith('choice')])
- unknown.difference_update(filtered)
-
- def get_urls(self):
- if not self.rawurl:
- rv = []
- for part in self.components:
- for arch in self.arches:
- rv.append("%s%s/os/%s/%s.db.tar.gz" %
- (self.url, part, arch, part))
- return rv
- else:
- raise Exception("PacSource : RAWUrl not supported (yet)")
- urls = property(get_urls)
-
- def read_files(self):
- bdeps = dict()
- bprov = dict()
-
- if self.recommended:
- depfnames = ['Depends', 'Pre-Depends', 'Recommends']
- else:
- depfnames = ['Depends', 'Pre-Depends']
-
- for fname in self.files:
- if not self.rawurl:
- barch = [x for x in fname.split('@') if x in self.arches][0]
- else:
- # RawURL entries assume that they only have one <Arch></Arch>
- # element and that it is the architecture of the source.
- barch = self.arches[0]
-
- if barch not in bdeps:
- bdeps[barch] = dict()
- bprov[barch] = dict()
- try:
- self.debug_log("Packages: try to read %s" % fname)
- tar = tarfile.open(fname, "r")
- reader = gzip.GzipFile(fname)
- except:
- self.logger.error("Packages: Failed to read file %s" % fname)
- raise
-
- for tarinfo in tar:
- if tarinfo.isdir():
- self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0])
- self.debug_log("Packages: added %s" %
- tarinfo.name.rsplit("-", 2)[0])
- tar.close()
-
- self.deps['global'] = dict()
- self.provides['global'] = dict()
- for barch in bdeps:
- self.deps[barch] = dict()
- self.provides[barch] = dict()
- for pkgname in self.pkgnames:
- pset = set()
- for barch in bdeps:
- if pkgname not in bdeps[barch]:
- bdeps[barch][pkgname] = []
- pset.add(tuple(bdeps[barch][pkgname]))
- if len(pset) == 1:
- self.deps['global'][pkgname] = pset.pop()
- else:
- for barch in bdeps:
- self.deps[barch][pkgname] = bdeps[barch][pkgname]
- provided = set()
- for bprovided in list(bprov.values()):
- provided.update(set(bprovided))
- for prov in provided:
- prset = set()
- for barch in bprov:
- if prov not in bprov[barch]:
- continue
- prset.add(tuple(bprov[barch].get(prov, ())))
- if len(prset) == 1:
- self.provides['global'][prov] = prset.pop()
- else:
- for barch in bprov:
- self.provides[barch][prov] = bprov[barch].get(prov, ())
- self.save_state()
-
- def is_package(self, _, pkg):
- return (pkg in self.pkgnames and
- pkg not in self.blacklist and
- (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py
deleted file mode 100644
index 7950f15e6..000000000
--- a/src/lib/Server/Plugins/Packages/PackagesConfig.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import Bcfg2.Server.Plugin
-
-class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig):
- _required = False
-
- def Index(self):
- """ Build local data structures """
- Bcfg2.Server.Plugin.SimpleConfig.Index(self)
-
- if hasattr(self.plugin, "sources") and self.plugin.sources.loaded:
- # only reload Packages plugin if sources have been loaded.
- # otherwise, this is getting called on server startup, and
- # we have to wait until all sources have been indexed
- # before we can call Packages.Reload()
- self.plugin.Reload()
diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Server/Plugins/Packages/PackagesSources.py
deleted file mode 100644
index da79c00e9..000000000
--- a/src/lib/Server/Plugins/Packages/PackagesSources.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import os
-import sys
-import lxml.etree
-import Bcfg2.Server.Plugin
-from Bcfg2.Server.Plugins.Packages.Source import SourceInitError
-
-class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked,
- Bcfg2.Server.Plugin.StructFile,
- Bcfg2.Server.Plugin.Debuggable):
- __identifier__ = None
-
- def __init__(self, filename, cachepath, fam, packages, config):
- Bcfg2.Server.Plugin.Debuggable.__init__(self)
- try:
- Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self,
- filename,
- fam)
- except OSError:
- err = sys.exc_info()[1]
- msg = "Packages: Failed to read configuration file: %s" % err
- if not os.path.exists(self.name):
- msg += " Have you created it?"
- self.logger.error(msg)
- raise Bcfg2.Server.Plugin.PluginInitError(msg)
- Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
- self.cachepath = cachepath
- self.config = config
- if not os.path.exists(self.cachepath):
- # create cache directory if needed
- try:
- os.makedirs(self.cachepath)
- except OSError:
- err = sys.exc_info()[1]
- self.logger.error("Could not create Packages cache at %s: %s" %
- (self.cachepath, err))
- self.pkg_obj = packages
- self.parsed = set()
- self.loaded = False
-
- def toggle_debug(self):
- Bcfg2.Server.Plugin.Debuggable.toggle_debug(self)
- for source in self.entries:
- source.toggle_debug()
-
- def HandleEvent(self, event=None):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event)
- if event.filename != self.name:
- self.parsed.add(os.path.basename(event.filename))
-
- if sorted(list(self.parsed)) == sorted(self.extras):
- self.logger.info("Reloading Packages plugin")
- self.pkg_obj.Reload()
- self.loaded = True
-
- def Index(self):
- Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self)
- self.entries = []
- for xsource in self.xdata.findall('.//Source'):
- source = self.source_from_xml(xsource)
- if source is not None:
- self.entries.append(source)
-
- def source_from_xml(self, xsource):
- """ create a *Source object from its XML representation in
- sources.xml """
- stype = xsource.get("type")
- if stype is None:
- self.logger.error("Packages: No type specified for source, "
- "skipping")
- return None
-
- try:
- module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
- stype.title()).Server.Plugins.Packages,
- stype.title())
- cls = getattr(module, "%sSource" % stype.title())
- except (ImportError, AttributeError):
- self.logger.error("Packages: Unknown source type %s" % stype)
- return None
-
- try:
- source = cls(self.cachepath, xsource, self.config)
- except SourceInitError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: %s" % err)
- source = None
-
- return source
-
- def __getitem__(self, key):
- return self.entries[key]
-
- def __repr__(self):
- return "PackagesSources: %s" % repr(self.entries)
-
- def __str__(self):
- return "PackagesSources: %s" % str(self.entries)
diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Server/Plugins/Packages/Source.py
deleted file mode 100644
index 1dfeecc40..000000000
--- a/src/lib/Server/Plugins/Packages/Source.py
+++ /dev/null
@@ -1,278 +0,0 @@
-import os
-import re
-import sys
-import base64
-import Bcfg2.Server.Plugin
-from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \
- HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \
- urlopen, file, cPickle
-
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
-
-def fetch_url(url):
- if '@' in url:
- mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
- if not mobj:
- raise ValueError
- user = mobj.group(2)
- passwd = mobj.group(3)
- url = mobj.group(1) + mobj.group(4)
- auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
- auth.add_password(None, url, user, passwd)
- install_opener(build_opener(auth))
- return urlopen(url).read()
-
-
-class SourceInitError(Exception):
- pass
-
-
-class Source(Bcfg2.Server.Plugin.Debuggable):
- mrepo_re = re.compile(r'/RPMS\.([^/]+)')
- pulprepo_re = re.compile(r'pulp/repos/([^/]+)')
- genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$')
- basegroups = []
-
- def __init__(self, basepath, xsource, config):
- Bcfg2.Server.Plugin.Debuggable.__init__(self)
- self.basepath = basepath
- self.xsource = xsource
- self.config = config
-
- try:
- self.version = xsource.find('Version').text
- except AttributeError:
- pass
-
- for key, tag in [('components', 'Component'), ('arches', 'Arch'),
- ('blacklist', 'Blacklist'),
- ('whitelist', 'Whitelist')]:
- self.__dict__[key] = [item.text for item in xsource.findall(tag)]
-
- self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
-
- self.recommended = xsource.get('recommended', 'false').lower() == 'true'
-
- self.rawurl = xsource.get('rawurl', '')
- if self.rawurl and not self.rawurl.endswith("/"):
- self.rawurl += "/"
- self.url = xsource.get('url', '')
- if self.url and not self.url.endswith("/"):
- self.url += "/"
- self.version = xsource.get('version', '')
-
- # build the set of conditions to see if this source applies to
- # a given set of metadata
- self.conditions = []
- self.groups = [] # provided for some limited backwards compat
- for el in xsource.iterancestors():
- if el.tag == "Group":
- if el.get("negate", "false").lower() == "true":
- self.conditions.append(lambda m, el=el:
- el.get("name") not in m.groups)
- else:
- self.groups.append(el.get("name"))
- self.conditions.append(lambda m, el=el:
- el.get("name") in m.groups)
- elif el.tag == "Client":
- if el.get("negate", "false").lower() == "true":
- self.conditions.append(lambda m, el=el:
- el.get("name") != m.hostname)
- else:
- self.conditions.append(lambda m, el=el:
- el.get("name") == m.hostname)
-
- self.deps = dict()
- self.provides = dict()
-
- self.cachefile = os.path.join(self.basepath,
- "cache-%s" % self.cachekey)
- self.url_map = []
-
- @property
- def cachekey(self):
- return md5(cPickle.dumps([self.version, self.components, self.url,
- self.rawurl, self.arches])).hexdigest()
-
- def get_relevant_groups(self, metadata):
- return sorted(list(set([g for g in metadata.groups
- if (g in self.basegroups or
- g in self.groups or
- g in self.arches)])))
-
- def load_state(self):
- pass
-
- def setup_data(self, force_update=False):
- should_read = True
- should_download = False
- if os.path.exists(self.cachefile):
- try:
- self.load_state()
- should_read = False
- except:
- self.logger.error("Packages: Cachefile %s load failed; "
- "falling back to file read" % self.cachefile)
- if should_read:
- try:
- self.read_files()
- except:
- self.logger.error("Packages: File read failed; "
- "falling back to file download")
- should_download = True
-
- if should_download or force_update:
- try:
- self.update()
- self.read_files()
- except:
- self.logger.error("Packages: Failed to load data for Source "
- "of %s. Some Packages will be missing." %
- self.urls)
-
- def get_repo_name(self, url_map):
- # try to find a sensible name for a repo
- if url_map['component']:
- rname = url_map['component']
- else:
- name = None
- for repo_re in (self.mrepo_re,
- self.pulprepo_re,
- self.genericrepo_re):
- match = repo_re.search(url_map['url'])
- if match:
- break
- if name is None:
- # couldn't figure out the name from the URL or URL map
- # (which probably means its a screwy URL), so we just
- # generate a random one
- name = base64.b64encode(os.urandom(16))[:-2]
- rname = "%s-%s" % (self.groups[0], name)
- # see yum/__init__.py in the yum source, lines 441-449, for
- # the source of this regex. yum doesn't like anything but
- # string.ascii_letters, string.digits, and [-_.:]. There
- # doesn't seem to be a reason for this, because yum.
- return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname)
-
- def __str__(self):
- if self.rawurl:
- return "%s at %s" % (self.__class__.__name__, self.rawurl)
- elif self.url:
- return "%s at %s" % (self.__class__.__name__, self.url)
- else:
- return self.__class__.__name__
-
- def get_urls(self):
- return []
- urls = property(get_urls)
-
- def get_files(self):
- return [self.escape_url(url) for url in self.urls]
- files = property(get_files)
-
- def get_vpkgs(self, metadata):
- agroups = ['global'] + [a for a in self.arches
- if a in metadata.groups]
- vdict = dict()
- for agrp in agroups:
- for key, value in list(self.provides[agrp].items()):
- if key not in vdict:
- vdict[key] = set(value)
- else:
- vdict[key].update(value)
- return vdict
-
- def is_virtual_package(self, metadata, package):
- """ called to determine if a package is a virtual package.
- this is only invoked if the package is not listed in the dict
- returned by get_vpkgs """
- return False
-
- def escape_url(self, url):
- return os.path.join(self.basepath, url.replace('/', '@'))
-
- def file_init(self):
- pass
-
- def read_files(self):
- pass
-
- def filter_unknown(self, unknown):
- pass
-
- def update(self):
- for url in self.urls:
- self.logger.info("Packages: Updating %s" % url)
- fname = self.escape_url(url)
- try:
- data = fetch_url(url)
- file(fname, 'w').write(data)
- except ValueError:
- self.logger.error("Packages: Bad url string %s" % url)
- raise
- except HTTPError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to fetch url %s. HTTP "
- "response code=%s" % (url, err.code))
- raise
-
- def applies(self, metadata):
- # check base groups
- if not self.magic_groups_match(metadata):
- return False
-
- # check Group/Client tags from sources.xml
- for condition in self.conditions:
- if not condition(metadata):
- return False
-
- return True
-
- def get_arches(self, metadata):
- return ['global'] + [a for a in self.arches if a in metadata.groups]
-
- def get_deps(self, metadata, pkgname):
- for arch in self.get_arches(metadata):
- if pkgname in self.deps[arch]:
- return self.deps[arch][pkgname]
- return []
-
- def get_provides(self, metadata, required):
- for arch in self.get_arches(metadata):
- if required in self.provides[arch]:
- return self.provides[arch][required]
- return []
-
- def is_package(self, metadata, _):
- return False
-
- def get_package(self, metadata, package):
- return package
-
- def get_group(self, metadata, group, ptype=None):
- return []
-
- def magic_groups_match(self, metadata):
- """ check to see if this source applies to the given host
- metadata by checking 'magic' (base) groups only, or if magic
- groups are off """
- # we always check that arch matches
- found_arch = False
- for arch in self.arches:
- if arch in metadata.groups:
- found_arch = True
- break
- if not found_arch:
- return False
-
- if self.config.getboolean("global", "magic_groups",
- default=True) == False:
- return True
- else:
- for group in self.basegroups:
- if group in metadata.groups:
- return True
- return False
diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py
deleted file mode 100644
index e13b28251..000000000
--- a/src/lib/Server/Plugins/Packages/Yum.py
+++ /dev/null
@@ -1,688 +0,0 @@
-import os
-import sys
-import time
-import copy
-import glob
-import socket
-import random
-import logging
-import threading
-import lxml.etree
-from UserDict import DictMixin
-from subprocess import Popen, PIPE, STDOUT
-import Bcfg2.Server.Plugin
-from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file
-from Bcfg2.Server.Plugins.Packages.Collection import Collection
-from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
- fetch_url
-
-logger = logging.getLogger(__name__)
-
-try:
- from pulp.client.consumer.config import ConsumerConfig
- from pulp.client.api.repository import RepositoryAPI
- from pulp.client.api.consumer import ConsumerAPI
- from pulp.client.api import server
- has_pulp = True
-except ImportError:
- has_pulp = False
-
-try:
- import yum
- has_yum = True
-except ImportError:
- has_yum = False
- logger.info("Packages: No yum libraries found; forcing use of internal "
- "dependency resolver")
-
-try:
- import json
-except ImportError:
- import simplejson as json
-
-XP = '{http://linux.duke.edu/metadata/common}'
-RP = '{http://linux.duke.edu/metadata/rpm}'
-RPO = '{http://linux.duke.edu/metadata/repo}'
-FL = '{http://linux.duke.edu/metadata/filelists}'
-
-PULPSERVER = None
-PULPCONFIG = None
-
-
-def _setup_pulp(config):
- global PULPSERVER, PULPCONFIG
- if not has_pulp:
- logger.error("Packages: Cannot create Pulp collection: Pulp libraries not "
- "found")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- if PULPSERVER is None:
- try:
- username = config.get("pulp", "username")
- password = config.get("pulp", "password")
- except ConfigParser.NoSectionError:
- logger.error("Packages: No [pulp] section found in Packages/packages.conf")
- raise Bcfg2.Server.Plugin.PluginInitError
- except ConfigParser.NoOptionError:
- err = sys.exc_info()[1]
- logger.error("Packages: Required option not found in "
- "Packages/packages.conf: %s" % err)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- PULPCONFIG = ConsumerConfig()
- serveropts = PULPCONFIG.server
-
- PULPSERVER = server.PulpServer(serveropts['host'],
- int(serveropts['port']),
- serveropts['scheme'],
- serveropts['path'])
- PULPSERVER.set_basic_auth_credentials(username, password)
- server.set_active_server(PULPSERVER)
- return PULPSERVER
-
-
-class YumCollection(Collection):
- # options that are included in the [yum] section but that should
- # not be included in the temporary yum.conf we write out
- option_blacklist = ["use_yum_libraries", "helper"]
-
- def __init__(self, metadata, sources, basepath, debug=False):
- Collection.__init__(self, metadata, sources, basepath, debug=debug)
- self.keypath = os.path.join(self.basepath, "keys")
-
- if len(sources):
- config = sources[0].config
- self.use_yum = has_yum and config.getboolean("yum",
- "use_yum_libraries",
- default=False)
- else:
- self.use_yum = False
-
- if self.use_yum:
- self.cachefile = os.path.join(self.cachepath,
- "cache-%s" % self.cachekey)
- if not os.path.exists(self.cachefile):
- os.mkdir(self.cachefile)
-
- self.configdir = os.path.join(self.basepath, "yum")
- if not os.path.exists(self.configdir):
- os.mkdir(self.configdir)
- self.cfgfile = os.path.join(self.configdir,
- "%s-yum.conf" % self.cachekey)
- self.write_config()
-
- self.helper = self.config.get("yum", "helper",
- default="/usr/sbin/bcfg2-yum-helper")
- if has_pulp:
- _setup_pulp(self.config)
-
- def write_config(self):
- if not os.path.exists(self.cfgfile):
- yumconf = self.get_config(raw=True)
- yumconf.add_section("main")
-
- mainopts = dict(cachedir=self.cachefile,
- keepcache="0",
- sslverify="0",
- debuglevel="0",
- reposdir="/dev/null")
- try:
- for opt in self.config.options("yum"):
- if opt not in self.option_blacklist:
- mainopts[opt] = self.config.get("yum", opt)
- except ConfigParser.NoSectionError:
- pass
-
- for opt, val in list(mainopts.items()):
- yumconf.set("main", opt, val)
-
- yumconf.write(open(self.cfgfile, 'w'))
-
- def get_config(self, raw=False):
- config = ConfigParser.SafeConfigParser()
- for source in self.sources:
- # get_urls() loads url_map as a side-effect
- source.get_urls()
- for url_map in source.url_map:
- if url_map['arch'] in self.metadata.groups:
- basereponame = source.get_repo_name(url_map)
- reponame = basereponame
-
- added = False
- while not added:
- try:
- config.add_section(reponame)
- added = True
- except ConfigParser.DuplicateSectionError:
- match = re.match("-(\d)", reponame)
- if match:
- rid = int(match.group(1)) + 1
- else:
- rid = 1
- reponame = "%s-%d" % (basereponame, rid)
-
- config.set(reponame, "name", reponame)
- config.set(reponame, "baseurl", url_map['url'])
- config.set(reponame, "enabled", "1")
- if len(source.gpgkeys):
- config.set(reponame, "gpgcheck", "1")
- config.set(reponame, "gpgkey",
- " ".join(source.gpgkeys))
- else:
- config.set(reponame, "gpgcheck", "0")
-
- if len(source.blacklist):
- config.set(reponame, "exclude",
- " ".join(source.blacklist))
- if len(source.whitelist):
- config.set(reponame, "includepkgs",
- " ".join(source.whitelist))
-
- if raw:
- return config
- else:
- # configparser only writes to file, so we have to use a
- # StringIO object to get the data out as a string
- buf = StringIO()
- config.write(buf)
- return "# This config was generated automatically by the Bcfg2 " \
- "Packages plugin\n\n" + buf.getvalue()
-
- def build_extra_structures(self, independent):
- """ build list of gpg keys to be added to the specification by
- validate_structures() """
- needkeys = set()
- for source in self.sources:
- for key in source.gpgkeys:
- needkeys.add(key)
-
- if len(needkeys):
- if has_yum:
- # this must be be has_yum, not use_yum, because
- # regardless of whether the user wants to use the yum
- # resolver we want to include gpg key data
- keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey",
- type=self.ptype, origin='Packages')
- else:
- self.logger.warning("GPGKeys were specified for yum sources in "
- "sources.xml, but no yum libraries were "
- "found")
- self.logger.warning("GPG key version/release data cannot be "
- "determined automatically")
- self.logger.warning("Install yum libraries, or manage GPG keys "
- "manually")
- keypkg = None
-
- for key in needkeys:
- # figure out the path of the key on the client
- keydir = self.config.get("global", "gpg_keypath",
- default="/etc/pki/rpm-gpg")
- remotekey = os.path.join(keydir, os.path.basename(key))
- localkey = os.path.join(self.keypath, os.path.basename(key))
- kdata = open(localkey).read()
-
- # copy the key to the client
- keypath = lxml.etree.Element("BoundPath", name=remotekey,
- encoding='ascii',
- owner='root', group='root',
- type='file', perms='0644',
- important='true')
- keypath.text = kdata
-
- # hook to add version/release info if possible
- self._add_gpg_instances(keypkg, kdata, localkey, remotekey)
- independent.append(keypath)
- if keypkg is not None:
- independent.append(keypkg)
-
- # see if there are any pulp sources to handle
- has_pulp_sources = False
- for source in self.sources:
- if source.pulp_id:
- has_pulp_sources = True
- break
-
- if has_pulp_sources:
- consumerapi = ConsumerAPI()
- consumer = self._get_pulp_consumer(consumerapi=consumerapi)
- if consumer is None:
- consumer = consumerapi.create(self.metadata.hostname,
- self.metadata.hostname)
- lxml.etree.SubElement(independent, "BoundAction",
- name="pulp-update", timing="pre",
- when="always", status="check",
- command="pulp-consumer consumer update")
-
- for source in self.sources:
- # each pulp source can only have one arch, so we don't
- # have to check the arch in url_map
- if (source.pulp_id and
- source.pulp_id not in consumer['repoids']):
- consumerapi.bind(self.metadata.hostname, source.pulp_id)
-
- crt = lxml.etree.SubElement(independent, "BoundPath",
- name="/etc/pki/consumer/cert.pem",
- type="file", owner="root",
- group="root", perms="0644")
- crt.text = consumerapi.certificate(self.metadata.hostname)
-
- def _get_pulp_consumer(self, consumerapi=None):
- if consumerapi is None:
- consumerapi = ConsumerAPI()
- consumer = None
- try:
- consumer = consumerapi.consumer(self.metadata.hostname)
- except server.ServerRequestError:
- # consumer does not exist
- pass
- except socket.error:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Could not contact Pulp server: %s" %
- err)
- except:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Unknown error querying Pulp server: %s"
- % err)
- return consumer
-
- def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey):
- """ add gpg keys to the specification to ensure they get
- installed """
- # this must be be has_yum, not use_yum, because regardless of
- # whether the user wants to use the yum resolver we want to
- # include gpg key data
- if not has_yum:
- return
-
- try:
- kinfo = yum.misc.getgpgkeyinfo(keydata)
- version = yum.misc.keyIdToRPMVer(kinfo['keyid'])
- release = yum.misc.keyIdToRPMVer(kinfo['timestamp'])
-
- lxml.etree.SubElement(keyentry, 'Instance',
- version=version,
- release=release,
- simplefile=remotekey)
- except ValueError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Could not read GPG key %s: %s" %
- (localkey, err))
-
- def is_package(self, package):
- if not self.use_yum:
- return Collection.is_package(self, package)
- elif isinstance(package, tuple):
- if package[1] is None and package[2] == (None, None, None):
- package = package[0]
- else:
- return None
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("is_package", package)
-
- def is_virtual_package(self, package):
- if not self.use_yum:
- return Collection.is_virtual_package(self, package)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("is_virtual_package", package)
-
- def get_deps(self, package):
- if not self.use_yum:
- return Collection.get_deps(self, package)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("get_deps", package)
-
- def get_provides(self, required, all=False, silent=False):
- if not self.use_yum:
- return Collection.get_provides(self, package)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("get_provides", package)
-
- def get_group(self, group, ptype="default"):
- if not self.use_yum:
- self.logger.warning("Packages: Package groups are not supported by "
- "Bcfg2's internal Yum dependency generator")
- return []
-
- if group.startswith("@"):
- group = group[1:]
-
- pkgs = self.call_helper("get_group", dict(group=group, type=ptype))
- return pkgs
-
- def complete(self, packagelist):
- if not self.use_yum:
- return Collection.complete(self, packagelist)
-
- packages = set()
- unknown = set(packagelist)
-
- if unknown:
- result = \
- self.call_helper("complete",
- dict(packages=list(unknown),
- groups=list(self.get_relevant_groups())))
- if result and "packages" in result and "unknown" in result:
- # we stringify every package because it gets returned
- # in unicode; set.update() doesn't work if some
- # elements are unicode and other are strings. (I.e.,
- # u'foo' and 'foo' get treated as unique elements.)
- packages.update([str(p) for p in result['packages']])
- unknown = set([str(p) for p in result['unknown']])
-
- self.filter_unknown(unknown)
-
- return packages, unknown
-
- def call_helper(self, command, input=None):
- """ Make a call to bcfg2-yum-helper. The yum libs have
- horrific memory leaks, so apparently the right way to get
- around that in long-running processes it to have a short-lived
- helper. No, seriously -- check out the yum-updatesd code.
- It's pure madness. """
- # it'd be nice if we could change this to be more verbose if
- # -v was given to bcfg2-server, but Collection objects don't
- # get the 'setup' variable, so we don't know how verbose
- # bcfg2-server is. It'd also be nice if we could tell yum to
- # log to syslog. So would a unicorn.
- cmd = [self.helper, "-c", self.cfgfile]
- if self.debug_flag:
- cmd.append("-v")
- cmd.append(command)
- self.debug_log("Packages: running %s" % " ".join(cmd))
- try:
- helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
- except OSError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to execute %s: %s" %
- (" ".join(cmd), err))
- return None
-
- if input:
- idata = json.dumps(input)
- (stdout, stderr) = helper.communicate(idata)
- else:
- (stdout, stderr) = helper.communicate()
- rv = helper.wait()
- if rv:
- self.logger.error("Packages: error running bcfg2-yum-helper "
- "(returned %d): %s" % (rv, stderr))
- elif self.debug_flag:
- self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" %
- stderr)
- try:
- return json.loads(stdout)
- except ValueError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: error reading bcfg2-yum-helper "
- "output: %s" % err)
- return None
-
- def setup_data(self, force_update=False):
- if not self.use_yum:
- return Collection.setup_data(self, force_update)
-
- if force_update:
- # we call this twice: one to clean up data from the old
- # config, and once to clean up data from the new config
- self.call_helper("clean")
-
- os.unlink(self.cfgfile)
- self.write_config()
-
- if force_update:
- self.call_helper("clean")
-
-
-class YumSource(Source):
- basegroups = ['yum', 'redhat', 'centos', 'fedora']
- ptype = 'yum'
-
- def __init__(self, basepath, xsource, config):
- Source.__init__(self, basepath, xsource, config)
- self.pulp_id = None
- if has_pulp and xsource.get("pulp_id"):
- self.pulp_id = xsource.get("pulp_id")
-
- _setup_pulp(self.config)
- repoapi = RepositoryAPI()
- try:
- self.repo = repoapi.repository(self.pulp_id)
- self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key)
- for key in repoapi.listkeys(self.pulp_id)]
- except server.ServerRequestError:
- err = sys.exc_info()[1]
- if err[0] == 401:
- msg = "Packages: Error authenticating to Pulp: %s" % err[1]
- elif err[0] == 404:
- msg = "Packages: Pulp repo id %s not found: %s" % \
- (self.pulp_id, err[1])
- else:
- msg = "Packages: Error %d fetching pulp repo %s: %s" % \
- (err[0], self.pulp_id, err[1])
- raise SourceInitError(msg)
- except socket.error:
- err = sys.exc_info()[1]
- raise SourceInitError("Could not contact Pulp server: %s" % err)
- except:
- err = sys.exc_info()[1]
- raise SourceInitError("Unknown error querying Pulp server: %s" %
- err)
- self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'],
- self.repo['relative_path'])
- self.arches = [self.repo['arch']]
-
- if not self.rawurl:
- self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
- else:
- self.baseurl = self.rawurl
- self.packages = dict()
- self.deps = dict([('global', dict())])
- self.provides = dict([('global', dict())])
- self.filemap = dict([(x, dict())
- for x in ['global'] + self.arches])
- self.needed_paths = set()
- self.file_to_arch = dict()
-
- self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries",
- default=False)
-
- def save_state(self):
- if not self.use_yum:
- cache = file(self.cachefile, 'wb')
- cPickle.dump((self.packages, self.deps, self.provides,
- self.filemap, self.url_map), cache, 2)
- cache.close()
-
-
- def load_state(self):
- if not self.use_yum:
- data = file(self.cachefile)
- (self.packages, self.deps, self.provides,
- self.filemap, self.url_map) = cPickle.load(data)
-
- def get_urls(self):
- surls = list()
- self.url_map = []
- for arch in self.arches:
- if self.url:
- usettings = [{'version':self.version, 'component':comp,
- 'arch':arch}
- for comp in self.components]
- else: # rawurl given
- usettings = [{'version':self.version, 'component':None,
- 'arch':arch}]
-
- for setting in usettings:
- setting['url'] = self.baseurl % setting
- self.url_map.append(copy.deepcopy(setting))
- surls.append((arch, [setting['url'] for setting in usettings]))
- urls = []
- for (sarch, surl_list) in surls:
- for surl in surl_list:
- urls.extend(self._get_urls_from_repodata(surl, sarch))
- return urls
- urls = property(get_urls)
-
- def _get_urls_from_repodata(self, url, arch):
- if self.use_yum:
- return [url]
-
- rmdurl = '%srepodata/repomd.xml' % url
- try:
- repomd = fetch_url(rmdurl)
- xdata = lxml.etree.XML(repomd)
- except ValueError:
- self.logger.error("Packages: Bad url string %s" % rmdurl)
- return []
- except HTTPError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to fetch url %s. code=%s" %
- (rmdurl, err.code))
- return []
- except lxml.etree.XMLSyntaxError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to process metadata at %s: %s" %
- (rmdurl, err))
- return []
-
- urls = []
- for elt in xdata.findall(RPO + 'data'):
- if elt.get('type') in ['filelists', 'primary']:
- floc = elt.find(RPO + 'location')
- fullurl = url + floc.get('href')
- urls.append(fullurl)
- self.file_to_arch[self.escape_url(fullurl)] = arch
- return urls
-
- def read_files(self):
- # we have to read primary.xml first, and filelists.xml afterwards;
- primaries = list()
- filelists = list()
- for fname in self.files:
- if fname.endswith('primary.xml.gz'):
- primaries.append(fname)
- elif fname.endswith('filelists.xml.gz'):
- filelists.append(fname)
-
- for fname in primaries:
- farch = self.file_to_arch[fname]
- fdata = lxml.etree.parse(fname).getroot()
- self.parse_primary(fdata, farch)
- for fname in filelists:
- farch = self.file_to_arch[fname]
- fdata = lxml.etree.parse(fname).getroot()
- self.parse_filelist(fdata, farch)
-
- # merge data
- sdata = list(self.packages.values())
- try:
- self.packages['global'] = copy.deepcopy(sdata.pop())
- except IndexError:
- logger.error("Packages: No packages in repo")
- while sdata:
- self.packages['global'] = \
- self.packages['global'].intersection(sdata.pop())
-
- for key in self.packages:
- if key == 'global':
- continue
- self.packages[key] = \
- self.packages[key].difference(self.packages['global'])
- self.save_state()
-
- def parse_filelist(self, data, arch):
- if arch not in self.filemap:
- self.filemap[arch] = dict()
- for pkg in data.findall(FL + 'package'):
- for fentry in pkg.findall(FL + 'file'):
- if fentry.text in self.needed_paths:
- if fentry.text in self.filemap[arch]:
- self.filemap[arch][fentry.text].add(pkg.get('name'))
- else:
- self.filemap[arch][fentry.text] = \
- set([pkg.get('name')])
-
- def parse_primary(self, data, arch):
- if arch not in self.packages:
- self.packages[arch] = set()
- if arch not in self.deps:
- self.deps[arch] = dict()
- if arch not in self.provides:
- self.provides[arch] = dict()
- for pkg in data.getchildren():
- if not pkg.tag.endswith('package'):
- continue
- pkgname = pkg.find(XP + 'name').text
- self.packages[arch].add(pkgname)
-
- pdata = pkg.find(XP + 'format')
- self.deps[arch][pkgname] = set()
- pre = pdata.find(RP + 'requires')
- if pre is not None:
- for entry in pre.getchildren():
- self.deps[arch][pkgname].add(entry.get('name'))
- if entry.get('name').startswith('/'):
- self.needed_paths.add(entry.get('name'))
- pro = pdata.find(RP + 'provides')
- if pro != None:
- for entry in pro.getchildren():
- prov = entry.get('name')
- if prov not in self.provides[arch]:
- self.provides[arch][prov] = list()
- self.provides[arch][prov].append(pkgname)
-
- def is_package(self, metadata, item):
- arch = [a for a in self.arches if a in metadata.groups]
- if not arch:
- return False
- return ((item in self.packages['global'] or
- item in self.packages[arch[0]]) and
- item not in self.blacklist and
- (len(self.whitelist) == 0 or item in self.whitelist))
-
- def get_vpkgs(self, metadata):
- if self.use_yum:
- return dict()
-
- rv = Source.get_vpkgs(self, metadata)
- for arch, fmdata in list(self.filemap.items()):
- if arch not in metadata.groups and arch != 'global':
- continue
- for filename, pkgs in list(fmdata.items()):
- rv[filename] = pkgs
- return rv
-
- def filter_unknown(self, unknown):
- if self.use_yum:
- filtered = set()
- for unk in unknown:
- try:
- if unk.startswith('rpmlib'):
- filtered.update(unk)
- except AttributeError:
- try:
- if unk[0].startswith('rpmlib'):
- filtered.update(unk)
- except (IndexError, AttributeError):
- pass
- else:
- filtered = set([u for u in unknown if u.startswith('rpmlib')])
- unknown.difference_update(filtered)
-
- def setup_data(self, force_update=False):
- if not self.use_yum:
- Source.setup_data(self, force_update=force_update)
-
- def get_repo_name(self, url_map):
- if self.pulp_id:
- return self.pulp_id
- else:
- return Source.get_repo_name(self, url_map)
diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Server/Plugins/Packages/__init__.py
deleted file mode 100644
index da5832e90..000000000
--- a/src/lib/Server/Plugins/Packages/__init__.py
+++ /dev/null
@@ -1,263 +0,0 @@
-import os
-import sys
-import time
-import copy
-import glob
-import shutil
-import lxml.etree
-import Bcfg2.Logger
-import Bcfg2.Server.Plugin
-from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen
-from Bcfg2.Server.Plugins.Packages import Collection
-from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
-from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig
-
-class Packages(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.StructureValidator,
- Bcfg2.Server.Plugin.Generator,
- Bcfg2.Server.Plugin.Connector):
- name = 'Packages'
- conflicts = ['Pkgmgr']
- experimental = True
- __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.StructureValidator.__init__(self)
- Bcfg2.Server.Plugin.Generator.__init__(self)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- Bcfg2.Server.Plugin.Probing.__init__(self)
-
- self.sentinels = set()
- self.cachepath = os.path.join(self.data, 'cache')
- self.keypath = os.path.join(self.data, 'keys')
- if not os.path.exists(self.keypath):
- # create key directory if needed
- os.makedirs(self.keypath)
-
- # set up config files
- self.config = PackagesConfig(self)
- self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
- self.cachepath, core.fam, self,
- self.config)
-
- def toggle_debug(self):
- Bcfg2.Server.Plugin.Plugin.toggle_debug(self)
- self.sources.toggle_debug()
-
- @property
- def disableResolver(self):
- try:
- return not self.config.getboolean("global", "resolver")
- except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
- return False
- except ValueError:
- # for historical reasons we also accept "enabled" and
- # "disabled", which are not handled according to the
- # Python docs but appear to be handled properly by
- # ConfigParser in at least some versions
- return self.config.get("global", "resolver",
- default="enabled").lower() == "disabled"
-
- @property
- def disableMetaData(self):
- try:
- return not self.config.getboolean("global", "resolver")
- except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
- return False
- except ValueError:
- # for historical reasons we also accept "enabled" and
- # "disabled"
- return self.config.get("global", "metadata",
- default="enabled").lower() == "disabled"
-
- def create_config(self, entry, metadata):
- """ create yum/apt config for the specified host """
- attrib = {'encoding': 'ascii',
- 'owner': 'root',
- 'group': 'root',
- 'type': 'file',
- 'perms': '0644'}
-
- collection = self._get_collection(metadata)
- entry.text = collection.get_config()
- for (key, value) in list(attrib.items()):
- entry.attrib.__setitem__(key, value)
-
- def HandleEntry(self, entry, metadata):
- if entry.tag == 'Package':
- collection = self._get_collection(metadata)
- entry.set('version', 'auto')
- entry.set('type', collection.ptype)
- elif entry.tag == 'Path':
- if (entry.get("name") == self.config.get("global", "yum_config",
- default="") or
- entry.get("name") == self.config.get("global", "apt_config",
- default="")):
- self.create_config(entry, metadata)
-
- def HandlesEntry(self, entry, metadata):
- if entry.tag == 'Package':
- if self.config.getboolean("global", "magic_groups",
- default=True) == True:
- collection = self._get_collection(metadata)
- if collection.magic_groups_match():
- return True
- else:
- return True
- elif entry.tag == 'Path':
- # managed entries for yum/apt configs
- if (entry.get("name") == self.config.get("global", "yum_config",
- default="") or
- entry.get("name") == self.config.get("global", "apt_config",
- default="")):
- return True
- return False
-
- def validate_structures(self, metadata, structures):
- '''Ensure client configurations include all needed prerequisites
-
- Arguments:
- metadata - client metadata instance
- structures - a list of structure-stage entry combinations
- '''
- collection = self._get_collection(metadata)
- indep = lxml.etree.Element('Independent')
- self._build_packages(metadata, indep, structures,
- collection=collection)
- collection.build_extra_structures(indep)
- structures.append(indep)
-
- def _build_packages(self, metadata, independent, structures,
- collection=None):
- """ build list of packages that need to be included in the
- specification by validate_structures() """
- if self.disableResolver:
- # Config requests no resolver
- return
-
- if collection is None:
- collection = self._get_collection(metadata)
- # initial is the set of packages that are explicitly specified
- # in the configuration
- initial = set()
- # base is the set of initial packages with groups expanded
- base = set()
- to_remove = []
- for struct in structures:
- for pkg in struct.xpath('//Package | //BoundPackage'):
- if pkg.get("name"):
- initial.add(pkg.get("name"))
- elif pkg.get("group"):
- try:
- if pkg.get("type"):
- gpkgs = collection.get_group(pkg.get("group"),
- ptype=pkg.get("type"))
- else:
- gpkgs = collection.get_group(pkg.get("group"))
- base.update(gpkgs)
- except TypeError:
- raise
- self.logger.error("Could not resolve group %s" %
- pkg.get("group"))
- to_remove.append(pkg)
- else:
- self.logger.error("Packages: Malformed Package: %s" %
- lxml.etree.tostring(pkg))
- base.update(initial)
- for el in to_remove:
- el.getparent().remove(el)
-
- packages, unknown = collection.complete(base)
- if unknown:
- self.logger.info("Packages: Got %d unknown entries" % len(unknown))
- self.logger.info("Packages: %s" % list(unknown))
- newpkgs = list(packages.difference(initial))
- self.debug_log("Packages: %d initial, %d complete, %d new" %
- (len(initial), len(packages), len(newpkgs)))
- newpkgs.sort()
- for pkg in newpkgs:
- lxml.etree.SubElement(independent, 'BoundPackage', name=pkg,
- version='auto', type=collection.ptype,
- origin='Packages')
-
- def Refresh(self):
- '''Packages.Refresh() => True|False\nReload configuration
- specification and download sources\n'''
- self._load_config(force_update=True)
- return True
-
- def Reload(self):
- '''Packages.Refresh() => True|False\nReload configuration
- specification and sources\n'''
- self._load_config()
- return True
-
- def _load_config(self, force_update=False):
- '''
- Load the configuration data and setup sources
-
- Keyword args:
- force_update Force downloading repo data
- '''
- self._load_sources(force_update)
- self._load_gpg_keys(force_update)
-
- def _load_sources(self, force_update):
- """ Load sources from the config """
- self.sentinels = set()
- cachefiles = set()
-
- for collection in list(Collection.collections.values()):
- cachefiles.update(collection.cachefiles)
- if not self.disableMetaData:
- collection.setup_data(force_update)
- self.sentinels.update(collection.basegroups)
-
- Collection.clear_cache()
-
- for source in self.sources:
- cachefiles.add(source.cachefile)
- if not self.disableMetaData:
- source.setup_data(force_update)
-
- for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
- if cfile not in cachefiles:
- try:
- if os.path.isdir(cfile):
- shutil.rmtree(cfile)
- else:
- os.unlink(cfile)
- except OSError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Could not remove cache file "
- "%s: %s" % (cfile, err))
-
- def _load_gpg_keys(self, force_update):
- """ Load gpg keys from the config """
- keyfiles = []
- keys = []
- for source in self.sources:
- for key in source.gpgkeys:
- localfile = os.path.join(self.keypath,
- os.path.basename(key.rstrip("/")))
- if localfile not in keyfiles:
- keyfiles.append(localfile)
- if ((force_update and key not in keys) or
- not os.path.exists(localfile)):
- self.logger.info("Packages: Downloading and parsing %s" % key)
- response = urlopen(key)
- open(localfile, 'w').write(response.read())
- keys.append(key)
-
- for kfile in glob.glob(os.path.join(self.keypath, "*")):
- if kfile not in keyfiles:
- os.unlink(kfile)
-
- def _get_collection(self, metadata):
- return Collection.factory(metadata, self.sources, self.data,
- debug=self.debug_flag)
-
- def get_additional_data(self, metadata):
- collection = self._get_collection(metadata)
- return dict(sources=collection.get_additional_data())
diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py
deleted file mode 100644
index e9254cdcc..000000000
--- a/src/lib/Server/Plugins/Pkgmgr.py
+++ /dev/null
@@ -1,169 +0,0 @@
-'''This module implements a package management scheme for all images'''
-
-import logging
-import re
-import Bcfg2.Server.Plugin
-import lxml
-try:
- set
-except NameError:
- from sets import Set as set
-
-logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr')
-
-
-class FuzzyDict(dict):
- fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)')
-
- def __getitem__(self, key):
- if isinstance(key, str):
- mdata = self.fuzzy.match(key)
- if mdata:
- return dict.__getitem__(self, mdata.groupdict()['name'])
- else:
- print("got non-string key %s" % str(key))
- return dict.__getitem__(self, key)
-
- def has_key(self, key):
- if isinstance(key, str):
- mdata = self.fuzzy.match(key)
- if self.fuzzy.match(key):
- return dict.has_key(self, mdata.groupdict()['name'])
- return dict.has_key(self, key)
-
- def get(self, key, default=None):
- try:
- return self.__getitem__(key)
- except:
- if default:
- return default
- raise
-
-
-class PNode(Bcfg2.Server.Plugin.INode):
- """PNode has a list of packages available at a
- particular group intersection.
- """
- splitters = {'rpm': re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \
- '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'),
- 'encap': re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')}
- ignore = ['Package']
-
- def Match(self, metadata, data, entry=lxml.etree.Element("None")):
- """Return a dictionary of package mappings."""
- if self.predicate(metadata, entry):
- for key in self.contents:
- try:
- data[key].update(self.contents[key])
- except:
- data[key] = FuzzyDict()
- data[key].update(self.contents[key])
- for child in self.children:
- child.Match(metadata, data)
-
- def __init__(self, data, pdict, parent=None):
- # copy local attributes to all child nodes if no local attribute exists
- if 'Package' not in pdict:
- pdict['Package'] = set()
- for child in data.getchildren():
- attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name'])
- for attr in attrs:
- try:
- child.set(attr, data.get(attr))
- except:
- # don't fail on things like comments and other immutable elements
- pass
- Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent)
- if 'Package' not in self.contents:
- self.contents['Package'] = FuzzyDict()
- for pkg in data.findall('./Package'):
- if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']:
- pdict['Package'].add(pkg.get('name'))
- if pkg.get('name') != None:
- self.contents['Package'][pkg.get('name')] = {}
- if pkg.getchildren():
- self.contents['Package'][pkg.get('name')]['__children__'] \
- = pkg.getchildren()
- if 'simplefile' in pkg.attrib:
- pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile')))
- self.contents['Package'][pkg.get('name')].update(pkg.attrib)
- else:
- if 'file' in pkg.attrib:
- if 'multiarch' in pkg.attrib:
- archs = pkg.get('multiarch').split()
- srcs = pkg.get('srcs', pkg.get('multiarch')).split()
- url = ' '.join(["%s/%s" % (pkg.get('uri'),
- pkg.get('file') % {'src':srcs[idx],
- 'arch':archs[idx]})
- for idx in range(len(archs))])
- pkg.set('url', url)
- else:
- pkg.set('url', '%s/%s' % (pkg.get('uri'),
- pkg.get('file')))
- if pkg.get('type') in self.splitters and pkg.get('file') != None:
- mdata = self.splitters[pkg.get('type')].match(pkg.get('file'))
- if not mdata:
- logger.error("Failed to match pkg %s" % pkg.get('file'))
- continue
- pkgname = mdata.group('name')
- self.contents['Package'][pkgname] = mdata.groupdict()
- self.contents['Package'][pkgname].update(pkg.attrib)
- if pkg.attrib.get('file'):
- self.contents['Package'][pkgname]['url'] = pkg.get('url')
- self.contents['Package'][pkgname]['type'] = pkg.get('type')
- if pkg.get('verify'):
- self.contents['Package'][pkgname]['verify'] = pkg.get('verify')
- if pkg.get('multiarch'):
- self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch')
- if pkgname not in pdict['Package']:
- pdict['Package'].add(pkgname)
- if pkg.getchildren():
- self.contents['Package'][pkgname]['__children__'] = pkg.getchildren()
- else:
- self.contents['Package'][pkg.get('name')].update(pkg.attrib)
-
-
-class PkgSrc(Bcfg2.Server.Plugin.XMLSrc):
- """PkgSrc files contain a PNode hierarchy that
- returns matching package entries.
- """
- __node__ = PNode
- __cacheobj__ = FuzzyDict
-
-
-class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
- """This is a generator that handles package assignments."""
- name = 'Pkgmgr'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- __child__ = PkgSrc
- __element__ = 'Package'
-
- def HandleEvent(self, event):
- '''Handle events and update dispatch table'''
- Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event)
- for src in list(self.entries.values()):
- for itype, children in list(src.items.items()):
- for child in children:
- try:
- self.Entries[itype][child] = self.BindEntry
- except KeyError:
- self.Entries[itype] = FuzzyDict([(child,
- self.BindEntry)])
-
- def BindEntry(self, entry, metadata):
- """Bind data for entry, and remove instances that are not requested."""
- pname = entry.get('name')
- Bcfg2.Server.Plugin.PrioDir.BindEntry(self, entry, metadata)
- if entry.findall('Instance'):
- mdata = FuzzyDict.fuzzy.match(pname)
- if mdata:
- arches = mdata.group('alist').split(',')
- [entry.remove(inst) for inst in \
- entry.findall('Instance') \
- if inst.get('arch') not in arches]
-
- def HandlesEntry(self, entry, metadata):
- return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys())
-
- def HandleEntry(self, entry, metadata):
- self.BindEntry(entry, metadata)
diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py
deleted file mode 100644
index 2af3635e9..000000000
--- a/src/lib/Server/Plugins/Probes.py
+++ /dev/null
@@ -1,285 +0,0 @@
-import time
-import lxml.etree
-import operator
-import re
-
-try:
- import json
- has_json = True
-except ImportError:
- try:
- import simplejson as json
- has_json = True
- except ImportError:
- has_json = False
-
-try:
- import syck
- has_syck = True
-except ImportError:
- has_syck = False
- try:
- import yaml
- has_yaml = True
- except ImportError:
- has_yaml = False
-
-import Bcfg2.Server.Plugin
-
-specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)")
-probe_matcher = re.compile("(.*/)?(?P<basename>\S+)")
-
-class ClientProbeDataSet(dict):
- """ dict of probe => [probe data] that records a for each host """
- def __init__(self, *args, **kwargs):
- if "timestamp" in kwargs and kwargs['timestamp'] is not None:
- self.timestamp = kwargs.pop("timestamp")
- else:
- self.timestamp = time.time()
- dict.__init__(self, *args, **kwargs)
-
-
-class ProbeData(object):
- """ a ProbeData object emulates a str object, but also has .xdata
- and .json properties to provide convenient ways to use ProbeData
- objects as XML or JSON data """
- def __init__(self, data):
- self.data = data
- self._xdata = None
- self._json = None
- self._yaml = None
-
- def __str__(self):
- return str(self.data)
-
- def __repr__(self):
- return repr(self.data)
-
- def __getattr__(self, name):
- """ make ProbeData act like a str object """
- return getattr(self.data, name)
-
- def __complex__(self):
- return complex(self.data)
-
- def __int__(self):
- return int(self.data)
-
- def __long__(self):
- return long(self.data)
-
- def __float__(self):
- return float(self.data)
-
- def __eq__(self, other):
- return str(self) == str(other)
-
- def __ne__(self, other):
- return str(self) != str(other)
-
- def __gt__(self, other):
- return str(self) > str(other)
-
- def __lt__(self, other):
- return str(self) < str(other)
-
- def __ge__(self, other):
- return self > other or self == other
-
- def __le__(self, other):
- return self < other or self == other
-
- @property
- def xdata(self):
- if self._xdata is None:
- try:
- self._xdata = lxml.etree.XML(self.data)
- except lxml.etree.XMLSyntaxError:
- pass
- return self._xdata
-
- @property
- def json(self):
- if self._json is None and has_json:
- try:
- self._json = json.loads(self.data)
- except ValueError:
- pass
- return self._json
-
- @property
- def yaml(self):
- if self._yaml is None:
- if has_yaml:
- try:
- self._yaml = yaml.load(self.data)
- except yaml.YAMLError:
- pass
- elif has_syck:
- try:
- self._yaml = syck.load(self.data)
- except syck.error:
- pass
- return self._yaml
-
-
-class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
- ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
-
- def __init__(self, path, fam, encoding, plugin_name):
- fpattern = '[0-9A-Za-z_\-]+'
- self.plugin_name = plugin_name
- Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
- Bcfg2.Server.Plugin.SpecificData,
- encoding)
- fam.AddMonitor(path, self)
- self.bangline = re.compile('^#!(?P<interpreter>.*)$')
-
- def HandleEvent(self, event):
- if event.filename != self.path:
- if (event.code2str == 'changed' and
- event.filename.endswith("probed.xml") and
- event.filename not in self.entries):
- # for some reason, probed.xml is particularly prone to
- # getting changed events before created events,
- # because gamin is the worst ever. anyhow, we
- # specifically handle it here to avoid a warning on
- # every single server startup.
- self.entry_init(event)
- return
- return self.handle_event(event)
-
- def get_probe_data(self, metadata):
- ret = []
- build = dict()
- candidates = self.get_matching(metadata)
- candidates.sort(key=operator.attrgetter('specific'))
- for entry in candidates:
- rem = specific_probe_matcher.match(entry.name)
- if not rem:
- rem = probe_matcher.match(entry.name)
- pname = rem.group('basename')
- if pname not in build:
- build[pname] = entry
-
- for (name, entry) in list(build.items()):
- probe = lxml.etree.Element('probe')
- probe.set('name', name.split('/')[-1])
- probe.set('source', self.plugin_name)
- probe.text = entry.data
- match = self.bangline.match(entry.data.split('\n')[0])
- if match:
- probe.set('interpreter', match.group('interpreter'))
- else:
- probe.set('interpreter', '/bin/sh')
- ret.append(probe)
- return ret
-
-
-class Probes(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Probing,
- Bcfg2.Server.Plugin.Connector):
- """A plugin to gather information from a client machine."""
- name = 'Probes'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- Bcfg2.Server.Plugin.Probing.__init__(self)
-
- try:
- self.probes = ProbeSet(self.data, core.fam, core.encoding,
- self.name)
- except:
- raise Bcfg2.Server.Plugin.PluginInitError
-
- self.probedata = dict()
- self.cgroups = dict()
- self.load_data()
-
- def write_data(self):
- """Write probe data out for use with bcfg2-info."""
- top = lxml.etree.Element("Probed")
- for client, probed in sorted(self.probedata.items()):
- cx = lxml.etree.SubElement(top, 'Client', name=client,
- timestamp=str(int(probed.timestamp)))
- for probe in sorted(probed):
- lxml.etree.SubElement(cx, 'Probe', name=probe,
- value=str(self.probedata[client][probe]))
- for group in sorted(self.cgroups[client]):
- lxml.etree.SubElement(cx, "Group", name=group)
- data = lxml.etree.tostring(top, encoding='UTF-8',
- xml_declaration=True,
- pretty_print='true')
- try:
- datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
- except IOError:
- self.logger.error("Failed to write probed.xml")
- datafile.write(data.decode('utf-8'))
-
- def load_data(self):
- try:
- data = lxml.etree.parse(self.data + '/probed.xml').getroot()
- except:
- self.logger.error("Failed to read file probed.xml")
- return
- self.probedata = {}
- self.cgroups = {}
- for client in data.getchildren():
- self.probedata[client.get('name')] = \
- ClientProbeDataSet(timestamp=client.get("timestamp"))
- self.cgroups[client.get('name')] = []
- for pdata in client:
- if (pdata.tag == 'Probe'):
- self.probedata[client.get('name')][pdata.get('name')] = \
- ProbeData(pdata.get('value'))
- elif (pdata.tag == 'Group'):
- self.cgroups[client.get('name')].append(pdata.get('name'))
-
- def GetProbes(self, meta, force=False):
- """Return a set of probes for execution on client."""
- return self.probes.get_probe_data(meta)
-
- def ReceiveData(self, client, datalist):
- self.cgroups[client.hostname] = []
- self.probedata[client.hostname] = ClientProbeDataSet()
- for data in datalist:
- self.ReceiveDataItem(client, data)
- self.write_data()
-
- def ReceiveDataItem(self, client, data):
- """Receive probe results pertaining to client."""
- if client.hostname not in self.cgroups:
- self.cgroups[client.hostname] = []
- if data.text == None:
- self.logger.error("Got null response to probe %s from %s" % \
- (data.get('name'), client.hostname))
- try:
- self.probedata[client.hostname].update({data.get('name'):
- ProbeData('')})
- except KeyError:
- self.probedata[client.hostname] = \
- ClientProbeDataSet([(data.get('name'), ProbeData(''))])
- return
- dlines = data.text.split('\n')
- self.logger.debug("%s:probe:%s:%s" % (client.hostname,
- data.get('name'), [line.strip() for line in dlines]))
- for line in dlines[:]:
- if line.split(':')[0] == 'group':
- newgroup = line.split(':')[1].strip()
- if newgroup not in self.cgroups[client.hostname]:
- self.cgroups[client.hostname].append(newgroup)
- dlines.remove(line)
- dobj = ProbeData("\n".join(dlines))
- try:
- self.probedata[client.hostname].update({data.get('name'): dobj})
- except KeyError:
- self.probedata[client.hostname] = \
- ClientProbeDataSet([(data.get('name'), dobj)])
-
- def get_additional_groups(self, meta):
- return self.cgroups.get(meta.hostname, list())
-
- def get_additional_data(self, meta):
- return self.probedata.get(meta.hostname, ClientProbeDataSet())
diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py
deleted file mode 100644
index 680881858..000000000
--- a/src/lib/Server/Plugins/Properties.py
+++ /dev/null
@@ -1,76 +0,0 @@
-import os
-import re
-import sys
-import copy
-import logging
-import lxml.etree
-import Bcfg2.Server.Plugin
-
-logger = logging.getLogger('Bcfg2.Plugins.Properties')
-
-class PropertyFile(Bcfg2.Server.Plugin.StructFile):
- """Class for properties files."""
- def write(self):
- """ Write the data in this data structure back to the property
- file """
- if self.validate_data():
- try:
- open(self.name,
- "wb").write(lxml.etree.tostring(self.xdata,
- pretty_print=True))
- return True
- except IOError:
- err = sys.exc_info()[1]
- logger.error("Failed to write %s: %s" % (self.name, err))
- return False
- else:
- return False
-
- def validate_data(self):
- """ ensure that the data in this object validates against the
- XML schema for this property file (if a schema exists) """
- schemafile = self.name.replace(".xml", ".xsd")
- if os.path.exists(schemafile):
- try:
- schema = lxml.etree.XMLSchema(file=schemafile)
- except:
- logger.error("Failed to process schema for %s" % self.name)
- return False
- else:
- # no schema exists
- return True
-
- if not schema.validate(self.xdata):
- logger.error("Data for %s fails to validate; run bcfg2-lint for "
- "more details" % self.name)
- return False
- else:
- return True
-
-
-class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
- __child__ = PropertyFile
- patterns = re.compile(r'.*\.xml$')
-
-
-class Properties(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Connector):
- """
- The properties plugin maps property
- files into client metadata instances.
- """
- name = 'Properties'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- try:
- self.store = PropDirectoryBacked(self.data, core.fam)
- except OSError:
- e = sys.exc_info()[1]
- self.logger.error("Error while creating Properties store: %s %s" %
- (e.strerror, e.filename))
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def get_additional_data(self, _):
- return copy.copy(self.store.entries)
diff --git a/src/lib/Server/Plugins/Rules.py b/src/lib/Server/Plugins/Rules.py
deleted file mode 100644
index 684f0c290..000000000
--- a/src/lib/Server/Plugins/Rules.py
+++ /dev/null
@@ -1,55 +0,0 @@
-"""This generator provides rule-based entry mappings."""
-
-import re
-import Bcfg2.Server.Plugin
-
-class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig):
- _required = False
-
-class Rules(Bcfg2.Server.Plugin.PrioDir):
- """This is a generator that handles service assignments."""
- name = 'Rules'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore)
- self.config = RulesConfig(self)
- self._regex_cache = dict()
-
- def HandlesEntry(self, entry, metadata):
- if entry.tag in self.Entries:
- return self._matches(entry, metadata,
- self.Entries[entry.tag].keys())
- return False
-
- def HandleEntry(self, entry, metadata):
- return self.BindEntry(entry, metadata)
-
- def BindEntry(self, entry, metadata):
- attrs = self.get_attrs(entry, metadata)
- for key, val in list(attrs.items()):
- if key not in entry.attrib:
- entry.attrib[key] = val
-
- def _matches(self, entry, metadata, rules):
- if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules):
- return True
- elif (entry.tag == "Path" and
- ((entry.get('name').endswith("/") and
- entry.get('name').rstrip("/") in rules) or
- (not entry.get('name').endswith("/") and
- entry.get('name') + '/' in rules))):
- # special case for Path tags:
- # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967
- return True
- elif self._regex_enabled:
- # attempt regular expression matching
- for rule in rules:
- if rule not in self._regex_cache:
- self._regex_cache[rule] = re.compile("%s$" % rule)
- if self._regex_cache[rule].match(entry.get('name')):
- return True
- return False
-
- def _regex_enabled(self):
- return self.config.getboolean("rules", "regex", default=False)
diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py
deleted file mode 100644
index 0ba08125e..000000000
--- a/src/lib/Server/Plugins/SGenshi.py
+++ /dev/null
@@ -1,97 +0,0 @@
-'''This module implements a templating generator based on Genshi'''
-
-import genshi.input
-import genshi.template
-import lxml.etree
-import logging
-import copy
-import sys
-import os.path
-
-import Bcfg2.Server.Plugin
-import Bcfg2.Server.Plugins.TGenshi
-
-logger = logging.getLogger('Bcfg2.Plugins.SGenshi')
-
-
-class SGenshiTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile,
- Bcfg2.Server.Plugin.StructFile):
- def __init__(self, name, specific, encoding):
- Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name,
- specific, encoding)
- Bcfg2.Server.Plugin.StructFile.__init__(self, name)
-
- def get_xml_value(self, metadata):
- if not hasattr(self, 'template'):
- logger.error("No parsed template information for %s" % (self.name))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- try:
- stream = self.template.generate(metadata=metadata).filter( \
- Bcfg2.Server.Plugins.TGenshi.removecomment)
- data = lxml.etree.XML(stream.render('xml', strip_whitespace=False))
- bundlename = os.path.splitext(os.path.basename(self.name))[0]
- bundle = lxml.etree.Element('Bundle', name=bundlename)
- for item in self.Match(metadata, data):
- bundle.append(copy.deepcopy(item))
- return bundle
- except LookupError:
- lerror = sys.exc_info()[1]
- logger.error('Genshi lookup error: %s' % lerror)
- except genshi.template.TemplateError:
- terror = sys.exc_info()[1]
- logger.error('Genshi template error: %s' % terror)
- raise
- except genshi.input.ParseError:
- perror = sys.exc_info()[1]
- logger.error('Genshi parse error: %s' % perror)
- raise
-
- def Match(self, metadata, xdata):
- """Return matching fragments of parsed template."""
- rv = []
- for child in xdata.getchildren():
- rv.extend(self._match(child, metadata))
- logger.debug("File %s got %d match(es)" % (self.name, len(rv)))
- return rv
-
-class SGenshiEntrySet(Bcfg2.Server.Plugin.EntrySet):
-
- def __init__(self, path, fam, encoding):
- fpattern = '\S+\.xml'
- Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
- SGenshiTemplateFile, encoding)
- fam.AddMonitor(path, self)
-
- def HandleEvent(self, event):
- '''passthrough event handler for old calling convention'''
- if event.filename != self.path:
- return self.handle_event(event)
-
- def BuildStructures(self, metadata):
- """Build SGenshi structures."""
- ret = []
- for entry in self.get_matching(metadata):
- try:
- ret.append(entry.get_xml_value(metadata))
- except:
- logger.error("SGenshi: Failed to template file %s" % entry.name)
- return ret
-
-
-class SGenshi(SGenshiEntrySet,
- Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Structure):
- """The SGenshi plugin provides templated structures."""
- name = 'SGenshi'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- deprecated = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Structure.__init__(self)
- try:
- SGenshiEntrySet.__init__(self, self.data, self.core.fam, core.encoding)
- except:
- logger.error("Failed to load %s repository; disabling %s" \
- % (self.name, self.name))
- raise Bcfg2.Server.Plugin.PluginInitError
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
deleted file mode 100644
index 2e247caa7..000000000
--- a/src/lib/Server/Plugins/SSHbase.py
+++ /dev/null
@@ -1,413 +0,0 @@
-'''This module manages ssh key files for bcfg2'''
-
-import binascii
-import re
-import os
-import socket
-import shutil
-import sys
-import tempfile
-from subprocess import Popen, PIPE
-import Bcfg2.Server.Plugin
-from Bcfg2.Bcfg2Py3k import u_str
-
-if sys.hexversion >= 0x03000000:
- from functools import reduce
-
-import logging
-logger = logging.getLogger(__name__)
-
-class KeyData(Bcfg2.Server.Plugin.SpecificData):
- def __init__(self, name, specific, encoding):
- Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
- encoding)
- self.encoding = encoding
-
- def bind_entry(self, entry, metadata):
- entry.set('type', 'file')
- if entry.get('encoding') == 'base64':
- entry.text = binascii.b2a_base64(self.data)
- else:
- try:
- entry.text = u_str(self.data, self.encoding)
- except UnicodeDecodeError:
- e = sys.exc_info()[1]
- logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
- logger.error("Please verify you are using the proper encoding.")
- raise Bcfg2.Server.Plugin.PluginExecutionError
- except ValueError:
- e = sys.exc_info()[1]
- logger.error("Error in specification for %s" %
- entry.get('name'))
- logger.error(str(e))
- logger.error("You need to specify base64 encoding for %s." %
- entry.get('name'))
- raise Bcfg2.Server.Plugin.PluginExecutionError
- if entry.text in ['', None]:
- entry.set('empty', 'true')
-
-class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet):
- def __init__(self, basename, path):
- if basename.startswith("ssh_host_key"):
- encoding = "base64"
- else:
- encoding = None
- Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData,
- encoding)
- self.metadata = {'owner': 'root',
- 'group': 'root',
- 'type': 'file'}
- if encoding is not None:
- self.metadata['encoding'] = encoding
- if basename.endswith('.pub'):
- self.metadata['perms'] = '0644'
- else:
- self.metadata['perms'] = '0600'
-
-
-class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet):
- def __init__(self, path):
- Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path,
- KeyData, None)
- self.metadata = {'owner': 'root',
- 'group': 'root',
- 'type': 'file',
- 'perms': '0644'}
-
-
-class SSHbase(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Generator,
- Bcfg2.Server.Plugin.PullTarget):
- """
- The sshbase generator manages ssh host keys (both v1 and v2)
- for hosts. It also manages the ssh_known_hosts file. It can
- integrate host keys from other management domains and similarly
- export its keys. The repository contains files in the following
- formats:
-
- ssh_host_key.H_(hostname) -> the v1 host private key for
- (hostname)
- ssh_host_key.pub.H_(hostname) -> the v1 host public key
- for (hostname)
- ssh_host_(ec)(dr)sa_key.H_(hostname) -> the v2 ssh host
- private key for (hostname)
- ssh_host_(ec)(dr)sa_key.pub.H_(hostname) -> the v2 ssh host
- public key for (hostname)
- ssh_known_hosts -> the current known hosts file. this
- is regenerated each time a new key is generated.
-
- """
- name = 'SSHbase'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- keypatterns = ["ssh_host_dsa_key",
- "ssh_host_ecdsa_key",
- "ssh_host_rsa_key",
- "ssh_host_key",
- "ssh_host_dsa_key.pub",
- "ssh_host_ecdsa_key.pub",
- "ssh_host_rsa_key.pub",
- "ssh_host_key.pub"]
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Generator.__init__(self)
- Bcfg2.Server.Plugin.PullTarget.__init__(self)
- self.ipcache = {}
- self.namecache = {}
- self.__skn = False
-
- # keep track of which bogus keys we've warned about, and only
- # do so once
- self.badnames = dict()
-
- core.fam.AddMonitor(self.data, self)
-
- self.static = dict()
- self.entries = dict()
- self.Entries['Path'] = dict()
-
- self.entries['/etc/ssh/ssh_known_hosts'] = KnownHostsEntrySet(self.data)
- self.Entries['Path']['/etc/ssh/ssh_known_hosts'] = self.build_skn
- for keypattern in self.keypatterns:
- self.entries["/etc/ssh/" + keypattern] = HostKeyEntrySet(keypattern,
- self.data)
- self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk
-
- def get_skn(self):
- """Build memory cache of the ssh known hosts file."""
- if not self.__skn:
- # if no metadata is registered yet, defer
- if len(self.core.metadata.query.all()) == 0:
- self.__skn = False
- return self.__skn
-
- skn = [s.data.decode().rstrip()
- for s in list(self.static.values())]
-
- mquery = self.core.metadata.query
-
- # build hostname cache
- names = dict()
- for cmeta in mquery.all():
- names[cmeta.hostname] = set([cmeta.hostname])
- names[cmeta.hostname].update(cmeta.aliases)
- newnames = set()
- newips = set()
- for name in names[cmeta.hostname]:
- newnames.add(name.split('.')[0])
- try:
- newips.add(self.get_ipcache_entry(name)[0])
- except:
- continue
- names[cmeta.hostname].update(newnames)
- names[cmeta.hostname].update(cmeta.addresses)
- names[cmeta.hostname].update(newips)
- # TODO: Only perform reverse lookups on IPs if an option is set.
- if True:
- for ip in newips:
- try:
- names[cmeta.hostname].update(self.get_namecache_entry(ip))
- except:
- continue
- names[cmeta.hostname] = sorted(names[cmeta.hostname])
-
- pubkeys = [pubk for pubk in list(self.entries.keys())
- if pubk.endswith('.pub')]
- pubkeys.sort()
- for pubkey in pubkeys:
- for entry in sorted(self.entries[pubkey].entries.values(),
- key=lambda e: e.specific.hostname or e.specific.group):
- specific = entry.specific
- hostnames = []
- if specific.hostname and specific.hostname in names:
- hostnames = names[specific.hostname]
- elif specific.group:
- hostnames = \
- reduce(lambda x, y: x + y,
- [names[cmeta.hostname]
- for cmeta in \
- mquery.by_groups([specific.group])], [])
- elif specific.all:
- # a generic key for all hosts? really?
- hostnames = reduce(lambda x, y: x + y,
- list(names.values()), [])
- if not hostnames:
- if specific.hostname:
- key = specific.hostname
- ktype = "host"
- elif specific.group:
- key = specific.group
- ktype = "group"
- else:
- # user has added a global SSH key, but
- # have no clients yet. don't warn about
- # this.
- continue
-
- if key not in self.badnames:
- self.badnames[key] = True
- self.logger.info("Ignoring key for unknown %s %s" %
- (ktype, key))
- continue
-
- skn.append("%s %s" % (','.join(hostnames),
- entry.data.decode().rstrip()))
-
- self.__skn = "\n".join(skn) + "\n"
- return self.__skn
-
- def set_skn(self, value):
- """Set backing data for skn."""
- self.__skn = value
- skn = property(get_skn, set_skn)
-
- def HandleEvent(self, event=None):
- """Local event handler that does skn regen on pubkey change."""
- # skip events we don't care about
- action = event.code2str()
- if action == "endExist" or event.filename == self.data:
- return
-
- for entry in list(self.entries.values()):
- if entry.specific.match(event.filename):
- entry.handle_event(event)
- if event.filename.endswith(".pub"):
- self.logger.info("New public key %s; invalidating "
- "ssh_known_hosts cache" % event.filename)
- self.skn = False
- return
-
- if event.filename in ['info', 'info.xml', ':info']:
- for entry in list(self.entries.values()):
- entry.handle_event(event)
- return
-
- if event.filename.endswith('.static'):
- self.logger.info("Static key %s %s; invalidating ssh_known_hosts "
- "cache" % (event.filename, action))
- if action == "deleted" and event.filename in self.static:
- del self.static[event.filename]
- self.skn = False
- else:
- self.static[event.filename] = \
- Bcfg2.Server.Plugin.FileBacked(os.path.join(self.data,
- event.filename))
- self.static[event.filename].HandleEvent(event)
- self.skn = False
- return
-
- self.logger.warn("SSHbase: Got unknown event %s %s" %
- (event.filename, action))
-
- def get_ipcache_entry(self, client):
- """Build a cache of dns results."""
- if client in self.ipcache:
- if self.ipcache[client]:
- return self.ipcache[client]
- else:
- raise socket.gaierror
- else:
- # need to add entry
- try:
- ipaddr = socket.gethostbyname(client)
- self.ipcache[client] = (ipaddr, client)
- return (ipaddr, client)
- except socket.gaierror:
- ipaddr = Popen(["getent", "hosts", client],
- stdout=PIPE).stdout.read().strip().split()
- if ipaddr:
- self.ipcache[client] = (ipaddr, client)
- return (ipaddr, client)
- self.ipcache[client] = False
- self.logger.error("Failed to find IP address for %s" % client)
- raise socket.gaierror
-
- def get_namecache_entry(self, cip):
- """Build a cache of name lookups from client IP addresses."""
- if cip in self.namecache:
- # lookup cached name from IP
- if self.namecache[cip]:
- return self.namecache[cip]
- else:
- raise socket.gaierror
- else:
- # add an entry that has not been cached
- try:
- rvlookup = socket.gethostbyaddr(cip)
- if rvlookup[0]:
- self.namecache[cip] = [rvlookup[0]]
- else:
- self.namecache[cip] = []
- self.namecache[cip].extend(rvlookup[1])
- return self.namecache[cip]
- except socket.gaierror:
- self.namecache[cip] = False
- self.logger.error("Failed to find any names associated with IP address %s" % cip)
- raise
-
- def build_skn(self, entry, metadata):
- """This function builds builds a host specific known_hosts file."""
- try:
- rv = self.entries[entry.get('name')].bind_entry(entry, metadata)
- except Bcfg2.Server.Plugin.PluginExecutionError:
- client = metadata.hostname
- entry.text = self.skn
- hostkeys = []
- for k in self.keypatterns:
- if k.endswith(".pub"):
- try:
- hostkeys.append(self.entries["/etc/ssh/" +
- k].best_matching(metadata))
- except Bcfg2.Server.Plugin.PluginExecutionError:
- pass
- hostkeys.sort()
- for hostkey in hostkeys:
- entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
- hostkey.data.decode())
- self.entries[entry.get('name')].bind_info_to_entry(entry, metadata)
-
- def build_hk(self, entry, metadata):
- """This binds host key data into entries."""
- try:
- self.entries[entry.get('name')].bind_entry(entry, metadata)
- except Bcfg2.Server.Plugin.PluginExecutionError:
- filename = entry.get('name').split('/')[-1]
- self.GenerateHostKeyPair(metadata.hostname, filename)
- # Service the FAM events queued up by the key generation
- # so the data structure entries will be available for
- # binding.
- #
- # NOTE: We wait for up to ten seconds. There is some
- # potential for race condition, because if the file
- # monitor doesn't get notified about the new key files in
- # time, those entries won't be available for binding. In
- # practice, this seems "good enough".
- tries = 0
- is_bound = False
- while not is_bound:
- if tries >= 10:
- self.logger.error("%s still not registered" % filename)
- raise Bcfg2.Server.Plugin.PluginExecutionError
- self.core.fam.handle_events_in_interval(1)
- tries += 1
- try:
- self.entries[entry.get('name')].bind_entry(entry, metadata)
- is_bound = True
- except Bcfg2.Server.Plugin.PluginExecutionError:
- pass
-
- def GenerateHostKeyPair(self, client, filename):
- """Generate new host key pair for client."""
- match = re.search(r'(ssh_host_(?:((?:ecd|d|r)sa)_)?key)', filename)
- if match:
- hostkey = "%s.H_%s" % (match.group(1), client)
- if match.group(2):
- keytype = match.group(2)
- else:
- keytype = 'rsa1'
- else:
- self.logger.error("Unknown key filename: %s" % filename)
- return
-
- fileloc = "%s/%s" % (self.data, hostkey)
- publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub',
- "H_%s" % client])
- tempdir = tempfile.mkdtemp()
- temploc = "%s/%s" % (tempdir, hostkey)
- cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "",
- "-t", keytype, "-C", "root@%s" % client]
- proc = Popen(cmd, stdout=PIPE, stdin=PIPE)
- proc.communicate()
- proc.wait()
-
- try:
- shutil.copy(temploc, fileloc)
- shutil.copy("%s.pub" % temploc, publoc)
- except IOError:
- err = sys.exc_info()[1]
- self.logger.error("Temporary SSH keys not found: %s" % err)
-
- try:
- os.unlink(temploc)
- os.unlink("%s.pub" % temploc)
- os.rmdir(tempdir)
- except OSError:
- err = sys.exc_info()[1]
- self.logger.error("Failed to unlink temporary ssh keys: %s" % err)
-
- def AcceptChoices(self, _, metadata):
- return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)]
-
- def AcceptPullData(self, specific, entry, log):
- """Per-plugin bcfg2-admin pull support."""
- # specific will always be host specific
- filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1],
- specific.hostname)
- try:
- open(filename, 'w').write(entry['text'])
- if log:
- print("Wrote file %s" % filename)
- except KeyError:
- self.logger.error("Failed to pull %s. This file does not currently "
- "exist on the client" % entry.get('name'))
diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Server/Plugins/SSLCA.py
deleted file mode 100644
index 0072dc62d..000000000
--- a/src/lib/Server/Plugins/SSLCA.py
+++ /dev/null
@@ -1,274 +0,0 @@
-import Bcfg2.Server.Plugin
-import Bcfg2.Options
-import lxml.etree
-import posixpath
-import tempfile
-import pipes
-import os
-from subprocess import Popen, PIPE, STDOUT
-# Compatibility import
-from Bcfg2.Bcfg2Py3k import ConfigParser
-
-
-class SSLCA(Bcfg2.Server.Plugin.GroupSpool):
- """
- The SSLCA generator handles the creation and
- management of ssl certificates and their keys.
- """
- name = 'SSLCA'
- __author__ = 'g.hagger@gmail.com'
- __child__ = Bcfg2.Server.Plugin.FileBacked
- key_specs = {}
- cert_specs = {}
- CAs = {}
-
- def HandleEvent(self, event=None):
- """
- Updates which files this plugin handles based upon filesystem events.
- Allows configuration items to be added/removed without server restarts.
- """
- action = event.code2str()
- if event.filename[0] == '/':
- return
- epath = "".join([self.data, self.handles[event.requestID],
- event.filename])
- if posixpath.isdir(epath):
- ident = self.handles[event.requestID] + event.filename
- else:
- ident = self.handles[event.requestID][:-1]
-
- fname = "".join([ident, '/', event.filename])
-
- if event.filename.endswith('.xml'):
- if action in ['exists', 'created', 'changed']:
- if event.filename.endswith('key.xml'):
- key_spec = dict(list(lxml.etree.parse(epath).find('Key').items()))
- self.key_specs[ident] = {
- 'bits': key_spec.get('bits', 2048),
- 'type': key_spec.get('type', 'rsa')
- }
- self.Entries['Path'][ident] = self.get_key
- elif event.filename.endswith('cert.xml'):
- cert_spec = dict(list(lxml.etree.parse(epath).find('Cert').items()))
- ca = cert_spec.get('ca', 'default')
- self.cert_specs[ident] = {
- 'ca': ca,
- 'format': cert_spec.get('format', 'pem'),
- 'key': cert_spec.get('key'),
- 'days': cert_spec.get('days', 365),
- 'C': cert_spec.get('c'),
- 'L': cert_spec.get('l'),
- 'ST': cert_spec.get('st'),
- 'OU': cert_spec.get('ou'),
- 'O': cert_spec.get('o'),
- 'emailAddress': cert_spec.get('emailaddress')
- }
- cp = ConfigParser.ConfigParser()
- cp.read(self.core.cfile)
- self.CAs[ca] = dict(cp.items('sslca_' + ca))
- self.Entries['Path'][ident] = self.get_cert
- if action == 'deleted':
- if ident in self.Entries['Path']:
- del self.Entries['Path'][ident]
- else:
- if action in ['exists', 'created']:
- if posixpath.isdir(epath):
- self.AddDirectoryMonitor(epath[len(self.data):])
- if ident not in self.entries and posixpath.isfile(epath):
- self.entries[fname] = self.__child__(epath)
- self.entries[fname].HandleEvent(event)
- if action == 'changed':
- self.entries[fname].HandleEvent(event)
- elif action == 'deleted':
- if fname in self.entries:
- del self.entries[fname]
- else:
- self.entries[fname].HandleEvent(event)
-
- def get_key(self, entry, metadata):
- """
- either grabs a prexisting key hostfile, or triggers the generation
- of a new key if one doesn't exist.
- """
- # set path type and permissions, otherwise bcfg2 won't bind the file
- permdata = {'owner': 'root',
- 'group': 'root',
- 'type': 'file',
- 'perms': '644'}
- [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
-
- # check if we already have a hostfile, or need to generate a new key
- # TODO: verify key fits the specs
- path = entry.get('name')
- filename = "".join([path, '/', path.rsplit('/', 1)[1],
- '.H_', metadata.hostname])
- if filename not in list(self.entries.keys()):
- key = self.build_key(filename, entry, metadata)
- open(self.data + filename, 'w').write(key)
- entry.text = key
- self.entries[filename] = self.__child__("%s%s" % (self.data,
- filename))
- self.entries[filename].HandleEvent()
- else:
- entry.text = self.entries[filename].data
-
- def build_key(self, filename, entry, metadata):
- """
- generates a new key according the the specification
- """
- type = self.key_specs[entry.get('name')]['type']
- bits = self.key_specs[entry.get('name')]['bits']
- if type == 'rsa':
- cmd = ["openssl", "genrsa", bits]
- elif type == 'dsa':
- cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits]
- key = Popen(cmd, stdout=PIPE).stdout.read()
- return key
-
- def get_cert(self, entry, metadata):
- """
- either grabs a prexisting cert hostfile, or triggers the generation
- of a new cert if one doesn't exist.
- """
- # set path type and permissions, otherwise bcfg2 won't bind the file
- permdata = {'owner': 'root',
- 'group': 'root',
- 'type': 'file',
- 'perms': '644'}
- [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
-
- path = entry.get('name')
- filename = "".join([path, '/', path.rsplit('/', 1)[1],
- '.H_', metadata.hostname])
-
- # first - ensure we have a key to work with
- key = self.cert_specs[entry.get('name')].get('key')
- key_filename = "".join([key, '/', key.rsplit('/', 1)[1],
- '.H_', metadata.hostname])
- if key_filename not in self.entries:
- e = lxml.etree.Element('Path')
- e.attrib['name'] = key
- self.core.Bind(e, metadata)
-
- # check if we have a valid hostfile
- if filename in list(self.entries.keys()) and self.verify_cert(filename,
- key_filename,
- entry):
- entry.text = self.entries[filename].data
- else:
- cert = self.build_cert(key_filename, entry, metadata)
- open(self.data + filename, 'w').write(cert)
- self.entries[filename] = self.__child__("%s%s" % (self.data,
- filename))
- self.entries[filename].HandleEvent()
- entry.text = cert
-
- def verify_cert(self, filename, key_filename, entry):
- if self.verify_cert_against_ca(filename, entry):
- if self.verify_cert_against_key(filename, key_filename):
- return True
- return False
-
- def verify_cert_against_ca(self, filename, entry):
- """
- check that a certificate validates against the ca cert,
- and that it has not expired.
- """
- chaincert = self.CAs[self.cert_specs[entry.get('name')]['ca']].get('chaincert')
- cert = self.data + filename
- res = Popen(["openssl", "verify", "-CAfile", chaincert, cert],
- stdout=PIPE, stderr=STDOUT).stdout.read()
- if res == cert + ": OK\n":
- return True
- return False
-
- def verify_cert_against_key(self, filename, key_filename):
- """
- check that a certificate validates against its private key.
- """
- cert = self.data + filename
- key = self.data + key_filename
- cmd = ("openssl x509 -noout -modulus -in %s | openssl md5" %
- pipes.quote(cert))
- cert_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read()
- cmd = ("openssl rsa -noout -modulus -in %s | openssl md5" %
- pipes.quote(key))
- key_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read()
- if cert_md5 == key_md5:
- return True
- return False
-
- def build_cert(self, key_filename, entry, metadata):
- """
- creates a new certificate according to the specification
- """
- req_config = self.build_req_config(entry, metadata)
- req = self.build_request(key_filename, req_config, entry)
- ca = self.cert_specs[entry.get('name')]['ca']
- ca_config = self.CAs[ca]['config']
- days = self.cert_specs[entry.get('name')]['days']
- passphrase = self.CAs[ca].get('passphrase')
- cmd = ["openssl", "ca", "-config", ca_config, "-in", req,
- "-days", days, "-batch"]
- if passphrase:
- cmd.extend(["-passin", "pass:%s" % passphrase])
- cert = Popen(cmd, stdout=PIPE).stdout.read()
- try:
- os.unlink(req_config)
- os.unlink(req)
- except OSError:
- self.logger.error("Failed to unlink temporary files")
- return cert
-
- def build_req_config(self, entry, metadata):
- """
- generates a temporary openssl configuration file that is
- used to generate the required certificate request
- """
- # create temp request config file
- conffile = open(tempfile.mkstemp()[1], 'w')
- cp = ConfigParser.ConfigParser({})
- cp.optionxform = str
- defaults = {
- 'req': {
- 'default_md': 'sha1',
- 'distinguished_name': 'req_distinguished_name',
- 'req_extensions': 'v3_req',
- 'x509_extensions': 'v3_req',
- 'prompt': 'no'
- },
- 'req_distinguished_name': {},
- 'v3_req': {
- 'subjectAltName': '@alt_names'
- },
- 'alt_names': {}
- }
- for section in list(defaults.keys()):
- cp.add_section(section)
- for key in defaults[section]:
- cp.set(section, key, defaults[section][key])
- x = 1
- altnames = list(metadata.aliases)
- altnames.append(metadata.hostname)
- for altname in altnames:
- cp.set('alt_names', 'DNS.' + str(x), altname)
- x += 1
- for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']:
- if self.cert_specs[entry.get('name')][item]:
- cp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item])
- cp.set('req_distinguished_name', 'CN', metadata.hostname)
- cp.write(conffile)
- conffile.close()
- return conffile.name
-
- def build_request(self, key_filename, req_config, entry):
- """
- creates the certificate request
- """
- req = tempfile.mkstemp()[1]
- days = self.cert_specs[entry.get('name')]['days']
- key = self.data + key_filename
- cmd = ["openssl", "req", "-new", "-config", req_config,
- "-days", days, "-key", key, "-text", "-out", req]
- res = Popen(cmd, stdout=PIPE).stdout.read()
- return req
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
deleted file mode 100644
index aeb3b9f74..000000000
--- a/src/lib/Server/Plugins/Snapshots.py
+++ /dev/null
@@ -1,142 +0,0 @@
-#import lxml.etree
-import logging
-import binascii
-import difflib
-#import sqlalchemy
-#import sqlalchemy.orm
-import Bcfg2.Server.Plugin
-import Bcfg2.Server.Snapshots
-import Bcfg2.Logger
-from Bcfg2.Server.Snapshots.model import Snapshot
-import sys
-import time
-import threading
-
-# Compatibility import
-from Bcfg2.Bcfg2Py3k import Queue
-
-logger = logging.getLogger('Snapshots')
-
-ftypes = ['ConfigFile', 'SymLink', 'Directory']
-datafields = {
- 'Package': ['version'],
- 'Path': ['type'],
- 'Service': ['status'],
- 'ConfigFile': ['owner', 'group', 'perms'],
- 'Directory': ['owner', 'group', 'perms'],
- 'SymLink': ['to'],
- }
-
-
-# py3k compatibility
-def u_str(string):
- if sys.hexversion >= 0x03000000:
- return string
- else:
- return unicode(string)
-
-
-def build_snap_ent(entry):
- basefields = []
- if entry.tag in ['Package', 'Service']:
- basefields += ['type']
- desired = dict([(key, u_str(entry.get(key))) for key in basefields])
- state = dict([(key, u_str(entry.get(key))) for key in basefields])
- desired.update([(key, u_str(entry.get(key))) for key in \
- datafields[entry.tag]])
- if entry.tag == 'ConfigFile' or \
- ((entry.tag == 'Path') and (entry.get('type') == 'file')):
- if entry.text == None:
- desired['contents'] = None
- else:
- if entry.get('encoding', 'ascii') == 'ascii':
- desired['contents'] = u_str(entry.text)
- else:
- desired['contents'] = u_str(binascii.a2b_base64(entry.text))
-
- if 'current_bfile' in entry.attrib:
- state['contents'] = u_str(binascii.a2b_base64( \
- entry.get('current_bfile')))
- elif 'current_bdiff' in entry.attrib:
- diff = binascii.a2b_base64(entry.get('current_bdiff'))
- state['contents'] = u_str( \
- '\n'.join(difflib.restore(diff.split('\n'), 1)))
-
- state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \
- for key in datafields[entry.tag]])
- if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false':
- state = None
- return [desired, state]
-
-
-class Snapshots(Bcfg2.Server.Plugin.Statistics,
- Bcfg2.Server.Plugin.Plugin):
- name = 'Snapshots'
- experimental = True
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Statistics.__init__(self)
- self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile)
- self.work_queue = Queue()
- self.loader = threading.Thread(target=self.load_snapshot)
- self.loader.start()
-
- def load_snapshot(self):
- while self.running:
- try:
- (metadata, data) = self.work_queue.get(block=True, timeout=5)
- except:
- continue
- self.statistics_from_old_stats(metadata, data)
-
- def process_statistics(self, metadata, data):
- return self.work_queue.put((metadata, data))
-
- def statistics_from_old_stats(self, metadata, xdata):
- # entries are name -> (modified, correct, start, desired, end)
- # not sure we can get all of this from old format stats
- t1 = time.time()
- entries = dict([('Package', dict()),
- ('Service', dict()), ('Path', dict())])
- extra = dict([('Package', dict()), ('Service', dict()),
- ('Path', dict())])
- bad = []
- state = xdata.find('.//Statistics')
- correct = state.get('state') == 'clean'
- revision = u_str(state.get('revision', '-1'))
- for entry in state.find('.//Bad'):
- data = [False, False, u_str(entry.get('name'))] \
- + build_snap_ent(entry)
- if entry.tag in ftypes:
- etag = 'Path'
- else:
- etag = entry.tag
- entries[etag][entry.get('name')] = data
- for entry in state.find('.//Modified'):
- if entry.tag in ftypes:
- etag = 'Path'
- else:
- etag = entry.tag
- if entry.get('name') in entries[etag]:
- data = [True, False, u_str(entry.get('name'))] + \
- build_snap_ent(entry)
- else:
- data = [True, False, u_str(entry.get('name'))] + \
- build_snap_ent(entry)
- for entry in state.find('.//Extra'):
- if entry.tag in datafields:
- data = build_snap_ent(entry)[1]
- ename = u_str(entry.get('name'))
- data['name'] = ename
- extra[entry.tag][ename] = data
- else:
- print("extra", entry.tag, entry.get('name'))
- t2 = time.time()
- snap = Snapshot.from_data(self.session, correct, revision,
- metadata, entries, extra)
- self.session.add(snap)
- self.session.commit()
- t3 = time.time()
- logger.info("Snapshot storage took %fs" % (t3 - t2))
- return True
diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Server/Plugins/Statistics.py
deleted file mode 100644
index 265ef95a8..000000000
--- a/src/lib/Server/Plugins/Statistics.py
+++ /dev/null
@@ -1,161 +0,0 @@
-'''This file manages the statistics collected by the BCFG2 Server'''
-
-import binascii
-import copy
-import difflib
-import logging
-from lxml.etree import XML, SubElement, Element, XMLSyntaxError
-import lxml.etree
-import os
-from time import asctime, localtime, time, strptime, mktime
-import threading
-
-import Bcfg2.Server.Plugin
-
-
-class StatisticsStore(object):
- """Manages the memory and file copy of statistics collected about client runs."""
- __min_write_delay__ = 0
-
- def __init__(self, filename):
- self.filename = filename
- self.element = Element('Dummy')
- self.dirty = 0
- self.lastwrite = 0
- self.logger = logging.getLogger('Bcfg2.Server.Statistics')
- self.ReadFromFile()
-
- def WriteBack(self, force=0):
- """Write statistics changes back to persistent store."""
- if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \
- or force:
- try:
- fout = open(self.filename + '.new', 'w')
- except IOError:
- ioerr = sys.exc_info()[1]
- self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr))
- else:
- fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True))
- fout.close()
- os.rename(self.filename + '.new', self.filename)
- self.dirty = 0
- self.lastwrite = time()
-
- def ReadFromFile(self):
- """Reads current state regarding statistics."""
- try:
- fin = open(self.filename, 'r')
- data = fin.read()
- fin.close()
- self.element = XML(data)
- self.dirty = 0
- except (IOError, XMLSyntaxError):
- self.logger.error("Creating new statistics file %s"%(self.filename))
- self.element = Element('ConfigStatistics')
- self.WriteBack()
- self.dirty = 0
-
- def updateStats(self, xml, client):
- """Updates the statistics of a current node with new data."""
-
- # Current policy:
- # - Keep anything less than 24 hours old
- # - Keep latest clean run for clean nodes
- # - Keep latest clean and dirty run for dirty nodes
- newstat = xml.find('Statistics')
-
- if newstat.get('state') == 'clean':
- node_dirty = 0
- else:
- node_dirty = 1
-
- # Find correct node entry in stats data
- # The following list comprehension should be guarenteed to return at
- # most one result
- nodes = [elem for elem in self.element.findall('Node') \
- if elem.get('name') == client]
- nummatch = len(nodes)
- if nummatch == 0:
- # Create an entry for this node
- node = SubElement(self.element, 'Node', name=client)
- elif nummatch == 1 and not node_dirty:
- # Delete old instance
- node = nodes[0]
- [node.remove(elem) for elem in node.findall('Statistics') \
- if self.isOlderThan24h(elem.get('time'))]
- elif nummatch == 1 and node_dirty:
- # Delete old dirty statistics entry
- node = nodes[0]
- [node.remove(elem) for elem in node.findall('Statistics') \
- if (elem.get('state') == 'dirty' \
- and self.isOlderThan24h(elem.get('time')))]
- else:
- # Shouldn't be reached
- self.logger.error("Duplicate node entry for %s"%(client))
-
- # Set current time for stats
- newstat.set('time', asctime(localtime()))
-
- # Add statistic
- node.append(copy.copy(newstat))
-
- # Set dirty
- self.dirty = 1
- self.WriteBack(force=1)
-
- def isOlderThan24h(self, testTime):
- """Helper function to determine if <time> string is older than 24 hours."""
- now = time()
- utime = mktime(strptime(testTime))
- secondsPerDay = 60*60*24
-
- return (now-utime) > secondsPerDay
-
-
-class Statistics(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.ThreadedStatistics,
- Bcfg2.Server.Plugin.PullSource):
- name = 'Statistics'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.PullSource.__init__(self)
- fpath = "%s/etc/statistics.xml" % datastore
- self.data_file = StatisticsStore(fpath)
-
- def handle_statistic(self, metadata, data):
- self.data_file.updateStats(data, metadata.hostname)
-
- def FindCurrent(self, client):
- rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0]
- maxtime = max([strptime(stat.get('time')) for stat \
- in rt.findall('Statistics')])
- return [stat for stat in rt.findall('Statistics') \
- if strptime(stat.get('time')) == maxtime][0]
-
- def GetExtra(self, client):
- return [(entry.tag, entry.get('name')) for entry \
- in self.FindCurrent(client).xpath('.//Extra/*')]
-
- def GetCurrentEntry(self, client, e_type, e_name):
- curr = self.FindCurrent(client)
- entry = curr.xpath('.//Bad/%s[@name="%s"]' % (e_type, e_name))
- if not entry:
- raise Bcfg2.Server.Plugin.PluginExecutionError
- cfentry = entry[-1]
-
- owner = cfentry.get('current_owner', cfentry.get('owner'))
- group = cfentry.get('current_group', cfentry.get('group'))
- perms = cfentry.get('current_perms', cfentry.get('perms'))
- if cfentry.get('sensitive') in ['true', 'True']:
- raise Bcfg2.Server.Plugin.PluginExecutionError
- elif 'current_bfile' in cfentry.attrib:
- contents = binascii.a2b_base64(cfentry.get('current_bfile'))
- elif 'current_bdiff' in cfentry.attrib:
- diff = binascii.a2b_base64(cfentry.get('current_bdiff'))
- contents = '\n'.join(difflib.restore(diff.split('\n'), 1))
- else:
- contents = None
-
- return (owner, group, perms, contents)
diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Server/Plugins/Svcmgr.py
deleted file mode 100644
index f4232ad5c..000000000
--- a/src/lib/Server/Plugins/Svcmgr.py
+++ /dev/null
@@ -1,10 +0,0 @@
-"""This generator provides service mappings."""
-
-import Bcfg2.Server.Plugin
-
-
-class Svcmgr(Bcfg2.Server.Plugin.PrioDir):
- """This is a generator that handles service assignments."""
- name = 'Svcmgr'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- deprecated = True
diff --git a/src/lib/Server/Plugins/Svn.py b/src/lib/Server/Plugins/Svn.py
deleted file mode 100644
index ae43388ea..000000000
--- a/src/lib/Server/Plugins/Svn.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-import pipes
-from subprocess import Popen, PIPE
-import Bcfg2.Server.Plugin
-
-# for debugging output only
-import logging
-logger = logging.getLogger('Bcfg2.Plugins.Svn')
-
-
-class Svn(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Svn is a version plugin for dealing with Bcfg2 repos."""
- name = 'Svn'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- self.core = core
- self.datastore = datastore
-
- # path to svn directory for bcfg2 repo
- svn_dir = "%s/.svn" % datastore
-
- # Read revision from bcfg2 repo
- if os.path.isdir(svn_dir):
- self.get_revision()
- else:
- logger.error("%s is not a directory" % svn_dir)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- logger.debug("Initialized svn plugin with svn directory = %s" % svn_dir)
-
- def get_revision(self):
- """Read svn revision information for the Bcfg2 repository."""
- try:
- data = Popen(("env LC_ALL=C svn info %s" %
- pipes.quote(self.datastore)), shell=True,
- stdout=PIPE).communicate()[0].split('\n')
- return [line.split(': ')[1] for line in data \
- if line[:9] == 'Revision:'][-1]
- except IndexError:
- logger.error("Failed to read svn info; disabling svn support")
- logger.error('''Ran command "svn info %s"''' % (self.datastore))
- logger.error("Got output: %s" % data)
- raise Bcfg2.Server.Plugin.PluginInitError
diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py
deleted file mode 100644
index 8d79348f8..000000000
--- a/src/lib/Server/Plugins/Svn2.py
+++ /dev/null
@@ -1,125 +0,0 @@
-try:
- import pysvn
- missing = False
-except:
- missing = True
-import Bcfg2.Server.Plugin
-
-class Svn2(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Version):
- """Svn is a version plugin for dealing with Bcfg2 repos."""
- name = 'Svn2'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- conflicts = ['Svn']
- experimental = True
- __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update','Commit']
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
-
- if missing:
- self.logger.error("Svn2: Missing PySvn")
- raise Bcfg2.Server.Plugin.PluginInitError
-
- self.client = pysvn.Client()
-
- self.core = core
- self.datastore = datastore
- self.svn_root = None
- self.revision = None
-
- # Read revision from bcfg2 repo
- revision = self.get_revision()
- if not self.revision:
- raise Bcfg2.Server.Plugin.PluginInitError
-
- self.logger.debug("Initialized svn plugin with svn root %s at revision %s"
- % (self.svn_root, revision))
-
- def get_revision(self):
- """Read svn revision information for the Bcfg2 repository."""
- try:
- info = self.client.info(self.datastore)
- self.revision = info.revision
- self.svn_root = info.url
- return str(self.revision.number)
- except:
- self.logger.error("Svn2: Failed to get revision", exc_info=1)
- self.revision = None
- return str(-1)
-
- def commit_data(self, file_list, comment=None):
- """Commit changes into the repository"""
- if not comment:
- comment = 'Svn2: autocommit'
-
- # First try to update
- if not self.Update():
- self.logger.error("Failed to update svn repository, refusing to commit changes")
- return
-
- #FIXME - look for conflicts?
-
- for fname in file_list:
- stat = self.client.status(fname)
- self.client.add([f.path for f in stat \
- if f.text_status == pysvn.wc_status_kind.unversioned])
- try:
- self.revision = self.client.checkin([self.datastore], comment,
- recurse=True)
- self.revision = self.client.update(self.datastore, recurse=True)[0]
- self.logger.info("Svn2: Commited changes. At %s" %
- self.revision.number)
- except Exception, err:
- # try to be smart about the error we got back
- details = None
- if "callback_ssl_server_trust_prompt" in str(err):
- details = "SVN server certificate is not trusted"
- elif "callback_get_login" in str(err):
- details = "SVN credentials not cached"
-
- if details is None:
- self.logger.error("Svn2: Failed to commit changes",
- exc_info=1)
- else:
- self.logger.error("Svn2: Failed to commit changes: %s" %
- details)
-
- def Update(self):
- '''Svn2.Update() => True|False\nUpdate svn working copy\n'''
- try:
- old_revision = self.revision.number
- self.revision = self.client.update(self.datastore, recurse=True)[0]
- except Exception, err:
- # try to be smart about the error we got back
- details = None
- if "callback_ssl_server_trust_prompt" in str(err):
- details = "SVN server certificate is not trusted"
- elif "callback_get_login" in str(err):
- details = "SVN credentials not cached"
-
- if details is None:
- self.logger.error("Svn2: Failed to update server repository",
- exc_info=1)
- else:
- self.logger.error("Svn2: Failed to update server repository: %s" %
- details)
- return False
-
- if old_revision == self.revision.number:
- self.logger.debug("repository is current")
- else:
- self.logger.info("Updated %s from revision %s to %s" % \
- (self.datastore, old_revision, self.revision.number))
- return True
-
- def Commit(self):
- """Svn2.Commit() => True|False\nCommit svn repository\n"""
- try:
- self.commit_changes([])
- return True
- except:
- return False
-
-
diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py
deleted file mode 100644
index 8879fdef1..000000000
--- a/src/lib/Server/Plugins/TCheetah.py
+++ /dev/null
@@ -1,80 +0,0 @@
-'''This module implements a templating generator based on Cheetah'''
-
-import binascii
-import logging
-import sys
-import traceback
-import Bcfg2.Server.Plugin
-# py3k compatibility
-if sys.hexversion >= 0x03000000:
- unicode = str
-
-logger = logging.getLogger('Bcfg2.Plugins.TCheetah')
-
-try:
- import Cheetah.Template
- import Cheetah.Parser
-except:
- logger.error("TCheetah: Failed to import Cheetah. Is it installed?")
- raise
-
-
-class TemplateFile:
- """Template file creates Cheetah template structures for the loaded file."""
-
- def __init__(self, name, specific, encoding):
- self.name = name
- self.specific = specific
- self.encoding = encoding
- self.template = None
- self.searchlist = dict()
-
- def handle_event(self, event):
- """Handle all fs events for this template."""
- if event.code2str() == 'deleted':
- return
- try:
- s = {'useStackFrames': False}
- self.template = Cheetah.Template.Template(open(self.name).read(),
- compilerSettings=s,
- searchList=self.searchlist)
- except Cheetah.Parser.ParseError:
- perror = sys.exc_info()[1]
- logger.error("Cheetah parse error for file %s" % (self.name))
- logger.error(perror.report())
-
- def bind_entry(self, entry, metadata):
- """Build literal file information."""
- self.template.metadata = metadata
- self.searchlist['metadata'] = metadata
- self.template.path = entry.get('realname', entry.get('name'))
- self.searchlist['path'] = entry.get('realname', entry.get('name'))
- self.template.source_path = self.name
- self.searchlist['source_path'] = self.name
-
- if entry.tag == 'Path':
- entry.set('type', 'file')
- try:
- if type(self.template) == unicode:
- entry.text = self.template
- else:
- if entry.get('encoding') == 'base64':
- # take care of case where file needs base64 encoding
- entry.text = binascii.b2a_base64(self.template)
- else:
- entry.text = unicode(str(self.template), self.encoding)
- except:
- (a, b, c) = sys.exc_info()
- msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1]
- logger.error(msg)
- logger.error("TCheetah template error for %s" % self.searchlist['path'])
- del a, b, c
- raise Bcfg2.Server.Plugin.PluginExecutionError
-
-
-class TCheetah(Bcfg2.Server.Plugin.GroupSpool):
- """The TCheetah generator implements a templating mechanism for configuration files."""
- name = 'TCheetah'
- __author__ = 'bcfg-dev@mcs.anl.gov'
- filename_pattern = 'template'
- es_child_cls = TemplateFile
diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py
deleted file mode 100644
index 3ba0f4272..000000000
--- a/src/lib/Server/Plugins/TGenshi.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""This module implements a templating generator based on Genshi."""
-
-import binascii
-import logging
-import sys
-import Bcfg2.Server.Plugin
-# py3k compatibility
-if sys.hexversion >= 0x03000000:
- unicode = str
-
-logger = logging.getLogger('Bcfg2.Plugins.TGenshi')
-
-# try to import genshi stuff
-try:
- import genshi.core
- import genshi.input
- from genshi.template import TemplateLoader, \
- TextTemplate, MarkupTemplate, TemplateError
-except ImportError:
- logger.error("TGenshi: Failed to import Genshi. Is it installed?")
- raise Bcfg2.Server.Plugin.PluginInitError
-try:
- from genshi.template import NewTextTemplate
- have_ntt = True
-except:
- have_ntt = False
-
-def removecomment(stream):
- """A genshi filter that removes comments from the stream."""
- for kind, data, pos in stream:
- if kind is genshi.core.COMMENT:
- continue
- yield kind, data, pos
-
-
-class TemplateFile:
- """Template file creates Genshi template structures for the loaded file."""
-
- def __init__(self, name, specific, encoding):
- self.name = name
- self.specific = specific
- self.encoding = encoding
- if self.specific.all:
- matchname = self.name
- elif self.specific.group:
- matchname = self.name[:self.name.find('.G')]
- else:
- matchname = self.name[:self.name.find('.H')]
- if matchname.endswith('.txt'):
- self.template_cls = TextTemplate
- elif matchname.endswith('.newtxt'):
- if not have_ntt:
- logger.error("Genshi NewTextTemplates not supported by this version of Genshi")
- else:
- self.template_cls = NewTextTemplate
- else:
- self.template_cls = MarkupTemplate
- self.HandleEvent = self.handle_event
-
- def handle_event(self, event=None):
- """Handle all fs events for this template."""
- if event and event.code2str() == 'deleted':
- return
- try:
- loader = TemplateLoader()
- try:
- self.template = loader.load(self.name, cls=self.template_cls,
- encoding=self.encoding)
- except LookupError:
- lerror = sys.exc_info()[1]
- logger.error('Genshi lookup error: %s' % lerror)
- except TemplateError:
- terror = sys.exc_info()[1]
- logger.error('Genshi template error: %s' % terror)
- except genshi.input.ParseError:
- perror = sys.exc_info()[1]
- logger.error('Genshi parse error: %s' % perror)
-
- def bind_entry(self, entry, metadata):
- """Build literal file information."""
- fname = entry.get('realname', entry.get('name'))
- if entry.tag == 'Path':
- entry.set('type', 'file')
- try:
- stream = self.template.generate( \
- name=fname, metadata=metadata,
- path=self.name).filter(removecomment)
- if have_ntt:
- ttypes = [TextTemplate, NewTextTemplate]
- else:
- ttypes = [TextTemplate]
- if True in [isinstance(self.template, t) for t in ttypes]:
- try:
- textdata = stream.render('text', strip_whitespace=False)
- except TypeError:
- textdata = stream.render('text')
- if type(textdata) == unicode:
- entry.text = textdata
- else:
- if entry.get('encoding') == 'base64':
- # take care of case where file needs base64 encoding
- entry.text = binascii.b2a_base64(textdata)
- else:
- entry.text = unicode(textdata, self.encoding)
- else:
- try:
- xmldata = stream.render('xml', strip_whitespace=False)
- except TypeError:
- xmldata = stream.render('xml')
- if type(xmldata) == unicode:
- entry.text = xmldata
- else:
- entry.text = unicode(xmldata, self.encoding)
- if entry.text == '':
- entry.set('empty', 'true')
- except TemplateError:
- terror = sys.exc_info()[1]
- logger.error('Genshi template error: %s' % terror)
- raise Bcfg2.Server.Plugin.PluginExecutionError
- except AttributeError:
- err = sys.exc_info()[1]
- logger.error('Genshi template loading error: %s' % err)
- raise Bcfg2.Server.Plugin.PluginExecutionError
-
-
-class TGenshi(Bcfg2.Server.Plugin.GroupSpool):
- """
- The TGenshi generator implements a templating
- mechanism for configuration files.
-
- """
- name = 'TGenshi'
- __author__ = 'jeff@ocjtech.us'
- filename_pattern = 'template\.(txt|newtxt|xml)'
- es_child_cls = TemplateFile
diff --git a/src/lib/Server/Plugins/TemplateHelper.py b/src/lib/Server/Plugins/TemplateHelper.py
deleted file mode 100644
index 42eafed56..000000000
--- a/src/lib/Server/Plugins/TemplateHelper.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import re
-import imp
-import sys
-import logging
-import Bcfg2.Server.Plugin
-
-logger = logging.getLogger(__name__)
-
-class HelperModule(Bcfg2.Server.Plugin.SpecificData):
- _module_name_re = re.compile(r'([^/]+?)\.py')
-
- def __init__(self, name, specific, encoding):
- Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
- encoding)
- match = self._module_name_re.search(self.name)
- if match:
- self._module_name = match.group(1)
- else:
- self._module_name = name
- self._attrs = []
-
- def handle_event(self, event):
- Bcfg2.Server.Plugin.SpecificData.handle_event(self, event)
- try:
- module = imp.load_source(self._module_name, self.name)
- except:
- err = sys.exc_info()[1]
- logger.error("TemplateHelper: Failed to import %s: %s" %
- (self.name, err))
- return
-
- if not hasattr(module, "__export__"):
- logger.error("TemplateHelper: %s has no __export__ list" %
- self.name)
- return
-
- for sym in module.__export__:
- if sym not in self._attrs and hasattr(self, sym):
- logger.warning("TemplateHelper: %s: %s is a reserved keyword, "
- "skipping export" % (self.name, sym))
- setattr(self, sym, getattr(module, sym))
- # remove old exports
- for sym in set(self._attrs) - set(module.__export__):
- delattr(self, sym)
-
- self._attrs = module.__export__
-
-
-class HelperSet(Bcfg2.Server.Plugin.EntrySet):
- ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$")
-
- def __init__(self, path, fam, encoding, plugin_name):
- fpattern = '[0-9A-Za-z_\-]+\.py'
- self.plugin_name = plugin_name
- Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
- HelperModule, encoding)
- fam.AddMonitor(path, self)
-
- def HandleEvent(self, event):
- if (event.filename != self.path and
- not self.ignore.match(event.filename)):
- return self.handle_event(event)
-
-
-class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Connector):
- """ A plugin to provide helper classes and functions to templates """
- name = 'TemplateHelper'
- __author__ = 'chris.a.st.pierre@gmail.com'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
-
- try:
- self.helpers = HelperSet(self.data, core.fam, core.encoding,
- self.name)
- except:
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def get_additional_data(self, metadata):
- return dict([(h._module_name, h)
- for h in list(self.helpers.entries.values())])
diff --git a/src/lib/Server/Plugins/Trigger.py b/src/lib/Server/Plugins/Trigger.py
deleted file mode 100644
index b0d21545c..000000000
--- a/src/lib/Server/Plugins/Trigger.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-import Bcfg2.Server.Plugin
-
-
-def async_run(prog, args):
- pid = os.fork()
- if pid:
- os.waitpid(pid, 0)
- else:
- dpid = os.fork()
- if not dpid:
- os.system(" ".join([prog] + args))
- os._exit(0)
-
-
-class Trigger(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Statistics):
- """Trigger is a plugin that calls external scripts (on the server)."""
- name = 'Trigger'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Statistics.__init__(self)
- try:
- os.stat(self.data)
- except:
- self.logger.error("Trigger: spool directory %s does not exist; "
- "unloading" % self.data)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- def process_statistics(self, metadata, _):
- args = [metadata.hostname, '-p', metadata.profile, '-g',
- ':'.join([g for g in metadata.groups])]
- for notifier in os.listdir(self.data):
- if ((notifier[-1] == '~') or
- (notifier[:2] == '.#') or
- (notifier[-4:] == '.swp') or
- (notifier in ['SCCS', '.svn', '4913'])):
- continue
- npath = self.data + '/' + notifier
- self.logger.debug("Running %s %s" % (npath, " ".join(args)))
- async_run(npath, args)
diff --git a/src/lib/Server/Plugins/__init__.py b/src/lib/Server/Plugins/__init__.py
deleted file mode 100644
index a36a9b766..000000000
--- a/src/lib/Server/Plugins/__init__.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""Imports for Bcfg2.Server.Plugins."""
-
-__all__ = [
- 'Account',
- 'Base',
- 'Bundler',
- 'Bzr',
- 'Cfg',
- 'Cvs',
- 'Darcs',
- 'Decisions',
- 'Fossil',
- 'Git',
- 'GroupPatterns',
- 'Hg',
- 'Hostbase',
- 'Metadata',
- 'NagiosGen',
- 'Ohai',
- 'Packages',
- 'Properties',
- 'Probes',
- 'Pkgmgr',
- 'Rules',
- 'SSHbase',
- 'Snapshots',
- 'Statistics',
- 'Svcmgr',
- 'Svn',
- 'TCheetah',
- 'Trigger',
- 'SGenshi',
- 'TGenshi',
- ]