summaryrefslogtreecommitdiffstats
path: root/src/lib/Bcfg2/Server/Plugins
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins')
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Account.py101
-rw-r--r--src/lib/Bcfg2/Server/Plugins/BB.py83
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Base.py43
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bundler.py99
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bzr.py35
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg.py293
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cvs.py46
-rw-r--r--src/lib/Bcfg2/Server/Plugins/DBStats.py113
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Darcs.py48
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Decisions.py66
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Defaults.py53
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Deps.py134
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Editor.py80
-rw-r--r--src/lib/Bcfg2/Server/Plugins/FileProbes.py230
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Fossil.py51
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Git.py44
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupPatterns.py124
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Guppy.py62
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Hg.py45
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Hostbase.py593
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Ldap.py245
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py866
-rw-r--r--src/lib/Bcfg2/Server/Plugins/NagiosGen.py151
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Ohai.py92
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py141
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Collection.py349
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Pac.py119
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py15
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py107
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py280
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py688
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py269
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Pkgmgr.py169
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py285
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Properties.py76
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Rules.py55
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SGenshi.py97
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSHbase.py413
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSLCA.py274
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Snapshots.py142
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Statistics.py161
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Svcmgr.py10
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Svn.py46
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Svn2.py126
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TCheetah.py80
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TGenshi.py135
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TemplateHelper.py83
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Trigger.py43
-rw-r--r--src/lib/Bcfg2/Server/Plugins/__init__.py34
49 files changed, 7894 insertions, 0 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Account.py b/src/lib/Bcfg2/Server/Plugins/Account.py
new file mode 100644
index 000000000..f2703dccb
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Account.py
@@ -0,0 +1,101 @@
+"""This handles authentication setup."""
+
+import Bcfg2.Server.Plugin
+
+
+class Account(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Generator):
+ """This module generates account config files,
+ based on an internal data repo:
+ static.(passwd|group|limits.conf) -> static entries
+ dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch)
+ useraccess -> users to be granted login access on some hosts
+ superusers -> users to be granted root privs on all hosts
+ rootlike -> users to be granted root privs on some hosts
+
+ """
+ name = 'Account'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Generator.__init__(self)
+ self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb,
+ '/etc/group': self.from_yp_cb,
+ '/etc/security/limits.conf': self.gen_limits_cb,
+ '/root/.ssh/authorized_keys': self.gen_root_keys_cb,
+ '/etc/sudoers': self.gen_sudoers}}
+ try:
+ self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data,
+ self.core.fam)
+ except:
+ self.logger.error("Failed to load repos: %s, %s" % \
+ (self.data, "%s/ssh" % (self.data)))
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def from_yp_cb(self, entry, metadata):
+ """Build password file from cached yp data."""
+ fname = entry.attrib['name'].split('/')[-1]
+ entry.text = self.repository.entries["static.%s" % (fname)].data
+ entry.text += self.repository.entries["dyn.%s" % (fname)].data
+ perms = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '0644'}
+ [entry.attrib.__setitem__(key, value) for (key, value) in \
+ list(perms.items())]
+
+ def gen_limits_cb(self, entry, metadata):
+ """Build limits entries based on current ACLs."""
+ entry.text = self.repository.entries["static.limits.conf"].data
+ superusers = self.repository.entries["superusers"].data.split()
+ useraccess = [line.split(':') for line in \
+ self.repository.entries["useraccess"].data.split()]
+ users = [user for (user, host) in \
+ useraccess if host == metadata.hostname.split('.')[0]]
+ perms = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '0600'}
+ [entry.attrib.__setitem__(key, value) for (key, value) in \
+ list(perms.items())]
+ entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users])
+ if "*" not in users:
+ entry.text += "* hard maxlogins 0\n"
+
+ def gen_root_keys_cb(self, entry, metadata):
+ """Build root authorized keys file based on current ACLs."""
+ superusers = self.repository.entries['superusers'].data.split()
+ try:
+ rootlike = [line.split(':', 1) for line in \
+ self.repository.entries['rootlike'].data.split()]
+ superusers += [user for (user, host) in rootlike \
+ if host == metadata.hostname.split('.')[0]]
+ except:
+ pass
+ rdata = self.repository.entries
+ entry.text = "".join([rdata["%s.key" % user].data for user \
+ in superusers if \
+ ("%s.key" % user) in rdata])
+ perms = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '0600'}
+ [entry.attrib.__setitem__(key, value) for (key, value) \
+ in list(perms.items())]
+
+ def gen_sudoers(self, entry, metadata):
+ """Build root authorized keys file based on current ACLs."""
+ superusers = self.repository.entries['superusers'].data.split()
+ try:
+ rootlike = [line.split(':', 1) for line in \
+ self.repository.entries['rootlike'].data.split()]
+ superusers += [user for (user, host) in rootlike \
+ if host == metadata.hostname.split('.')[0]]
+ except:
+ pass
+ entry.text = self.repository.entries['static.sudoers'].data
+ entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \
+ for uname in superusers])
+ perms = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '0440'}
+ [entry.attrib.__setitem__(key, value) for (key, value) \
+ in list(perms.items())]
diff --git a/src/lib/Bcfg2/Server/Plugins/BB.py b/src/lib/Bcfg2/Server/Plugins/BB.py
new file mode 100644
index 000000000..c015ec47c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/BB.py
@@ -0,0 +1,83 @@
+import lxml.etree
+import Bcfg2.Server.Plugin
+import glob
+import os
+import socket
+
+#manage boot symlinks
+ #add statistics check to do build->boot mods
+
+#map profiles: first array is not empty we replace the -p with a determined profile.
+logger = Bcfg2.Server.Plugin.logger
+
+class BBfile(Bcfg2.Server.Plugin.XMLFileBacked):
+ """Class for bb files."""
+ def Index(self):
+ """Build data into an xml object."""
+
+ try:
+ self.data = lxml.etree.XML(self.data)
+ except lxml.etree.XMLSyntaxError:
+ Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % self.name)
+ return
+ self.tftppath = self.data.get('tftp', '/tftpboot')
+ self.macs = {}
+ self.users = {}
+ self.actions = {}
+ self.bootlinks = []
+
+ for node in self.data.findall('Node'):
+ iface = node.find('Interface')
+ if iface != None:
+ mac = "01-%s" % (iface.get('mac'.replace(':','-').lower()))
+ self.actions[node.get('name')] = node.get('action')
+ self.bootlinks.append((mac, node.get('action')))
+ try:
+ ip = socket.gethostbyname(node.get('name'))
+ except:
+ logger.error("failed host resolution for %s" % node.get('name'))
+
+ self.macs[node.get('name')] = (iface.get('mac'), ip)
+ else:
+ logger.error("%s" % lxml.etree.tostring(node))
+ self.users[node.get('name')] = node.get('user',"").split(':')
+
+ def enforce_bootlinks(self):
+ for mac, target in self.bootlinks:
+ path = self.tftppath + '/' + mac
+ if not os.path.islink(path):
+ logger.error("Boot file %s not a link" % path)
+ if target != os.readlink(path):
+ try:
+ os.unlink(path)
+ os.symlink(target, path)
+ except:
+ logger.error("Failed to modify link %s" % path)
+
+class BBDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
+ __child__ = BBfile
+
+
+class BB(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ """The BB plugin maps users to machines and metadata to machines."""
+ name = 'BB'
+ deprecated = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.store = BBDirectoryBacked(self.data, core.fam)
+
+ def get_additional_data(self, metadata):
+
+ users = {}
+ for user in self.store.entries['bb.xml'].users.get(metadata.hostname.split(".")[0], []):
+ pubkeys = []
+ for fname in glob.glob('/home/%s/.ssh/*.pub'%user):
+ pubkeys.append(open(fname).read())
+
+ users[user] = pubkeys
+
+ return dict([('users', users),
+ ('macs', self.store.entries['bb.xml'].macs)])
diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py
new file mode 100644
index 000000000..389ca7a95
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Base.py
@@ -0,0 +1,43 @@
+"""This module sets up a base list of configuration entries."""
+
+import copy
+import lxml.etree
+import sys
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ from functools import reduce
+
+import Bcfg2.Server.Plugin
+
+
+class Base(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Structure,
+ Bcfg2.Server.Plugin.XMLDirectoryBacked):
+ """This Structure is good for the pile of independent configs
+ needed for most actual systems.
+ """
+ name = 'Base'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ __child__ = Bcfg2.Server.Plugin.StructFile
+ deprecated = True
+
+ """Base creates independent clauses based on client metadata."""
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Structure.__init__(self)
+ try:
+ Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
+ self.data,
+ self.core.fam)
+ except OSError:
+ self.logger.error("Failed to load Base repository")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def BuildStructures(self, metadata):
+ """Build structures for client described by metadata."""
+ ret = lxml.etree.Element("Independent", version='2.0')
+ fragments = reduce(lambda x, y: x + y,
+ [base.Match(metadata) for base
+ in list(self.entries.values())], [])
+ [ret.append(copy.copy(frag)) for frag in fragments]
+ return [ret]
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
new file mode 100644
index 000000000..ccb99481e
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -0,0 +1,99 @@
+"""This provides bundle clauses with translation functionality."""
+
+import copy
+import lxml.etree
+import os
+import os.path
+import re
+import sys
+
+import Bcfg2.Server.Plugin
+
+try:
+ import genshi.template
+ import genshi.template.base
+ import Bcfg2.Server.Plugins.SGenshi
+ have_genshi = True
+except:
+ have_genshi = False
+
+
+class BundleFile(Bcfg2.Server.Plugin.StructFile):
+
+ def get_xml_value(self, metadata):
+ bundlename = os.path.splitext(os.path.basename(self.name))[0]
+ bundle = lxml.etree.Element('Bundle', name=bundlename)
+ [bundle.append(copy.copy(item)) for item in self.Match(metadata)]
+ return bundle
+
+
+class Bundler(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Structure,
+ Bcfg2.Server.Plugin.XMLDirectoryBacked):
+ """The bundler creates dependent clauses based on the
+ bundle/translation scheme from Bcfg1.
+ """
+ name = 'Bundler'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$')
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Structure.__init__(self)
+ self.encoding = core.encoding
+ self.__child__ = self.template_dispatch
+ try:
+ Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
+ self.data,
+ self.core.fam)
+ except OSError:
+ self.logger.error("Failed to load Bundle repository")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def template_dispatch(self, name):
+ bundle = lxml.etree.parse(name)
+ nsmap = bundle.getroot().nsmap
+ if name.endswith('.xml'):
+ if have_genshi and \
+ (nsmap == {'py': 'http://genshi.edgewall.org/'}):
+ # allow for genshi bundles with .xml extensions
+ spec = Bcfg2.Server.Plugin.Specificity()
+ return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name,
+ spec,
+ self.encoding)
+ else:
+ return BundleFile(name)
+ elif name.endswith('.genshi'):
+ if have_genshi:
+ spec = Bcfg2.Server.Plugin.Specificity()
+ return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name,
+ spec,
+ self.encoding)
+
+ def BuildStructures(self, metadata):
+ """Build all structures for client (metadata)."""
+ bundleset = []
+
+ bundle_entries = {}
+ for key, item in self.entries.items():
+ bundle_entries.setdefault(self.patterns.match(os.path.basename(key)).group('name'),
+ []).append(item)
+
+ for bundlename in metadata.bundles:
+ try:
+ entries = bundle_entries[bundlename]
+ except KeyError:
+ self.logger.error("Bundler: Bundle %s does not exist" %
+ bundlename)
+ continue
+ try:
+ bundleset.append(entries[0].get_xml_value(metadata))
+ except genshi.template.base.TemplateError:
+ t = sys.exc_info()[1]
+ self.logger.error("Bundler: Failed to template genshi bundle %s"
+ % bundlename)
+ self.logger.error(t)
+ except:
+ self.logger.error("Bundler: Unexpected bundler error for %s" %
+ bundlename, exc_info=1)
+ return bundleset
diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py
new file mode 100644
index 000000000..a71021cb5
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py
@@ -0,0 +1,35 @@
+import Bcfg2.Server.Plugin
+from bzrlib.workingtree import WorkingTree
+from bzrlib import errors
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Bzr')
+
+class Bzr(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Bzr is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Bzr'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ self.core = core
+ self.datastore = datastore
+
+ # Read revision from bcfg2 repo
+ revision = self.get_revision()
+
+ logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision})
+
+ def get_revision(self):
+ """Read Bazaar revision information for the Bcfg2 repository."""
+ try:
+ working_tree = WorkingTree.open(self.datastore)
+ revision = str(working_tree.branch.revno())
+ if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns():
+ revision += "+"
+ except errors.NotBranchError:
+ logger.error("Failed to read Bazaar branch; disabling Bazaar support")
+ raise Bcfg2.Server.Plugin.PluginInitError
+ return revision
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg.py b/src/lib/Bcfg2/Server/Plugins/Cfg.py
new file mode 100644
index 000000000..81904d082
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg.py
@@ -0,0 +1,293 @@
+"""This module implements a config file repository."""
+
+import binascii
+import logging
+import lxml
+import operator
+import os
+import os.path
+import re
+import stat
+import sys
+import tempfile
+from subprocess import Popen, PIPE
+from Bcfg2.Bcfg2Py3k import u_str
+
+import Bcfg2.Server.Plugin
+
+try:
+ import genshi.core
+ import genshi.input
+ from genshi.template import TemplateLoader, NewTextTemplate
+ have_genshi = True
+except:
+ have_genshi = False
+
+try:
+ import Cheetah.Template
+ import Cheetah.Parser
+ have_cheetah = True
+except:
+ have_cheetah = False
+
+# setup logging
+logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+
+
+# snipped from TGenshi
+def removecomment(stream):
+ """A genshi filter that removes comments from the stream."""
+ for kind, data, pos in stream:
+ if kind is genshi.core.COMMENT:
+ continue
+ yield kind, data, pos
+
+
+def process_delta(data, delta):
+ if not delta.specific.delta:
+ return data
+ if delta.specific.delta == 'cat':
+ datalines = data.strip().split('\n')
+ for line in delta.data.split('\n'):
+ if not line:
+ continue
+ if line[0] == '+':
+ datalines.append(line[1:])
+ elif line[0] == '-':
+ if line[1:] in datalines:
+ datalines.remove(line[1:])
+ return "\n".join(datalines) + "\n"
+ elif delta.specific.delta == 'diff':
+ basehandle, basename = tempfile.mkstemp()
+ basefile = open(basename, 'w')
+ basefile.write(data)
+ basefile.close()
+ os.close(basehandle)
+
+ cmd = ["patch", "-u", "-f", basefile.name]
+ patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ stderr = patch.communicate(input=delta.data)[1]
+ ret = patch.wait()
+ output = open(basefile.name, 'r').read()
+ os.unlink(basefile.name)
+ if ret >> 8 != 0:
+ logger.error("Error applying diff %s: %s" % (delta.name, stderr))
+ raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta)
+ return output
+
+
+class CfgMatcher:
+
+ def __init__(self, fname):
+ name = re.escape(fname)
+ self.basefile_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))((?P<genshi>\\.genshi)|(?P<cheetah>\\.cheetah))?$' % name)
+ self.delta_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % name)
+ self.cat_count = fname.count(".cat")
+ self.diff_count = fname.count(".diff")
+
+ def match(self, fname):
+ if fname.count(".cat") > self.cat_count \
+ or fname.count('.diff') > self.diff_count:
+ return self.delta_reg.match(fname)
+ return self.basefile_reg.match(fname)
+
+
+class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
+
+ def __init__(self, basename, path, entry_type, encoding):
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
+ entry_type, encoding)
+ self.specific = CfgMatcher(path.split('/')[-1])
+ path = path
+
+ def debug_log(self, message, flag=None):
+ if (flag is None and self.debug_flag) or flag:
+ logger.error(message)
+
+ def sort_by_specific(self, one, other):
+ return cmp(one.specific, other.specific)
+
+ def get_pertinent_entries(self, entry, metadata):
+ """return a list of all entries pertinent
+ to a client => [base, delta1, delta2]
+ """
+ matching = [ent for ent in list(self.entries.values()) if \
+ ent.specific.matches(metadata)]
+ matching.sort(key=operator.attrgetter('specific'))
+ # base entries which apply to a client
+ # (e.g. foo, foo.G##_groupname, foo.H_hostname)
+ base_files = [matching.index(m) for m in matching
+ if not m.specific.delta]
+ if not base_files:
+ msg = "No base file found for %s" % entry.get('name')
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ base = min(base_files)
+ used = matching[:base + 1]
+ used.reverse()
+ return used
+
+ def bind_entry(self, entry, metadata):
+ self.bind_info_to_entry(entry, metadata)
+ used = self.get_pertinent_entries(entry, metadata)
+ basefile = used.pop(0)
+ if entry.get('perms').lower() == 'inherit':
+ # use on-disk permissions
+ fname = os.path.join(self.path, entry.get('name'))
+ entry.set('perms',
+ str(oct(stat.S_IMODE(os.stat(fname).st_mode))))
+ if entry.tag == 'Path':
+ entry.set('type', 'file')
+ if basefile.name.endswith(".genshi"):
+ if not have_genshi:
+ msg = "Cfg: Genshi is not available: %s" % entry.get("name")
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ try:
+ template_cls = NewTextTemplate
+ loader = TemplateLoader()
+ template = loader.load(basefile.name, cls=template_cls,
+ encoding=self.encoding)
+ fname = entry.get('realname', entry.get('name'))
+ stream = template.generate(name=fname,
+ metadata=metadata,
+ path=basefile.name).filter(removecomment)
+ try:
+ data = stream.render('text', encoding=self.encoding,
+ strip_whitespace=False)
+ except TypeError:
+ data = stream.render('text', encoding=self.encoding)
+ if data == '':
+ entry.set('empty', 'true')
+ except Exception:
+ msg = "Cfg: genshi exception (%s): %s" % (entry.get("name"),
+ sys.exc_info()[1])
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ elif basefile.name.endswith(".cheetah"):
+ if not have_cheetah:
+ msg = "Cfg: Cheetah is not available: %s" % entry.get("name")
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ try:
+ fname = entry.get('realname', entry.get('name'))
+ s = {'useStackFrames': False}
+ template = Cheetah.Template.Template(open(basefile.name).read(),
+ compilerSettings=s)
+ template.metadata = metadata
+ template.path = fname
+ template.source_path = basefile.name
+ data = template.respond()
+ if data == '':
+ entry.set('empty', 'true')
+ except Exception:
+ msg = "Cfg: cheetah exception (%s): %s" % (entry.get("name"),
+ sys.exc_info()[1])
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ else:
+ data = basefile.data
+ for delta in used:
+ data = process_delta(data, delta)
+ if entry.get('encoding') == 'base64':
+ entry.text = binascii.b2a_base64(data)
+ else:
+ try:
+ entry.text = u_str(data, self.encoding)
+ except UnicodeDecodeError:
+ msg = "Failed to decode %s: %s" % (entry.get('name'),
+ sys.exc_info()[1])
+ logger.error(msg)
+ logger.error("Please verify you are using the proper encoding.")
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ except ValueError:
+ msg = "Error in specification for %s: %s" % (entry.get('name'),
+ sys.exc_info()[1])
+ logger.error(msg)
+ logger.error("You need to specify base64 encoding for %s." %
+ entry.get('name'))
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ if entry.text in ['', None]:
+ entry.set('empty', 'true')
+
+ def list_accept_choices(self, entry, metadata):
+ '''return a list of candidate pull locations'''
+ used = self.get_pertinent_entries(entry, metadata)
+ ret = []
+ if used:
+ ret.append(used[0].specific)
+ if not ret[0].hostname:
+ ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname))
+ return ret
+
+ def build_filename(self, specific):
+ bfname = self.path + '/' + self.path.split('/')[-1]
+ if specific.all:
+ return bfname
+ elif specific.group:
+ return "%s.G%02d_%s" % (bfname, specific.prio, specific.group)
+ elif specific.hostname:
+ return "%s.H_%s" % (bfname, specific.hostname)
+
+ def write_update(self, specific, new_entry, log):
+ if 'text' in new_entry:
+ name = self.build_filename(specific)
+ if os.path.exists("%s.genshi" % name):
+ msg = "Cfg: Unable to pull data for genshi types"
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ elif os.path.exists("%s.cheetah" % name):
+ msg = "Cfg: Unable to pull data for cheetah types"
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ try:
+ etext = new_entry['text'].encode(self.encoding)
+ except:
+ msg = "Cfg: Cannot encode content of %s as %s" % (name,
+ self.encoding)
+ logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+ open(name, 'w').write(etext)
+ self.debug_log("Wrote file %s" % name, flag=log)
+ badattr = [attr for attr in ['owner', 'group', 'perms']
+ if attr in new_entry]
+ if badattr:
+ # check for info files and inform user of their removal
+ if os.path.exists(self.path + "/:info"):
+ logger.info("Removing :info file and replacing with "
+ "info.xml")
+ os.remove(self.path + "/:info")
+ if os.path.exists(self.path + "/info"):
+ logger.info("Removing info file and replacing with "
+ "info.xml")
+ os.remove(self.path + "/info")
+ metadata_updates = {}
+ metadata_updates.update(self.metadata)
+ for attr in badattr:
+ metadata_updates[attr] = new_entry.get(attr)
+ infoxml = lxml.etree.Element('FileInfo')
+ infotag = lxml.etree.SubElement(infoxml, 'Info')
+ [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \
+ for attr in metadata_updates]
+ ofile = open(self.path + "/info.xml", "w")
+ ofile.write(lxml.etree.tostring(infoxml, pretty_print=True))
+ ofile.close()
+ self.debug_log("Wrote file %s" % (self.path + "/info.xml"),
+ flag=log)
+
+
+class Cfg(Bcfg2.Server.Plugin.GroupSpool,
+ Bcfg2.Server.Plugin.PullTarget):
+ """This generator in the configuration file repository for Bcfg2."""
+ name = 'Cfg'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ es_cls = CfgEntrySet
+ es_child_cls = Bcfg2.Server.Plugin.SpecificData
+
+ def AcceptChoices(self, entry, metadata):
+ return self.entries[entry.get('name')].list_accept_choices(entry, metadata)
+
+ def AcceptPullData(self, specific, new_entry, log):
+ return self.entries[new_entry.get('name')].write_update(specific,
+ new_entry,
+ log)
diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py
new file mode 100644
index 000000000..6ce72acd2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py
@@ -0,0 +1,46 @@
+import os
+from subprocess import Popen, PIPE
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Cvs')
+
+class Cvs(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """CVS is a version plugin for dealing with Bcfg2 repository."""
+ name = 'Cvs'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ self.core = core
+ self.datastore = datastore
+
+ # path to cvs directory for Bcfg2 repo
+ cvs_dir = "%s/CVSROOT" % datastore
+
+ # Read revision from Bcfg2 repo
+ if os.path.isdir(cvs_dir):
+ self.get_revision()
+ else:
+ logger.error("%s is not a directory" % cvs_dir)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir)
+
+ def get_revision(self):
+ """Read cvs revision information for the Bcfg2 repository."""
+ try:
+ data = Popen("env LC_ALL=C cvs log",
+ shell=True,
+ cwd=self.datastore,
+ stdout=PIPE).stdout.readlines()
+ revision = data[3].strip('\n')
+ except IndexError:
+ logger.error("Failed to read cvs log; disabling cvs support")
+ logger.error('''Ran command "cvs log %s"''' % (self.datastore))
+ logger.error("Got output: %s" % data)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py
new file mode 100644
index 000000000..95395f74e
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py
@@ -0,0 +1,113 @@
+import binascii
+import difflib
+import logging
+import lxml.etree
+import platform
+import time
+
+try:
+ from django.core.exceptions import MultipleObjectsReturned
+except ImportError:
+ pass
+
+import Bcfg2.Server.Plugin
+import Bcfg2.Server.Reports.importscript
+from Bcfg2.Server.Reports.reports.models import Client
+import Bcfg2.Server.Reports.settings
+from Bcfg2.Server.Reports.updatefix import update_database
+# for debugging output only
+logger = logging.getLogger('Bcfg2.Plugins.DBStats')
+
+class DBStats(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.ThreadedStatistics,
+ Bcfg2.Server.Plugin.PullSource):
+ name = 'DBStats'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.PullSource.__init__(self)
+ self.cpath = "%s/Metadata/clients.xml" % datastore
+ self.core = core
+ logger.debug("Searching for new models to add to the statistics database")
+ try:
+ update_database()
+ except Exception:
+ inst = sys.exc_info()[1]
+ logger.debug(str(inst))
+ logger.debug(str(type(inst)))
+
+ def handle_statistic(self, metadata, data):
+ newstats = data.find("Statistics")
+ newstats.set('time', time.asctime(time.localtime()))
+ # ick
+ data = lxml.etree.tostring(newstats)
+ ndx = lxml.etree.XML(data)
+ e = lxml.etree.Element('Node', name=metadata.hostname)
+ e.append(ndx)
+ container = lxml.etree.Element("ConfigStatistics")
+ container.append(e)
+
+ # FIXME need to build a metadata interface to expose a list of clients
+ start = time.time()
+ for i in [1, 2, 3]:
+ try:
+ Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
+ container,
+ self.core.encoding,
+ 0,
+ logger,
+ True,
+ platform.node())
+ logger.info("Imported data for %s in %s seconds" \
+ % (metadata.hostname, time.time() - start))
+ return
+ except MultipleObjectsReturned:
+ e = sys.exc_info()[1]
+ logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \
+ (metadata.hostname, e))
+ logger.error("DBStats: Data is inconsistent")
+ break
+ except:
+ logger.error("DBStats: Failed to write to db (lock); retrying",
+ exc_info=1)
+ logger.error("DBStats: Retry limit failed for %s; aborting operation" \
+ % metadata.hostname)
+
+ def GetExtra(self, client):
+ c_inst = Client.objects.filter(name=client)[0]
+ return [(a.entry.kind, a.entry.name) for a in
+ c_inst.current_interaction.extra()]
+
+ def GetCurrentEntry(self, client, e_type, e_name):
+ try:
+ c_inst = Client.objects.filter(name=client)[0]
+ except IndexError:
+ self.logger.error("Unknown client: %s" % client)
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ result = c_inst.current_interaction.bad().filter(entry__kind=e_type,
+ entry__name=e_name)
+ if not result:
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ entry = result[0]
+ ret = []
+ data = ('owner', 'group', 'perms')
+ for t in data:
+ if getattr(entry.reason, "current_%s" % t) == '':
+ ret.append(getattr(entry.reason, t))
+ else:
+ ret.append(getattr(entry.reason, "current_%s" % t))
+ if entry.reason.is_sensitive:
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ elif entry.reason.current_diff != '':
+ if entry.reason.is_binary:
+ ret.append(binascii.a2b_base64(entry.reason.current_diff))
+ else:
+ ret.append('\n'.join(difflib.restore(\
+ entry.reason.current_diff.split('\n'), 1)))
+ elif entry.reason.is_binary:
+ # If len is zero the object was too large to store
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ else:
+ ret.append(None)
+ return ret
diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py
new file mode 100644
index 000000000..9fb9ff4f1
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py
@@ -0,0 +1,48 @@
+import os
+from subprocess import Popen, PIPE
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Darcs')
+
+class Darcs(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Darcs is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Darcs'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Version.__init__(self)
+ self.core = core
+ self.datastore = datastore
+
+ # path to darcs directory for bcfg2 repo
+ darcs_dir = "%s/_darcs" % datastore
+
+ # Read changeset from bcfg2 repo
+ if os.path.isdir(darcs_dir):
+ self.get_revision()
+ else:
+ logger.error("%s is not present." % darcs_dir)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir)
+
+ def get_revision(self):
+ """Read Darcs changeset information for the Bcfg2 repository."""
+ try:
+ data = Popen("env LC_ALL=C darcs changes",
+ shell=True,
+ cwd=self.datastore,
+ stdout=PIPE).stdout.readlines()
+ revision = data[0].strip('\n')
+ except:
+ logger.error("Failed to read darcs repository; disabling Darcs support")
+ logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore))
+ logger.error("Got output: %s" % data)
+ raise Bcfg2.Server.Plugin.PluginInitError
+ return revision
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py
new file mode 100644
index 000000000..b432474f2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py
@@ -0,0 +1,66 @@
+import logging
+import lxml.etree
+import sys
+
+import Bcfg2.Server.Plugin
+logger = logging.getLogger('Bcfg2.Plugins.Decisions')
+
+class DecisionFile(Bcfg2.Server.Plugin.SpecificData):
+ def handle_event(self, event):
+ Bcfg2.Server.Plugin.SpecificData.handle_event(self, event)
+ self.contents = lxml.etree.XML(self.data)
+
+ def get_decisions(self):
+ return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')]
+
+class DecisionSet(Bcfg2.Server.Plugin.EntrySet):
+ def __init__(self, path, fam, encoding):
+ """Container for decision specification files.
+
+ Arguments:
+ - `path`: repository path
+ - `fam`: reference to the file monitor
+ - `encoding`: XML character encoding
+
+ """
+ pattern = '(white|black)list'
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, pattern, path, \
+ DecisionFile, encoding)
+ try:
+ fam.AddMonitor(path, self)
+ except OSError:
+ e = sys.exc_info()[1]
+ logger.error('Adding filemonitor for %s failed. '
+ 'Make sure directory exists' % path)
+ raise Bcfg2.Server.Plugin.PluginInitError(e)
+
+ def HandleEvent(self, event):
+ if event.filename != self.path:
+ return self.handle_event(event)
+
+ def GetDecisions(self, metadata, mode):
+ ret = []
+ candidates = [c for c in self.get_matching(metadata)
+ if c.name.split('/')[-1].startswith(mode)]
+ for c in candidates:
+ ret += c.get_decisions()
+ return ret
+
+class Decisions(DecisionSet,
+ Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Decision):
+ name = 'Decisions'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ """Decisions plugins
+
+ Arguments:
+ - `core`: Bcfg2.Core instance
+ - `datastore`: File repository location
+
+ """
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Decision.__init__(self)
+ DecisionSet.__init__(self, self.data, core.fam, core.encoding)
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py
new file mode 100644
index 000000000..718192e2a
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py
@@ -0,0 +1,53 @@
+"""This generator provides rule-based entry mappings."""
+
+import re
+import Bcfg2.Server.Plugin
+import Bcfg2.Server.Plugins.Rules
+
+class Defaults(Bcfg2.Server.Plugins.Rules.Rules,
+ Bcfg2.Server.Plugin.StructureValidator):
+ """Set default attributes on bound entries"""
+ name = 'Defaults'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ # Rules is a Generator that happens to implement all of the
+ # functionality we want, so we overload it, but Defaults should
+ # _not_ handle any entries; it does its stuff in the structure
+ # validation phase. so we overload Handle(s)Entry and HandleEvent
+ # to ensure that Defaults handles no entries, even though it's a
+ # Generator.
+
+ def HandlesEntry(self, entry, metadata):
+ return False
+
+ def HandleEntry(self, entry, metadata):
+ raise PluginExecutionError
+
+ def HandleEvent(self, event):
+ Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event)
+
+ def validate_structures(self, metadata, structures):
+ """ Apply defaults """
+ for struct in structures:
+ for entry in struct.iter():
+ if entry.tag.startswith("Bound"):
+ is_bound = True
+ entry.tag = entry.tag[5:]
+ else:
+ is_bound = False
+ try:
+ try:
+ self.BindEntry(entry, metadata)
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ # either no matching defaults (which is okay),
+ # or multiple matching defaults (which is not
+ # okay, but is logged). either way, we don't
+ # care about the error.
+ pass
+ finally:
+ if is_bound:
+ entry.tag = "Bound" + entry.tag
+
+ def _regex_enabled(self):
+ """ Defaults depends on regex matching, so force it enabled """
+ return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py
new file mode 100644
index 000000000..9b848baae
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Deps.py
@@ -0,0 +1,134 @@
+"""This plugin provides automatic dependency handling."""
+
+import lxml.etree
+
+import Bcfg2.Server.Plugin
+
+
+class DNode(Bcfg2.Server.Plugin.INode):
+ """DNode provides supports for single predicate types for dependencies."""
+ raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"}
+ containers = ['Group']
+
+ def __init__(self, data, idict, parent=None):
+ self.data = data
+ self.contents = {}
+ if parent == None:
+ self.predicate = lambda x, d: True
+ else:
+ predicate = parent.predicate
+ if data.tag in list(self.raw.keys()):
+ self.predicate = eval(self.raw[data.tag] %
+ {'name': data.get('name')},
+ {'predicate': predicate})
+ else:
+ raise Exception
+ mytype = self.__class__
+ self.children = []
+ for item in data.getchildren():
+ if item.tag in self.containers:
+ self.children.append(mytype(item, idict, self))
+ else:
+ data = [(child.tag, child.get('name'))
+ for child in item.getchildren()]
+ try:
+ self.contents[item.tag][item.get('name')] = data
+ except KeyError:
+ self.contents[item.tag] = {item.get('name'): data}
+
+
+class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc):
+ __node__ = DNode
+
+
+class Deps(Bcfg2.Server.Plugin.PrioDir,
+ Bcfg2.Server.Plugin.StructureValidator):
+ name = 'Deps'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ __child__ = DepXMLSrc
+
+ # Override the default sort_order (of 500) so that this plugin
+ # gets handled after others running at the default. In particular,
+ # we want to run after Packages, so we can see the final set of
+ # packages that will be installed on the client.
+ sort_order = 750
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.StructureValidator.__init__(self)
+ self.cache = {}
+
+ def HandleEvent(self, event):
+ self.cache = {}
+ Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event)
+
+ def validate_structures(self, metadata, structures):
+ """Examine the passed structures and append any additional
+ prerequisite entries as defined by the files in Deps.
+ """
+ entries = []
+ for structure in structures:
+ for entry in structure.getchildren():
+ tag = entry.tag
+ if tag.startswith('Bound'):
+ tag = tag[5:]
+ if (tag, entry.get('name')) not in entries \
+ and not isinstance(entry, lxml.etree._Comment):
+ entries.append((tag, entry.get('name')))
+ entries.sort()
+ entries = tuple(entries)
+ gdata = list(metadata.groups)
+ gdata.sort()
+ gdata = tuple(gdata)
+
+ # Check to see if we have cached the prereqs already
+ if (entries, gdata) in self.cache:
+ prereqs = self.cache[(entries, gdata)]
+ else:
+ prereqs = self.calculate_prereqs(metadata, entries)
+ self.cache[(entries, gdata)] = prereqs
+
+ newstruct = lxml.etree.Element("Independent")
+ for tag, name in prereqs:
+ try:
+ lxml.etree.SubElement(newstruct, tag, name=name)
+ except:
+ self.logger.error("Failed to add dep entry for %s:%s" % (tag, name))
+ structures.append(newstruct)
+
+
+ def calculate_prereqs(self, metadata, entries):
+ """Calculate the prerequisites defined in Deps for the passed
+ set of entries.
+ """
+ prereqs = []
+ [src.Cache(metadata) for src in self.entries.values()]
+
+ toexamine = list(entries[:])
+ while toexamine:
+ entry = toexamine.pop()
+ matching = [src for src in list(self.entries.values())
+ if src.cache and entry[0] in src.cache[1]
+ and entry[1] in src.cache[1][entry[0]]]
+ if len(matching) > 1:
+ prio = [int(src.priority) for src in matching]
+ if prio.count(max(prio)) > 1:
+ self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" %
+ (entry[0].lower(), metadata.hostname, entry[1]))
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ index = prio.index(max(prio))
+ matching = [matching[index]]
+ elif len(matching) == 1:
+ for prq in matching[0].cache[1][entry[0]][entry[1]]:
+ # XML comments seem to show up in the cache as a
+ # tuple with item 0 being callable. The logic
+ # below filters them out. Would be better to
+ # exclude them when we load the cache in the first
+ # place.
+ if prq not in prereqs and prq not in entries and not callable(prq[0]):
+ toexamine.append(prq)
+ prereqs.append(prq)
+ else:
+ continue
+
+ return prereqs
diff --git a/src/lib/Bcfg2/Server/Plugins/Editor.py b/src/lib/Bcfg2/Server/Plugins/Editor.py
new file mode 100644
index 000000000..c0d2cfbad
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Editor.py
@@ -0,0 +1,80 @@
+import Bcfg2.Server.Plugin
+import re
+import lxml.etree
+
+
+def linesub(pattern, repl, filestring):
+ """Substitutes instances of pattern with repl in filestring."""
+ if filestring == None:
+ filestring = ''
+ output = list()
+ fileread = filestring.split('\n')
+ for line in fileread:
+ output.append(re.sub(pattern, repl, filestring))
+ return '\n'.join(output)
+
+
+class EditDirectives(Bcfg2.Server.Plugin.SpecificData):
+ """This object handles the editing directives."""
+ def ProcessDirectives(self, input):
+ """Processes a list of edit directives on input."""
+ temp = input
+ for directive in self.data.split('\n'):
+ directive = directive.split(',')
+ temp = linesub(directive[0], directive[1], temp)
+ return temp
+
+
+class EditEntrySet(Bcfg2.Server.Plugin.EntrySet):
+ def __init__(self, basename, path, entry_type, encoding):
+ self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1])
+ Bcfg2.Server.Plugin.EntrySet.__init__(self,
+ basename,
+ path,
+ entry_type,
+ encoding)
+ self.inputs = dict()
+
+ def bind_entry(self, entry, metadata):
+ client = metadata.hostname
+ filename = entry.get('name')
+ permdata = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '0644'}
+ [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
+ entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client))
+ if not entry.text:
+ entry.set('empty', 'true')
+ try:
+ f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w')
+ f.write(entry.text)
+ f.close()
+ except:
+ pass
+
+ def get_client_data(self, client):
+ return self.inputs[client]
+
+
+class Editor(Bcfg2.Server.Plugin.GroupSpool,
+ Bcfg2.Server.Plugin.Probing):
+ name = 'Editor'
+ __author__ = 'bcfg2-dev@mcs.anl.gov'
+ filename_pattern = 'edits'
+ es_child_cls = EditDirectives
+ es_cls = EditEntrySet
+
+ def GetProbes(self, _):
+ '''Return a set of probes for execution on client'''
+ probelist = list()
+ for name in list(self.entries.keys()):
+ probe = lxml.etree.Element('probe')
+ probe.set('name', name)
+ probe.set('source', "Editor")
+ probe.text = "cat %s" % name
+ probelist.append(probe)
+ return probelist
+
+ def ReceiveData(self, client, datalist):
+ for data in datalist:
+ self.entries[data.get('name')].inputs[client.hostname] = data.text
diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
new file mode 100644
index 000000000..5beec7be0
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
@@ -0,0 +1,230 @@
+""" This module allows you to probe a client for a file, which is then
+added to the specification. On subsequent runs, the file will be
+replaced on the client if it is missing; if it has changed on the
+client, it can either be updated in the specification or replaced on
+the client """
+
+import os
+import sys
+import errno
+import binascii
+import lxml.etree
+import Bcfg2.Options
+import Bcfg2.Server.Plugin
+
+probecode = """#!/usr/bin/env python
+
+import os
+import pwd
+import grp
+import binascii
+import lxml.etree
+
+path = "%s"
+
+if not os.path.exists(path):
+ print "%%s does not exist" %% path
+ raise SystemExit(1)
+
+stat = os.stat(path)
+data = lxml.etree.Element("ProbedFileData",
+ name=path,
+ owner=pwd.getpwuid(stat[4])[0],
+ group=grp.getgrgid(stat[5])[0],
+ perms=oct(stat[0] & 07777))
+data.text = binascii.b2a_base64(open(path).read())
+print lxml.etree.tostring(data)
+"""
+
+class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked,
+ Bcfg2.Server.Plugin.StructFile):
+ """ Config file handler for FileProbes """
+ def __init__(self, filename, fam):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
+ Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
+
+
+class FileProbes(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Probing):
+ """ This module allows you to probe a client for a file, which is then
+ added to the specification. On subsequent runs, the file will be
+ replaced on the client if it is missing; if it has changed on the
+ client, it can either be updated in the specification or replaced on
+ the client """
+
+ name = 'FileProbes'
+ experimental = True
+ __author__ = 'chris.a.st.pierre@gmail.com'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Probing.__init__(self)
+ self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'),
+ core.fam)
+ self.entries = dict()
+ self.probes = dict()
+
+ def GetProbes(self, metadata):
+ """Return a set of probes for execution on client."""
+ if metadata.hostname not in self.probes:
+ cfg = self.core.plugins['Cfg']
+ self.entries[metadata.hostname] = dict()
+ self.probes[metadata.hostname] = []
+ for entry in self.config.Match(metadata):
+ path = entry.get("name")
+ # do not probe for files that are already in Cfg and
+ # for which update is false; we can't possibly do
+ # anything with the data we get from such a probe
+ try:
+ if (entry.get('update', 'false').lower() == "false" and
+ cfg.entries[path].get_pertinent_entries(entry,
+ metadata)):
+ continue
+ except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError):
+ pass
+ self.entries[metadata.hostname][path] = entry
+ probe = lxml.etree.Element('probe', name=path,
+ source=self.name,
+ interpreter="/usr/bin/env python")
+ probe.text = probecode % path
+ self.probes[metadata.hostname].append(probe)
+ self.debug_log("Adding file probe for %s to %s" %
+ (path, metadata.hostname))
+ return self.probes[metadata.hostname]
+
+ def ReceiveData(self, metadata, datalist):
+ """Receive data from probe."""
+ self.debug_log("Receiving file probe data from %s" % metadata.hostname)
+
+ for data in datalist:
+ if data.text is None:
+ self.logger.error("Got null response to %s file probe from %s" %
+ (data.get('name'), metadata.hostname))
+ else:
+ try:
+ self.write_data(lxml.etree.XML(data.text), metadata)
+ except lxml.etree.XMLSyntaxError:
+ # if we didn't get XML back from the probe, assume
+ # it's an error message
+ self.logger.error(data.text)
+
+ def write_data(self, data, metadata):
+ """Write the probed file data to the bcfg2 specification."""
+ filename = data.get("name")
+ contents = binascii.a2b_base64(data.text)
+ entry = self.entries[metadata.hostname][filename]
+ cfg = self.core.plugins['Cfg']
+ specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname)
+ # we can't use os.path.join() for this because specific
+ # already has a leading /, which confuses os.path.join()
+ fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific))
+
+ create = False
+ try:
+ cfg.entries[filename].bind_entry(entry, metadata)
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ create = True
+
+ # get current entry data
+ if entry.text and entry.get("encoding") == "base64":
+ entrydata = binascii.a2b_base64(entry.text)
+ else:
+ entrydata = entry.text
+
+ if create:
+ self.logger.info("Writing new probed file %s" % fileloc)
+ self.write_file(fileloc, contents)
+ self.verify_file(filename, contents, metadata)
+ infoxml = os.path.join("%s%s" % (cfg.data, filename), "info.xml")
+ self.write_infoxml(infoxml, entry, data)
+ elif entrydata == contents:
+ self.debug_log("Existing %s contents match probed contents" %
+ filename)
+ return
+ elif (entry.get('update', 'false').lower() == "true"):
+ self.logger.info("Writing updated probed file %s" % fileloc)
+ self.write_file(fileloc, contents)
+ self.verify_file(filename, contents, metadata)
+ else:
+ self.logger.info("Skipping updated probed file %s" % fileloc)
+ return
+
+ def write_file(self, fileloc, contents):
+ try:
+ os.makedirs(os.path.dirname(fileloc))
+ except OSError:
+ err = sys.exc_info()[1]
+ if err.errno == errno.EEXIST:
+ pass
+ else:
+ self.logger.error("Could not create parent directories for %s: "
+ "%s" % (fileloc, err))
+ return
+
+ try:
+ open(fileloc, 'wb').write(contents)
+ except IOError:
+ err = sys.exc_info()[1]
+ self.logger.error("Could not write %s: %s" % (fileloc, err))
+ return
+
+ def verify_file(self, filename, contents, metadata):
+ # Service the FAM events queued up by the key generation so
+ # the data structure entries will be available for binding.
+ #
+ # NOTE: We wait for up to ten seconds. There is some potential
+ # for race condition, because if the file monitor doesn't get
+ # notified about the new key files in time, those entries
+ # won't be available for binding. In practice, this seems
+ # "good enough".
+ entry = self.entries[metadata.hostname][filename]
+ cfg = self.core.plugins['Cfg']
+ tries = 0
+ updated = False
+ while not updated:
+ if tries >= 10:
+ self.logger.error("%s still not registered" % filename)
+ return
+ self.core.fam.handle_events_in_interval(1)
+ try:
+ cfg.entries[filename].bind_entry(entry, metadata)
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ tries += 1
+ continue
+
+ # get current entry data
+ if entry.get("encoding") == "base64":
+ entrydata = binascii.a2b_base64(entry.text)
+ else:
+ entrydata = entry.text
+ if entrydata == contents:
+ updated = True
+ tries += 1
+
+ def write_infoxml(self, infoxml, entry, data):
+ """ write an info.xml for the file """
+ if os.path.exists(infoxml):
+ return
+
+ self.logger.info("Writing info.xml at %s for %s" %
+ (infoxml, data.get("name")))
+ info = \
+ lxml.etree.Element("Info",
+ owner=data.get("owner",
+ Bcfg2.Options.MDATA_OWNER.value),
+ group=data.get("group",
+ Bcfg2.Options.MDATA_GROUP.value),
+ perms=data.get("perms",
+ Bcfg2.Options.MDATA_PERMS.value),
+ encoding=entry.get("encoding",
+ Bcfg2.Options.ENCODING.value))
+
+ root = lxml.etree.Element("FileInfo")
+ root.append(info)
+ try:
+ open(infoxml, "w").write(lxml.etree.tostring(root,
+ pretty_print=True))
+ except IOError:
+ err = sys.exc_info()[1]
+ self.logger.error("Could not write %s: %s" % (fileloc, err))
+ return
diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py
new file mode 100644
index 000000000..1b1627688
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py
@@ -0,0 +1,51 @@
+import os
+from subprocess import Popen, PIPE
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Fossil')
+
+class Fossil(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Fossil is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Fossil'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ self.core = core
+ self.datastore = datastore
+
+ # path to fossil file for bcfg2 repo
+ fossil_file = "%s/_FOSSIL_" % datastore
+
+ # Read revision from bcfg2 repo
+ if os.path.isfile(fossil_file):
+ revision = self.get_revision()
+ elif not os.path.isdir(datastore):
+ logger.error("%s is not a directory" % datastore)
+ raise Bcfg2.Server.Plugin.PluginInitError
+ else:
+ logger.error("%s is not a file" % fossil_file)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \
+ % {'ffile': fossil_file, 'frev': revision})
+
+ def get_revision(self):
+ """Read fossil revision information for the Bcfg2 repository."""
+ try:
+ data = Popen("env LC_ALL=C fossil info",
+ shell=True,
+ cwd=self.datastore,
+ stdout=PIPE).stdout.readlines()
+ revline = [line.split(': ')[1].strip() for line in data if \
+ line.split(': ')[0].strip() == 'checkout'][-1]
+ revision = revline.split(' ')[0]
+ except IndexError:
+ logger.error("Failed to read fossil info; disabling fossil support")
+ logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore))
+ logger.error("Got output: %s" % data)
+ raise Bcfg2.Server.Plugin.PluginInitError
+ return revision
diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py
new file mode 100644
index 000000000..8f8ea87f1
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Git.py
@@ -0,0 +1,44 @@
+"""The Git plugin provides a revision interface for Bcfg2 repos using git."""
+
+import os
+from dulwich.repo import Repo
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Git')
+
+
+class Git(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Git is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Git'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Version.__init__(self)
+ self.core = core
+ self.datastore = datastore
+
+ # path to git directory for bcfg2 repo
+ git_dir = "%s/.git" % datastore
+
+ # Read revision from bcfg2 repo
+ if os.path.isdir(git_dir):
+ self.get_revision()
+ else:
+ logger.error("%s is not a directory" % git_dir)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized git plugin with git directory %s" % git_dir)
+
+ def get_revision(self):
+ """Read git revision information for the Bcfg2 repository."""
+ try:
+ repo = Repo(self.datastore)
+ revision = repo.head()
+ except:
+ logger.error("Failed to read git repository; disabling git support")
+ raise Bcfg2.Server.Plugin.PluginInitError
+ return revision
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
new file mode 100644
index 000000000..58b4d4afb
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -0,0 +1,124 @@
+import re
+import logging
+import lxml.etree
+import Bcfg2.Server.Plugin
+
+class PackedDigitRange(object):
+ def __init__(self, digit_range):
+ self.sparse = list()
+ self.ranges = list()
+ for item in digit_range.split(','):
+ if '-' in item:
+ self.ranges.append(tuple([int(x) for x in item.split('-')]))
+ else:
+ self.sparse.append(int(item))
+
+ def includes(self, other):
+ iother = int(other)
+ if iother in self.sparse:
+ return True
+ for (start, end) in self.ranges:
+ if iother in range(start, end + 1):
+ return True
+ return False
+
+
+class PatternMap(object):
+ range_finder = r'\[\[[\d\-,]+\]\]'
+
+ def __init__(self, pattern, rangestr, groups):
+ self.pattern = pattern
+ self.rangestr = rangestr
+ self.groups = groups
+ if pattern != None:
+ self.re = re.compile(pattern)
+ self.process = self.process_re
+ elif rangestr != None:
+ if '\\' in rangestr:
+ raise Exception("Backslashes are not allowed in NameRanges")
+ self.process = self.process_range
+ self.re = re.compile('^' + re.sub(self.range_finder, '(\d+)',
+ rangestr))
+ dmatcher = re.compile(re.sub(self.range_finder,
+ r'\[\[([\d\-,]+)\]\]',
+ rangestr))
+ self.dranges = [PackedDigitRange(x)
+ for x in dmatcher.match(rangestr).groups()]
+ else:
+ raise Exception("No pattern or range given")
+
+ def process_range(self, name):
+ match = self.re.match(name)
+ if not match:
+ return None
+ digits = match.groups()
+ for i in range(len(digits)):
+ if not self.dranges[i].includes(digits[i]):
+ return None
+ return self.groups
+
+ def process_re(self, name):
+ match = self.re.match(name)
+ if not match:
+ return None
+ ret = list()
+ sub = match.groups()
+ for group in self.groups:
+ newg = group
+ for idx in range(len(sub)):
+ newg = newg.replace('$%s' % (idx + 1), sub[idx])
+ ret.append(newg)
+ return ret
+
+
+class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked):
+ __identifier__ = None
+
+ def __init__(self, filename, fam):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
+ self.patterns = []
+ self.logger = logging.getLogger(self.__class__.__name__)
+
+ def Index(self):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self)
+ self.patterns = []
+ for entry in self.xdata.xpath('//GroupPattern'):
+ try:
+ groups = [g.text for g in entry.findall('Group')]
+ for pat_ent in entry.findall('NamePattern'):
+ pat = pat_ent.text
+ self.patterns.append(PatternMap(pat, None, groups))
+ for range_ent in entry.findall('NameRange'):
+ rng = range_ent.text
+ self.patterns.append(PatternMap(None, rng, groups))
+ except:
+ self.logger.error("GroupPatterns: Failed to initialize pattern "
+ "%s" % entry.get('pattern'))
+
+ def process_patterns(self, hostname):
+ ret = []
+ for pattern in self.patterns:
+ try:
+ gn = pattern.process(hostname)
+ if gn is not None:
+ ret.extend(gn)
+ except:
+ self.logger.error("GroupPatterns: Failed to process pattern %s "
+ "for %s" % (pattern.pattern, hostname),
+ exc_info=1)
+ return ret
+
+
+class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ name = "GroupPatterns"
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.config = PatternFile(self.data + '/config.xml',
+ core.fam)
+
+ def get_additional_groups(self, metadata):
+ return self.config.process_patterns(metadata.hostname)
diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py
new file mode 100644
index 000000000..eea92f30f
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py
@@ -0,0 +1,62 @@
+"""
+This plugin is used to trace memory leaks within the bcfg2-server
+process using Guppy. By default the remote debugger is started
+when this plugin is enabled. The debugger can be shutoff in a running
+process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using
+"bcfg2-admin xcmd Guppy.Enable".
+
+To attach the console run:
+
+python -c "from guppy import hpy;hpy().monitor()"
+
+For example:
+
+# python -c "from guppy import hpy;hpy().monitor()"
+<Monitor>
+*** Connection 1 opened ***
+<Monitor> lc
+CID PID ARGV
+ 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid']
+<Monitor> sc 1
+Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN>
+<Annex> int
+Remote interactive console. To return to Annex, type '-'.
+>>> hp.heap()
+...
+
+
+"""
+import re
+import Bcfg2.Server.Plugin
+
+class Guppy(Bcfg2.Server.Plugin.Plugin):
+ """Guppy is a debugging plugin to help trace memory leaks"""
+ name = 'Guppy'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ experimental = True
+ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable']
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+
+ self.Enable()
+
+ def Enable(self):
+ """Enable remote debugging"""
+ try:
+ from guppy.heapy import Remote
+ Remote.on()
+ except:
+ self.logger.error("Failed to create Heapy context")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def Disable(self):
+ """Disable remote debugging"""
+ try:
+ from guppy.heapy import Remote
+ Remote.off()
+ except:
+ self.logger.error("Failed to disable Heapy")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py
new file mode 100644
index 000000000..0c3537613
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Hg.py
@@ -0,0 +1,45 @@
+import os
+from mercurial import ui, hg
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Mercurial')
+
+class Hg(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Mercurial is a version plugin for dealing with Bcfg2 repository."""
+ name = 'Mercurial'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Version.__init__(self)
+ self.core = core
+ self.datastore = datastore
+
+ # path to hg directory for Bcfg2 repo
+ hg_dir = "%s/.hg" % datastore
+
+ # Read changeset from bcfg2 repo
+ if os.path.isdir(hg_dir):
+ self.get_revision()
+ else:
+ logger.error("%s is not present." % hg_dir)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir)
+
+ def get_revision(self):
+ """Read hg revision information for the Bcfg2 repository."""
+ try:
+ repo_path = "%s/" % self.datastore
+ repo = hg.repository(ui.ui(), repo_path)
+ tip = repo.changelog.tip()
+ revision = repo.changelog.rev(tip)
+ except:
+ logger.error("Failed to read hg repository; disabling mercurial support")
+ raise Bcfg2.Server.Plugin.PluginInitError
+ return revision
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py
new file mode 100644
index 000000000..e9c1c1cff
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Hostbase.py
@@ -0,0 +1,593 @@
+"""
+This file provides the Hostbase plugin.
+It manages dns/dhcp/nis host information
+"""
+
+import os
+os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings'
+from lxml.etree import Element, SubElement
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError
+from time import strftime
+from sets import Set
+from django.template import Context, loader
+from django.db import connection
+import re
+# Compatibility imports
+from Bcfg2.Bcfg2Py3k import StringIO
+
+
+class Hostbase(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Structure,
+ Bcfg2.Server.Plugin.Generator):
+ """The Hostbase plugin handles host/network info."""
+ name = 'Hostbase'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ filepath = '/my/adm/hostbase/files/bind'
+
+ def __init__(self, core, datastore):
+
+ self.ready = False
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Structure.__init__(self)
+ Bcfg2.Server.Plugin.Generator.__init__(self)
+ files = ['zone.tmpl',
+ 'reversesoa.tmpl',
+ 'named.tmpl',
+ 'reverseappend.tmpl',
+ 'dhcpd.tmpl',
+ 'hosts.tmpl',
+ 'hostsappend.tmpl']
+ self.filedata = {}
+ self.dnsservers = []
+ self.dhcpservers = []
+ self.templates = {'zone': loader.get_template('zone.tmpl'),
+ 'reversesoa': loader.get_template('reversesoa.tmpl'),
+ 'named': loader.get_template('named.tmpl'),
+ 'namedviews': loader.get_template('namedviews.tmpl'),
+ 'reverseapp': loader.get_template('reverseappend.tmpl'),
+ 'dhcp': loader.get_template('dhcpd.tmpl'),
+ 'hosts': loader.get_template('hosts.tmpl'),
+ 'hostsapp': loader.get_template('hostsappend.tmpl'),
+ }
+ self.Entries['ConfigFile'] = {}
+ self.__rmi__ = ['rebuildState']
+ try:
+ self.rebuildState(None)
+ except:
+ raise PluginInitError
+
+ def FetchFile(self, entry, metadata):
+ """Return prebuilt file data."""
+ fname = entry.get('name').split('/')[-1]
+ if not fname in self.filedata:
+ raise PluginExecutionError
+ perms = {'owner': 'root',
+ 'group': 'root',
+ 'perms': '644'}
+ [entry.attrib.__setitem__(key, value)
+ for (key, value) in list(perms.items())]
+ entry.text = self.filedata[fname]
+
+ def BuildStructures(self, metadata):
+ """Build hostbase bundle."""
+ if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers:
+ return []
+ output = Element("Bundle", name='hostbase')
+ if metadata.hostname in self.dnsservers:
+ for configfile in self.Entries['ConfigFile']:
+ if re.search('/etc/bind/', configfile):
+ SubElement(output, "ConfigFile", name=configfile)
+ if metadata.hostname in self.dhcpservers:
+ SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf")
+ return [output]
+
+ def rebuildState(self, _):
+ """Pre-cache all state information for hostbase config files
+ callable as an XMLRPC function.
+
+ """
+ self.buildZones()
+ self.buildDHCP()
+ self.buildHosts()
+ self.buildHostsLPD()
+ self.buildPrinters()
+ self.buildNetgroups()
+ return True
+
+ def buildZones(self):
+ """Pre-build and stash zone files."""
+ cursor = connection.cursor()
+
+ cursor.execute("SELECT id, serial FROM hostbase_zone")
+ zones = cursor.fetchall()
+
+ for zone in zones:
+ # update the serial number for all zone files
+ todaydate = (strftime('%Y%m%d'))
+ try:
+ if todaydate == str(zone[1])[:8]:
+ serial = zone[1] + 1
+ else:
+ serial = int(todaydate) * 100
+ except (KeyError):
+ serial = int(todaydate) * 100
+ cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0]))
+
+ cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'")
+ zones = cursor.fetchall()
+
+ iplist = []
+ hosts = {}
+
+ for zone in zones:
+ zonefile = StringIO()
+ externalzonefile = StringIO()
+ cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z
+ INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id
+ WHERE z.zone_id = \'%s\'""" % zone[0])
+ nameservers = cursor.fetchall()
+ cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z
+ INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id
+ WHERE z.zone_id = \'%s\'""" % zone[0])
+ addresses = cursor.fetchall()
+ cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z
+ INNER JOIN hostbase_mx m ON z.mx_id = m.id
+ WHERE z.zone_id = \'%s\'""" % zone[0])
+ mxs = cursor.fetchall()
+ context = Context({
+ 'zone': zone,
+ 'nameservers': nameservers,
+ 'addresses': addresses,
+ 'mxs': mxs
+ })
+ zonefile.write(self.templates['zone'].render(context))
+ externalzonefile.write(self.templates['zone'].render(context))
+
+ querystring = """SELECT h.hostname, p.ip_addr,
+ n.name, c.cname, m.priority, m.mx, n.dns_view
+ FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id)
+ INNER JOIN hostbase_name n ON p.id = n.ip_id)
+ INNER JOIN hostbase_name_mxs x ON n.id = x.name_id)
+ INNER JOIN hostbase_mx m ON m.id = x.mx_id)
+ LEFT JOIN hostbase_cname c ON n.id = c.name_id
+ WHERE n.name LIKE '%%%%%s'
+ AND h.status = 'active'
+ ORDER BY h.hostname, n.name, p.ip_addr
+ """ % zone[1]
+ cursor.execute(querystring)
+ zonehosts = cursor.fetchall()
+ prevhost = (None, None, None, None)
+ cnames = StringIO()
+ cnamesexternal = StringIO()
+ for host in zonehosts:
+ if not host[2].split(".", 1)[1] == zone[1]:
+ zonefile.write(cnames.getvalue())
+ externalzonefile.write(cnamesexternal.getvalue())
+ cnames = StringIO()
+ cnamesexternal = StringIO()
+ continue
+ if not prevhost[1] == host[1] or not prevhost[2] == host[2]:
+ zonefile.write(cnames.getvalue())
+ externalzonefile.write(cnamesexternal.getvalue())
+ cnames = StringIO()
+ cnamesexternal = StringIO()
+ zonefile.write("%-32s%-10s%-32s\n" %
+ (host[2].split(".", 1)[0], 'A', host[1]))
+ zonefile.write("%-32s%-10s%-3s%s.\n" %
+ ('', 'MX', host[4], host[5]))
+ if host[6] == 'global':
+ externalzonefile.write("%-32s%-10s%-32s\n" %
+ (host[2].split(".", 1)[0], 'A', host[1]))
+ externalzonefile.write("%-32s%-10s%-3s%s.\n" %
+ ('', 'MX', host[4], host[5]))
+ elif not prevhost[5] == host[5]:
+ zonefile.write("%-32s%-10s%-3s%s.\n" %
+ ('', 'MX', host[4], host[5]))
+ if host[6] == 'global':
+ externalzonefile.write("%-32s%-10s%-3s%s.\n" %
+ ('', 'MX', host[4], host[5]))
+
+ if host[3]:
+ try:
+ if host[3].split(".", 1)[1] == zone[1]:
+ cnames.write("%-32s%-10s%-32s\n" %
+ (host[3].split(".", 1)[0],
+ 'CNAME', host[2].split(".", 1)[0]))
+ if host[6] == 'global':
+ cnamesexternal.write("%-32s%-10s%-32s\n" %
+ (host[3].split(".", 1)[0],
+ 'CNAME', host[2].split(".", 1)[0]))
+ else:
+ cnames.write("%-32s%-10s%-32s\n" %
+ (host[3] + ".",
+ 'CNAME',
+ host[2].split(".", 1)[0]))
+ if host[6] == 'global':
+ cnamesexternal.write("%-32s%-10s%-32s\n" %
+ (host[3] + ".",
+ 'CNAME',
+ host[2].split(".", 1)[0]))
+
+ except:
+ pass
+ prevhost = host
+ zonefile.write(cnames.getvalue())
+ externalzonefile.write(cnamesexternal.getvalue())
+ zonefile.write("\n\n%s" % zone[9])
+ externalzonefile.write("\n\n%s" % zone[9])
+ self.filedata[zone[1]] = zonefile.getvalue()
+ self.filedata[zone[1] + ".external"] = externalzonefile.getvalue()
+ zonefile.close()
+ externalzonefile.close()
+ self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile
+ self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile
+
+ cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'")
+ reversezones = cursor.fetchall()
+
+ reversenames = []
+ for reversezone in reversezones:
+ cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z
+ INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id
+ WHERE z.zone_id = \'%s\'""" % reversezone[0])
+ reverse_nameservers = cursor.fetchall()
+
+ context = Context({
+ 'inaddr': reversezone[1].rstrip('.rev'),
+ 'zone': reversezone,
+ 'nameservers': reverse_nameservers,
+ })
+
+ self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context)
+ self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context)
+ self.filedata[reversezone[1]] += reversezone[9]
+ self.filedata[reversezone[1] + '.external'] += reversezone[9]
+
+ subnet = reversezone[1].split(".")
+ subnet.reverse()
+ reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:])))
+
+ for filename in reversenames:
+ cursor.execute("""
+ SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h
+ INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id)
+ INNER JOIN hostbase_name n ON n.ip_id = p.id
+ WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr
+ """ % filename[1])
+ reversehosts = cursor.fetchall()
+ zonefile = StringIO()
+ externalzonefile = StringIO()
+ if len(filename[0].split(".")) == 2:
+ originlist = []
+ [originlist.append((".".join([ip[1].split(".")[2], filename[0]]),
+ ".".join([filename[1], ip[1].split(".")[2]])))
+ for ip in reversehosts
+ if (".".join([ip[1].split(".")[2], filename[0]]),
+ ".".join([filename[1], ip[1].split(".")[2]])) not in originlist]
+ for origin in originlist:
+ hosts = [(host[1].split("."), host[0])
+ for host in reversehosts
+ if host[1].rstrip('0123456789').rstrip('.') == origin[1]]
+ hosts_external = [(host[1].split("."), host[0])
+ for host in reversehosts
+ if (host[1].rstrip('0123456789').rstrip('.') == origin[1]
+ and host[2] == 'global')]
+ context = Context({
+ 'hosts': hosts,
+ 'inaddr': origin[0],
+ 'fileorigin': filename[0],
+ })
+ zonefile.write(self.templates['reverseapp'].render(context))
+ context = Context({
+ 'hosts': hosts_external,
+ 'inaddr': origin[0],
+ 'fileorigin': filename[0],
+ })
+ externalzonefile.write(self.templates['reverseapp'].render(context))
+ else:
+ originlist = [filename[0]]
+ hosts = [(host[1].split("."), host[0])
+ for host in reversehosts
+ if (host[1].split("."), host[0]) not in hosts]
+ hosts_external = [(host[1].split("."), host[0])
+ for host in reversehosts
+ if ((host[1].split("."), host[0]) not in hosts_external
+ and host[2] == 'global')]
+ context = Context({
+ 'hosts': hosts,
+ 'inaddr': filename[0],
+ 'fileorigin': None,
+ })
+ zonefile.write(self.templates['reverseapp'].render(context))
+ context = Context({
+ 'hosts': hosts_external,
+ 'inaddr': filename[0],
+ 'fileorigin': None,
+ })
+ externalzonefile.write(self.templates['reverseapp'].render(context))
+ self.filedata['%s.rev' % filename[0]] += zonefile.getvalue()
+ self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue()
+ zonefile.close()
+ externalzonefile.close()
+ self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile
+ self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile
+
+ ## here's where the named.conf file gets written
+ context = Context({
+ 'zones': zones,
+ 'reverses': reversenames,
+ })
+ self.filedata['named.conf'] = self.templates['named'].render(context)
+ self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile
+ self.filedata['named.conf.views'] = self.templates['namedviews'].render(context)
+ self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile
+
+ def buildDHCP(self):
+ """Pre-build dhcpd.conf and stash in the filedata table."""
+
+ # fetches all the hosts with DHCP == True
+ cursor = connection.cursor()
+ cursor.execute("""
+ SELECT hostname, mac_addr, ip_addr
+ FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip ip ON i.id = ip.interface_id
+ WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> ''
+ AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown'
+ ORDER BY h.hostname, i.mac_addr
+ """)
+
+ dhcphosts = cursor.fetchall()
+ count = 0
+ hosts = []
+ hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]]
+ if len(dhcphosts) > 1:
+ for x in range(1, len(dhcphosts)):
+ # if an interface has 2 or more ip addresses
+ # adds the ip to the current interface
+ if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]:
+ hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]])
+ # if a host has 2 or more interfaces
+ # writes the current one and grabs the next
+ elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]:
+ hosts.append(hostdata)
+ count += 1
+ hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]]
+ # new host found, writes current data to the template
+ else:
+ hosts.append(hostdata)
+ count = 0
+ hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]]
+ #makes sure the last of the data gets written out
+ if hostdata not in hosts:
+ hosts.append(hostdata)
+
+ context = Context({
+ 'hosts': hosts,
+ 'numips': len(hosts),
+ })
+
+ self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context)
+ self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile
+
+ def buildHosts(self):
+ """Pre-build and stash /etc/hosts file."""
+
+ append_data = []
+
+ cursor = connection.cursor()
+ cursor.execute("""
+ SELECT hostname FROM hostbase_host ORDER BY hostname
+ """)
+ hostbase = cursor.fetchall()
+ domains = [host[0].split(".", 1)[1] for host in hostbase]
+ domains_set = Set(domains)
+ domain_data = [(domain, domains.count(domain)) for domain in domains_set]
+ domain_data.sort()
+
+ cursor.execute("""
+ SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr
+ """)
+ ips = cursor.fetchall()
+ three_octets = [ip[0].rstrip('0123456789').rstrip('.') \
+ for ip in ips]
+ three_octets_set = Set(three_octets)
+ three_octets_data = [(octet, three_octets.count(octet)) \
+ for octet in three_octets_set]
+ three_octets_data.sort()
+
+ for three_octet in three_octets_data:
+ querystring = """SELECT h.hostname, h.primary_user,
+ p.ip_addr, n.name, c.cname
+ FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id)
+ INNER JOIN hostbase_name n ON p.id = n.ip_id)
+ LEFT JOIN hostbase_cname c ON n.id = c.name_id
+ WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0]
+ cursor.execute(querystring)
+ tosort = list(cursor.fetchall())
+ tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1])))
+ append_data.append((three_octet, tuple(tosort)))
+
+ two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets]
+ two_octets_set = Set(two_octets)
+ two_octets_data = [(octet, two_octets.count(octet))
+ for octet in two_octets_set]
+ two_octets_data.sort()
+
+ context = Context({
+ 'domain_data': domain_data,
+ 'three_octets_data': three_octets_data,
+ 'two_octets_data': two_octets_data,
+ 'three_octets': three_octets,
+ 'num_ips': len(three_octets),
+ })
+
+ self.filedata['hosts'] = self.templates['hosts'].render(context)
+
+ for subnet in append_data:
+ ips = []
+ simple = True
+ namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]]
+ cnamelist = []
+ if subnet[1][0][4]:
+ cnamelist.append(subnet[1][0][4].split('.', 1)[0])
+ simple = False
+ appenddata = subnet[1][0]
+ for ip in subnet[1][1:]:
+ if appenddata[2] == ip[2]:
+ namelist.append(ip[3].split('.', 1)[0])
+ if ip[4]:
+ cnamelist.append(ip[4].split('.', 1)[0])
+ simple = False
+ appenddata = ip
+ else:
+ if appenddata[0] == ip[0]:
+ simple = False
+ ips.append((appenddata[2], appenddata[0], Set(namelist),
+ cnamelist, simple, appenddata[1]))
+ appenddata = ip
+ simple = True
+ namelist = [ip[3].split('.', 1)[0]]
+ cnamelist = []
+ if ip[4]:
+ cnamelist.append(ip[4].split('.', 1)[0])
+ simple = False
+ ips.append((appenddata[2], appenddata[0], Set(namelist),
+ cnamelist, simple, appenddata[1]))
+ context = Context({
+ 'subnet': subnet[0],
+ 'ips': ips,
+ })
+ self.filedata['hosts'] += self.templates['hostsapp'].render(context)
+ self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile
+
+ def buildPrinters(self):
+ """The /mcs/etc/printers.data file"""
+ header = """# This file is automatically generated. DO NOT EDIT IT!
+#
+Name Room User Type Notes
+============== ========== ============================== ======================== ====================
+"""
+
+ cursor = connection.cursor()
+ # fetches all the printers from the database
+ cursor.execute("""
+ SELECT printq, location, primary_user, comments
+ FROM hostbase_host
+ WHERE whatami='printer' AND printq <> '' AND status = 'active'
+ ORDER BY printq
+ """)
+ printers = cursor.fetchall()
+
+ printersfile = header
+ for printer in printers:
+ # splits up the printq line and gets the
+ # correct description out of the comments section
+ temp = printer[3].split('\n')
+ for printq in re.split(',[ ]*', printer[0]):
+ if len(temp) > 1:
+ printersfile += ("%-16s%-12s%-32s%-26s%s\n" %
+ (printq, printer[1], printer[2], temp[1], temp[0]))
+ else:
+ printersfile += ("%-16s%-12s%-32s%-26s%s\n" %
+ (printq, printer[1], printer[2], '', printer[3]))
+ self.filedata['printers.data'] = printersfile
+ self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile
+
+ def buildHostsLPD(self):
+ """Creates the /mcs/etc/hosts.lpd file"""
+
+ # this header needs to be changed to be more generic
+ header = """+@machines
++@all-machines
+achilles.ctd.anl.gov
+raven.ops.anl.gov
+seagull.hr.anl.gov
+parrot.ops.anl.gov
+condor.ops.anl.gov
+delphi.esh.anl.gov
+anlcv1.ctd.anl.gov
+anlvms.ctd.anl.gov
+olivia.ctd.anl.gov\n\n"""
+
+ cursor = connection.cursor()
+ cursor.execute("""
+ SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active'
+ ORDER BY hostname""")
+ redmachines = list(cursor.fetchall())
+ cursor.execute("""
+ SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id
+ WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active'
+ """)
+ redmachines.extend(list(cursor.fetchall()))
+ cursor.execute("""
+ SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active'
+ ORDER BY hostname""")
+ winmachines = list(cursor.fetchall())
+ cursor.execute("""
+ SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id
+ WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active'
+ """)
+ winmachines.__add__(list(cursor.fetchall()))
+ hostslpdfile = header
+ for machine in redmachines:
+ hostslpdfile += machine[0] + "\n"
+ hostslpdfile += "\n"
+ for machine in winmachines:
+ hostslpdfile += machine[0] + "\n"
+ self.filedata['hosts.lpd'] = hostslpdfile
+ self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile
+
+ def buildNetgroups(self):
+ """Makes the *-machine files"""
+ header = """###################################################################
+# This file lists hosts in the '%s' machine netgroup, it is
+# automatically generated. DO NOT EDIT THIS FILE!
+#
+# Number of hosts in '%s' machine netgroup: %i
+#\n\n"""
+
+ cursor = connection.cursor()
+ # fetches all the hosts that with valid netgroup entries
+ cursor.execute("""
+ SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h
+ INNER JOIN hostbase_interface i ON h.id = i.host_id)
+ INNER JOIN hostbase_ip p ON i.id = p.interface_id)
+ INNER JOIN hostbase_name n ON p.id = n.ip_id
+ WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active'
+ ORDER BY h.netgroup, h.hostname
+ """)
+ nameslist = cursor.fetchall()
+ # gets the first host and initializes the hash
+ hostdata = nameslist[0]
+ netgroups = {hostdata[2]: [hostdata[0]]}
+ for row in nameslist:
+ # if new netgroup, create it
+ if row[2] not in netgroups:
+ netgroups.update({row[2]: []})
+ # if it belongs in the netgroup and has multiple interfaces, put them in
+ if hostdata[0] == row[0] and row[3]:
+ netgroups[row[2]].append(row[1])
+ hostdata = row
+ # if its a new host, write the old one to the hash
+ elif hostdata[0] != row[0]:
+ netgroups[row[2]].append(row[0])
+ hostdata = row
+
+ for netgroup in netgroups:
+ fileoutput = StringIO()
+ fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup])))
+ for each in netgroups[netgroup]:
+ fileoutput.write(each + "\n")
+ self.filedata['%s-machines' % netgroup] = fileoutput.getvalue()
+ fileoutput.close()
+ self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile
+
+ cursor.execute("""
+ UPDATE hostbase_host SET dirty=0
+ """)
diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
new file mode 100644
index 000000000..29abf5b13
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -0,0 +1,245 @@
+import imp
+import logging
+import sys
+import time
+import traceback
+import Bcfg2.Options
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger('Bcfg2.Plugins.Ldap')
+
+try:
+ import ldap
+except:
+ logger.error("Unable to load ldap module. Is python-ldap installed?")
+ raise ImportError
+
+# time in seconds between retries after failed LDAP connection
+RETRY_DELAY = 5
+# how many times to try reaching the LDAP server if a connection is broken
+# at the very minimum, one retry is needed to handle a restarted LDAP daemon
+RETRY_COUNT = 3
+
+SCOPE_MAP = {
+ "base" : ldap.SCOPE_BASE,
+ "one" : ldap.SCOPE_ONELEVEL,
+ "sub" : ldap.SCOPE_SUBTREE,
+}
+
+LDAP_QUERIES = []
+
+def register_query(query):
+ LDAP_QUERIES.append(query)
+
+class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
+ """
+ Config file for the Ldap plugin
+
+ The config file cannot be 'parsed' in the traditional sense as we would
+ need some serious type checking ugliness to just get the LdapQuery
+ subclasses. The alternative would be to have the user create a list with
+ a predefined name that contains all queries.
+ The approach implemented here is having the user call a registering
+ decorator that updates a global variable in this module.
+ """
+ def __init__(self, filename, fam):
+ self.filename = filename
+ Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename)
+ fam.AddMonitor(self.filename, self)
+
+ def Index(self):
+ """
+ Reregisters the queries in the config file
+
+ The config will take care of actually registering the queries,
+ so we just load it once and don't keep it.
+ """
+ global LDAP_QUERIES
+ LDAP_QUERIES = []
+ imp.load_source("ldap_cfg", self.filename)
+
+class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector):
+ """
+ The Ldap plugin allows adding data from an LDAP server to your metadata.
+ """
+ name = "Ldap"
+ experimental = True
+ debug_flag = False
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.config = ConfigFile(self.data + "/config.py", core.fam)
+
+ def debug_log(self, message, flag = None):
+ if (flag is None) and self.debug_flag or flag:
+ self.logger.error(message)
+
+ def get_additional_data(self, metadata):
+ query = None
+ try:
+ data = {}
+ self.debug_log("LdapPlugin debug: found queries " +
+ str(LDAP_QUERIES))
+ for QueryClass in LDAP_QUERIES:
+ query = QueryClass()
+ if query.is_applicable(metadata):
+ self.debug_log("LdapPlugin debug: processing query '" +
+ query.name + "'")
+ data[query.name] = query.get_result(metadata)
+ else:
+ self.debug_log("LdapPlugin debug: query '" + query.name +
+ "' not applicable to host '" + metadata.hostname + "'")
+ return data
+ except Exception:
+ if hasattr(query, "name"):
+ Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
+ "Exception during processing of query named '" +
+ str(query.name) +
+ "', query results will be empty" +
+ " and may cause bind failures")
+ for line in traceback.format_exception(sys.exc_info()[0],
+ sys.exc_info()[1],
+ sys.exc_info()[2]):
+ Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
+ line.replace("\n", ""))
+ return {}
+
+class LdapConnection(object):
+ """
+ Connection to an LDAP server.
+ """
+ def __init__(self, host = "localhost", port = 389,
+ binddn = None, bindpw = None):
+ self.host = host
+ self.port = port
+ self.binddn = binddn
+ self.bindpw = bindpw
+ self.conn = None
+
+ def __del__(self):
+ if self.conn:
+ self.conn.unbind()
+
+ def init_conn(self):
+ self.conn = ldap.initialize(self.url)
+ if self.binddn is not None and self.bindpw is not None:
+ self.conn.simple_bind_s(self.binddn, self.bindpw)
+
+ def run_query(self, query):
+ result = None
+ for attempt in range(RETRY_COUNT + 1):
+ if attempt >= 1:
+ Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
+ "LDAP server down (retry " + str(attempt) + "/" +
+ str(RETRY_COUNT) + ")")
+ try:
+ if not self.conn:
+ self.init_conn()
+ result = self.conn.search_s(
+ query.base,
+ SCOPE_MAP[query.scope],
+ query.filter,
+ query.attrs,
+ )
+ break
+ except ldap.SERVER_DOWN:
+ self.conn = None
+ time.sleep(RETRY_DELAY)
+ return result
+
+ @property
+ def url(self):
+ return "ldap://" + self.host + ":" + str(self.port)
+
+class LdapQuery(object):
+ """
+ Query referencing an LdapConnection and providing several
+ methods for query manipulation.
+ """
+
+ name = "unknown"
+ base = ""
+ scope = "sub"
+ filter = "(objectClass=*)"
+ attrs = None
+ connection = None
+ result = None
+
+ def __unicode__(self):
+ return "LdapQuery:" + self.name
+
+ def is_applicable(self, metadata):
+ """
+ Overrideable method to determine if the query is to be executed for
+ the given metadata object.
+ Defaults to true.
+ """
+ return True
+
+ def prepare_query(self, metadata):
+ """
+ Overrideable method to alter the query based on metadata.
+ Defaults to doing nothing.
+
+ In most cases, you will do something like
+
+ self.filter = "(cn=" + metadata.hostname + ")"
+
+ here.
+ """
+ pass
+
+ def process_result(self, metadata):
+ """
+ Overrideable method to post-process the query result.
+ Defaults to returning the unaltered result.
+ """
+ return self.result
+
+ def get_result(self, metadata):
+ """
+ Method to handle preparing, executing and processing the query.
+ """
+ if isinstance(self.connection, LdapConnection):
+ self.prepare_query(metadata)
+ self.result = self.connection.run_query(self)
+ self.result = self.process_result(metadata)
+ return self.result
+ else:
+ Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
+ "No valid connection defined for query " + str(self))
+ return None
+
+class LdapSubQuery(LdapQuery):
+ """
+ SubQueries are meant for internal use only and are not added
+ to the metadata object. They are useful for situations where
+ you need to run more than one query to obtain some data.
+ """
+ def prepare_query(self, metadata, **kwargs):
+ """
+ Overrideable method to alter the query based on metadata.
+ Defaults to doing nothing.
+ """
+ pass
+
+ def process_result(self, metadata, **kwargs):
+ """
+ Overrideable method to post-process the query result.
+ Defaults to returning the unaltered result.
+ """
+ return self.result
+
+ def get_result(self, metadata, **kwargs):
+ """
+ Method to handle preparing, executing and processing the query.
+ """
+ if isinstance(self.connection, LdapConnection):
+ self.prepare_query(metadata, **kwargs)
+ self.result = self.connection.run_query(self)
+ return self.process_result(metadata, **kwargs)
+ else:
+ Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " +
+ "No valid connection defined for query " + str(self))
+ return None
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
new file mode 100644
index 000000000..970126b80
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -0,0 +1,866 @@
+"""
+This file stores persistent metadata for the Bcfg2 Configuration Repository.
+"""
+
+import copy
+import fcntl
+import lxml.etree
+import os
+import os.path
+import socket
+import sys
+import time
+
+import Bcfg2.Server.FileMonitor
+import Bcfg2.Server.Plugin
+
+
+def locked(fd):
+ """Aquire a lock on a file"""
+ try:
+ fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError:
+ return True
+ return False
+
+
+class MetadataConsistencyError(Exception):
+ """This error gets raised when metadata is internally inconsistent."""
+ pass
+
+
+class MetadataRuntimeError(Exception):
+ """This error is raised when the metadata engine
+ is called prior to reading enough data.
+ """
+ pass
+
+
+class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked):
+ """Handles xml config files and all XInclude statements"""
+ def __init__(self, metadata, watch_clients, basefile):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self,
+ os.path.join(metadata.data,
+ basefile),
+ metadata.core.fam)
+ self.metadata = metadata
+ self.basefile = basefile
+ self.should_monitor = watch_clients
+ self.data = None
+ self.basedata = None
+ self.basedir = metadata.data
+ self.logger = metadata.logger
+ self.pseudo_monitor = isinstance(metadata.core.fam,
+ Bcfg2.Server.FileMonitor.Pseudo)
+
+ @property
+ def xdata(self):
+ if not self.data:
+ raise MetadataRuntimeError
+ return self.data
+
+ @property
+ def base_xdata(self):
+ if not self.basedata:
+ raise MetadataRuntimeError
+ return self.basedata
+
+ def add_monitor(self, fpath, fname):
+ """Add a fam monitor for an included file"""
+ if self.should_monitor:
+ self.metadata.core.fam.AddMonitor(fpath, self.metadata)
+ self.extras.append(fname)
+
+ def load_xml(self):
+ """Load changes from XML"""
+ try:
+ xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile))
+ except lxml.etree.XMLSyntaxError:
+ self.logger.error('Failed to parse %s' % self.basefile)
+ return
+ self.extras = []
+ self.basedata = copy.copy(xdata)
+ self._follow_xincludes(xdata=xdata)
+ if self.extras:
+ try:
+ xdata.xinclude()
+ except lxml.etree.XIncludeError:
+ self.logger.error("Failed to process XInclude for file %s" %
+ self.basefile)
+ self.data = xdata
+
+ def write(self):
+ """Write changes to xml back to disk."""
+ self.write_xml(os.path.join(self.basedir, self.basefile),
+ self.basedata)
+
+ def write_xml(self, fname, xmltree):
+ """Write changes to xml back to disk."""
+ tmpfile = "%s.new" % fname
+ try:
+ datafile = open(tmpfile, 'w')
+ except IOError:
+ e = sys.exc_info()[1]
+ self.logger.error("Failed to write %s: %s" % (tmpfile, e))
+ raise MetadataRuntimeError
+ # prep data
+ dataroot = xmltree.getroot()
+ newcontents = lxml.etree.tostring(dataroot, pretty_print=True)
+
+ fd = datafile.fileno()
+ while locked(fd) == True:
+ pass
+ try:
+ datafile.write(newcontents)
+ except:
+ fcntl.lockf(fd, fcntl.LOCK_UN)
+ self.logger.error("Metadata: Failed to write new xml data to %s" %
+ tmpfile, exc_info=1)
+ os.unlink(tmpfile)
+ raise MetadataRuntimeError
+ datafile.close()
+
+ # check if clients.xml is a symlink
+ if os.path.islink(fname):
+ fname = os.readlink(fname)
+
+ try:
+ os.rename(tmpfile, fname)
+ except:
+ self.logger.error("Metadata: Failed to rename %s" % tmpfile)
+ raise MetadataRuntimeError
+
+ def find_xml_for_xpath(self, xpath):
+ """Find and load xml file containing the xpath query"""
+ if self.pseudo_monitor:
+ # Reload xml if we don't have a real monitor
+ self.load_xml()
+ cli = self.basedata.xpath(xpath)
+ if len(cli) > 0:
+ return {'filename': os.path.join(self.basedir, self.basefile),
+ 'xmltree': self.basedata,
+ 'xquery': cli}
+ else:
+ """Try to find the data in included files"""
+ for included in self.extras:
+ try:
+ xdata = lxml.etree.parse(os.path.join(self.basedir,
+ included))
+ cli = xdata.xpath(xpath)
+ if len(cli) > 0:
+ return {'filename': os.path.join(self.basedir,
+ included),
+ 'xmltree': xdata,
+ 'xquery': cli}
+ except lxml.etree.XMLSyntaxError:
+ self.logger.error('Failed to parse %s' % (included))
+ return {}
+
+ def HandleEvent(self, event):
+ """Handle fam events"""
+ filename = event.filename.split('/')[-1]
+ if filename in self.extras:
+ if event.code2str() == 'exists':
+ return False
+ elif filename != self.basefile:
+ return False
+ if event.code2str() == 'endExist':
+ return False
+ self.load_xml()
+ return True
+
+
+class ClientMetadata(object):
+ """This object contains client metadata."""
+ def __init__(self, client, profile, groups, bundles,
+ aliases, addresses, categories, uuid, password, query):
+ self.hostname = client
+ self.profile = profile
+ self.bundles = bundles
+ self.aliases = aliases
+ self.addresses = addresses
+ self.groups = groups
+ self.categories = categories
+ self.uuid = uuid
+ self.password = password
+ self.connectors = []
+ self.query = query
+
+ def inGroup(self, group):
+ """Test to see if client is a member of group."""
+ return group in self.groups
+
+ def group_in_category(self, category):
+ for grp in self.query.all_groups_in_category(category):
+ if grp in self.groups:
+ return grp
+ return ''
+
+
+class MetadataQuery(object):
+ def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category):
+ # resolver is set later
+ self.by_name = by_name
+ self.names_by_groups = by_groups
+ self.names_by_profiles = by_profiles
+ self.all_clients = get_clients
+ self.all_groups = all_groups
+ self.all_groups_in_category = all_groups_in_category
+
+ def by_groups(self, groups):
+ return [self.by_name(name) for name in self.names_by_groups(groups)]
+
+ def by_profiles(self, profiles):
+ return [self.by_name(name) for name in self.names_by_profiles(profiles)]
+
+ def all(self):
+ return [self.by_name(name) for name in self.all_clients()]
+
+
+class Metadata(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Metadata,
+ Bcfg2.Server.Plugin.Statistics):
+ """This class contains data for bcfg2 server metadata."""
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ name = "Metadata"
+ sort_order = 500
+
+ def __init__(self, core, datastore, watch_clients=True):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Metadata.__init__(self)
+ Bcfg2.Server.Plugin.Statistics.__init__(self)
+ if watch_clients:
+ try:
+ core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self)
+ core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self)
+ except:
+ print("Unable to add file monitor for groups.xml or clients.xml")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml')
+ self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml')
+ self.states = {}
+ if watch_clients:
+ self.states = {"groups.xml": False,
+ "clients.xml": False}
+ self.addresses = {}
+ self.auth = dict()
+ self.clients = {}
+ self.aliases = {}
+ self.groups = {}
+ self.cgroups = {}
+ self.public = []
+ self.private = []
+ self.profiles = []
+ self.categories = {}
+ self.bad_clients = {}
+ self.uuid = {}
+ self.secure = []
+ self.floating = []
+ self.passwords = {}
+ self.session_cache = {}
+ self.default = None
+ self.pdirty = False
+ self.extra = {'groups.xml': [],
+ 'clients.xml': []}
+ self.password = core.password
+ self.query = MetadataQuery(core.build_metadata,
+ lambda: list(self.clients.keys()),
+ self.get_client_names_by_groups,
+ self.get_client_names_by_profiles,
+ self.get_all_group_names,
+ self.get_all_groups_in_category)
+
+ @classmethod
+ def init_repo(cls, repo, groups, os_selection, clients):
+ path = os.path.join(repo, cls.name)
+ os.makedirs(path)
+ open(os.path.join(repo, "Metadata", "groups.xml"),
+ "w").write(groups % os_selection)
+ open(os.path.join(repo, "Metadata", "clients.xml"),
+ "w").write(clients % socket.getfqdn())
+
+ def get_groups(self):
+ '''return groups xml tree'''
+ groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"))
+ root = groups_tree.getroot()
+ return root
+
+ def _search_xdata(self, tag, name, tree, alias=False):
+ for node in tree.findall("//%s" % tag):
+ if node.get("name") == name:
+ return node
+ elif alias:
+ for child in node:
+ if (child.tag == "Alias" and
+ child.attrib["name"] == name):
+ return node
+ return None
+
+ def search_group(self, group_name, tree):
+ """Find a group."""
+ return self._search_xdata("Group", group_name, tree)
+
+ def search_bundle(self, bundle_name, tree):
+ """Find a bundle."""
+ return self._search_xdata("Bundle", bundle_name, tree)
+
+ def search_client(self, client_name, tree):
+ return self._search_xdata("Client", client_name, tree, alias=True)
+
+ def _add_xdata(self, config, tag, name, attribs=None, alias=False):
+ node = self._search_xdata(tag, name, config.xdata, alias=alias)
+ if node != None:
+ self.logger.error("%s \"%s\" already exists" % (tag, name))
+ raise MetadataConsistencyError
+ element = lxml.etree.SubElement(config.base_xdata.getroot(),
+ tag, name=name)
+ if attribs:
+ for key, val in list(attribs.items()):
+ element.set(key, val)
+ config.write()
+
+ def add_group(self, group_name, attribs):
+ """Add group to groups.xml."""
+ return self._add_xdata(self.groups_xml, "Group", group_name,
+ attribs=attribs)
+
+ def add_bundle(self, bundle_name):
+ """Add bundle to groups.xml."""
+ return self._add_xdata(self.groups_xml, "Bundle", bundle_name)
+
+ def add_client(self, client_name, attribs):
+ """Add client to clients.xml."""
+ return self._add_xdata(self.clients_xml, "Client", client_name,
+ attribs=attribs, alias=True)
+
+ def _update_xdata(self, config, tag, name, attribs, alias=False):
+ node = self._search_xdata(tag, name, config.xdata, alias=alias)
+ if node == None:
+ self.logger.error("%s \"%s\" does not exist" % (tag, name))
+ raise MetadataConsistencyError
+ xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
+ (tag, node.get('name')))
+ if not xdict:
+ self.logger.error("Unexpected error finding %s \"%s\"" %
+ (tag, name))
+ raise MetadataConsistencyError
+ for key, val in list(attribs.items()):
+ xdict['xquery'][0].set(key, val)
+ config.write_xml(xdict['filename'], xdict['xmltree'])
+
+ def update_group(self, group_name, attribs):
+ """Update a groups attributes."""
+ return self._update_xdata(self.groups_xml, "Group", group_name, attribs)
+
+ def update_client(self, client_name, attribs):
+ """Update a clients attributes."""
+ return self._update_xdata(self.clients_xml, "Client", client_name,
+ attribs, alias=True)
+
+ def _remove_xdata(self, config, tag, name, alias=False):
+ node = self._search_xdata(tag, name, config.xdata)
+ if node == None:
+ self.logger.error("%s \"%s\" does not exist" % (tag, name))
+ raise MetadataConsistencyError
+ xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
+ (tag, node.get('name')))
+ if not xdict:
+ self.logger.error("Unexpected error finding %s \"%s\"" %
+ (tag, name))
+ raise MetadataConsistencyError
+ xdict['xquery'][0].getparent().remove(xdict['xquery'][0])
+ self.groups_xml.write_xml(xdict['filename'], xdict['xmltree'])
+
+ def remove_group(self, group_name):
+ """Remove a group."""
+ return self._remove_xdata(self.groups_xml, "Group", group_name)
+
+ def remove_bundle(self, bundle_name):
+ """Remove a bundle."""
+ return self._remove_xdata(self.groups_xml, "Bundle", bundle_name)
+
+ def _handle_clients_xml_event(self, event):
+ xdata = self.clients_xml.xdata
+ self.clients = {}
+ self.aliases = {}
+ self.raliases = {}
+ self.bad_clients = {}
+ self.secure = []
+ self.floating = []
+ self.addresses = {}
+ self.raddresses = {}
+ for client in xdata.findall('.//Client'):
+ clname = client.get('name').lower()
+ if 'address' in client.attrib:
+ caddr = client.get('address')
+ if caddr in self.addresses:
+ self.addresses[caddr].append(clname)
+ else:
+ self.addresses[caddr] = [clname]
+ if clname not in self.raddresses:
+ self.raddresses[clname] = set()
+ self.raddresses[clname].add(caddr)
+ if 'auth' in client.attrib:
+ self.auth[client.get('name')] = client.get('auth',
+ 'cert+password')
+ if 'uuid' in client.attrib:
+ self.uuid[client.get('uuid')] = clname
+ if client.get('secure', 'false') == 'true':
+ self.secure.append(clname)
+ if client.get('location', 'fixed') == 'floating':
+ self.floating.append(clname)
+ if 'password' in client.attrib:
+ self.passwords[clname] = client.get('password')
+
+ self.raliases[clname] = set()
+ for alias in client.findall('Alias'):
+ self.aliases.update({alias.get('name'): clname})
+ self.raliases[clname].add(alias.get('name'))
+ if 'address' not in alias.attrib:
+ continue
+ if alias.get('address') in self.addresses:
+ self.addresses[alias.get('address')].append(clname)
+ else:
+ self.addresses[alias.get('address')] = [clname]
+ if clname not in self.raddresses:
+ self.raddresses[clname] = set()
+ self.raddresses[clname].add(alias.get('address'))
+ self.clients.update({clname: client.get('profile')})
+ self.states['clients.xml'] = True
+
+ def _handle_groups_xml_event(self, event):
+ xdata = self.groups_xml.xdata
+ self.public = []
+ self.private = []
+ self.profiles = []
+ self.groups = {}
+ grouptmp = {}
+ self.categories = {}
+ groupseen = list()
+ for group in xdata.xpath('//Groups/Group'):
+ if group.get('name') not in groupseen:
+ groupseen.append(group.get('name'))
+ else:
+ self.logger.error("Metadata: Group %s defined multiply" %
+ group.get('name'))
+ grouptmp[group.get('name')] = \
+ ([item.get('name') for item in group.findall('./Bundle')],
+ [item.get('name') for item in group.findall('./Group')])
+ grouptmp[group.get('name')][1].append(group.get('name'))
+ if group.get('default', 'false') == 'true':
+ self.default = group.get('name')
+ if group.get('profile', 'false') == 'true':
+ self.profiles.append(group.get('name'))
+ if group.get('public', 'false') == 'true':
+ self.public.append(group.get('name'))
+ elif group.get('public', 'true') == 'false':
+ self.private.append(group.get('name'))
+ if 'category' in group.attrib:
+ self.categories[group.get('name')] = group.get('category')
+
+ for group in grouptmp:
+ # self.groups[group] => (bundles, groups, categories)
+ self.groups[group] = (set(), set(), {})
+ tocheck = [group]
+ group_cat = self.groups[group][2]
+ while tocheck:
+ now = tocheck.pop()
+ self.groups[group][1].add(now)
+ if now in grouptmp:
+ (bundles, groups) = grouptmp[now]
+ for ggg in groups:
+ if ggg in self.groups[group][1]:
+ continue
+ if (ggg not in self.categories or \
+ self.categories[ggg] not in self.groups[group][2]):
+ self.groups[group][1].add(ggg)
+ tocheck.append(ggg)
+ if ggg in self.categories:
+ group_cat[self.categories[ggg]] = ggg
+ elif ggg in self.categories:
+ self.logger.info("Group %s: %s cat-suppressed %s" % \
+ (group,
+ group_cat[self.categories[ggg]],
+ ggg))
+ [self.groups[group][0].add(bund) for bund in bundles]
+ self.states['groups.xml'] = True
+
+ def HandleEvent(self, event):
+ """Handle update events for data files."""
+ if self.clients_xml.HandleEvent(event):
+ self._handle_clients_xml_event(event)
+ elif self.groups_xml.HandleEvent(event):
+ self._handle_groups_xml_event(event)
+
+ if False not in list(self.states.values()):
+ # check that all client groups are real and complete
+ real = list(self.groups.keys())
+ for client in list(self.clients.keys()):
+ if self.clients[client] not in self.profiles:
+ self.logger.error("Client %s set as nonexistent or "
+ "incomplete group %s" %
+ (client, self.clients[client]))
+ self.logger.error("Removing client mapping for %s" % client)
+ self.bad_clients[client] = self.clients[client]
+ del self.clients[client]
+ for bclient in list(self.bad_clients.keys()):
+ if self.bad_clients[bclient] in self.profiles:
+ self.logger.info("Restored profile mapping for client %s" %
+ bclient)
+ self.clients[bclient] = self.bad_clients[bclient]
+ del self.bad_clients[bclient]
+
+ def set_profile(self, client, profile, addresspair):
+ """Set group parameter for provided client."""
+ self.logger.info("Asserting client %s profile to %s" % (client, profile))
+ if False in list(self.states.values()):
+ raise MetadataRuntimeError
+ if profile not in self.public:
+ self.logger.error("Failed to set client %s to private group %s" %
+ (client, profile))
+ raise MetadataConsistencyError
+ if client in self.clients:
+ self.logger.info("Changing %s group from %s to %s" %
+ (client, self.clients[client], profile))
+ self.update_client(client, dict(profile=profile))
+ else:
+ self.logger.info("Creating new client: %s, profile %s" %
+ (client, profile))
+ if addresspair in self.session_cache:
+ # we are working with a uuid'd client
+ self.add_client(self.session_cache[addresspair][1],
+ dict(uuid=client, profile=profile,
+ address=addresspair[0]))
+ else:
+ self.add_client(client, dict(profile=profile))
+ self.clients[client] = profile
+ self.clients_xml.write()
+
+ def resolve_client(self, addresspair, cleanup_cache=False):
+ """Lookup address locally or in DNS to get a hostname."""
+ if addresspair in self.session_cache:
+ # client _was_ cached, so there can be some expired
+ # entries. we need to clean them up to avoid potentially
+ # infinite memory swell
+ cache_ttl = 90
+ if cleanup_cache:
+ # remove entries for this client's IP address with
+ # _any_ port numbers - perhaps a priority queue could
+ # be faster?
+ curtime = time.time()
+ for addrpair in self.session_cache.keys():
+ if addresspair[0] == addrpair[0]:
+ (stamp, _) = self.session_cache[addrpair]
+ if curtime - stamp > cache_ttl:
+ del self.session_cache[addrpair]
+ # return the cached data
+ try:
+ (stamp, uuid) = self.session_cache[addresspair]
+ if time.time() - stamp < cache_ttl:
+ return self.session_cache[addresspair][1]
+ except KeyError:
+ # we cleaned all cached data for this client in cleanup_cache
+ pass
+ address = addresspair[0]
+ if address in self.addresses:
+ if len(self.addresses[address]) != 1:
+ self.logger.error("Address %s has multiple reverse assignments; "
+ "a uuid must be used" % (address))
+ raise MetadataConsistencyError
+ return self.addresses[address][0]
+ try:
+ cname = socket.gethostbyaddr(address)[0].lower()
+ if cname in self.aliases:
+ return self.aliases[cname]
+ return cname
+ except socket.herror:
+ warning = "address resolution error for %s" % (address)
+ self.logger.warning(warning)
+ raise MetadataConsistencyError
+
+ def get_initial_metadata(self, client):
+ """Return the metadata for a given client."""
+ if False in list(self.states.values()):
+ raise MetadataRuntimeError
+ client = client.lower()
+ if client in self.aliases:
+ client = self.aliases[client]
+ if client in self.clients:
+ profile = self.clients[client]
+ (bundles, groups, categories) = self.groups[profile]
+ else:
+ if self.default == None:
+ self.logger.error("Cannot set group for client %s; "
+ "no default group set" % client)
+ raise MetadataConsistencyError
+ self.set_profile(client, self.default, (None, None))
+ profile = self.default
+ [bundles, groups, categories] = self.groups[self.default]
+ aliases = self.raliases.get(client, set())
+ addresses = self.raddresses.get(client, set())
+ newgroups = set(groups)
+ newbundles = set(bundles)
+ newcategories = {}
+ newcategories.update(categories)
+ if client in self.passwords:
+ password = self.passwords[client]
+ else:
+ password = None
+ uuids = [item for item, value in list(self.uuid.items())
+ if value == client]
+ if uuids:
+ uuid = uuids[0]
+ else:
+ uuid = None
+ for group in self.cgroups.get(client, []):
+ if group in self.groups:
+ nbundles, ngroups, ncategories = self.groups[group]
+ else:
+ nbundles, ngroups, ncategories = ([], [group], {})
+ [newbundles.add(b) for b in nbundles if b not in newbundles]
+ [newgroups.add(g) for g in ngroups if g not in newgroups]
+ newcategories.update(ncategories)
+ return ClientMetadata(client, profile, newgroups, newbundles, aliases,
+ addresses, newcategories, uuid, password,
+ self.query)
+
+ def get_all_group_names(self):
+ all_groups = set()
+ [all_groups.update(g[1]) for g in list(self.groups.values())]
+ return all_groups
+
+ def get_all_groups_in_category(self, category):
+ all_groups = set()
+ [all_groups.add(g) for g in self.categories \
+ if self.categories[g] == category]
+ return all_groups
+
+ def get_client_names_by_profiles(self, profiles):
+ return [client for client, profile in list(self.clients.items()) \
+ if profile in profiles]
+
+ def get_client_names_by_groups(self, groups):
+ mdata = [self.core.build_metadata(client)
+ for client in list(self.clients.keys())]
+ return [md.hostname for md in mdata if md.groups.issuperset(groups)]
+
+ def merge_additional_groups(self, imd, groups):
+ for group in groups:
+ if (group in self.categories and
+ self.categories[group] in imd.categories):
+ continue
+ newbundles, newgroups, _ = self.groups.get(group,
+ (list(),
+ [group],
+ dict()))
+ for newbundle in newbundles:
+ if newbundle not in imd.bundles:
+ imd.bundles.add(newbundle)
+ for newgroup in newgroups:
+ if newgroup not in imd.groups:
+ if (newgroup in self.categories and
+ self.categories[newgroup] in imd.categories):
+ continue
+ if newgroup in self.private:
+ self.logger.error("Refusing to add dynamic membership "
+ "in private group %s for client %s" %
+ (newgroup, imd.hostname))
+ continue
+ imd.groups.add(newgroup)
+
+ def merge_additional_data(self, imd, source, data):
+ if not hasattr(imd, source):
+ setattr(imd, source, data)
+ imd.connectors.append(source)
+
+ def validate_client_address(self, client, addresspair):
+ """Check address against client."""
+ address = addresspair[0]
+ if client in self.floating:
+ self.debug_log("Client %s is floating" % client)
+ return True
+ if address in self.addresses:
+ if client in self.addresses[address]:
+ self.debug_log("Client %s matches address %s" %
+ (client, address))
+ return True
+ else:
+ self.logger.error("Got request for non-float client %s from %s" %
+ (client, address))
+ return False
+ resolved = self.resolve_client(addresspair)
+ if resolved.lower() == client.lower():
+ return True
+ else:
+ self.logger.error("Got request for %s from incorrect address %s" %
+ (client, address))
+ self.logger.error("Resolved to %s" % resolved)
+ return False
+
+ def AuthenticateConnection(self, cert, user, password, address):
+ """This function checks auth creds."""
+ if cert:
+ id_method = 'cert'
+ certinfo = dict([x[0] for x in cert['subject']])
+ # look at cert.cN
+ client = certinfo['commonName']
+ self.debug_log("Got cN %s; using as client name" % client)
+ auth_type = self.auth.get(client, 'cert+password')
+ elif user == 'root':
+ id_method = 'address'
+ try:
+ client = self.resolve_client(address)
+ except MetadataConsistencyError:
+ self.logger.error("Client %s failed to resolve; metadata problem"
+ % address[0])
+ return False
+ else:
+ id_method = 'uuid'
+ # user maps to client
+ if user not in self.uuid:
+ client = user
+ self.uuid[user] = user
+ else:
+ client = self.uuid[user]
+
+ # we have the client name
+ self.debug_log("Authenticating client %s" % client)
+
+ # next we validate the address
+ if id_method == 'uuid':
+ addr_is_valid = True
+ else:
+ addr_is_valid = self.validate_client_address(client, address)
+
+ if not addr_is_valid:
+ return False
+
+ if id_method == 'cert' and auth_type != 'cert+password':
+ # remember the cert-derived client name for this connection
+ if client in self.floating:
+ self.session_cache[address] = (time.time(), client)
+ # we are done if cert+password not required
+ return True
+
+ if client not in self.passwords:
+ if client in self.secure:
+ self.logger.error("Client %s in secure mode but has no password"
+ % address[0])
+ return False
+ if password != self.password:
+ self.logger.error("Client %s used incorrect global password" %
+ address[0])
+ return False
+ if client not in self.secure:
+ if client in self.passwords:
+ plist = [self.password, self.passwords[client]]
+ else:
+ plist = [self.password]
+ if password not in plist:
+ self.logger.error("Client %s failed to use either allowed "
+ "password" % address[0])
+ return False
+ else:
+ # client in secure mode and has a client password
+ if password != self.passwords[client]:
+ self.logger.error("Client %s failed to use client password in "
+ "secure mode" % address[0])
+ return False
+ # populate the session cache
+ if user.decode('utf-8') != 'root':
+ self.session_cache[address] = (time.time(), client)
+ return True
+
+ def process_statistics(self, meta, _):
+ """Hook into statistics interface to toggle clients in bootstrap mode."""
+ client = meta.hostname
+ if client in self.auth and self.auth[client] == 'bootstrap':
+ self.update_client(client, dict(auth='cert'))
+
+ def viz(self, hosts, bundles, key, only_client, colors):
+ """Admin mode viz support."""
+ if only_client:
+ clientmeta = self.core.build_metadata(only_client)
+
+ def include_client(client):
+ return not only_client or client != only_client
+
+ def include_bundle(bundle):
+ return not only_client or bundle in clientmeta.bundles
+
+ def include_group(group):
+ return not only_client or group in clientmeta.groups
+
+ groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"))
+ try:
+ groups_tree.xinclude()
+ except lxml.etree.XIncludeError:
+ self.logger.error("Failed to process XInclude for file %s: %s" %
+ (dest, sys.exc_info()[1]))
+ groups = groups_tree.getroot()
+ categories = {'default': 'grey83'}
+ viz_str = []
+ egroups = groups.findall("Group") + groups.findall('.//Groups/Group')
+ for group in egroups:
+ if not group.get('category') in categories:
+ categories[group.get('category')] = colors.pop()
+ group.set('color', categories[group.get('category')])
+ if None in categories:
+ del categories[None]
+ if hosts:
+ instances = {}
+ clients = self.clients
+ for client, profile in list(clients.items()):
+ if include_client(client):
+ continue
+ if profile in instances:
+ instances[profile].append(client)
+ else:
+ instances[profile] = [client]
+ for profile, clist in list(instances.items()):
+ clist.sort()
+ viz_str.append('"%s-instances" [ label="%s", shape="record" ];' %
+ (profile, '|'.join(clist)))
+ viz_str.append('"%s-instances" -> "group-%s";' %
+ (profile, profile))
+ if bundles:
+ bundles = []
+ [bundles.append(bund.get('name')) \
+ for bund in groups.findall('.//Bundle') \
+ if bund.get('name') not in bundles \
+ and include_bundle(bund.get('name'))]
+ bundles.sort()
+ for bundle in bundles:
+ viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' %
+ (bundle, bundle))
+ gseen = []
+ for group in egroups:
+ if group.get('profile', 'false') == 'true':
+ style = "filled, bold"
+ else:
+ style = "filled"
+ gseen.append(group.get('name'))
+ if include_group(group.get('name')):
+ viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' %
+ (group.get('name'), group.get('name'), style,
+ group.get('color')))
+ if bundles:
+ for bundle in group.findall('Bundle'):
+ viz_str.append('"group-%s" -> "bundle-%s";' %
+ (group.get('name'), bundle.get('name')))
+ gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];'
+ for group in egroups:
+ for parent in group.findall('Group'):
+ if parent.get('name') not in gseen and include_group(parent.get('name')):
+ viz_str.append(gfmt % (parent.get('name'),
+ parent.get('name')))
+ gseen.append(parent.get("name"))
+ if include_group(group.get('name')):
+ viz_str.append('"group-%s" -> "group-%s";' %
+ (group.get('name'), parent.get('name')))
+ if key:
+ for category in categories:
+ viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' %
+ (category, category, categories[category]))
+ return "\n".join("\t" + s for s in viz_str)
diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
new file mode 100644
index 000000000..4dbd57d16
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
@@ -0,0 +1,151 @@
+'''This module implements a Nagios configuration generator'''
+
+import os
+import re
+import sys
+import glob
+import socket
+import logging
+import lxml.etree
+
+import Bcfg2.Server.Plugin
+
+LOGGER = logging.getLogger('Bcfg2.Plugins.NagiosGen')
+
+line_fmt = '\t%-32s %s'
+
+class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked,
+ Bcfg2.Server.Plugin.StructFile):
+ def __init__(self, filename, fam):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam)
+ Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
+
+
+class NagiosGen(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Generator):
+ """NagiosGen is a Bcfg2 plugin that dynamically generates
+ Nagios configuration file based on Bcfg2 data.
+ """
+ name = 'NagiosGen'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Generator.__init__(self)
+ self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'),
+ core.fam)
+ self.Entries = {'Path':
+ {'/etc/nagiosgen.status': self.createhostconfig,
+ '/etc/nagios/nagiosgen.cfg': self.createserverconfig}}
+
+ self.client_attrib = {'encoding': 'ascii',
+ 'owner': 'root',
+ 'group': 'root',
+ 'type': 'file',
+ 'perms': '0400'}
+ self.server_attrib = {'encoding': 'ascii',
+ 'owner': 'nagios',
+ 'group': 'nagios',
+ 'type': 'file',
+ 'perms': '0440'}
+
+ def createhostconfig(self, entry, metadata):
+ """Build host specific configuration file."""
+ host_address = socket.gethostbyname(metadata.hostname)
+ host_groups = [grp for grp in metadata.groups
+ if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))]
+ host_config = ['define host {',
+ line_fmt % ('host_name', metadata.hostname),
+ line_fmt % ('alias', metadata.hostname),
+ line_fmt % ('address', host_address)]
+
+ if host_groups:
+ host_config.append(line_fmt % ("hostgroups",
+ ",".join(host_groups)))
+
+ # read the old-style Properties config, but emit a warning.
+ xtra = dict()
+ props = None
+ if (hasattr(metadata, 'Properties') and
+ 'NagiosGen.xml' in metadata.Properties):
+ props = metadata.Properties['NagiosGen.xml'].data
+ if props is not None:
+ LOGGER.warn("Parsing deprecated Properties/NagiosGen.xml. "
+ "Update to the new-style config with "
+ "nagiosgen-convert.py.")
+ xtra = dict((el.tag, el.text)
+ for el in props.find(metadata.hostname))
+ # hold off on parsing the defaults until we've checked for
+ # a new-style config
+
+ # read the old-style parents.xml, but emit a warning
+ pfile = os.path.join(self.data, "parents.xml")
+ if os.path.exists(pfile):
+ LOGGER.warn("Parsing deprecated NagiosGen/parents.xml. "
+ "Update to the new-style config with "
+ "nagiosgen-convert.py.")
+ parents = lxml.etree.parse(pfile)
+ for el in parents.xpath("//Depend[@name='%s']" % metadata.hostname):
+ if 'parent' in xtra:
+ xtra['parent'] += "," + el.get("on")
+ else:
+ xtra['parent'] = el.get("on")
+
+ # read the new-style config and overwrite the old-style config
+ for el in self.config.Match(metadata):
+ if el.tag == 'Option':
+ xtra[el.get("name")] = el.text
+
+ # if we haven't found anything in the new- or old-style
+ # configs, finally read defaults from old-style config
+ if not xtra and props is not None:
+ xtra = dict((el.tag, el.text) for el in props.find('default'))
+
+ if xtra:
+ host_config.extend([line_fmt % (opt, val)
+ for opt, val in list(xtra.items())])
+ else:
+ host_config.append(line_fmt % ('use', 'default'))
+
+ host_config.append('}')
+ entry.text = "%s\n" % "\n".join(host_config)
+ [entry.attrib.__setitem__(key, value)
+ for (key, value) in list(self.client_attrib.items())]
+ try:
+ fileh = open("%s/%s-host.cfg" %
+ (self.data, metadata.hostname), 'w')
+ fileh.write(entry.text)
+ fileh.close()
+ except OSError:
+ ioerr = sys.exc_info()[1]
+ LOGGER.error("Failed to write %s/%s-host.cfg" %
+ (self.data, metadata.hostname))
+ LOGGER.error(ioerr)
+
+ def createserverconfig(self, entry, _):
+ """Build monolithic server configuration file."""
+ host_configs = glob.glob('%s/*-host.cfg' % self.data)
+ group_configs = glob.glob('%s/*-group.cfg' % self.data)
+ host_data = []
+ group_data = []
+ for host in host_configs:
+ host_data.append(open(host, 'r').read())
+
+ for group in group_configs:
+ group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group)
+ if "\n".join(host_data).find(group_name) != -1:
+ groupfile = open(group, 'r')
+ group_data.append(groupfile.read())
+ groupfile.close()
+
+ entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data))
+ [entry.attrib.__setitem__(key, value)
+ for (key, value) in list(self.server_attrib.items())]
+ try:
+ fileh = open("%s/nagiosgen.cfg" % self.data, 'w')
+ fileh.write(entry.text)
+ fileh.close()
+ except OSError:
+ ioerr = sys.exc_info()[1]
+ LOGGER.error("Failed to write %s/nagiosgen.cfg" % self.data)
+ LOGGER.error(ioerr)
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
new file mode 100644
index 000000000..5fff20d98
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -0,0 +1,92 @@
+import lxml.etree
+import os
+
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Ohai')
+
+import Bcfg2.Server.Plugin
+
+try:
+ import json
+except:
+ # FIXME: can be removed when server prereq is >= python 2.6
+ # necessary for clients without the in-tree json module
+ try:
+ import simplejson as json
+ except:
+ logger.error("Unable to load any json modules. Make sure "
+ "python-simplejson is installed.")
+ raise ImportError
+
+
+probecode = """#!/bin/sh
+
+export PATH=$PATH:/sbin:/usr/sbin
+
+if type ohai >& /dev/null; then
+ ohai
+else
+ # an empty dict, so "'foo' in metadata.Ohai" tests succeed
+ echo '{}'
+fi
+"""
+
+class OhaiCache(object):
+ def __init__(self, dirname):
+ self.dirname = dirname
+ self.cache = dict()
+
+ def __setitem__(self, item, value):
+ if value == None:
+ # simply return if the client returned nothing
+ return
+ self.cache[item] = json.loads(value)
+ file("%s/%s.json" % (self.dirname, item), 'w').write(value)
+
+ def __getitem__(self, item):
+ if item not in self.cache:
+ try:
+ data = open("%s/%s.json" % (self.dirname, item)).read()
+ except:
+ raise KeyError(item)
+ self.cache[item] = json.loads(data)
+ return self.cache[item]
+
+ def __iter__(self):
+ data = list(self.cache.keys())
+ data.extend([x[:-5] for x in os.listdir(self.dirname)])
+ return data.__iter__()
+
+
+class Ohai(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Probing,
+ Bcfg2.Server.Plugin.Connector):
+ """The Ohai plugin is used to detect information
+ about the client operating system.
+ """
+ name = 'Ohai'
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Probing.__init__(self)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai',
+ interpreter='/bin/sh')
+ self.probe.text = probecode
+ try:
+ os.stat(self.data)
+ except:
+ os.makedirs(self.data)
+ self.cache = OhaiCache(self.data)
+
+ def GetProbes(self, meta, force=False):
+ return [self.probe]
+
+ def ReceiveData(self, meta, datalist):
+ self.cache[meta.hostname] = datalist[0].text
+
+ def get_additional_data(self, meta):
+ if meta.hostname in self.cache:
+ return self.cache[meta.hostname]
+ return dict()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
new file mode 100644
index 000000000..cbe2b4f2c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -0,0 +1,141 @@
+import re
+import gzip
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import Source
+from Bcfg2.Bcfg2Py3k import cPickle, file
+
+class AptCollection(Collection):
+ def get_group(self, group):
+ self.logger.warning("Packages: Package groups are not supported by APT")
+ return []
+
+class AptSource(Source):
+ basegroups = ['apt', 'debian', 'ubuntu', 'nexenta']
+ ptype = 'deb'
+ essentialpkgs = set()
+
+ def __init__(self, basepath, xsource, config):
+ Source.__init__(self, basepath, xsource, config)
+ self.pkgnames = set()
+
+ self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
+ 'version': self.version,
+ 'components': self.components, 'arches': self.arches}]
+
+ def save_state(self):
+ cache = file(self.cachefile, 'wb')
+ cPickle.dump((self.pkgnames, self.deps, self.provides),
+ cache, 2)
+ cache.close()
+
+ def load_state(self):
+ data = file(self.cachefile)
+ self.pkgnames, self.deps, self.provides = cPickle.load(data)
+
+ def filter_unknown(self, unknown):
+ filtered = set([u for u in unknown if u.startswith('choice')])
+ unknown.difference_update(filtered)
+
+ def get_urls(self):
+ if not self.rawurl:
+ rv = []
+ for part in self.components:
+ for arch in self.arches:
+ rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
+ (self.url, self.version, part, arch))
+ return rv
+ else:
+ return ["%sPackages.gz" % self.rawurl]
+ urls = property(get_urls)
+
+ def read_files(self):
+ bdeps = dict()
+ bprov = dict()
+ depfnames = ['Depends', 'Pre-Depends']
+ if self.recommended:
+ depfnames.append('Recommends')
+ for fname in self.files:
+ if not self.rawurl:
+ barch = [x
+ for x in fname.split('@')
+ if x.startswith('binary-')][0][7:]
+ else:
+ # RawURL entries assume that they only have one <Arch></Arch>
+ # element and that it is the architecture of the source.
+ barch = self.arches[0]
+ if barch not in bdeps:
+ bdeps[barch] = dict()
+ bprov[barch] = dict()
+ try:
+ reader = gzip.GzipFile(fname)
+ except:
+ self.logger.error("Packages: Failed to read file %s" % fname)
+ raise
+ for line in reader.readlines():
+ words = str(line.strip()).split(':', 1)
+ if words[0] == 'Package':
+ pkgname = words[1].strip().rstrip()
+ self.pkgnames.add(pkgname)
+ bdeps[barch][pkgname] = []
+ elif words[0] == 'Essential' and self.essential:
+ self.essentialpkgs.add(pkgname)
+ elif words[0] in depfnames:
+ vindex = 0
+ for dep in words[1].split(','):
+ if '|' in dep:
+ cdeps = [re.sub('\s+', '',
+ re.sub('\(.*\)', '', cdep))
+ for cdep in dep.split('|')]
+ dyn_dname = "choice-%s-%s-%s" % (pkgname,
+ barch,
+ vindex)
+ vindex += 1
+ bdeps[barch][pkgname].append(dyn_dname)
+ bprov[barch][dyn_dname] = set(cdeps)
+ else:
+ raw_dep = re.sub('\(.*\)', '', dep)
+ raw_dep = raw_dep.rstrip().strip()
+ bdeps[barch][pkgname].append(raw_dep)
+ elif words[0] == 'Provides':
+ for pkg in words[1].split(','):
+ dname = pkg.rstrip().strip()
+ if dname not in bprov[barch]:
+ bprov[barch][dname] = set()
+ bprov[barch][dname].add(pkgname)
+
+ self.deps['global'] = dict()
+ self.provides['global'] = dict()
+ for barch in bdeps:
+ self.deps[barch] = dict()
+ self.provides[barch] = dict()
+ for pkgname in self.pkgnames:
+ pset = set()
+ for barch in bdeps:
+ if pkgname not in bdeps[barch]:
+ bdeps[barch][pkgname] = []
+ pset.add(tuple(bdeps[barch][pkgname]))
+ if len(pset) == 1:
+ self.deps['global'][pkgname] = pset.pop()
+ else:
+ for barch in bdeps:
+ self.deps[barch][pkgname] = bdeps[barch][pkgname]
+ provided = set()
+ for bprovided in list(bprov.values()):
+ provided.update(set(bprovided))
+ for prov in provided:
+ prset = set()
+ for barch in bprov:
+ if prov not in bprov[barch]:
+ continue
+ prset.add(tuple(bprov[barch].get(prov, ())))
+ if len(prset) == 1:
+ self.provides['global'][prov] = prset.pop()
+ else:
+ for barch in bprov:
+ self.provides[barch][prov] = bprov[barch].get(prov, ())
+ self.save_state()
+
+ def is_package(self, _, pkg):
+ return (pkg in self.pkgnames and
+ pkg not in self.blacklist and
+ (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
new file mode 100644
index 000000000..959dac03b
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -0,0 +1,349 @@
+import copy
+import logging
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger(__name__)
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+
+# we have to cache Collection objects so that calling Packages.Refresh
+# or .Reload can tell the collection objects to clean up their cache,
+# but we don't actually use the cache to return a Collection object
+# when one is requested, because that prevents new machines from
+# working, since a Collection object gets created by
+# get_additional_data(), which is called for all clients at server
+# startup. (It would also prevent machines that change groups from
+# working properly; e.g., if you reinstall a machine with a new OS,
+# then returning a cached Collection object would give the wrong
+# sources to that client.)
+collections = dict()
+
+class Collection(Bcfg2.Server.Plugin.Debuggable):
+ def __init__(self, metadata, sources, basepath, debug=False):
+ """ don't call this directly; use the factory function """
+ Bcfg2.Server.Plugin.Debuggable.__init__(self)
+ self.debug_flag = debug
+ self.metadata = metadata
+ self.sources = sources
+ self.basepath = basepath
+ self.virt_pkgs = dict()
+
+ try:
+ self.config = sources[0].config
+ self.cachepath = sources[0].basepath
+ self.ptype = sources[0].ptype
+ except IndexError:
+ self.config = None
+ self.cachepath = None
+ self.ptype = "unknown"
+
+ self.cachefile = None
+
+ @property
+ def cachekey(self):
+ return md5(self.get_config()).hexdigest()
+
+ def get_config(self):
+ self.logger.error("Packages: Cannot generate config for host with "
+ "multiple source types (%s)" % self.metadata.hostname)
+ return ""
+
+ def get_relevant_groups(self):
+ groups = []
+ for source in self.sources:
+ groups.extend(source.get_relevant_groups(self.metadata))
+ return sorted(list(set(groups)))
+
+ @property
+ def basegroups(self):
+ groups = set()
+ for source in self.sources:
+ groups.update(source.basegroups)
+ return list(groups)
+
+ @property
+ def cachefiles(self):
+ cachefiles = set([self.cachefile])
+ for source in self.sources:
+ cachefiles.add(source.cachefile)
+ return list(cachefiles)
+
+ def get_group(self, group, ptype=None):
+ for source in self.sources:
+ pkgs = source.get_group(self.metadata, group, ptype=ptype)
+ if pkgs:
+ return pkgs
+ self.logger.warning("Packages: '%s' is not a valid group" % group)
+ return []
+
+ def is_package(self, package):
+ for source in self.sources:
+ if source.is_package(self.metadata, package):
+ return True
+ return False
+
+ def is_virtual_package(self, package):
+ for source in self.sources:
+ if source.is_virtual_package(self.metadata, package):
+ return True
+ return False
+
+ def get_deps(self, package):
+ for source in self.sources:
+ if source.is_package(self.metadata, package):
+ return source.get_deps(self.metadata, package)
+ return []
+
+ def get_essential(self):
+ essential = set()
+ for source in self.sources:
+ essential |= source.essentialpkgs
+ return essential
+
+ def get_provides(self, package):
+ for source in self.sources:
+ providers = source.get_provides(self.metadata, package)
+ if providers:
+ return providers
+ return []
+
+ def get_vpkgs(self):
+ """ get virtual packages """
+ vpkgs = dict()
+ for source in self.sources:
+ s_vpkgs = source.get_vpkgs(self.metadata)
+ for name, prov_set in list(s_vpkgs.items()):
+ if name not in vpkgs:
+ vpkgs[name] = set(prov_set)
+ else:
+ vpkgs[name].update(prov_set)
+ return vpkgs
+
+ def filter_unknown(self, unknown):
+ for source in self.sources:
+ source.filter_unknown(unknown)
+
+ def magic_groups_match(self):
+ for source in self.sources:
+ if source.magic_groups_match(self.metadata):
+ return True
+
+ def build_extra_structures(self, independent):
+ pass
+
+ def get_additional_data(self):
+ sdata = []
+ for source in self.sources:
+ sdata.extend(copy.deepcopy(source.url_map))
+ return sdata
+
+ def setup_data(self, force_update=False):
+ """ do any collection-level data setup tasks """
+ pass
+
+ def complete(self, packagelist):
+ '''Build the transitive closure of all package dependencies
+
+ Arguments:
+ packageslist - set of package names
+ returns => (set(packages), set(unsatisfied requirements))
+ '''
+
+ # setup vpkg cache
+ pgrps = tuple(self.get_relevant_groups())
+ if pgrps not in self.virt_pkgs:
+ self.virt_pkgs[pgrps] = self.get_vpkgs()
+ vpkg_cache = self.virt_pkgs[pgrps]
+
+ # unclassified is set of unsatisfied requirements (may be pkg
+ # for vpkg)
+ unclassified = set(packagelist)
+ vpkgs = set()
+ both = set()
+ pkgs = set(packagelist)
+
+ packages = set()
+ examined = set()
+ unknown = set()
+
+ final_pass = False
+ really_done = False
+ # do while unclassified or vpkgs or both or pkgs
+ while unclassified or pkgs or both or final_pass:
+ if really_done:
+ break
+ if len(unclassified) + len(pkgs) + len(both) == 0:
+ # one more pass then exit
+ really_done = True
+
+ while unclassified:
+ current = unclassified.pop()
+ examined.add(current)
+ is_pkg = False
+ if self.is_package(current):
+ is_pkg = True
+
+ is_vpkg = current in vpkg_cache
+
+ if is_pkg and is_vpkg:
+ both.add(current)
+ elif is_pkg and not is_vpkg:
+ pkgs.add(current)
+ elif is_vpkg and not is_pkg:
+ vpkgs.add(current)
+ elif not is_vpkg and not is_pkg:
+ unknown.add(current)
+
+ while pkgs:
+ # direct packages; current can be added, and all deps
+ # should be resolved
+ current = pkgs.pop()
+ self.debug_log("Packages: handling package requirement %s" %
+ current)
+ packages.add(current)
+ deps = self.get_deps(current)
+ newdeps = set(deps).difference(examined)
+ if newdeps:
+ self.debug_log("Packages: Package %s added requirements %s"
+ % (current, newdeps))
+ unclassified.update(newdeps)
+
+ satisfied_vpkgs = set()
+ for current in vpkgs:
+ # virtual dependencies, satisfied if one of N in the
+ # config, or can be forced if only one provider
+ if len(vpkg_cache[current]) == 1:
+ self.debug_log("Packages: requirement %s satisfied by %s" %
+ (current, vpkg_cache[current]))
+ unclassified.update(vpkg_cache[current].difference(examined))
+ satisfied_vpkgs.add(current)
+ else:
+ satisfiers = [item for item in vpkg_cache[current]
+ if item in packages]
+ self.debug_log("Packages: requirement %s satisfied by %s" %
+ (current, satisfiers))
+ satisfied_vpkgs.add(current)
+ vpkgs.difference_update(satisfied_vpkgs)
+
+ satisfied_both = set()
+ for current in both:
+ # packages that are both have virtual providers as
+ # well as a package with that name. allow use of virt
+ # through explicit specification, then fall back to
+ # forcing current on last pass
+ satisfiers = [item for item in vpkg_cache[current]
+ if item in packages]
+ if satisfiers:
+ self.debug_log("Packages: requirement %s satisfied by %s" %
+ (current, satisfiers))
+ satisfied_both.add(current)
+ elif current in packagelist or final_pass:
+ pkgs.add(current)
+ satisfied_both.add(current)
+ both.difference_update(satisfied_both)
+
+ if len(unclassified) + len(pkgs) == 0:
+ final_pass = True
+ else:
+ final_pass = False
+
+ self.filter_unknown(unknown)
+
+ return packages, unknown
+
+ def __len__(self):
+ return len(self.sources)
+
+ def __getitem__(self, item):
+ return self.sources[item]
+
+ def __setitem__(self, item, value):
+ self.sources[item] = value
+
+ def __delitem__(self, item):
+ del self.sources[item]
+
+ def append(self, item):
+ self.sources.append(item)
+
+ def count(self):
+ return self.sources.count()
+
+ def index(self, item):
+ return self.sources.index(item)
+
+ def extend(self, items):
+ self.sources.extend(items)
+
+ def insert(self, index, item):
+ self.sources.insert(index, item)
+
+ def pop(self, index=None):
+ self.sources.pop(index)
+
+ def remove(self, item):
+ self.sources.remove(item)
+
+ def sort(self, cmp=None, key=None, reverse=False):
+ self.sources.sort(cmp, key, reverse)
+
+def clear_cache():
+ global collections
+ collections = dict()
+
+def factory(metadata, sources, basepath, debug=False):
+ global collections
+
+ if not sources.loaded:
+ # if sources.xml has not received a FAM event yet, defer;
+ # instantiate a dummy Collection object
+ return Collection(metadata, [], basepath)
+
+ sclasses = set()
+ relevant = list()
+
+ for source in sources:
+ if source.applies(metadata):
+ relevant.append(source)
+ sclasses.update([source.__class__])
+
+ if len(sclasses) > 1:
+ logger.warning("Packages: Multiple source types found for %s: %s" %
+ ",".join([s.__name__ for s in sclasses]))
+ cclass = Collection
+ elif len(sclasses) == 0:
+ # you'd think this should be a warning, but it happens all the
+ # freaking time if you have a) machines in your clients.xml
+ # that do not have the proper groups set up yet (e.g., if you
+ # have multiple Bcfg2 servers and Packages-relevant groups set
+ # by probes); and b) templates that query all or multiple
+ # machines (e.g., with metadata.query.all_clients())
+ if debug:
+ logger.error("Packages: No sources found for %s" %
+ metadata.hostname)
+ cclass = Collection
+ else:
+ stype = sclasses.pop().__name__.replace("Source", "")
+ try:
+ module = \
+ getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
+ stype.title()).Server.Plugins.Packages,
+ stype.title())
+ cclass = getattr(module, "%sCollection" % stype.title())
+ except ImportError:
+ logger.error("Packages: Unknown source type %s" % stype)
+ except AttributeError:
+ logger.warning("Packages: No collection class found for %s sources"
+ % stype)
+
+ if debug:
+ logger.error("Packages: Using %s for Collection of sources for %s" %
+ (cclass.__name__, metadata.hostname))
+
+ collection = cclass(metadata, relevant, basepath, debug=debug)
+ collections[metadata.hostname] = collection
+ return collection
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
new file mode 100644
index 000000000..99a090739
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -0,0 +1,119 @@
+import gzip
+import tarfile
+from Bcfg2.Bcfg2Py3k import cPickle, file
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import Source
+
+class PacCollection(Collection):
+ def get_group(self, group):
+ self.logger.warning("Packages: Package groups are not supported by Pacman")
+ return []
+
+class PacSource(Source):
+ basegroups = ['arch', 'parabola']
+ ptype = 'pacman'
+
+ def __init__(self, basepath, xsource, config):
+ Source.__init__(self, basepath, xsource, config)
+ self.pkgnames = set()
+
+ self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
+ 'version': self.version,
+ 'components': self.components, 'arches': self.arches}]
+
+ def save_state(self):
+ cache = file(self.cachefile, 'wb')
+ cPickle.dump((self.pkgnames, self.deps, self.provides),
+ cache, 2)
+ cache.close()
+
+ def load_state(self):
+ data = file(self.cachefile)
+ self.pkgnames, self.deps, self.provides = cPickle.load(data)
+
+ def filter_unknown(self, unknown):
+ filtered = set([u for u in unknown if u.startswith('choice')])
+ unknown.difference_update(filtered)
+
+ def get_urls(self):
+ if not self.rawurl:
+ rv = []
+ for part in self.components:
+ for arch in self.arches:
+ rv.append("%s%s/os/%s/%s.db.tar.gz" %
+ (self.url, part, arch, part))
+ return rv
+ else:
+ raise Exception("PacSource : RAWUrl not supported (yet)")
+ urls = property(get_urls)
+
+ def read_files(self):
+ bdeps = dict()
+ bprov = dict()
+
+ depfnames = ['Depends', 'Pre-Depends']
+ if self.recommended:
+ depfnames.append('Recommends')
+
+ for fname in self.files:
+ if not self.rawurl:
+ barch = [x for x in fname.split('@') if x in self.arches][0]
+ else:
+ # RawURL entries assume that they only have one <Arch></Arch>
+ # element and that it is the architecture of the source.
+ barch = self.arches[0]
+
+ if barch not in bdeps:
+ bdeps[barch] = dict()
+ bprov[barch] = dict()
+ try:
+ self.debug_log("Packages: try to read %s" % fname)
+ tar = tarfile.open(fname, "r")
+ reader = gzip.GzipFile(fname)
+ except:
+ self.logger.error("Packages: Failed to read file %s" % fname)
+ raise
+
+ for tarinfo in tar:
+ if tarinfo.isdir():
+ self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0])
+ self.debug_log("Packages: added %s" %
+ tarinfo.name.rsplit("-", 2)[0])
+ tar.close()
+
+ self.deps['global'] = dict()
+ self.provides['global'] = dict()
+ for barch in bdeps:
+ self.deps[barch] = dict()
+ self.provides[barch] = dict()
+ for pkgname in self.pkgnames:
+ pset = set()
+ for barch in bdeps:
+ if pkgname not in bdeps[barch]:
+ bdeps[barch][pkgname] = []
+ pset.add(tuple(bdeps[barch][pkgname]))
+ if len(pset) == 1:
+ self.deps['global'][pkgname] = pset.pop()
+ else:
+ for barch in bdeps:
+ self.deps[barch][pkgname] = bdeps[barch][pkgname]
+ provided = set()
+ for bprovided in list(bprov.values()):
+ provided.update(set(bprovided))
+ for prov in provided:
+ prset = set()
+ for barch in bprov:
+ if prov not in bprov[barch]:
+ continue
+ prset.add(tuple(bprov[barch].get(prov, ())))
+ if len(prset) == 1:
+ self.provides['global'][prov] = prset.pop()
+ else:
+ for barch in bprov:
+ self.provides[barch][prov] = bprov[barch].get(prov, ())
+ self.save_state()
+
+ def is_package(self, _, pkg):
+ return (pkg in self.pkgnames and
+ pkg not in self.blacklist and
+ (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py
new file mode 100644
index 000000000..3846c06ce
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py
@@ -0,0 +1,15 @@
+import Bcfg2.Server.Plugin
+
+class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig):
+ _required = False
+
+ def Index(self):
+ """ Build local data structures """
+ Bcfg2.Server.Plugin.SimpleConfig.Index(self)
+
+ if hasattr(self.plugin, "sources") and self.plugin.sources.loaded:
+ # only reload Packages plugin if sources have been loaded.
+ # otherwise, this is getting called on server startup, and
+ # we have to wait until all sources have been indexed
+ # before we can call Packages.Reload()
+ self.plugin.Reload()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
new file mode 100644
index 000000000..8d0067b6a
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -0,0 +1,107 @@
+import os
+import sys
+import lxml.etree
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugins.Packages.Source import SourceInitError
+
+class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked,
+ Bcfg2.Server.Plugin.StructFile,
+ Bcfg2.Server.Plugin.Debuggable):
+ __identifier__ = None
+
+ def __init__(self, filename, cachepath, fam, packages, config):
+ Bcfg2.Server.Plugin.Debuggable.__init__(self)
+ try:
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self,
+ filename,
+ fam)
+ except OSError:
+ err = sys.exc_info()[1]
+ msg = "Packages: Failed to read configuration file: %s" % err
+ if not os.path.exists(self.name):
+ msg += " Have you created it?"
+ self.logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginInitError(msg)
+ Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
+ self.cachepath = cachepath
+ self.config = config
+ if not os.path.exists(self.cachepath):
+ # create cache directory if needed
+ try:
+ os.makedirs(self.cachepath)
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Could not create Packages cache at %s: %s" %
+ (self.cachepath, err))
+ self.pkg_obj = packages
+ self.parsed = set()
+ self.loaded = False
+
+ def toggle_debug(self):
+ Bcfg2.Server.Plugin.Debuggable.toggle_debug(self)
+ for source in self.entries:
+ source.toggle_debug()
+
+ def HandleEvent(self, event=None):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event)
+ if event and event.filename != self.name:
+ for fname in self.extras:
+ fpath = None
+ if fname.startswith("/"):
+ fpath = os.path.abspath(fname)
+ else:
+ fpath = \
+ os.path.abspath(os.path.join(os.path.dirname(self.name),
+ fname))
+ if fpath == os.path.abspath(event.filename):
+ self.parsed.add(fname)
+ break
+
+ if sorted(list(self.parsed)) == sorted(self.extras):
+ self.logger.info("Reloading Packages plugin")
+ self.pkg_obj.Reload()
+ self.loaded = True
+
+ def Index(self):
+ Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self)
+ self.entries = []
+ for xsource in self.xdata.findall('.//Source'):
+ source = self.source_from_xml(xsource)
+ if source is not None:
+ self.entries.append(source)
+
+ def source_from_xml(self, xsource):
+ """ create a *Source object from its XML representation in
+ sources.xml """
+ stype = xsource.get("type")
+ if stype is None:
+ self.logger.error("Packages: No type specified for source, "
+ "skipping")
+ return None
+
+ try:
+ module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
+ stype.title()).Server.Plugins.Packages,
+ stype.title())
+ cls = getattr(module, "%sSource" % stype.title())
+ except (ImportError, AttributeError):
+ self.logger.error("Packages: Unknown source type %s" % stype)
+ return None
+
+ try:
+ source = cls(self.cachepath, xsource, self.config)
+ except SourceInitError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: %s" % err)
+ source = None
+
+ return source
+
+ def __getitem__(self, key):
+ return self.entries[key]
+
+ def __repr__(self):
+ return "PackagesSources: %s" % repr(self.entries)
+
+ def __str__(self):
+ return "PackagesSources: %s" % str(self.entries)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
new file mode 100644
index 000000000..a7c246940
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -0,0 +1,280 @@
+import os
+import re
+import sys
+import base64
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \
+ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \
+ urlopen, file, cPickle
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+
+def fetch_url(url):
+ if '@' in url:
+ mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
+ if not mobj:
+ raise ValueError
+ user = mobj.group(2)
+ passwd = mobj.group(3)
+ url = mobj.group(1) + mobj.group(4)
+ auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
+ auth.add_password(None, url, user, passwd)
+ install_opener(build_opener(auth))
+ return urlopen(url).read()
+
+
+class SourceInitError(Exception):
+ pass
+
+
+class Source(Bcfg2.Server.Plugin.Debuggable):
+ mrepo_re = re.compile(r'/RPMS\.([^/]+)')
+ pulprepo_re = re.compile(r'pulp/repos/([^/]+)')
+ genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$')
+ basegroups = []
+
+ def __init__(self, basepath, xsource, config):
+ Bcfg2.Server.Plugin.Debuggable.__init__(self)
+ self.basepath = basepath
+ self.xsource = xsource
+ self.config = config
+
+ try:
+ self.version = xsource.find('Version').text
+ except AttributeError:
+ pass
+
+ for key, tag in [('components', 'Component'), ('arches', 'Arch'),
+ ('blacklist', 'Blacklist'),
+ ('whitelist', 'Whitelist')]:
+ self.__dict__[key] = [item.text for item in xsource.findall(tag)]
+
+ self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
+
+ self.essential = xsource.get('essential', 'true').lower() == 'true'
+ self.recommended = xsource.get('recommended', 'false').lower() == 'true'
+
+ self.rawurl = xsource.get('rawurl', '')
+ if self.rawurl and not self.rawurl.endswith("/"):
+ self.rawurl += "/"
+ self.url = xsource.get('url', '')
+ if self.url and not self.url.endswith("/"):
+ self.url += "/"
+ self.version = xsource.get('version', '')
+
+ # build the set of conditions to see if this source applies to
+ # a given set of metadata
+ self.conditions = []
+ self.groups = [] # provided for some limited backwards compat
+ for el in xsource.iterancestors():
+ if el.tag == "Group":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") not in m.groups)
+ else:
+ self.groups.append(el.get("name"))
+ self.conditions.append(lambda m, el=el:
+ el.get("name") in m.groups)
+ elif el.tag == "Client":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") != m.hostname)
+ else:
+ self.conditions.append(lambda m, el=el:
+ el.get("name") == m.hostname)
+
+ self.deps = dict()
+ self.provides = dict()
+
+ self.cachefile = os.path.join(self.basepath,
+ "cache-%s" % self.cachekey)
+ self.url_map = []
+
+ @property
+ def cachekey(self):
+ return md5(cPickle.dumps([self.version, self.components, self.url,
+ self.rawurl, self.arches])).hexdigest()
+
+ def get_relevant_groups(self, metadata):
+ return sorted(list(set([g for g in metadata.groups
+ if (g in self.basegroups or
+ g in self.groups or
+ g in self.arches)])))
+
+ def load_state(self):
+ pass
+
+ def setup_data(self, force_update=False):
+ should_read = True
+ should_download = False
+ if os.path.exists(self.cachefile):
+ try:
+ self.load_state()
+ should_read = False
+ except:
+ self.logger.error("Packages: Cachefile %s load failed; "
+ "falling back to file read" % self.cachefile)
+ if should_read:
+ try:
+ self.read_files()
+ except:
+ self.logger.error("Packages: File read failed; "
+ "falling back to file download")
+ should_download = True
+
+ if should_download or force_update:
+ try:
+ self.update()
+ self.read_files()
+ except:
+ self.logger.error("Packages: Failed to load data for Source "
+ "of %s. Some Packages will be missing." %
+ self.urls)
+
+ def get_repo_name(self, url_map):
+ # try to find a sensible name for a repo
+ if url_map['components']:
+ # use the first component as the name
+ rname = url_map['components'][0]
+ else:
+ name = None
+ for repo_re in (self.mrepo_re,
+ self.pulprepo_re,
+ self.genericrepo_re):
+ match = repo_re.search(url_map['url'])
+ if match:
+ break
+ if name is None:
+ # couldn't figure out the name from the URL or URL map
+ # (which probably means its a screwy URL), so we just
+ # generate a random one
+ name = base64.b64encode(os.urandom(16))[:-2]
+ rname = "%s-%s" % (self.groups[0], name)
+ # see yum/__init__.py in the yum source, lines 441-449, for
+ # the source of this regex. yum doesn't like anything but
+ # string.ascii_letters, string.digits, and [-_.:]. There
+ # doesn't seem to be a reason for this, because yum.
+ return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname)
+
+ def __str__(self):
+ if self.rawurl:
+ return "%s at %s" % (self.__class__.__name__, self.rawurl)
+ elif self.url:
+ return "%s at %s" % (self.__class__.__name__, self.url)
+ else:
+ return self.__class__.__name__
+
+ def get_urls(self):
+ return []
+ urls = property(get_urls)
+
+ def get_files(self):
+ return [self.escape_url(url) for url in self.urls]
+ files = property(get_files)
+
+ def get_vpkgs(self, metadata):
+ agroups = ['global'] + [a for a in self.arches
+ if a in metadata.groups]
+ vdict = dict()
+ for agrp in agroups:
+ for key, value in list(self.provides[agrp].items()):
+ if key not in vdict:
+ vdict[key] = set(value)
+ else:
+ vdict[key].update(value)
+ return vdict
+
+ def is_virtual_package(self, metadata, package):
+ """ called to determine if a package is a virtual package.
+ this is only invoked if the package is not listed in the dict
+ returned by get_vpkgs """
+ return False
+
+ def escape_url(self, url):
+ return os.path.join(self.basepath, url.replace('/', '@'))
+
+ def file_init(self):
+ pass
+
+ def read_files(self):
+ pass
+
+ def filter_unknown(self, unknown):
+ pass
+
+ def update(self):
+ for url in self.urls:
+ self.logger.info("Packages: Updating %s" % url)
+ fname = self.escape_url(url)
+ try:
+ data = fetch_url(url)
+ file(fname, 'w').write(data)
+ except ValueError:
+ self.logger.error("Packages: Bad url string %s" % url)
+ raise
+ except HTTPError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Failed to fetch url %s. HTTP "
+ "response code=%s" % (url, err.code))
+ raise
+
+ def applies(self, metadata):
+ # check base groups
+ if not self.magic_groups_match(metadata):
+ return False
+
+ # check Group/Client tags from sources.xml
+ for condition in self.conditions:
+ if not condition(metadata):
+ return False
+
+ return True
+
+ def get_arches(self, metadata):
+ return ['global'] + [a for a in self.arches if a in metadata.groups]
+
+ def get_deps(self, metadata, pkgname):
+ for arch in self.get_arches(metadata):
+ if pkgname in self.deps[arch]:
+ return self.deps[arch][pkgname]
+ return []
+
+ def get_provides(self, metadata, required):
+ for arch in self.get_arches(metadata):
+ if required in self.provides[arch]:
+ return self.provides[arch][required]
+ return []
+
+ def is_package(self, metadata, _):
+ return False
+
+ def get_package(self, metadata, package):
+ return package
+
+ def get_group(self, metadata, group, ptype=None):
+ return []
+
+ def magic_groups_match(self, metadata):
+ """ check to see if this source applies to the given host
+ metadata by checking 'magic' (base) groups only, or if magic
+ groups are off """
+ # we always check that arch matches
+ found_arch = False
+ for arch in self.arches:
+ if arch in metadata.groups:
+ found_arch = True
+ break
+ if not found_arch:
+ return False
+
+ if self.config.getboolean("global", "magic_groups",
+ default=True) == False:
+ return True
+ else:
+ for group in self.basegroups:
+ if group in metadata.groups:
+ return True
+ return False
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
new file mode 100644
index 000000000..b39b6aed2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -0,0 +1,688 @@
+import os
+import sys
+import time
+import copy
+import glob
+import socket
+import random
+import logging
+import threading
+import lxml.etree
+from UserDict import DictMixin
+from subprocess import Popen, PIPE, STDOUT
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
+ fetch_url
+
+logger = logging.getLogger(__name__)
+
+try:
+ from pulp.client.consumer.config import ConsumerConfig
+ from pulp.client.api.repository import RepositoryAPI
+ from pulp.client.api.consumer import ConsumerAPI
+ from pulp.client.api import server
+ has_pulp = True
+except ImportError:
+ has_pulp = False
+
+try:
+ import yum
+ has_yum = True
+except ImportError:
+ has_yum = False
+ logger.info("Packages: No yum libraries found; forcing use of internal "
+ "dependency resolver")
+
+try:
+ import json
+except ImportError:
+ import simplejson as json
+
+XP = '{http://linux.duke.edu/metadata/common}'
+RP = '{http://linux.duke.edu/metadata/rpm}'
+RPO = '{http://linux.duke.edu/metadata/repo}'
+FL = '{http://linux.duke.edu/metadata/filelists}'
+
+PULPSERVER = None
+PULPCONFIG = None
+
+
+def _setup_pulp(config):
+ global PULPSERVER, PULPCONFIG
+ if not has_pulp:
+ logger.error("Packages: Cannot create Pulp collection: Pulp libraries not "
+ "found")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ if PULPSERVER is None:
+ try:
+ username = config.get("pulp", "username")
+ password = config.get("pulp", "password")
+ except ConfigParser.NoSectionError:
+ logger.error("Packages: No [pulp] section found in Packages/packages.conf")
+ raise Bcfg2.Server.Plugin.PluginInitError
+ except ConfigParser.NoOptionError:
+ err = sys.exc_info()[1]
+ logger.error("Packages: Required option not found in "
+ "Packages/packages.conf: %s" % err)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ PULPCONFIG = ConsumerConfig()
+ serveropts = PULPCONFIG.server
+
+ PULPSERVER = server.PulpServer(serveropts['host'],
+ int(serveropts['port']),
+ serveropts['scheme'],
+ serveropts['path'])
+ PULPSERVER.set_basic_auth_credentials(username, password)
+ server.set_active_server(PULPSERVER)
+ return PULPSERVER
+
+
+class YumCollection(Collection):
+ # options that are included in the [yum] section but that should
+ # not be included in the temporary yum.conf we write out
+ option_blacklist = ["use_yum_libraries", "helper"]
+
+ def __init__(self, metadata, sources, basepath, debug=False):
+ Collection.__init__(self, metadata, sources, basepath, debug=debug)
+ self.keypath = os.path.join(self.basepath, "keys")
+
+ if len(sources):
+ config = sources[0].config
+ self.use_yum = has_yum and config.getboolean("yum",
+ "use_yum_libraries",
+ default=False)
+ else:
+ self.use_yum = False
+
+ if self.use_yum:
+ self.cachefile = os.path.join(self.cachepath,
+ "cache-%s" % self.cachekey)
+ if not os.path.exists(self.cachefile):
+ os.mkdir(self.cachefile)
+
+ self.configdir = os.path.join(self.basepath, "yum")
+ if not os.path.exists(self.configdir):
+ os.mkdir(self.configdir)
+ self.cfgfile = os.path.join(self.configdir,
+ "%s-yum.conf" % self.cachekey)
+ self.write_config()
+
+ self.helper = self.config.get("yum", "helper",
+ default="/usr/sbin/bcfg2-yum-helper")
+ if has_pulp:
+ _setup_pulp(self.config)
+
+ def write_config(self):
+ if not os.path.exists(self.cfgfile):
+ yumconf = self.get_config(raw=True)
+ yumconf.add_section("main")
+
+ mainopts = dict(cachedir=self.cachefile,
+ keepcache="0",
+ sslverify="0",
+ debuglevel="0",
+ reposdir="/dev/null")
+ try:
+ for opt in self.config.options("yum"):
+ if opt not in self.option_blacklist:
+ mainopts[opt] = self.config.get("yum", opt)
+ except ConfigParser.NoSectionError:
+ pass
+
+ for opt, val in list(mainopts.items()):
+ yumconf.set("main", opt, val)
+
+ yumconf.write(open(self.cfgfile, 'w'))
+
+ def get_config(self, raw=False):
+ config = ConfigParser.SafeConfigParser()
+ for source in self.sources:
+ # get_urls() loads url_map as a side-effect
+ source.get_urls()
+ for url_map in source.url_map:
+ if url_map['arch'] in self.metadata.groups:
+ basereponame = source.get_repo_name(url_map)
+ reponame = basereponame
+
+ added = False
+ while not added:
+ try:
+ config.add_section(reponame)
+ added = True
+ except ConfigParser.DuplicateSectionError:
+ match = re.match("-(\d)", reponame)
+ if match:
+ rid = int(match.group(1)) + 1
+ else:
+ rid = 1
+ reponame = "%s-%d" % (basereponame, rid)
+
+ config.set(reponame, "name", reponame)
+ config.set(reponame, "baseurl", url_map['url'])
+ config.set(reponame, "enabled", "1")
+ if len(source.gpgkeys):
+ config.set(reponame, "gpgcheck", "1")
+ config.set(reponame, "gpgkey",
+ " ".join(source.gpgkeys))
+ else:
+ config.set(reponame, "gpgcheck", "0")
+
+ if len(source.blacklist):
+ config.set(reponame, "exclude",
+ " ".join(source.blacklist))
+ if len(source.whitelist):
+ config.set(reponame, "includepkgs",
+ " ".join(source.whitelist))
+
+ if raw:
+ return config
+ else:
+ # configparser only writes to file, so we have to use a
+ # StringIO object to get the data out as a string
+ buf = StringIO()
+ config.write(buf)
+ return "# This config was generated automatically by the Bcfg2 " \
+ "Packages plugin\n\n" + buf.getvalue()
+
+ def build_extra_structures(self, independent):
+ """ build list of gpg keys to be added to the specification by
+ validate_structures() """
+ needkeys = set()
+ for source in self.sources:
+ for key in source.gpgkeys:
+ needkeys.add(key)
+
+ if len(needkeys):
+ if has_yum:
+ # this must be be has_yum, not use_yum, because
+ # regardless of whether the user wants to use the yum
+ # resolver we want to include gpg key data
+ keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey",
+ type=self.ptype, origin='Packages')
+ else:
+ self.logger.warning("GPGKeys were specified for yum sources in "
+ "sources.xml, but no yum libraries were "
+ "found")
+ self.logger.warning("GPG key version/release data cannot be "
+ "determined automatically")
+ self.logger.warning("Install yum libraries, or manage GPG keys "
+ "manually")
+ keypkg = None
+
+ for key in needkeys:
+ # figure out the path of the key on the client
+ keydir = self.config.get("global", "gpg_keypath",
+ default="/etc/pki/rpm-gpg")
+ remotekey = os.path.join(keydir, os.path.basename(key))
+ localkey = os.path.join(self.keypath, os.path.basename(key))
+ kdata = open(localkey).read()
+
+ # copy the key to the client
+ keypath = lxml.etree.Element("BoundPath", name=remotekey,
+ encoding='ascii',
+ owner='root', group='root',
+ type='file', perms='0644',
+ important='true')
+ keypath.text = kdata
+
+ # hook to add version/release info if possible
+ self._add_gpg_instances(keypkg, kdata, localkey, remotekey)
+ independent.append(keypath)
+ if keypkg is not None:
+ independent.append(keypkg)
+
+ # see if there are any pulp sources to handle
+ has_pulp_sources = False
+ for source in self.sources:
+ if source.pulp_id:
+ has_pulp_sources = True
+ break
+
+ if has_pulp_sources:
+ consumerapi = ConsumerAPI()
+ consumer = self._get_pulp_consumer(consumerapi=consumerapi)
+ if consumer is None:
+ consumer = consumerapi.create(self.metadata.hostname,
+ self.metadata.hostname)
+ lxml.etree.SubElement(independent, "BoundAction",
+ name="pulp-update", timing="pre",
+ when="always", status="check",
+ command="pulp-consumer consumer update")
+
+ for source in self.sources:
+ # each pulp source can only have one arch, so we don't
+ # have to check the arch in url_map
+ if (source.pulp_id and
+ source.pulp_id not in consumer['repoids']):
+ consumerapi.bind(self.metadata.hostname, source.pulp_id)
+
+ crt = lxml.etree.SubElement(independent, "BoundPath",
+ name="/etc/pki/consumer/cert.pem",
+ type="file", owner="root",
+ group="root", perms="0644")
+ crt.text = consumerapi.certificate(self.metadata.hostname)
+
+ def _get_pulp_consumer(self, consumerapi=None):
+ if consumerapi is None:
+ consumerapi = ConsumerAPI()
+ consumer = None
+ try:
+ consumer = consumerapi.consumer(self.metadata.hostname)
+ except server.ServerRequestError:
+ # consumer does not exist
+ pass
+ except socket.error:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Could not contact Pulp server: %s" %
+ err)
+ except:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Unknown error querying Pulp server: %s"
+ % err)
+ return consumer
+
+ def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey):
+ """ add gpg keys to the specification to ensure they get
+ installed """
+ # this must be be has_yum, not use_yum, because regardless of
+ # whether the user wants to use the yum resolver we want to
+ # include gpg key data
+ if not has_yum:
+ return
+
+ try:
+ kinfo = yum.misc.getgpgkeyinfo(keydata)
+ version = yum.misc.keyIdToRPMVer(kinfo['keyid'])
+ release = yum.misc.keyIdToRPMVer(kinfo['timestamp'])
+
+ lxml.etree.SubElement(keyentry, 'Instance',
+ version=version,
+ release=release,
+ simplefile=remotekey)
+ except ValueError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Could not read GPG key %s: %s" %
+ (localkey, err))
+
+ def is_package(self, package):
+ if not self.use_yum:
+ return Collection.is_package(self, package)
+ elif isinstance(package, tuple):
+ if package[1] is None and package[2] == (None, None, None):
+ package = package[0]
+ else:
+ return None
+ else:
+ # this should really never get called; it's just provided
+ # for API completeness
+ return self.call_helper("is_package", package)
+
+ def is_virtual_package(self, package):
+ if not self.use_yum:
+ return Collection.is_virtual_package(self, package)
+ else:
+ # this should really never get called; it's just provided
+ # for API completeness
+ return self.call_helper("is_virtual_package", package)
+
+ def get_deps(self, package):
+ if not self.use_yum:
+ return Collection.get_deps(self, package)
+ else:
+ # this should really never get called; it's just provided
+ # for API completeness
+ return self.call_helper("get_deps", package)
+
+ def get_provides(self, required, all=False, silent=False):
+ if not self.use_yum:
+ return Collection.get_provides(self, package)
+ else:
+ # this should really never get called; it's just provided
+ # for API completeness
+ return self.call_helper("get_provides", package)
+
+ def get_group(self, group, ptype="default"):
+ if not self.use_yum:
+ self.logger.warning("Packages: Package groups are not supported by "
+ "Bcfg2's internal Yum dependency generator")
+ return []
+
+ if group.startswith("@"):
+ group = group[1:]
+
+ pkgs = self.call_helper("get_group", dict(group=group, type=ptype))
+ return pkgs
+
+ def complete(self, packagelist):
+ if not self.use_yum:
+ return Collection.complete(self, packagelist)
+
+ packages = set()
+ unknown = set(packagelist)
+
+ if unknown:
+ result = \
+ self.call_helper("complete",
+ dict(packages=list(unknown),
+ groups=list(self.get_relevant_groups())))
+ if result and "packages" in result and "unknown" in result:
+ # we stringify every package because it gets returned
+ # in unicode; set.update() doesn't work if some
+ # elements are unicode and other are strings. (I.e.,
+ # u'foo' and 'foo' get treated as unique elements.)
+ packages.update([str(p) for p in result['packages']])
+ unknown = set([str(p) for p in result['unknown']])
+
+ self.filter_unknown(unknown)
+
+ return packages, unknown
+
+ def call_helper(self, command, input=None):
+ """ Make a call to bcfg2-yum-helper. The yum libs have
+ horrific memory leaks, so apparently the right way to get
+ around that in long-running processes it to have a short-lived
+ helper. No, seriously -- check out the yum-updatesd code.
+ It's pure madness. """
+ # it'd be nice if we could change this to be more verbose if
+ # -v was given to bcfg2-server, but Collection objects don't
+ # get the 'setup' variable, so we don't know how verbose
+ # bcfg2-server is. It'd also be nice if we could tell yum to
+ # log to syslog. So would a unicorn.
+ cmd = [self.helper, "-c", self.cfgfile]
+ if self.debug_flag:
+ cmd.append("-v")
+ cmd.append(command)
+ self.debug_log("Packages: running %s" % " ".join(cmd))
+ try:
+ helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Failed to execute %s: %s" %
+ (" ".join(cmd), err))
+ return None
+
+ if input:
+ idata = json.dumps(input)
+ (stdout, stderr) = helper.communicate(idata)
+ else:
+ (stdout, stderr) = helper.communicate()
+ rv = helper.wait()
+ if rv:
+ self.logger.error("Packages: error running bcfg2-yum-helper "
+ "(returned %d): %s" % (rv, stderr))
+ elif self.debug_flag:
+ self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" %
+ stderr)
+ try:
+ return json.loads(stdout)
+ except ValueError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: error reading bcfg2-yum-helper "
+ "output: %s" % err)
+ return None
+
+ def setup_data(self, force_update=False):
+ if not self.use_yum:
+ return Collection.setup_data(self, force_update)
+
+ if force_update:
+ # we call this twice: one to clean up data from the old
+ # config, and once to clean up data from the new config
+ self.call_helper("clean")
+
+ os.unlink(self.cfgfile)
+ self.write_config()
+
+ if force_update:
+ self.call_helper("clean")
+
+
+class YumSource(Source):
+ basegroups = ['yum', 'redhat', 'centos', 'fedora']
+ ptype = 'yum'
+
+ def __init__(self, basepath, xsource, config):
+ Source.__init__(self, basepath, xsource, config)
+ self.pulp_id = None
+ if has_pulp and xsource.get("pulp_id"):
+ self.pulp_id = xsource.get("pulp_id")
+
+ _setup_pulp(self.config)
+ repoapi = RepositoryAPI()
+ try:
+ self.repo = repoapi.repository(self.pulp_id)
+ self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key)
+ for key in repoapi.listkeys(self.pulp_id)]
+ except server.ServerRequestError:
+ err = sys.exc_info()[1]
+ if err[0] == 401:
+ msg = "Packages: Error authenticating to Pulp: %s" % err[1]
+ elif err[0] == 404:
+ msg = "Packages: Pulp repo id %s not found: %s" % \
+ (self.pulp_id, err[1])
+ else:
+ msg = "Packages: Error %d fetching pulp repo %s: %s" % \
+ (err[0], self.pulp_id, err[1])
+ raise SourceInitError(msg)
+ except socket.error:
+ err = sys.exc_info()[1]
+ raise SourceInitError("Could not contact Pulp server: %s" % err)
+ except:
+ err = sys.exc_info()[1]
+ raise SourceInitError("Unknown error querying Pulp server: %s" %
+ err)
+ self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'],
+ self.repo['relative_path'])
+ self.arches = [self.repo['arch']]
+
+ if not self.rawurl:
+ self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
+ else:
+ self.baseurl = self.rawurl
+ self.packages = dict()
+ self.deps = dict([('global', dict())])
+ self.provides = dict([('global', dict())])
+ self.filemap = dict([(x, dict())
+ for x in ['global'] + self.arches])
+ self.needed_paths = set()
+ self.file_to_arch = dict()
+
+ self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries",
+ default=False)
+
+ def save_state(self):
+ if not self.use_yum:
+ cache = file(self.cachefile, 'wb')
+ cPickle.dump((self.packages, self.deps, self.provides,
+ self.filemap, self.url_map), cache, 2)
+ cache.close()
+
+
+ def load_state(self):
+ if not self.use_yum:
+ data = file(self.cachefile)
+ (self.packages, self.deps, self.provides,
+ self.filemap, self.url_map) = cPickle.load(data)
+
+ def get_urls(self):
+ surls = list()
+ self.url_map = []
+ for arch in self.arches:
+ if self.url:
+ usettings = [{'version':self.version, 'component':comp,
+ 'arch':arch}
+ for comp in self.components]
+ else: # rawurl given
+ usettings = [{'version':self.version, 'component':None,
+ 'arch':arch}]
+
+ for setting in usettings:
+ setting['url'] = self.baseurl % setting
+ self.url_map.append(copy.deepcopy(setting))
+ surls.append((arch, [setting['url'] for setting in usettings]))
+ urls = []
+ for (sarch, surl_list) in surls:
+ for surl in surl_list:
+ urls.extend(self._get_urls_from_repodata(surl, sarch))
+ return urls
+ urls = property(get_urls)
+
+ def _get_urls_from_repodata(self, url, arch):
+ if self.use_yum:
+ return [url]
+
+ rmdurl = '%srepodata/repomd.xml' % url
+ try:
+ repomd = fetch_url(rmdurl)
+ xdata = lxml.etree.XML(repomd)
+ except ValueError:
+ self.logger.error("Packages: Bad url string %s" % rmdurl)
+ return []
+ except HTTPError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Failed to fetch url %s. code=%s" %
+ (rmdurl, err.code))
+ return []
+ except lxml.etree.XMLSyntaxError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Failed to process metadata at %s: %s" %
+ (rmdurl, err))
+ return []
+
+ urls = []
+ for elt in xdata.findall(RPO + 'data'):
+ if elt.get('type') in ['filelists', 'primary']:
+ floc = elt.find(RPO + 'location')
+ fullurl = url + floc.get('href')
+ urls.append(fullurl)
+ self.file_to_arch[self.escape_url(fullurl)] = arch
+ return urls
+
+ def read_files(self):
+ # we have to read primary.xml first, and filelists.xml afterwards;
+ primaries = list()
+ filelists = list()
+ for fname in self.files:
+ if fname.endswith('primary.xml.gz'):
+ primaries.append(fname)
+ elif fname.endswith('filelists.xml.gz'):
+ filelists.append(fname)
+
+ for fname in primaries:
+ farch = self.file_to_arch[fname]
+ fdata = lxml.etree.parse(fname).getroot()
+ self.parse_primary(fdata, farch)
+ for fname in filelists:
+ farch = self.file_to_arch[fname]
+ fdata = lxml.etree.parse(fname).getroot()
+ self.parse_filelist(fdata, farch)
+
+ # merge data
+ sdata = list(self.packages.values())
+ try:
+ self.packages['global'] = copy.deepcopy(sdata.pop())
+ except IndexError:
+ logger.error("Packages: No packages in repo")
+ while sdata:
+ self.packages['global'] = \
+ self.packages['global'].intersection(sdata.pop())
+
+ for key in self.packages:
+ if key == 'global':
+ continue
+ self.packages[key] = \
+ self.packages[key].difference(self.packages['global'])
+ self.save_state()
+
+ def parse_filelist(self, data, arch):
+ if arch not in self.filemap:
+ self.filemap[arch] = dict()
+ for pkg in data.findall(FL + 'package'):
+ for fentry in pkg.findall(FL + 'file'):
+ if fentry.text in self.needed_paths:
+ if fentry.text in self.filemap[arch]:
+ self.filemap[arch][fentry.text].add(pkg.get('name'))
+ else:
+ self.filemap[arch][fentry.text] = \
+ set([pkg.get('name')])
+
+ def parse_primary(self, data, arch):
+ if arch not in self.packages:
+ self.packages[arch] = set()
+ if arch not in self.deps:
+ self.deps[arch] = dict()
+ if arch not in self.provides:
+ self.provides[arch] = dict()
+ for pkg in data.getchildren():
+ if not pkg.tag.endswith('package'):
+ continue
+ pkgname = pkg.find(XP + 'name').text
+ self.packages[arch].add(pkgname)
+
+ pdata = pkg.find(XP + 'format')
+ self.deps[arch][pkgname] = set()
+ pre = pdata.find(RP + 'requires')
+ if pre is not None:
+ for entry in pre.getchildren():
+ self.deps[arch][pkgname].add(entry.get('name'))
+ if entry.get('name').startswith('/'):
+ self.needed_paths.add(entry.get('name'))
+ pro = pdata.find(RP + 'provides')
+ if pro != None:
+ for entry in pro.getchildren():
+ prov = entry.get('name')
+ if prov not in self.provides[arch]:
+ self.provides[arch][prov] = list()
+ self.provides[arch][prov].append(pkgname)
+
+ def is_package(self, metadata, item):
+ arch = [a for a in self.arches if a in metadata.groups]
+ if not arch:
+ return False
+ return ((item in self.packages['global'] or
+ item in self.packages[arch[0]]) and
+ item not in self.blacklist and
+ (len(self.whitelist) == 0 or item in self.whitelist))
+
+ def get_vpkgs(self, metadata):
+ if self.use_yum:
+ return dict()
+
+ rv = Source.get_vpkgs(self, metadata)
+ for arch, fmdata in list(self.filemap.items()):
+ if arch not in metadata.groups and arch != 'global':
+ continue
+ for filename, pkgs in list(fmdata.items()):
+ rv[filename] = pkgs
+ return rv
+
+ def filter_unknown(self, unknown):
+ if self.use_yum:
+ filtered = set()
+ for unk in unknown:
+ try:
+ if unk.startswith('rpmlib'):
+ filtered.update(unk)
+ except AttributeError:
+ try:
+ if unk[0].startswith('rpmlib'):
+ filtered.update(unk)
+ except (IndexError, AttributeError):
+ pass
+ else:
+ filtered = set([u for u in unknown if u.startswith('rpmlib')])
+ unknown.difference_update(filtered)
+
+ def setup_data(self, force_update=False):
+ if not self.use_yum:
+ Source.setup_data(self, force_update=force_update)
+
+ def get_repo_name(self, url_map):
+ if self.pulp_id:
+ return self.pulp_id
+ else:
+ return Source.get_repo_name(self, url_map)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
new file mode 100644
index 000000000..e4793a28d
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -0,0 +1,269 @@
+import os
+import sys
+import time
+import copy
+import glob
+import shutil
+import lxml.etree
+import Bcfg2.Logger
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen
+from Bcfg2.Server.Plugins.Packages import Collection
+from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
+from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig
+
+class Packages(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.StructureValidator,
+ Bcfg2.Server.Plugin.Generator,
+ Bcfg2.Server.Plugin.Connector):
+ name = 'Packages'
+ conflicts = ['Pkgmgr']
+ experimental = True
+ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.StructureValidator.__init__(self)
+ Bcfg2.Server.Plugin.Generator.__init__(self)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ Bcfg2.Server.Plugin.Probing.__init__(self)
+
+ self.sentinels = set()
+ self.cachepath = os.path.join(self.data, 'cache')
+ self.keypath = os.path.join(self.data, 'keys')
+ if not os.path.exists(self.keypath):
+ # create key directory if needed
+ os.makedirs(self.keypath)
+
+ # set up config files
+ self.config = PackagesConfig(self)
+ self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
+ self.cachepath, core.fam, self,
+ self.config)
+
+ def toggle_debug(self):
+ Bcfg2.Server.Plugin.Plugin.toggle_debug(self)
+ self.sources.toggle_debug()
+
+ @property
+ def disableResolver(self):
+ try:
+ return not self.config.getboolean("global", "resolver")
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ return False
+ except ValueError:
+ # for historical reasons we also accept "enabled" and
+ # "disabled", which are not handled according to the
+ # Python docs but appear to be handled properly by
+ # ConfigParser in at least some versions
+ return self.config.get("global", "resolver",
+ default="enabled").lower() == "disabled"
+
+ @property
+ def disableMetaData(self):
+ try:
+ return not self.config.getboolean("global", "resolver")
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ return False
+ except ValueError:
+ # for historical reasons we also accept "enabled" and
+ # "disabled"
+ return self.config.get("global", "metadata",
+ default="enabled").lower() == "disabled"
+
+ def create_config(self, entry, metadata):
+ """ create yum/apt config for the specified host """
+ attrib = {'encoding': 'ascii',
+ 'owner': 'root',
+ 'group': 'root',
+ 'type': 'file',
+ 'perms': '0644'}
+
+ collection = self._get_collection(metadata)
+ entry.text = collection.get_config()
+ for (key, value) in list(attrib.items()):
+ entry.attrib.__setitem__(key, value)
+
+ def HandleEntry(self, entry, metadata):
+ if entry.tag == 'Package':
+ collection = self._get_collection(metadata)
+ entry.set('version', 'auto')
+ entry.set('version', self.config.get("global",
+ "version",
+ default="auto"))
+ entry.set('type', collection.ptype)
+ elif entry.tag == 'Path':
+ if (entry.get("name") == self.config.get("global", "yum_config",
+ default="") or
+ entry.get("name") == self.config.get("global", "apt_config",
+ default="")):
+ self.create_config(entry, metadata)
+
+ def HandlesEntry(self, entry, metadata):
+ if entry.tag == 'Package':
+ if self.config.getboolean("global", "magic_groups",
+ default=True) == True:
+ collection = self._get_collection(metadata)
+ if collection.magic_groups_match():
+ return True
+ else:
+ return True
+ elif entry.tag == 'Path':
+ # managed entries for yum/apt configs
+ if (entry.get("name") == self.config.get("global", "yum_config",
+ default="") or
+ entry.get("name") == self.config.get("global", "apt_config",
+ default="")):
+ return True
+ return False
+
+ def validate_structures(self, metadata, structures):
+ '''Ensure client configurations include all needed prerequisites
+
+ Arguments:
+ metadata - client metadata instance
+ structures - a list of structure-stage entry combinations
+ '''
+ collection = self._get_collection(metadata)
+ indep = lxml.etree.Element('Independent')
+ self._build_packages(metadata, indep, structures,
+ collection=collection)
+ collection.build_extra_structures(indep)
+ structures.append(indep)
+
+ def _build_packages(self, metadata, independent, structures,
+ collection=None):
+ """ build list of packages that need to be included in the
+ specification by validate_structures() """
+ if self.disableResolver:
+ # Config requests no resolver
+ return
+
+ if collection is None:
+ collection = self._get_collection(metadata)
+ # initial is the set of packages that are explicitly specified
+ # in the configuration
+ initial = set()
+ # base is the set of initial packages with groups expanded
+ base = set()
+ # essential pkgs are those marked as such by the distribution
+ essential = collection.get_essential()
+ to_remove = []
+ for struct in structures:
+ for pkg in struct.xpath('//Package | //BoundPackage'):
+ if pkg.get("name"):
+ initial.add(pkg.get("name"))
+ elif pkg.get("group"):
+ try:
+ if pkg.get("type"):
+ gpkgs = collection.get_group(pkg.get("group"),
+ ptype=pkg.get("type"))
+ else:
+ gpkgs = collection.get_group(pkg.get("group"))
+ base.update(gpkgs)
+ except TypeError:
+ raise
+ self.logger.error("Could not resolve group %s" %
+ pkg.get("group"))
+ to_remove.append(pkg)
+ else:
+ self.logger.error("Packages: Malformed Package: %s" %
+ lxml.etree.tostring(pkg))
+ base.update(initial | essential)
+ for el in to_remove:
+ el.getparent().remove(el)
+
+ packages, unknown = collection.complete(base)
+ if unknown:
+ self.logger.info("Packages: Got %d unknown entries" % len(unknown))
+ self.logger.info("Packages: %s" % list(unknown))
+ newpkgs = list(packages.difference(initial))
+ self.debug_log("Packages: %d initial, %d complete, %d new" %
+ (len(initial), len(packages), len(newpkgs)))
+ newpkgs.sort()
+ for pkg in newpkgs:
+ lxml.etree.SubElement(independent, 'BoundPackage', name=pkg,
+ version=self.config.get("global", "version",
+ default="auto"),
+ type=collection.ptype, origin='Packages')
+
+ def Refresh(self):
+ '''Packages.Refresh() => True|False\nReload configuration
+ specification and download sources\n'''
+ self._load_config(force_update=True)
+ return True
+
+ def Reload(self):
+ '''Packages.Refresh() => True|False\nReload configuration
+ specification and sources\n'''
+ self._load_config()
+ return True
+
+ def _load_config(self, force_update=False):
+ '''
+ Load the configuration data and setup sources
+
+ Keyword args:
+ force_update Force downloading repo data
+ '''
+ self._load_sources(force_update)
+ self._load_gpg_keys(force_update)
+
+ def _load_sources(self, force_update):
+ """ Load sources from the config """
+ self.sentinels = set()
+ cachefiles = set()
+
+ for collection in list(Collection.collections.values()):
+ cachefiles.update(collection.cachefiles)
+ if not self.disableMetaData:
+ collection.setup_data(force_update)
+ self.sentinels.update(collection.basegroups)
+
+ Collection.clear_cache()
+
+ for source in self.sources:
+ cachefiles.add(source.cachefile)
+ if not self.disableMetaData:
+ source.setup_data(force_update)
+
+ for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
+ if cfile not in cachefiles:
+ try:
+ if os.path.isdir(cfile):
+ shutil.rmtree(cfile)
+ else:
+ os.unlink(cfile)
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Could not remove cache file "
+ "%s: %s" % (cfile, err))
+
+ def _load_gpg_keys(self, force_update):
+ """ Load gpg keys from the config """
+ keyfiles = []
+ keys = []
+ for source in self.sources:
+ for key in source.gpgkeys:
+ localfile = os.path.join(self.keypath,
+ os.path.basename(key.rstrip("/")))
+ if localfile not in keyfiles:
+ keyfiles.append(localfile)
+ if ((force_update and key not in keys) or
+ not os.path.exists(localfile)):
+ self.logger.info("Packages: Downloading and parsing %s" % key)
+ response = urlopen(key)
+ open(localfile, 'w').write(response.read())
+ keys.append(key)
+
+ for kfile in glob.glob(os.path.join(self.keypath, "*")):
+ if kfile not in keyfiles:
+ os.unlink(kfile)
+
+ def _get_collection(self, metadata):
+ return Collection.factory(metadata, self.sources, self.data,
+ debug=self.debug_flag)
+
+ def get_additional_data(self, metadata):
+ collection = self._get_collection(metadata)
+ return dict(sources=collection.get_additional_data())
diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
new file mode 100644
index 000000000..e9254cdcc
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
@@ -0,0 +1,169 @@
+'''This module implements a package management scheme for all images'''
+
+import logging
+import re
+import Bcfg2.Server.Plugin
+import lxml
+try:
+ set
+except NameError:
+ from sets import Set as set
+
+logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr')
+
+
+class FuzzyDict(dict):
+ fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)')
+
+ def __getitem__(self, key):
+ if isinstance(key, str):
+ mdata = self.fuzzy.match(key)
+ if mdata:
+ return dict.__getitem__(self, mdata.groupdict()['name'])
+ else:
+ print("got non-string key %s" % str(key))
+ return dict.__getitem__(self, key)
+
+ def has_key(self, key):
+ if isinstance(key, str):
+ mdata = self.fuzzy.match(key)
+ if self.fuzzy.match(key):
+ return dict.has_key(self, mdata.groupdict()['name'])
+ return dict.has_key(self, key)
+
+ def get(self, key, default=None):
+ try:
+ return self.__getitem__(key)
+ except:
+ if default:
+ return default
+ raise
+
+
+class PNode(Bcfg2.Server.Plugin.INode):
+ """PNode has a list of packages available at a
+ particular group intersection.
+ """
+ splitters = {'rpm': re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \
+ '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'),
+ 'encap': re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')}
+ ignore = ['Package']
+
+ def Match(self, metadata, data, entry=lxml.etree.Element("None")):
+ """Return a dictionary of package mappings."""
+ if self.predicate(metadata, entry):
+ for key in self.contents:
+ try:
+ data[key].update(self.contents[key])
+ except:
+ data[key] = FuzzyDict()
+ data[key].update(self.contents[key])
+ for child in self.children:
+ child.Match(metadata, data)
+
+ def __init__(self, data, pdict, parent=None):
+ # copy local attributes to all child nodes if no local attribute exists
+ if 'Package' not in pdict:
+ pdict['Package'] = set()
+ for child in data.getchildren():
+ attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name'])
+ for attr in attrs:
+ try:
+ child.set(attr, data.get(attr))
+ except:
+ # don't fail on things like comments and other immutable elements
+ pass
+ Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent)
+ if 'Package' not in self.contents:
+ self.contents['Package'] = FuzzyDict()
+ for pkg in data.findall('./Package'):
+ if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']:
+ pdict['Package'].add(pkg.get('name'))
+ if pkg.get('name') != None:
+ self.contents['Package'][pkg.get('name')] = {}
+ if pkg.getchildren():
+ self.contents['Package'][pkg.get('name')]['__children__'] \
+ = pkg.getchildren()
+ if 'simplefile' in pkg.attrib:
+ pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile')))
+ self.contents['Package'][pkg.get('name')].update(pkg.attrib)
+ else:
+ if 'file' in pkg.attrib:
+ if 'multiarch' in pkg.attrib:
+ archs = pkg.get('multiarch').split()
+ srcs = pkg.get('srcs', pkg.get('multiarch')).split()
+ url = ' '.join(["%s/%s" % (pkg.get('uri'),
+ pkg.get('file') % {'src':srcs[idx],
+ 'arch':archs[idx]})
+ for idx in range(len(archs))])
+ pkg.set('url', url)
+ else:
+ pkg.set('url', '%s/%s' % (pkg.get('uri'),
+ pkg.get('file')))
+ if pkg.get('type') in self.splitters and pkg.get('file') != None:
+ mdata = self.splitters[pkg.get('type')].match(pkg.get('file'))
+ if not mdata:
+ logger.error("Failed to match pkg %s" % pkg.get('file'))
+ continue
+ pkgname = mdata.group('name')
+ self.contents['Package'][pkgname] = mdata.groupdict()
+ self.contents['Package'][pkgname].update(pkg.attrib)
+ if pkg.attrib.get('file'):
+ self.contents['Package'][pkgname]['url'] = pkg.get('url')
+ self.contents['Package'][pkgname]['type'] = pkg.get('type')
+ if pkg.get('verify'):
+ self.contents['Package'][pkgname]['verify'] = pkg.get('verify')
+ if pkg.get('multiarch'):
+ self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch')
+ if pkgname not in pdict['Package']:
+ pdict['Package'].add(pkgname)
+ if pkg.getchildren():
+ self.contents['Package'][pkgname]['__children__'] = pkg.getchildren()
+ else:
+ self.contents['Package'][pkg.get('name')].update(pkg.attrib)
+
+
+class PkgSrc(Bcfg2.Server.Plugin.XMLSrc):
+ """PkgSrc files contain a PNode hierarchy that
+ returns matching package entries.
+ """
+ __node__ = PNode
+ __cacheobj__ = FuzzyDict
+
+
+class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
+ """This is a generator that handles package assignments."""
+ name = 'Pkgmgr'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ __child__ = PkgSrc
+ __element__ = 'Package'
+
+ def HandleEvent(self, event):
+ '''Handle events and update dispatch table'''
+ Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event)
+ for src in list(self.entries.values()):
+ for itype, children in list(src.items.items()):
+ for child in children:
+ try:
+ self.Entries[itype][child] = self.BindEntry
+ except KeyError:
+ self.Entries[itype] = FuzzyDict([(child,
+ self.BindEntry)])
+
+ def BindEntry(self, entry, metadata):
+ """Bind data for entry, and remove instances that are not requested."""
+ pname = entry.get('name')
+ Bcfg2.Server.Plugin.PrioDir.BindEntry(self, entry, metadata)
+ if entry.findall('Instance'):
+ mdata = FuzzyDict.fuzzy.match(pname)
+ if mdata:
+ arches = mdata.group('alist').split(',')
+ [entry.remove(inst) for inst in \
+ entry.findall('Instance') \
+ if inst.get('arch') not in arches]
+
+ def HandlesEntry(self, entry, metadata):
+ return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys())
+
+ def HandleEntry(self, entry, metadata):
+ self.BindEntry(entry, metadata)
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
new file mode 100644
index 000000000..af908eee8
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -0,0 +1,285 @@
+import time
+import lxml.etree
+import operator
+import re
+
+try:
+ import json
+ has_json = True
+except ImportError:
+ try:
+ import simplejson as json
+ has_json = True
+ except ImportError:
+ has_json = False
+
+try:
+ import syck
+ has_syck = True
+except ImportError:
+ has_syck = False
+ try:
+ import yaml
+ has_yaml = True
+ except ImportError:
+ has_yaml = False
+
+import Bcfg2.Server.Plugin
+
+specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)")
+probe_matcher = re.compile("(.*/)?(?P<basename>\S+)")
+
+class ClientProbeDataSet(dict):
+ """ dict of probe => [probe data] that records a for each host """
+ def __init__(self, *args, **kwargs):
+ if "timestamp" in kwargs and kwargs['timestamp'] is not None:
+ self.timestamp = kwargs.pop("timestamp")
+ else:
+ self.timestamp = time.time()
+ dict.__init__(self, *args, **kwargs)
+
+
+class ProbeData(object):
+ """ a ProbeData object emulates a str object, but also has .xdata
+ and .json properties to provide convenient ways to use ProbeData
+ objects as XML or JSON data """
+ def __init__(self, data):
+ self.data = data
+ self._xdata = None
+ self._json = None
+ self._yaml = None
+
+ def __str__(self):
+ return str(self.data)
+
+ def __repr__(self):
+ return repr(self.data)
+
+ def __getattr__(self, name):
+ """ make ProbeData act like a str object """
+ return getattr(self.data, name)
+
+ def __complex__(self):
+ return complex(self.data)
+
+ def __int__(self):
+ return int(self.data)
+
+ def __long__(self):
+ return long(self.data)
+
+ def __float__(self):
+ return float(self.data)
+
+ def __eq__(self, other):
+ return str(self) == str(other)
+
+ def __ne__(self, other):
+ return str(self) != str(other)
+
+ def __gt__(self, other):
+ return str(self) > str(other)
+
+ def __lt__(self, other):
+ return str(self) < str(other)
+
+ def __ge__(self, other):
+ return self > other or self == other
+
+ def __le__(self, other):
+ return self < other or self == other
+
+ @property
+ def xdata(self):
+ if self._xdata is None:
+ try:
+ self._xdata = lxml.etree.XML(self.data)
+ except lxml.etree.XMLSyntaxError:
+ pass
+ return self._xdata
+
+ @property
+ def json(self):
+ if self._json is None and has_json:
+ try:
+ self._json = json.loads(self.data)
+ except ValueError:
+ pass
+ return self._json
+
+ @property
+ def yaml(self):
+ if self._yaml is None:
+ if has_yaml:
+ try:
+ self._yaml = yaml.load(self.data)
+ except yaml.YAMLError:
+ pass
+ elif has_syck:
+ try:
+ self._yaml = syck.load(self.data)
+ except syck.error:
+ pass
+ return self._yaml
+
+
+class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
+ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
+
+ def __init__(self, path, fam, encoding, plugin_name):
+ fpattern = '[0-9A-Za-z_\-]+'
+ self.plugin_name = plugin_name
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
+ Bcfg2.Server.Plugin.SpecificData,
+ encoding)
+ fam.AddMonitor(path, self)
+ self.bangline = re.compile('^#!(?P<interpreter>.*)$')
+
+ def HandleEvent(self, event):
+ if event.filename != self.path:
+ if (event.code2str == 'changed' and
+ event.filename.endswith("probed.xml") and
+ event.filename not in self.entries):
+ # for some reason, probed.xml is particularly prone to
+ # getting changed events before created events,
+ # because gamin is the worst ever. anyhow, we
+ # specifically handle it here to avoid a warning on
+ # every single server startup.
+ self.entry_init(event)
+ return
+ return self.handle_event(event)
+
+ def get_probe_data(self, metadata):
+ ret = []
+ build = dict()
+ candidates = self.get_matching(metadata)
+ candidates.sort(key=operator.attrgetter('specific'))
+ for entry in candidates:
+ rem = specific_probe_matcher.match(entry.name)
+ if not rem:
+ rem = probe_matcher.match(entry.name)
+ pname = rem.group('basename')
+ if pname not in build:
+ build[pname] = entry
+
+ for (name, entry) in list(build.items()):
+ probe = lxml.etree.Element('probe')
+ probe.set('name', name.split('/')[-1])
+ probe.set('source', self.plugin_name)
+ probe.text = entry.data
+ match = self.bangline.match(entry.data.split('\n')[0])
+ if match:
+ probe.set('interpreter', match.group('interpreter'))
+ else:
+ probe.set('interpreter', '/bin/sh')
+ ret.append(probe)
+ return ret
+
+
+class Probes(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Probing,
+ Bcfg2.Server.Plugin.Connector):
+ """A plugin to gather information from a client machine."""
+ name = 'Probes'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ Bcfg2.Server.Plugin.Probing.__init__(self)
+
+ try:
+ self.probes = ProbeSet(self.data, core.fam, core.encoding,
+ self.name)
+ except:
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ self.probedata = dict()
+ self.cgroups = dict()
+ self.load_data()
+
+ def write_data(self):
+ """Write probe data out for use with bcfg2-info."""
+ top = lxml.etree.Element("Probed")
+ for client, probed in sorted(self.probedata.items()):
+ cx = lxml.etree.SubElement(top, 'Client', name=client,
+ timestamp=str(int(probed.timestamp)))
+ for probe in sorted(probed):
+ lxml.etree.SubElement(cx, 'Probe', name=probe,
+ value=str(self.probedata[client][probe]))
+ for group in sorted(self.cgroups[client]):
+ lxml.etree.SubElement(cx, "Group", name=group)
+ data = lxml.etree.tostring(top, encoding='UTF-8',
+ xml_declaration=True,
+ pretty_print='true')
+ try:
+ datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
+ except IOError:
+ self.logger.error("Failed to write probed.xml")
+ datafile.write(data.decode('utf-8'))
+
+ def load_data(self):
+ try:
+ data = lxml.etree.parse(self.data + '/probed.xml').getroot()
+ except:
+ self.logger.error("Failed to read file probed.xml")
+ return
+ self.probedata = {}
+ self.cgroups = {}
+ for client in data.getchildren():
+ self.probedata[client.get('name')] = \
+ ClientProbeDataSet(timestamp=client.get("timestamp"))
+ self.cgroups[client.get('name')] = []
+ for pdata in client:
+ if (pdata.tag == 'Probe'):
+ self.probedata[client.get('name')][pdata.get('name')] = \
+ ProbeData(pdata.get('value'))
+ elif (pdata.tag == 'Group'):
+ self.cgroups[client.get('name')].append(pdata.get('name'))
+
+ def GetProbes(self, meta, force=False):
+ """Return a set of probes for execution on client."""
+ return self.probes.get_probe_data(meta)
+
+ def ReceiveData(self, client, datalist):
+ self.cgroups[client.hostname] = []
+ self.probedata[client.hostname] = ClientProbeDataSet()
+ for data in datalist:
+ self.ReceiveDataItem(client, data)
+ self.write_data()
+
+ def ReceiveDataItem(self, client, data):
+ """Receive probe results pertaining to client."""
+ if client.hostname not in self.cgroups:
+ self.cgroups[client.hostname] = []
+ if data.text == None:
+ self.logger.error("Got null response to probe %s from %s" % \
+ (data.get('name'), client.hostname))
+ try:
+ self.probedata[client.hostname].update({data.get('name'):
+ ProbeData('')})
+ except KeyError:
+ self.probedata[client.hostname] = \
+ ClientProbeDataSet([(data.get('name'), ProbeData(''))])
+ return
+ dlines = data.text.split('\n')
+ self.logger.debug("%s:probe:%s:%s" % (client.hostname,
+ data.get('name'), [line.strip() for line in dlines]))
+ for line in dlines[:]:
+ if line.split(':')[0] == 'group':
+ newgroup = line.split(':')[1].strip()
+ if newgroup not in self.cgroups[client.hostname]:
+ self.cgroups[client.hostname].append(newgroup)
+ dlines.remove(line)
+ dobj = ProbeData("\n".join(dlines))
+ try:
+ self.probedata[client.hostname].update({data.get('name'): dobj})
+ except KeyError:
+ self.probedata[client.hostname] = \
+ ClientProbeDataSet([(data.get('name'), dobj)])
+
+ def get_additional_groups(self, meta):
+ return self.cgroups.get(meta.hostname, list())
+
+ def get_additional_data(self, meta):
+ return self.probedata.get(meta.hostname, ClientProbeDataSet())
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
new file mode 100644
index 000000000..680881858
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -0,0 +1,76 @@
+import os
+import re
+import sys
+import copy
+import logging
+import lxml.etree
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger('Bcfg2.Plugins.Properties')
+
+class PropertyFile(Bcfg2.Server.Plugin.StructFile):
+ """Class for properties files."""
+ def write(self):
+ """ Write the data in this data structure back to the property
+ file """
+ if self.validate_data():
+ try:
+ open(self.name,
+ "wb").write(lxml.etree.tostring(self.xdata,
+ pretty_print=True))
+ return True
+ except IOError:
+ err = sys.exc_info()[1]
+ logger.error("Failed to write %s: %s" % (self.name, err))
+ return False
+ else:
+ return False
+
+ def validate_data(self):
+ """ ensure that the data in this object validates against the
+ XML schema for this property file (if a schema exists) """
+ schemafile = self.name.replace(".xml", ".xsd")
+ if os.path.exists(schemafile):
+ try:
+ schema = lxml.etree.XMLSchema(file=schemafile)
+ except:
+ logger.error("Failed to process schema for %s" % self.name)
+ return False
+ else:
+ # no schema exists
+ return True
+
+ if not schema.validate(self.xdata):
+ logger.error("Data for %s fails to validate; run bcfg2-lint for "
+ "more details" % self.name)
+ return False
+ else:
+ return True
+
+
+class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
+ __child__ = PropertyFile
+ patterns = re.compile(r'.*\.xml$')
+
+
+class Properties(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ """
+ The properties plugin maps property
+ files into client metadata instances.
+ """
+ name = 'Properties'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ try:
+ self.store = PropDirectoryBacked(self.data, core.fam)
+ except OSError:
+ e = sys.exc_info()[1]
+ self.logger.error("Error while creating Properties store: %s %s" %
+ (e.strerror, e.filename))
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def get_additional_data(self, _):
+ return copy.copy(self.store.entries)
diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py
new file mode 100644
index 000000000..b80ef351a
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Rules.py
@@ -0,0 +1,55 @@
+"""This generator provides rule-based entry mappings."""
+
+import re
+import Bcfg2.Server.Plugin
+
+class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig):
+ _required = False
+
+class Rules(Bcfg2.Server.Plugin.PrioDir):
+ """This is a generator that handles service assignments."""
+ name = 'Rules'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore)
+ self.config = RulesConfig(self)
+ self._regex_cache = dict()
+
+ def HandlesEntry(self, entry, metadata):
+ if entry.tag in self.Entries:
+ return self._matches(entry, metadata,
+ self.Entries[entry.tag].keys())
+ return False
+
+ def HandleEntry(self, entry, metadata):
+ return self.BindEntry(entry, metadata)
+
+ def BindEntry(self, entry, metadata):
+ attrs = self.get_attrs(entry, metadata)
+ for key, val in list(attrs.items()):
+ if key not in entry.attrib:
+ entry.attrib[key] = val
+
+ def _matches(self, entry, metadata, rules):
+ if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules):
+ return True
+ elif (entry.tag == "Path" and
+ ((entry.get('name').endswith("/") and
+ entry.get('name').rstrip("/") in rules) or
+ (not entry.get('name').endswith("/") and
+ entry.get('name') + '/' in rules))):
+ # special case for Path tags:
+ # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967
+ return True
+ elif self._regex_enabled:
+ # attempt regular expression matching
+ for rule in rules:
+ if rule not in self._regex_cache:
+ self._regex_cache[rule] = re.compile("%s$" % rule)
+ if self._regex_cache[rule].match(entry.get('name')):
+ return True
+ return False
+
+ def _regex_enabled(self):
+ return self.config.getboolean("rules", "regex", default=False)
diff --git a/src/lib/Bcfg2/Server/Plugins/SGenshi.py b/src/lib/Bcfg2/Server/Plugins/SGenshi.py
new file mode 100644
index 000000000..0ba08125e
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/SGenshi.py
@@ -0,0 +1,97 @@
+'''This module implements a templating generator based on Genshi'''
+
+import genshi.input
+import genshi.template
+import lxml.etree
+import logging
+import copy
+import sys
+import os.path
+
+import Bcfg2.Server.Plugin
+import Bcfg2.Server.Plugins.TGenshi
+
+logger = logging.getLogger('Bcfg2.Plugins.SGenshi')
+
+
+class SGenshiTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile,
+ Bcfg2.Server.Plugin.StructFile):
+ def __init__(self, name, specific, encoding):
+ Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name,
+ specific, encoding)
+ Bcfg2.Server.Plugin.StructFile.__init__(self, name)
+
+ def get_xml_value(self, metadata):
+ if not hasattr(self, 'template'):
+ logger.error("No parsed template information for %s" % (self.name))
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ try:
+ stream = self.template.generate(metadata=metadata).filter( \
+ Bcfg2.Server.Plugins.TGenshi.removecomment)
+ data = lxml.etree.XML(stream.render('xml', strip_whitespace=False))
+ bundlename = os.path.splitext(os.path.basename(self.name))[0]
+ bundle = lxml.etree.Element('Bundle', name=bundlename)
+ for item in self.Match(metadata, data):
+ bundle.append(copy.deepcopy(item))
+ return bundle
+ except LookupError:
+ lerror = sys.exc_info()[1]
+ logger.error('Genshi lookup error: %s' % lerror)
+ except genshi.template.TemplateError:
+ terror = sys.exc_info()[1]
+ logger.error('Genshi template error: %s' % terror)
+ raise
+ except genshi.input.ParseError:
+ perror = sys.exc_info()[1]
+ logger.error('Genshi parse error: %s' % perror)
+ raise
+
+ def Match(self, metadata, xdata):
+ """Return matching fragments of parsed template."""
+ rv = []
+ for child in xdata.getchildren():
+ rv.extend(self._match(child, metadata))
+ logger.debug("File %s got %d match(es)" % (self.name, len(rv)))
+ return rv
+
+class SGenshiEntrySet(Bcfg2.Server.Plugin.EntrySet):
+
+ def __init__(self, path, fam, encoding):
+ fpattern = '\S+\.xml'
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
+ SGenshiTemplateFile, encoding)
+ fam.AddMonitor(path, self)
+
+ def HandleEvent(self, event):
+ '''passthrough event handler for old calling convention'''
+ if event.filename != self.path:
+ return self.handle_event(event)
+
+ def BuildStructures(self, metadata):
+ """Build SGenshi structures."""
+ ret = []
+ for entry in self.get_matching(metadata):
+ try:
+ ret.append(entry.get_xml_value(metadata))
+ except:
+ logger.error("SGenshi: Failed to template file %s" % entry.name)
+ return ret
+
+
+class SGenshi(SGenshiEntrySet,
+ Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Structure):
+ """The SGenshi plugin provides templated structures."""
+ name = 'SGenshi'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ deprecated = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Structure.__init__(self)
+ try:
+ SGenshiEntrySet.__init__(self, self.data, self.core.fam, core.encoding)
+ except:
+ logger.error("Failed to load %s repository; disabling %s" \
+ % (self.name, self.name))
+ raise Bcfg2.Server.Plugin.PluginInitError
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
new file mode 100644
index 000000000..ac281ad1a
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -0,0 +1,413 @@
+'''This module manages ssh key files for bcfg2'''
+
+import binascii
+import re
+import os
+import socket
+import shutil
+import sys
+import tempfile
+from subprocess import Popen, PIPE
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import u_str
+
+if sys.hexversion >= 0x03000000:
+ from functools import reduce
+
+import logging
+logger = logging.getLogger(__name__)
+
+class KeyData(Bcfg2.Server.Plugin.SpecificData):
+ def __init__(self, name, specific, encoding):
+ Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
+ encoding)
+ self.encoding = encoding
+
+ def bind_entry(self, entry, metadata):
+ entry.set('type', 'file')
+ if entry.get('encoding') == 'base64':
+ entry.text = binascii.b2a_base64(self.data)
+ else:
+ try:
+ entry.text = u_str(self.data, self.encoding)
+ except UnicodeDecodeError:
+ e = sys.exc_info()[1]
+ logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
+ logger.error("Please verify you are using the proper encoding.")
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ except ValueError:
+ e = sys.exc_info()[1]
+ logger.error("Error in specification for %s" %
+ entry.get('name'))
+ logger.error(str(e))
+ logger.error("You need to specify base64 encoding for %s." %
+ entry.get('name'))
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ if entry.text in ['', None]:
+ entry.set('empty', 'true')
+
+class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet):
+ def __init__(self, basename, path):
+ if basename.startswith("ssh_host_key"):
+ encoding = "base64"
+ else:
+ encoding = None
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData,
+ encoding)
+ self.metadata = {'owner': 'root',
+ 'group': 'root',
+ 'type': 'file'}
+ if encoding is not None:
+ self.metadata['encoding'] = encoding
+ if basename.endswith('.pub'):
+ self.metadata['perms'] = '0644'
+ else:
+ self.metadata['perms'] = '0600'
+
+
+class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet):
+ def __init__(self, path):
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path,
+ KeyData, None)
+ self.metadata = {'owner': 'root',
+ 'group': 'root',
+ 'type': 'file',
+ 'perms': '0644'}
+
+
+class SSHbase(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Generator,
+ Bcfg2.Server.Plugin.PullTarget):
+ """
+ The sshbase generator manages ssh host keys (both v1 and v2)
+ for hosts. It also manages the ssh_known_hosts file. It can
+ integrate host keys from other management domains and similarly
+ export its keys. The repository contains files in the following
+ formats:
+
+ ssh_host_key.H_(hostname) -> the v1 host private key for
+ (hostname)
+ ssh_host_key.pub.H_(hostname) -> the v1 host public key
+ for (hostname)
+ ssh_host_(ec)(dr)sa_key.H_(hostname) -> the v2 ssh host
+ private key for (hostname)
+ ssh_host_(ec)(dr)sa_key.pub.H_(hostname) -> the v2 ssh host
+ public key for (hostname)
+ ssh_known_hosts -> the current known hosts file. this
+ is regenerated each time a new key is generated.
+
+ """
+ name = 'SSHbase'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ keypatterns = ["ssh_host_dsa_key",
+ "ssh_host_ecdsa_key",
+ "ssh_host_rsa_key",
+ "ssh_host_key",
+ "ssh_host_dsa_key.pub",
+ "ssh_host_ecdsa_key.pub",
+ "ssh_host_rsa_key.pub",
+ "ssh_host_key.pub"]
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Generator.__init__(self)
+ Bcfg2.Server.Plugin.PullTarget.__init__(self)
+ self.ipcache = {}
+ self.namecache = {}
+ self.__skn = False
+
+ # keep track of which bogus keys we've warned about, and only
+ # do so once
+ self.badnames = dict()
+
+ core.fam.AddMonitor(self.data, self)
+
+ self.static = dict()
+ self.entries = dict()
+ self.Entries['Path'] = dict()
+
+ self.entries['/etc/ssh/ssh_known_hosts'] = KnownHostsEntrySet(self.data)
+ self.Entries['Path']['/etc/ssh/ssh_known_hosts'] = self.build_skn
+ for keypattern in self.keypatterns:
+ self.entries["/etc/ssh/" + keypattern] = HostKeyEntrySet(keypattern,
+ self.data)
+ self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk
+
+ def get_skn(self):
+ """Build memory cache of the ssh known hosts file."""
+ if not self.__skn:
+ # if no metadata is registered yet, defer
+ if len(self.core.metadata.query.all()) == 0:
+ self.__skn = False
+ return self.__skn
+
+ skn = [s.data.decode().rstrip()
+ for s in list(self.static.values())]
+
+ mquery = self.core.metadata.query
+
+ # build hostname cache
+ names = dict()
+ for cmeta in mquery.all():
+ names[cmeta.hostname] = set([cmeta.hostname])
+ names[cmeta.hostname].update(cmeta.aliases)
+ newnames = set()
+ newips = set()
+ for name in names[cmeta.hostname]:
+ newnames.add(name.split('.')[0])
+ try:
+ newips.add(self.get_ipcache_entry(name)[0])
+ except:
+ continue
+ names[cmeta.hostname].update(newnames)
+ names[cmeta.hostname].update(cmeta.addresses)
+ names[cmeta.hostname].update(newips)
+ # TODO: Only perform reverse lookups on IPs if an option is set.
+ if True:
+ for ip in newips:
+ try:
+ names[cmeta.hostname].update(self.get_namecache_entry(ip))
+ except:
+ continue
+ names[cmeta.hostname] = sorted(names[cmeta.hostname])
+
+ pubkeys = [pubk for pubk in list(self.entries.keys())
+ if pubk.endswith('.pub')]
+ pubkeys.sort()
+ for pubkey in pubkeys:
+ for entry in sorted(self.entries[pubkey].entries.values(),
+ key=lambda e: e.specific.hostname or e.specific.group):
+ specific = entry.specific
+ hostnames = []
+ if specific.hostname and specific.hostname in names:
+ hostnames = names[specific.hostname]
+ elif specific.group:
+ hostnames = \
+ reduce(lambda x, y: x + y,
+ [names[cmeta.hostname]
+ for cmeta in \
+ mquery.by_groups([specific.group])], [])
+ elif specific.all:
+ # a generic key for all hosts? really?
+ hostnames = reduce(lambda x, y: x + y,
+ list(names.values()), [])
+ if not hostnames:
+ if specific.hostname:
+ key = specific.hostname
+ ktype = "host"
+ elif specific.group:
+ key = specific.group
+ ktype = "group"
+ else:
+ # user has added a global SSH key, but
+ # have no clients yet. don't warn about
+ # this.
+ continue
+
+ if key not in self.badnames:
+ self.badnames[key] = True
+ self.logger.info("Ignoring key for unknown %s %s" %
+ (ktype, key))
+ continue
+
+ skn.append("%s %s" % (','.join(hostnames),
+ entry.data.decode().rstrip()))
+
+ self.__skn = "\n".join(skn) + "\n"
+ return self.__skn
+
+ def set_skn(self, value):
+ """Set backing data for skn."""
+ self.__skn = value
+ skn = property(get_skn, set_skn)
+
+ def HandleEvent(self, event=None):
+ """Local event handler that does skn regen on pubkey change."""
+ # skip events we don't care about
+ action = event.code2str()
+ if action == "endExist" or event.filename == self.data:
+ return
+
+ for entry in list(self.entries.values()):
+ if entry.specific.match(event.filename):
+ entry.handle_event(event)
+ if event.filename.endswith(".pub"):
+ self.logger.info("New public key %s; invalidating "
+ "ssh_known_hosts cache" % event.filename)
+ self.skn = False
+ return
+
+ if event.filename in ['info', 'info.xml', ':info']:
+ for entry in list(self.entries.values()):
+ entry.handle_event(event)
+ return
+
+ if event.filename.endswith('.static'):
+ self.logger.info("Static key %s %s; invalidating ssh_known_hosts "
+ "cache" % (event.filename, action))
+ if action == "deleted" and event.filename in self.static:
+ del self.static[event.filename]
+ self.skn = False
+ else:
+ self.static[event.filename] = \
+ Bcfg2.Server.Plugin.FileBacked(os.path.join(self.data,
+ event.filename))
+ self.static[event.filename].HandleEvent(event)
+ self.skn = False
+ return
+
+ self.logger.warn("SSHbase: Got unknown event %s %s" %
+ (event.filename, action))
+
+ def get_ipcache_entry(self, client):
+ """Build a cache of dns results."""
+ if client in self.ipcache:
+ if self.ipcache[client]:
+ return self.ipcache[client]
+ else:
+ raise socket.gaierror
+ else:
+ # need to add entry
+ try:
+ ipaddr = socket.gethostbyname(client)
+ self.ipcache[client] = (ipaddr, client)
+ return (ipaddr, client)
+ except socket.gaierror:
+ ipaddr = Popen(["getent", "hosts", client],
+ stdout=PIPE).stdout.read().strip().split()
+ if ipaddr:
+ self.ipcache[client] = (ipaddr, client)
+ return (ipaddr, client)
+ self.ipcache[client] = False
+ self.logger.error("Failed to find IP address for %s" % client)
+ raise socket.gaierror
+
+ def get_namecache_entry(self, cip):
+ """Build a cache of name lookups from client IP addresses."""
+ if cip in self.namecache:
+ # lookup cached name from IP
+ if self.namecache[cip]:
+ return self.namecache[cip]
+ else:
+ raise socket.gaierror
+ else:
+ # add an entry that has not been cached
+ try:
+ rvlookup = socket.gethostbyaddr(cip)
+ if rvlookup[0]:
+ self.namecache[cip] = [rvlookup[0]]
+ else:
+ self.namecache[cip] = []
+ self.namecache[cip].extend(rvlookup[1])
+ return self.namecache[cip]
+ except socket.gaierror:
+ self.namecache[cip] = False
+ self.logger.error("Failed to find any names associated with IP address %s" % cip)
+ raise
+
+ def build_skn(self, entry, metadata):
+ """This function builds builds a host specific known_hosts file."""
+ try:
+ rv = self.entries[entry.get('name')].bind_entry(entry, metadata)
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ client = metadata.hostname
+ entry.text = self.skn
+ hostkeys = []
+ for k in self.keypatterns:
+ if k.endswith(".pub"):
+ try:
+ hostkeys.append(self.entries["/etc/ssh/" +
+ k].best_matching(metadata))
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ pass
+ hostkeys.sort()
+ for hostkey in hostkeys:
+ entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
+ hostkey.data.decode())
+ self.entries[entry.get('name')].bind_info_to_entry(entry, metadata)
+
+ def build_hk(self, entry, metadata):
+ """This binds host key data into entries."""
+ try:
+ self.entries[entry.get('name')].bind_entry(entry, metadata)
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ filename = entry.get('name').split('/')[-1]
+ self.GenerateHostKeyPair(metadata.hostname, filename)
+ # Service the FAM events queued up by the key generation
+ # so the data structure entries will be available for
+ # binding.
+ #
+ # NOTE: We wait for up to ten seconds. There is some
+ # potential for race condition, because if the file
+ # monitor doesn't get notified about the new key files in
+ # time, those entries won't be available for binding. In
+ # practice, this seems "good enough".
+ tries = 0
+ is_bound = False
+ while not is_bound:
+ if tries >= 10:
+ self.logger.error("%s still not registered" % filename)
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ self.core.fam.handle_events_in_interval(1)
+ tries += 1
+ try:
+ self.entries[entry.get('name')].bind_entry(entry, metadata)
+ is_bound = True
+ except Bcfg2.Server.Plugin.PluginExecutionError:
+ pass
+
+ def GenerateHostKeyPair(self, client, filename):
+ """Generate new host key pair for client."""
+ match = re.search(r'(ssh_host_(?:((?:ecd|d|r)sa)_)?key)', filename)
+ if match:
+ hostkey = "%s.H_%s" % (match.group(1), client)
+ if match.group(2):
+ keytype = match.group(2)
+ else:
+ keytype = 'rsa1'
+ else:
+ self.logger.error("Unknown key filename: %s" % filename)
+ return
+
+ fileloc = "%s/%s" % (self.data, hostkey)
+ publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub',
+ "H_%s" % client])
+ tempdir = tempfile.mkdtemp()
+ temploc = "%s/%s" % (tempdir, hostkey)
+ cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "",
+ "-t", keytype, "-C", "root@%s" % client]
+ proc = Popen(cmd, stdout=PIPE, stdin=PIPE)
+ proc.communicate()
+ proc.wait()
+
+ try:
+ shutil.copy(temploc, fileloc)
+ shutil.copy("%s.pub" % temploc, publoc)
+ except IOError:
+ err = sys.exc_info()[1]
+ self.logger.error("Temporary SSH keys not found: %s" % err)
+
+ try:
+ os.unlink(temploc)
+ os.unlink("%s.pub" % temploc)
+ os.rmdir(tempdir)
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to unlink temporary ssh keys: %s" % err)
+
+ def AcceptChoices(self, _, metadata):
+ return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)]
+
+ def AcceptPullData(self, specific, entry, log):
+ """Per-plugin bcfg2-admin pull support."""
+ # specific will always be host specific
+ filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1],
+ specific.hostname)
+ try:
+ open(filename, 'w').write(entry['text'])
+ if log:
+ print("Wrote file %s" % filename)
+ except KeyError:
+ self.logger.error("Failed to pull %s. This file does not currently "
+ "exist on the client" % entry.get('name'))
diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
new file mode 100644
index 000000000..0072dc62d
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
@@ -0,0 +1,274 @@
+import Bcfg2.Server.Plugin
+import Bcfg2.Options
+import lxml.etree
+import posixpath
+import tempfile
+import pipes
+import os
+from subprocess import Popen, PIPE, STDOUT
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
+
+
+class SSLCA(Bcfg2.Server.Plugin.GroupSpool):
+ """
+ The SSLCA generator handles the creation and
+ management of ssl certificates and their keys.
+ """
+ name = 'SSLCA'
+ __author__ = 'g.hagger@gmail.com'
+ __child__ = Bcfg2.Server.Plugin.FileBacked
+ key_specs = {}
+ cert_specs = {}
+ CAs = {}
+
+ def HandleEvent(self, event=None):
+ """
+ Updates which files this plugin handles based upon filesystem events.
+ Allows configuration items to be added/removed without server restarts.
+ """
+ action = event.code2str()
+ if event.filename[0] == '/':
+ return
+ epath = "".join([self.data, self.handles[event.requestID],
+ event.filename])
+ if posixpath.isdir(epath):
+ ident = self.handles[event.requestID] + event.filename
+ else:
+ ident = self.handles[event.requestID][:-1]
+
+ fname = "".join([ident, '/', event.filename])
+
+ if event.filename.endswith('.xml'):
+ if action in ['exists', 'created', 'changed']:
+ if event.filename.endswith('key.xml'):
+ key_spec = dict(list(lxml.etree.parse(epath).find('Key').items()))
+ self.key_specs[ident] = {
+ 'bits': key_spec.get('bits', 2048),
+ 'type': key_spec.get('type', 'rsa')
+ }
+ self.Entries['Path'][ident] = self.get_key
+ elif event.filename.endswith('cert.xml'):
+ cert_spec = dict(list(lxml.etree.parse(epath).find('Cert').items()))
+ ca = cert_spec.get('ca', 'default')
+ self.cert_specs[ident] = {
+ 'ca': ca,
+ 'format': cert_spec.get('format', 'pem'),
+ 'key': cert_spec.get('key'),
+ 'days': cert_spec.get('days', 365),
+ 'C': cert_spec.get('c'),
+ 'L': cert_spec.get('l'),
+ 'ST': cert_spec.get('st'),
+ 'OU': cert_spec.get('ou'),
+ 'O': cert_spec.get('o'),
+ 'emailAddress': cert_spec.get('emailaddress')
+ }
+ cp = ConfigParser.ConfigParser()
+ cp.read(self.core.cfile)
+ self.CAs[ca] = dict(cp.items('sslca_' + ca))
+ self.Entries['Path'][ident] = self.get_cert
+ if action == 'deleted':
+ if ident in self.Entries['Path']:
+ del self.Entries['Path'][ident]
+ else:
+ if action in ['exists', 'created']:
+ if posixpath.isdir(epath):
+ self.AddDirectoryMonitor(epath[len(self.data):])
+ if ident not in self.entries and posixpath.isfile(epath):
+ self.entries[fname] = self.__child__(epath)
+ self.entries[fname].HandleEvent(event)
+ if action == 'changed':
+ self.entries[fname].HandleEvent(event)
+ elif action == 'deleted':
+ if fname in self.entries:
+ del self.entries[fname]
+ else:
+ self.entries[fname].HandleEvent(event)
+
+ def get_key(self, entry, metadata):
+ """
+ either grabs a prexisting key hostfile, or triggers the generation
+ of a new key if one doesn't exist.
+ """
+ # set path type and permissions, otherwise bcfg2 won't bind the file
+ permdata = {'owner': 'root',
+ 'group': 'root',
+ 'type': 'file',
+ 'perms': '644'}
+ [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
+
+ # check if we already have a hostfile, or need to generate a new key
+ # TODO: verify key fits the specs
+ path = entry.get('name')
+ filename = "".join([path, '/', path.rsplit('/', 1)[1],
+ '.H_', metadata.hostname])
+ if filename not in list(self.entries.keys()):
+ key = self.build_key(filename, entry, metadata)
+ open(self.data + filename, 'w').write(key)
+ entry.text = key
+ self.entries[filename] = self.__child__("%s%s" % (self.data,
+ filename))
+ self.entries[filename].HandleEvent()
+ else:
+ entry.text = self.entries[filename].data
+
+ def build_key(self, filename, entry, metadata):
+ """
+ generates a new key according the the specification
+ """
+ type = self.key_specs[entry.get('name')]['type']
+ bits = self.key_specs[entry.get('name')]['bits']
+ if type == 'rsa':
+ cmd = ["openssl", "genrsa", bits]
+ elif type == 'dsa':
+ cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits]
+ key = Popen(cmd, stdout=PIPE).stdout.read()
+ return key
+
+ def get_cert(self, entry, metadata):
+ """
+ either grabs a prexisting cert hostfile, or triggers the generation
+ of a new cert if one doesn't exist.
+ """
+ # set path type and permissions, otherwise bcfg2 won't bind the file
+ permdata = {'owner': 'root',
+ 'group': 'root',
+ 'type': 'file',
+ 'perms': '644'}
+ [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
+
+ path = entry.get('name')
+ filename = "".join([path, '/', path.rsplit('/', 1)[1],
+ '.H_', metadata.hostname])
+
+ # first - ensure we have a key to work with
+ key = self.cert_specs[entry.get('name')].get('key')
+ key_filename = "".join([key, '/', key.rsplit('/', 1)[1],
+ '.H_', metadata.hostname])
+ if key_filename not in self.entries:
+ e = lxml.etree.Element('Path')
+ e.attrib['name'] = key
+ self.core.Bind(e, metadata)
+
+ # check if we have a valid hostfile
+ if filename in list(self.entries.keys()) and self.verify_cert(filename,
+ key_filename,
+ entry):
+ entry.text = self.entries[filename].data
+ else:
+ cert = self.build_cert(key_filename, entry, metadata)
+ open(self.data + filename, 'w').write(cert)
+ self.entries[filename] = self.__child__("%s%s" % (self.data,
+ filename))
+ self.entries[filename].HandleEvent()
+ entry.text = cert
+
+ def verify_cert(self, filename, key_filename, entry):
+ if self.verify_cert_against_ca(filename, entry):
+ if self.verify_cert_against_key(filename, key_filename):
+ return True
+ return False
+
+ def verify_cert_against_ca(self, filename, entry):
+ """
+ check that a certificate validates against the ca cert,
+ and that it has not expired.
+ """
+ chaincert = self.CAs[self.cert_specs[entry.get('name')]['ca']].get('chaincert')
+ cert = self.data + filename
+ res = Popen(["openssl", "verify", "-CAfile", chaincert, cert],
+ stdout=PIPE, stderr=STDOUT).stdout.read()
+ if res == cert + ": OK\n":
+ return True
+ return False
+
+ def verify_cert_against_key(self, filename, key_filename):
+ """
+ check that a certificate validates against its private key.
+ """
+ cert = self.data + filename
+ key = self.data + key_filename
+ cmd = ("openssl x509 -noout -modulus -in %s | openssl md5" %
+ pipes.quote(cert))
+ cert_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read()
+ cmd = ("openssl rsa -noout -modulus -in %s | openssl md5" %
+ pipes.quote(key))
+ key_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read()
+ if cert_md5 == key_md5:
+ return True
+ return False
+
+ def build_cert(self, key_filename, entry, metadata):
+ """
+ creates a new certificate according to the specification
+ """
+ req_config = self.build_req_config(entry, metadata)
+ req = self.build_request(key_filename, req_config, entry)
+ ca = self.cert_specs[entry.get('name')]['ca']
+ ca_config = self.CAs[ca]['config']
+ days = self.cert_specs[entry.get('name')]['days']
+ passphrase = self.CAs[ca].get('passphrase')
+ cmd = ["openssl", "ca", "-config", ca_config, "-in", req,
+ "-days", days, "-batch"]
+ if passphrase:
+ cmd.extend(["-passin", "pass:%s" % passphrase])
+ cert = Popen(cmd, stdout=PIPE).stdout.read()
+ try:
+ os.unlink(req_config)
+ os.unlink(req)
+ except OSError:
+ self.logger.error("Failed to unlink temporary files")
+ return cert
+
+ def build_req_config(self, entry, metadata):
+ """
+ generates a temporary openssl configuration file that is
+ used to generate the required certificate request
+ """
+ # create temp request config file
+ conffile = open(tempfile.mkstemp()[1], 'w')
+ cp = ConfigParser.ConfigParser({})
+ cp.optionxform = str
+ defaults = {
+ 'req': {
+ 'default_md': 'sha1',
+ 'distinguished_name': 'req_distinguished_name',
+ 'req_extensions': 'v3_req',
+ 'x509_extensions': 'v3_req',
+ 'prompt': 'no'
+ },
+ 'req_distinguished_name': {},
+ 'v3_req': {
+ 'subjectAltName': '@alt_names'
+ },
+ 'alt_names': {}
+ }
+ for section in list(defaults.keys()):
+ cp.add_section(section)
+ for key in defaults[section]:
+ cp.set(section, key, defaults[section][key])
+ x = 1
+ altnames = list(metadata.aliases)
+ altnames.append(metadata.hostname)
+ for altname in altnames:
+ cp.set('alt_names', 'DNS.' + str(x), altname)
+ x += 1
+ for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']:
+ if self.cert_specs[entry.get('name')][item]:
+ cp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item])
+ cp.set('req_distinguished_name', 'CN', metadata.hostname)
+ cp.write(conffile)
+ conffile.close()
+ return conffile.name
+
+ def build_request(self, key_filename, req_config, entry):
+ """
+ creates the certificate request
+ """
+ req = tempfile.mkstemp()[1]
+ days = self.cert_specs[entry.get('name')]['days']
+ key = self.data + key_filename
+ cmd = ["openssl", "req", "-new", "-config", req_config,
+ "-days", days, "-key", key, "-text", "-out", req]
+ res = Popen(cmd, stdout=PIPE).stdout.read()
+ return req
diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py
new file mode 100644
index 000000000..aeb3b9f74
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py
@@ -0,0 +1,142 @@
+#import lxml.etree
+import logging
+import binascii
+import difflib
+#import sqlalchemy
+#import sqlalchemy.orm
+import Bcfg2.Server.Plugin
+import Bcfg2.Server.Snapshots
+import Bcfg2.Logger
+from Bcfg2.Server.Snapshots.model import Snapshot
+import sys
+import time
+import threading
+
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import Queue
+
+logger = logging.getLogger('Snapshots')
+
+ftypes = ['ConfigFile', 'SymLink', 'Directory']
+datafields = {
+ 'Package': ['version'],
+ 'Path': ['type'],
+ 'Service': ['status'],
+ 'ConfigFile': ['owner', 'group', 'perms'],
+ 'Directory': ['owner', 'group', 'perms'],
+ 'SymLink': ['to'],
+ }
+
+
+# py3k compatibility
+def u_str(string):
+ if sys.hexversion >= 0x03000000:
+ return string
+ else:
+ return unicode(string)
+
+
+def build_snap_ent(entry):
+ basefields = []
+ if entry.tag in ['Package', 'Service']:
+ basefields += ['type']
+ desired = dict([(key, u_str(entry.get(key))) for key in basefields])
+ state = dict([(key, u_str(entry.get(key))) for key in basefields])
+ desired.update([(key, u_str(entry.get(key))) for key in \
+ datafields[entry.tag]])
+ if entry.tag == 'ConfigFile' or \
+ ((entry.tag == 'Path') and (entry.get('type') == 'file')):
+ if entry.text == None:
+ desired['contents'] = None
+ else:
+ if entry.get('encoding', 'ascii') == 'ascii':
+ desired['contents'] = u_str(entry.text)
+ else:
+ desired['contents'] = u_str(binascii.a2b_base64(entry.text))
+
+ if 'current_bfile' in entry.attrib:
+ state['contents'] = u_str(binascii.a2b_base64( \
+ entry.get('current_bfile')))
+ elif 'current_bdiff' in entry.attrib:
+ diff = binascii.a2b_base64(entry.get('current_bdiff'))
+ state['contents'] = u_str( \
+ '\n'.join(difflib.restore(diff.split('\n'), 1)))
+
+ state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \
+ for key in datafields[entry.tag]])
+ if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false':
+ state = None
+ return [desired, state]
+
+
+class Snapshots(Bcfg2.Server.Plugin.Statistics,
+ Bcfg2.Server.Plugin.Plugin):
+ name = 'Snapshots'
+ experimental = True
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Statistics.__init__(self)
+ self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile)
+ self.work_queue = Queue()
+ self.loader = threading.Thread(target=self.load_snapshot)
+ self.loader.start()
+
+ def load_snapshot(self):
+ while self.running:
+ try:
+ (metadata, data) = self.work_queue.get(block=True, timeout=5)
+ except:
+ continue
+ self.statistics_from_old_stats(metadata, data)
+
+ def process_statistics(self, metadata, data):
+ return self.work_queue.put((metadata, data))
+
+ def statistics_from_old_stats(self, metadata, xdata):
+ # entries are name -> (modified, correct, start, desired, end)
+ # not sure we can get all of this from old format stats
+ t1 = time.time()
+ entries = dict([('Package', dict()),
+ ('Service', dict()), ('Path', dict())])
+ extra = dict([('Package', dict()), ('Service', dict()),
+ ('Path', dict())])
+ bad = []
+ state = xdata.find('.//Statistics')
+ correct = state.get('state') == 'clean'
+ revision = u_str(state.get('revision', '-1'))
+ for entry in state.find('.//Bad'):
+ data = [False, False, u_str(entry.get('name'))] \
+ + build_snap_ent(entry)
+ if entry.tag in ftypes:
+ etag = 'Path'
+ else:
+ etag = entry.tag
+ entries[etag][entry.get('name')] = data
+ for entry in state.find('.//Modified'):
+ if entry.tag in ftypes:
+ etag = 'Path'
+ else:
+ etag = entry.tag
+ if entry.get('name') in entries[etag]:
+ data = [True, False, u_str(entry.get('name'))] + \
+ build_snap_ent(entry)
+ else:
+ data = [True, False, u_str(entry.get('name'))] + \
+ build_snap_ent(entry)
+ for entry in state.find('.//Extra'):
+ if entry.tag in datafields:
+ data = build_snap_ent(entry)[1]
+ ename = u_str(entry.get('name'))
+ data['name'] = ename
+ extra[entry.tag][ename] = data
+ else:
+ print("extra", entry.tag, entry.get('name'))
+ t2 = time.time()
+ snap = Snapshot.from_data(self.session, correct, revision,
+ metadata, entries, extra)
+ self.session.add(snap)
+ self.session.commit()
+ t3 = time.time()
+ logger.info("Snapshot storage took %fs" % (t3 - t2))
+ return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py
new file mode 100644
index 000000000..265ef95a8
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Statistics.py
@@ -0,0 +1,161 @@
+'''This file manages the statistics collected by the BCFG2 Server'''
+
+import binascii
+import copy
+import difflib
+import logging
+from lxml.etree import XML, SubElement, Element, XMLSyntaxError
+import lxml.etree
+import os
+from time import asctime, localtime, time, strptime, mktime
+import threading
+
+import Bcfg2.Server.Plugin
+
+
+class StatisticsStore(object):
+ """Manages the memory and file copy of statistics collected about client runs."""
+ __min_write_delay__ = 0
+
+ def __init__(self, filename):
+ self.filename = filename
+ self.element = Element('Dummy')
+ self.dirty = 0
+ self.lastwrite = 0
+ self.logger = logging.getLogger('Bcfg2.Server.Statistics')
+ self.ReadFromFile()
+
+ def WriteBack(self, force=0):
+ """Write statistics changes back to persistent store."""
+ if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \
+ or force:
+ try:
+ fout = open(self.filename + '.new', 'w')
+ except IOError:
+ ioerr = sys.exc_info()[1]
+ self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr))
+ else:
+ fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True))
+ fout.close()
+ os.rename(self.filename + '.new', self.filename)
+ self.dirty = 0
+ self.lastwrite = time()
+
+ def ReadFromFile(self):
+ """Reads current state regarding statistics."""
+ try:
+ fin = open(self.filename, 'r')
+ data = fin.read()
+ fin.close()
+ self.element = XML(data)
+ self.dirty = 0
+ except (IOError, XMLSyntaxError):
+ self.logger.error("Creating new statistics file %s"%(self.filename))
+ self.element = Element('ConfigStatistics')
+ self.WriteBack()
+ self.dirty = 0
+
+ def updateStats(self, xml, client):
+ """Updates the statistics of a current node with new data."""
+
+ # Current policy:
+ # - Keep anything less than 24 hours old
+ # - Keep latest clean run for clean nodes
+ # - Keep latest clean and dirty run for dirty nodes
+ newstat = xml.find('Statistics')
+
+ if newstat.get('state') == 'clean':
+ node_dirty = 0
+ else:
+ node_dirty = 1
+
+ # Find correct node entry in stats data
+ # The following list comprehension should be guarenteed to return at
+ # most one result
+ nodes = [elem for elem in self.element.findall('Node') \
+ if elem.get('name') == client]
+ nummatch = len(nodes)
+ if nummatch == 0:
+ # Create an entry for this node
+ node = SubElement(self.element, 'Node', name=client)
+ elif nummatch == 1 and not node_dirty:
+ # Delete old instance
+ node = nodes[0]
+ [node.remove(elem) for elem in node.findall('Statistics') \
+ if self.isOlderThan24h(elem.get('time'))]
+ elif nummatch == 1 and node_dirty:
+ # Delete old dirty statistics entry
+ node = nodes[0]
+ [node.remove(elem) for elem in node.findall('Statistics') \
+ if (elem.get('state') == 'dirty' \
+ and self.isOlderThan24h(elem.get('time')))]
+ else:
+ # Shouldn't be reached
+ self.logger.error("Duplicate node entry for %s"%(client))
+
+ # Set current time for stats
+ newstat.set('time', asctime(localtime()))
+
+ # Add statistic
+ node.append(copy.copy(newstat))
+
+ # Set dirty
+ self.dirty = 1
+ self.WriteBack(force=1)
+
+ def isOlderThan24h(self, testTime):
+ """Helper function to determine if <time> string is older than 24 hours."""
+ now = time()
+ utime = mktime(strptime(testTime))
+ secondsPerDay = 60*60*24
+
+ return (now-utime) > secondsPerDay
+
+
+class Statistics(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.ThreadedStatistics,
+ Bcfg2.Server.Plugin.PullSource):
+ name = 'Statistics'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.PullSource.__init__(self)
+ fpath = "%s/etc/statistics.xml" % datastore
+ self.data_file = StatisticsStore(fpath)
+
+ def handle_statistic(self, metadata, data):
+ self.data_file.updateStats(data, metadata.hostname)
+
+ def FindCurrent(self, client):
+ rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0]
+ maxtime = max([strptime(stat.get('time')) for stat \
+ in rt.findall('Statistics')])
+ return [stat for stat in rt.findall('Statistics') \
+ if strptime(stat.get('time')) == maxtime][0]
+
+ def GetExtra(self, client):
+ return [(entry.tag, entry.get('name')) for entry \
+ in self.FindCurrent(client).xpath('.//Extra/*')]
+
+ def GetCurrentEntry(self, client, e_type, e_name):
+ curr = self.FindCurrent(client)
+ entry = curr.xpath('.//Bad/%s[@name="%s"]' % (e_type, e_name))
+ if not entry:
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ cfentry = entry[-1]
+
+ owner = cfentry.get('current_owner', cfentry.get('owner'))
+ group = cfentry.get('current_group', cfentry.get('group'))
+ perms = cfentry.get('current_perms', cfentry.get('perms'))
+ if cfentry.get('sensitive') in ['true', 'True']:
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+ elif 'current_bfile' in cfentry.attrib:
+ contents = binascii.a2b_base64(cfentry.get('current_bfile'))
+ elif 'current_bdiff' in cfentry.attrib:
+ diff = binascii.a2b_base64(cfentry.get('current_bdiff'))
+ contents = '\n'.join(difflib.restore(diff.split('\n'), 1))
+ else:
+ contents = None
+
+ return (owner, group, perms, contents)
diff --git a/src/lib/Bcfg2/Server/Plugins/Svcmgr.py b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py
new file mode 100644
index 000000000..f4232ad5c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py
@@ -0,0 +1,10 @@
+"""This generator provides service mappings."""
+
+import Bcfg2.Server.Plugin
+
+
+class Svcmgr(Bcfg2.Server.Plugin.PrioDir):
+ """This is a generator that handles service assignments."""
+ name = 'Svcmgr'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ deprecated = True
diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py
new file mode 100644
index 000000000..ae43388ea
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Svn.py
@@ -0,0 +1,46 @@
+import os
+import pipes
+from subprocess import Popen, PIPE
+import Bcfg2.Server.Plugin
+
+# for debugging output only
+import logging
+logger = logging.getLogger('Bcfg2.Plugins.Svn')
+
+
+class Svn(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Svn is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Svn'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ self.core = core
+ self.datastore = datastore
+
+ # path to svn directory for bcfg2 repo
+ svn_dir = "%s/.svn" % datastore
+
+ # Read revision from bcfg2 repo
+ if os.path.isdir(svn_dir):
+ self.get_revision()
+ else:
+ logger.error("%s is not a directory" % svn_dir)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ logger.debug("Initialized svn plugin with svn directory = %s" % svn_dir)
+
+ def get_revision(self):
+ """Read svn revision information for the Bcfg2 repository."""
+ try:
+ data = Popen(("env LC_ALL=C svn info %s" %
+ pipes.quote(self.datastore)), shell=True,
+ stdout=PIPE).communicate()[0].split('\n')
+ return [line.split(': ')[1] for line in data \
+ if line[:9] == 'Revision:'][-1]
+ except IndexError:
+ logger.error("Failed to read svn info; disabling svn support")
+ logger.error('''Ran command "svn info %s"''' % (self.datastore))
+ logger.error("Got output: %s" % data)
+ raise Bcfg2.Server.Plugin.PluginInitError
diff --git a/src/lib/Bcfg2/Server/Plugins/Svn2.py b/src/lib/Bcfg2/Server/Plugins/Svn2.py
new file mode 100644
index 000000000..e4df9574f
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Svn2.py
@@ -0,0 +1,126 @@
+try:
+ import pysvn
+ missing = False
+except:
+ missing = True
+import Bcfg2.Server.Plugin
+
+class Svn2(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Version):
+ """Svn is a version plugin for dealing with Bcfg2 repos."""
+ name = 'Svn2'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ conflicts = ['Svn']
+ experimental = True
+ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update','Commit']
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+
+ if missing:
+ self.logger.error("Svn2: Missing PySvn")
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ self.client = pysvn.Client()
+
+ self.core = core
+ self.datastore = datastore
+ self.svn_root = None
+ self.revision = None
+
+ # Read revision from bcfg2 repo
+ revision = self.get_revision()
+ if not self.revision:
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ self.logger.debug("Initialized svn plugin with svn root %s at revision %s"
+ % (self.svn_root, revision))
+
+ def get_revision(self):
+ """Read svn revision information for the Bcfg2 repository."""
+ try:
+ info = self.client.info(self.datastore)
+ self.revision = info.revision
+ self.svn_root = info.url
+ return str(self.revision.number)
+ except:
+ self.logger.error("Svn2: Failed to get revision", exc_info=1)
+ self.revision = None
+ return str(-1)
+
+ def commit_data(self, file_list, comment=None):
+ """Commit changes into the repository"""
+ if not comment:
+ comment = 'Svn2: autocommit'
+
+ # First try to update
+ if not self.Update():
+ self.logger.error("Failed to update svn repository, refusing to commit changes")
+ return
+
+ #FIXME - look for conflicts?
+
+ for fname in file_list:
+ stat = self.client.status(fname)
+ self.client.add([f.path for f in stat \
+ if f.text_status == pysvn.wc_status_kind.unversioned])
+ try:
+ self.revision = self.client.checkin([self.datastore], comment,
+ recurse=True)
+ self.revision = self.client.update(self.datastore, recurse=True)[0]
+ self.logger.info("Svn2: Commited changes. At %s" %
+ self.revision.number)
+ except Exception:
+ err = sys.exc_info()[1]
+ # try to be smart about the error we got back
+ details = None
+ if "callback_ssl_server_trust_prompt" in str(err):
+ details = "SVN server certificate is not trusted"
+ elif "callback_get_login" in str(err):
+ details = "SVN credentials not cached"
+
+ if details is None:
+ self.logger.error("Svn2: Failed to commit changes",
+ exc_info=1)
+ else:
+ self.logger.error("Svn2: Failed to commit changes: %s" %
+ details)
+
+ def Update(self):
+ '''Svn2.Update() => True|False\nUpdate svn working copy\n'''
+ try:
+ old_revision = self.revision.number
+ self.revision = self.client.update(self.datastore, recurse=True)[0]
+ except Exception, err:
+ # try to be smart about the error we got back
+ details = None
+ if "callback_ssl_server_trust_prompt" in str(err):
+ details = "SVN server certificate is not trusted"
+ elif "callback_get_login" in str(err):
+ details = "SVN credentials not cached"
+
+ if details is None:
+ self.logger.error("Svn2: Failed to update server repository",
+ exc_info=1)
+ else:
+ self.logger.error("Svn2: Failed to update server repository: %s" %
+ details)
+ return False
+
+ if old_revision == self.revision.number:
+ self.logger.debug("repository is current")
+ else:
+ self.logger.info("Updated %s from revision %s to %s" % \
+ (self.datastore, old_revision, self.revision.number))
+ return True
+
+ def Commit(self):
+ """Svn2.Commit() => True|False\nCommit svn repository\n"""
+ try:
+ self.commit_changes([])
+ return True
+ except:
+ return False
+
+
diff --git a/src/lib/Bcfg2/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py
new file mode 100644
index 000000000..8879fdef1
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/TCheetah.py
@@ -0,0 +1,80 @@
+'''This module implements a templating generator based on Cheetah'''
+
+import binascii
+import logging
+import sys
+import traceback
+import Bcfg2.Server.Plugin
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ unicode = str
+
+logger = logging.getLogger('Bcfg2.Plugins.TCheetah')
+
+try:
+ import Cheetah.Template
+ import Cheetah.Parser
+except:
+ logger.error("TCheetah: Failed to import Cheetah. Is it installed?")
+ raise
+
+
+class TemplateFile:
+ """Template file creates Cheetah template structures for the loaded file."""
+
+ def __init__(self, name, specific, encoding):
+ self.name = name
+ self.specific = specific
+ self.encoding = encoding
+ self.template = None
+ self.searchlist = dict()
+
+ def handle_event(self, event):
+ """Handle all fs events for this template."""
+ if event.code2str() == 'deleted':
+ return
+ try:
+ s = {'useStackFrames': False}
+ self.template = Cheetah.Template.Template(open(self.name).read(),
+ compilerSettings=s,
+ searchList=self.searchlist)
+ except Cheetah.Parser.ParseError:
+ perror = sys.exc_info()[1]
+ logger.error("Cheetah parse error for file %s" % (self.name))
+ logger.error(perror.report())
+
+ def bind_entry(self, entry, metadata):
+ """Build literal file information."""
+ self.template.metadata = metadata
+ self.searchlist['metadata'] = metadata
+ self.template.path = entry.get('realname', entry.get('name'))
+ self.searchlist['path'] = entry.get('realname', entry.get('name'))
+ self.template.source_path = self.name
+ self.searchlist['source_path'] = self.name
+
+ if entry.tag == 'Path':
+ entry.set('type', 'file')
+ try:
+ if type(self.template) == unicode:
+ entry.text = self.template
+ else:
+ if entry.get('encoding') == 'base64':
+ # take care of case where file needs base64 encoding
+ entry.text = binascii.b2a_base64(self.template)
+ else:
+ entry.text = unicode(str(self.template), self.encoding)
+ except:
+ (a, b, c) = sys.exc_info()
+ msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1]
+ logger.error(msg)
+ logger.error("TCheetah template error for %s" % self.searchlist['path'])
+ del a, b, c
+ raise Bcfg2.Server.Plugin.PluginExecutionError
+
+
+class TCheetah(Bcfg2.Server.Plugin.GroupSpool):
+ """The TCheetah generator implements a templating mechanism for configuration files."""
+ name = 'TCheetah'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+ filename_pattern = 'template'
+ es_child_cls = TemplateFile
diff --git a/src/lib/Bcfg2/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py
new file mode 100644
index 000000000..c4dd40614
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/TGenshi.py
@@ -0,0 +1,135 @@
+"""This module implements a templating generator based on Genshi."""
+
+import binascii
+import logging
+import sys
+import Bcfg2.Server.Plugin
+# py3k compatibility
+if sys.hexversion >= 0x03000000:
+ unicode = str
+
+logger = logging.getLogger('Bcfg2.Plugins.TGenshi')
+
+# try to import genshi stuff
+try:
+ import genshi.core
+ import genshi.input
+ from genshi.template import TemplateLoader, \
+ TextTemplate, MarkupTemplate, TemplateError
+except ImportError:
+ logger.error("TGenshi: Failed to import Genshi. Is it installed?")
+ raise Bcfg2.Server.Plugin.PluginInitError
+try:
+ from genshi.template import NewTextTemplate
+ have_ntt = True
+except:
+ have_ntt = False
+
+def removecomment(stream):
+ """A genshi filter that removes comments from the stream."""
+ for kind, data, pos in stream:
+ if kind is genshi.core.COMMENT:
+ continue
+ yield kind, data, pos
+
+
+class TemplateFile:
+ """Template file creates Genshi template structures for the loaded file."""
+
+ def __init__(self, name, specific, encoding):
+ self.name = name
+ self.specific = specific
+ self.encoding = encoding
+ if self.specific.all:
+ matchname = self.name
+ elif self.specific.group:
+ matchname = self.name[:self.name.find('.G')]
+ else:
+ matchname = self.name[:self.name.find('.H')]
+ if matchname.endswith('.txt'):
+ self.template_cls = TextTemplate
+ elif matchname.endswith('.newtxt'):
+ if not have_ntt:
+ logger.error("Genshi NewTextTemplates not supported by this version of Genshi")
+ else:
+ self.template_cls = NewTextTemplate
+ else:
+ self.template_cls = MarkupTemplate
+ self.HandleEvent = self.handle_event
+
+ def handle_event(self, event=None):
+ """Handle all fs events for this template."""
+ if event and event.code2str() == 'deleted':
+ return
+ try:
+ loader = TemplateLoader()
+ try:
+ self.template = loader.load(self.name, cls=self.template_cls,
+ encoding=self.encoding)
+ except LookupError:
+ lerror = sys.exc_info()[1]
+ logger.error('Genshi lookup error: %s' % lerror)
+ except TemplateError:
+ terror = sys.exc_info()[1]
+ logger.error('Genshi template error: %s' % terror)
+ except genshi.input.ParseError:
+ perror = sys.exc_info()[1]
+ logger.error('Genshi parse error: %s' % perror)
+
+ def bind_entry(self, entry, metadata):
+ """Build literal file information."""
+ fname = entry.get('realname', entry.get('name'))
+ if entry.tag == 'Path':
+ entry.set('type', 'file')
+ try:
+ stream = self.template.generate( \
+ name=fname, metadata=metadata,
+ path=self.name).filter(removecomment)
+ if have_ntt:
+ ttypes = [TextTemplate, NewTextTemplate]
+ else:
+ ttypes = [TextTemplate]
+ if True in [isinstance(self.template, t) for t in ttypes]:
+ try:
+ textdata = stream.render('text', strip_whitespace=False)
+ except TypeError:
+ textdata = stream.render('text')
+ if type(textdata) == unicode:
+ entry.text = textdata
+ else:
+ if entry.get('encoding') == 'base64':
+ # take care of case where file needs base64 encoding
+ entry.text = binascii.b2a_base64(textdata)
+ else:
+ entry.text = unicode(textdata, self.encoding)
+ else:
+ try:
+ xmldata = stream.render('xml', strip_whitespace=False)
+ except TypeError:
+ xmldata = stream.render('xml')
+ if type(xmldata) == unicode:
+ entry.text = xmldata
+ else:
+ entry.text = unicode(xmldata, self.encoding)
+ if entry.text == '':
+ entry.set('empty', 'true')
+ except TemplateError:
+ err = sys.exc_info()[1]
+ logger.exception('Genshi template error')
+ raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err)
+ except AttributeError:
+ err = sys.exc_info()[1]
+ logger.exception('Genshi template loading error')
+ raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err)
+
+
+class TGenshi(Bcfg2.Server.Plugin.GroupSpool):
+ """
+ The TGenshi generator implements a templating
+ mechanism for configuration files.
+
+ """
+ name = 'TGenshi'
+ __author__ = 'jeff@ocjtech.us'
+ filename_pattern = 'template\.(txt|newtxt|xml)'
+ es_child_cls = TemplateFile
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
new file mode 100644
index 000000000..2c0ee03e0
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -0,0 +1,83 @@
+import re
+import imp
+import sys
+import logging
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger(__name__)
+
+class HelperModule(Bcfg2.Server.Plugin.SpecificData):
+ _module_name_re = re.compile(r'([^/]+?)\.py')
+
+ def __init__(self, name, specific, encoding):
+ Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
+ encoding)
+ match = self._module_name_re.search(self.name)
+ if match:
+ self._module_name = match.group(1)
+ else:
+ self._module_name = name
+ self._attrs = []
+
+ def handle_event(self, event):
+ Bcfg2.Server.Plugin.SpecificData.handle_event(self, event)
+ try:
+ module = imp.load_source(self._module_name, self.name)
+ except:
+ err = sys.exc_info()[1]
+ logger.error("TemplateHelper: Failed to import %s: %s" %
+ (self.name, err))
+ return
+
+ if not hasattr(module, "__export__"):
+ logger.error("TemplateHelper: %s has no __export__ list" %
+ self.name)
+ return
+
+ for sym in module.__export__:
+ if sym not in self._attrs and hasattr(self, sym):
+ logger.warning("TemplateHelper: %s: %s is a reserved keyword, "
+ "skipping export" % (self.name, sym))
+ setattr(self, sym, getattr(module, sym))
+ # remove old exports
+ for sym in set(self._attrs) - set(module.__export__):
+ delattr(self, sym)
+
+ self._attrs = module.__export__
+
+
+class HelperSet(Bcfg2.Server.Plugin.EntrySet):
+ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$")
+
+ def __init__(self, path, fam, encoding, plugin_name):
+ fpattern = '[0-9A-Za-z_\-]+\.py'
+ self.plugin_name = plugin_name
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path,
+ HelperModule, encoding)
+ fam.AddMonitor(path, self)
+
+ def HandleEvent(self, event):
+ if (event.filename != self.path and
+ not self.ignore.match(event.filename)):
+ return self.handle_event(event)
+
+
+class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ """ A plugin to provide helper classes and functions to templates """
+ name = 'TemplateHelper'
+ __author__ = 'chris.a.st.pierre@gmail.com'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+
+ try:
+ self.helpers = HelperSet(self.data, core.fam, core.encoding,
+ self.name)
+ except:
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def get_additional_data(self, metadata):
+ return dict([(h._module_name, h)
+ for h in list(self.helpers.entries.values())])
diff --git a/src/lib/Bcfg2/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py
new file mode 100644
index 000000000..b0d21545c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py
@@ -0,0 +1,43 @@
+import os
+import Bcfg2.Server.Plugin
+
+
+def async_run(prog, args):
+ pid = os.fork()
+ if pid:
+ os.waitpid(pid, 0)
+ else:
+ dpid = os.fork()
+ if not dpid:
+ os.system(" ".join([prog] + args))
+ os._exit(0)
+
+
+class Trigger(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Statistics):
+ """Trigger is a plugin that calls external scripts (on the server)."""
+ name = 'Trigger'
+ __author__ = 'bcfg-dev@mcs.anl.gov'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Statistics.__init__(self)
+ try:
+ os.stat(self.data)
+ except:
+ self.logger.error("Trigger: spool directory %s does not exist; "
+ "unloading" % self.data)
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def process_statistics(self, metadata, _):
+ args = [metadata.hostname, '-p', metadata.profile, '-g',
+ ':'.join([g for g in metadata.groups])]
+ for notifier in os.listdir(self.data):
+ if ((notifier[-1] == '~') or
+ (notifier[:2] == '.#') or
+ (notifier[-4:] == '.swp') or
+ (notifier in ['SCCS', '.svn', '4913'])):
+ continue
+ npath = self.data + '/' + notifier
+ self.logger.debug("Running %s %s" % (npath, " ".join(args)))
+ async_run(npath, args)
diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py
new file mode 100644
index 000000000..f9f1b4e52
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/__init__.py
@@ -0,0 +1,34 @@
+"""Imports for Bcfg2.Server.Plugins."""
+
+__all__ = [
+ 'Account',
+ 'Base',
+ 'Bundler',
+ 'Bzr',
+ 'Cfg',
+ 'Cvs',
+ 'Darcs',
+ 'Decisions',
+ 'Fossil',
+ 'Git',
+ 'GroupPatterns',
+ 'Hg',
+ 'Hostbase',
+ 'Metadata',
+ 'NagiosGen',
+ 'Ohai',
+ 'Packages',
+ 'Properties',
+ 'Probes',
+ 'Pkgmgr',
+ 'Rules',
+ 'SSHbase',
+ 'Snapshots',
+ 'Statistics',
+ 'Svcmgr',
+ 'Svn',
+ 'TCheetah',
+ 'Trigger',
+ 'SGenshi',
+ 'TGenshi',
+ ]