diff options
Diffstat (limited to 'src/lib/Server/Plugins')
| -rw-r--r-- | src/lib/Server/Plugins/Guppy.py | 63 | ||||
| -rw-r--r-- | src/lib/Server/Plugins/Metadata.py | 18 | ||||
| -rw-r--r-- | src/lib/Server/Plugins/Packages.py | 112 |
3 files changed, 191 insertions, 2 deletions
diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Server/Plugins/Guppy.py new file mode 100644 index 000000000..b217378d6 --- /dev/null +++ b/src/lib/Server/Plugins/Guppy.py @@ -0,0 +1,63 @@ +""" +This plugin is used to trace memory leaks within the bcfg2-server +process using Guppy. By default the remote debugger is started +when this plugin is enabled. The debugger can be shutoff in a running +process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using +"bcfg2-admin xcmd Guppy.Enable". + +To attach the console run: + +python -c "from guppy import hpy;hpy().monitor()" + +For example: + +# python -c "from guppy import hpy;hpy().monitor()" +<Monitor> +*** Connection 1 opened *** +<Monitor> lc +CID PID ARGV + 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid'] +<Monitor> sc 1 +Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN> +<Annex> int +Remote interactive console. To return to Annex, type '-'. +>>> hp.heap() +... + + +""" +import re +import Bcfg2.Server.Plugin + +class Guppy(Bcfg2.Server.Plugin.Plugin): + """Guppy is a debugging plugin to help trace memory leaks""" + name = 'Guppy' + __version__ = '$Id$' + __author__ = 'bcfg-dev@mcs.anl.gov' + + experimental = True + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + + self.Enable() + + def Enable(self): + """Enable remote debugging""" + try: + from guppy.heapy import Remote + Remote.on() + except: + self.logger.error("Failed to create Heapy context") + raise Bcfg2.Server.Plugin.PluginInitError + + def Disable(self): + """Disable remote debugging""" + try: + from guppy.heapy import Remote + Remote.off() + except: + self.logger.error("Failed to disable Heapy") + raise Bcfg2.Server.Plugin.PluginInitError + diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 3161a69e3..81fd3e173 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -39,14 +39,21 @@ class ClientMetadata(object): """Test to see if client is a member of group.""" return group in self.groups + def group_in_category(self, category): + for grp in self.query.all_groups_in_category(category): + if grp in self.groups: + return grp + return '' + class MetadataQuery(object): - def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups): + def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later self.by_name = by_name self.names_by_groups = by_groups self.names_by_profiles = by_profiles self.all_clients = get_clients self.all_groups = all_groups + self.all_groups_in_category = all_groups_in_category def by_groups(self, groups): return [self.by_name(name) for name in self.names_by_groups(groups)] @@ -105,7 +112,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, lambda:self.clients.keys(), self.get_client_names_by_groups, self.get_client_names_by_profiles, - self.get_all_group_names) + self.get_all_group_names, + self.get_all_groups_in_category) @classmethod def init_repo(cls, repo, groups, os_selection, clients): @@ -592,6 +600,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, [all_groups.update(g[1]) for g in self.groups.values()] return all_groups + def get_all_groups_in_category(self, category): + all_groups = set() + [all_groups.add(g) for g in self.categories \ + if self.categories[g] == category] + return all_groups + def get_client_names_by_profiles(self, profiles): return [client for client, profile in self.clients.iteritems() \ if profile in profiles] diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index b83b7444f..194330723 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -1,6 +1,7 @@ import cPickle import copy import gzip +import tarfile import glob import logging import lxml.etree @@ -467,6 +468,114 @@ class APTSource(Source): pkg not in self.blacklist and \ (len(self.whitelist) == 0 or pkg in self.whitelist) +class PACSource(Source): + basegroups = ['arch', 'parabola'] + ptype = 'pacman' + + def __init__(self, basepath, url, version, arches, components, groups, + rawurl, blacklist, whitelist, recommended): + Source.__init__(self, basepath, url, version, arches, components, groups, + rawurl, blacklist, whitelist, recommended) + self.pkgnames = set() + + self.url_map = [{'rawurl': self.rawurl, 'url': self.url, 'version': self.version, \ + 'components': self.components, 'arches': self.arches, 'groups': self.groups}] + + def save_state(self): + cache = file(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides), + cache, 2) + cache.close() + + def load_state(self): + data = file(self.cachefile) + self.pkgnames, self.deps, self.provides = cPickle.load(data) + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + if not self.rawurl: + return ["%s/%s/os/%s/%s.db.tar.gz" % \ + (self.url, part, arch, part) for part in self.components \ + for arch in self.arches] + else: + raise Exception("PACSource : RAWUrl not supported (yet)") + urls = property(get_urls) + + + def read_files(self): + bdeps = dict() + bprov = dict() + + if self.recommended: + depfnames = ['Depends', 'Pre-Depends', 'Recommends'] + else: + depfnames = ['Depends', 'Pre-Depends'] + + for fname in self.files: + if not self.rawurl: + barch = [x for x in fname.split('@') if x in self.arches][0] + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + + if barch not in bdeps: + bdeps[barch] = dict() + bprov[barch] = dict() + try: + print "try to read : " + fname + tar = tarfile.open(fname, "r") + reader = gzip.GzipFile(fname) + except: + print("Failed to read file %s" % fname) + raise + + for tarinfo in tar: + if tarinfo.isdir(): + self.pkgnames.add(tarinfo.name.rsplit("-",2)[0]) + print "added : " + tarinfo.name.rsplit("-",2)[0] + tar.close() + + self.deps['global'] = dict() + self.provides['global'] = dict() + for barch in bdeps: + self.deps[barch] = dict() + self.provides[barch] = dict() + for pkgname in self.pkgnames: + pset = set() + for barch in bdeps: + if pkgname not in bdeps[barch]: + bdeps[barch][pkgname] = [] + pset.add(tuple(bdeps[barch][pkgname])) + if len(pset) == 1: + self.deps['global'][pkgname] = pset.pop() + else: + for barch in bdeps: + self.deps[barch][pkgname] = bdeps[barch][pkgname] + provided = set() + for bprovided in bprov.values(): + provided.update(set(bprovided)) + for prov in provided: + prset = set() + for barch in bprov: + if prov not in bprov[barch]: + continue + prset.add(tuple(bprov[barch].get(prov, ()))) + if len(prset) == 1: + self.provides['global'][prov] = prset.pop() + else: + for barch in bprov: + self.provides[barch][prov] = bprov[barch].get(prov, ()) + self.save_state() + + def is_package(self, _, pkg): + return pkg in self.pkgnames and \ + pkg not in self.blacklist and \ + (len(self.whitelist) == 0 or pkg in self.whitelist) + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -742,6 +851,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.sources.append(APTSource(self.cachepath, **source_from_xml(s))) for s in xdata.findall('.//YUMSource'): self.sources.append(YUMSource(self.cachepath, **source_from_xml(s))) + for s in xdata.findall('.//PACSource'): + self.sources.append(PACSource(self.cachepath, **source_from_xml(s))) + cachefiles = [] for source in self.sources: cachefiles.append(source.cachefile) |