diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Client/Tools/Pacman.py | 82 | ||||
-rw-r--r-- | src/lib/Client/Tools/YUMng.py | 7 | ||||
-rw-r--r-- | src/lib/Options.py | 2 | ||||
-rw-r--r-- | src/lib/Server/Hostbase/settings.py | 7 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Guppy.py | 63 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Metadata.py | 18 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Packages.py | 112 | ||||
-rw-r--r-- | src/lib/Server/Reports/settings.py | 3 | ||||
-rw-r--r-- | src/lib/Server/Reports/updatefix.py | 8 | ||||
-rwxr-xr-x | src/sbin/bcfg2 | 17 | ||||
-rwxr-xr-x | src/sbin/bcfg2-info | 2 |
11 files changed, 311 insertions, 10 deletions
diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py new file mode 100644 index 000000000..be3fb0c94 --- /dev/null +++ b/src/lib/Client/Tools/Pacman.py @@ -0,0 +1,82 @@ +"""This is the bcfg2 support for pacman""" + +import Bcfg2.Client.Tools + + +class Pacman(Bcfg2.Client.Tools.PkgTool): + '''Archlinux package support''' + name = 'Pacman' + __execs__ = ["/usr/bin/pacman"] + __handles__ = [('Package', 'pacman')] + __req__ = {'Package': ['name', 'version']} + pkgtype = 'pacman' + pkgtool = ("/usr/bin/pacman --needed --noconfirm --noprogressbar") + + def __init__(self, logger, setup, config): + Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + self.installed = {} + self.RefreshPackages() + + def RefreshPackages(self): + '''Refresh memory hashes of packages''' + pkgcache = self.cmd.run("/usr/bin/pacman -Q")[1] + self.installed = {} + for pkg in pkgcache: + pkgname = pkg.split(' ')[0].strip() + version = pkg.split(' ')[1].strip() + #self.logger.info(" pkgname: %s, version: %s" % (pkgname, version)) + self.installed[pkgname] = version + + def VerifyPackage(self, entry, modlist): + '''Verify Package status for entry''' + + self.logger.info("VerifyPackage : %s : %s" % entry.get('name'), + entry.get('version')) + + if not 'version' in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.attrib['name'])) + return False + + if entry.attrib['name'] in self.installed: + if entry.attrib['version'] == 'auto': + return True + elif self.installed[entry.attrib['name']] == entry.attrib['version']: + #if not self.setup['quick'] and \ + # entry.get('verify', 'true') == 'true': + #FIXME: need to figure out if pacman + # allows you to verify packages + return True + else: + entry.set('current_version', self.installed[entry.get('name')]) + self.logger.info("attribname: %s" % (entry.attrib['name'])) + self.logger.info("attribname: %s" % (entry.attrib['name'])) + return False + entry.set('current_exists', 'false') + self.logger.info("attribname: %s" % (entry.attrib['name'])) + return False + + def RemovePackages(self, packages): + '''Remove extra packages''' + names = [pkg.get('name') for pkg in packages] + self.logger.info("Removing packages: %s" % " ".join(names)) + self.cmd.run("%s --noconfirm --noprogressbar -R %s" % \ + (self.pkgtool, " ".join(names))) + self.RefreshPackages() + self.extra = self.FindExtraPackages() + + def Install(self, packages, states): + ''' + Pacman Install + ''' + pkgline = "" + for pkg in packages: + pkgline += " " + pkg.get('name') + + print "packages : " + pkgline + + try: + self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline)) + self.cmd.run("%s -S %s" % (self.pkgtool, pkgline)) + except Exception, e: + self.logger.error("Error occurred during installation: %s" % e) diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 9cdcdca40..f0d906717 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -210,6 +210,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): "version_fail_action", "upgrade").lower() == "upgrade" self.doReinst = CP.get(self.name, "verify_fail_action", "reinstall").lower() == "reinstall" + self.verifyFlags = CP.get(self.name, "verify_flags", + "").lower().replace(' ', ',') self.installOnlyPkgs = self.yb.conf.installonlypkgs if 'gpg-pubkey' not in self.installOnlyPkgs: @@ -225,6 +227,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): % self.doReinst) self.logger.debug("YUMng: installOnlyPkgs: %s" \ % str(self.installOnlyPkgs)) + self.logger.debug("YUMng: verify_flags: %s" % self.verifyFlags) def _fixAutoVersion(self, entry): # old style entry; synthesize Instances from current installed @@ -436,6 +439,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): stat['verify_fail'] = False stat['pkg'] = entry stat['modlist'] = modlist + verify_flags = inst.get('verify_flags', self.verifyFlags) + verify_flags = verify_flags.lower().replace(' ', ',').split(',') if len(POs) == 0: # Package not installed @@ -505,6 +510,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): for p in probs: if p.type == 'missing' and os.path.islink(fn): continue + elif 'no' + p.type in verify_flags: + continue if p.type not in ['missingok', 'ghost']: tmp.append((p.type, p.message)) if tmp != []: diff --git a/src/lib/Options.py b/src/lib/Options.py index b467a776d..1dcad6427 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -290,6 +290,8 @@ CLIENT_REMOVE = Option('force removal of additional configuration items', default=False, cmd='-r', odesc="<entry type|all>") CLIENT_BUNDLE = Option('only configure the given bundle(s)', default=[], cmd='-b', odesc='<bundle:bundle>', cook=colon_split) +CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, + cmd='-Q') CLIENT_INDEP = Option('only configure the given bundle(s)', default=False, cmd='-z') CLIENT_KEVLAR = Option('run in kevlar (bulletproof) mode', default=False, diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index dadf98d24..a42fd5b2e 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -44,8 +44,11 @@ DATABASE_HOST = options['database_host'] # Set to empty string for default. Not used with sqlite3. DATABASE_PORT = int(options['database_port']) # Local time zone for this installation. All choices can be found here: -# http://www.postgresql.org/docs/current/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE -TIME_ZONE = 'America/Chicago' +# http://docs.djangoproject.com/en/dev/ref/settings/#time-zone +try: + TIME_ZONE = c.get('statistics', 'time_zone') +except: + TIME_ZONE = None # enter the defauly MX record machines will get in Hostbase # this setting may move elsewhere eventually diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Server/Plugins/Guppy.py new file mode 100644 index 000000000..b217378d6 --- /dev/null +++ b/src/lib/Server/Plugins/Guppy.py @@ -0,0 +1,63 @@ +""" +This plugin is used to trace memory leaks within the bcfg2-server +process using Guppy. By default the remote debugger is started +when this plugin is enabled. The debugger can be shutoff in a running +process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using +"bcfg2-admin xcmd Guppy.Enable". + +To attach the console run: + +python -c "from guppy import hpy;hpy().monitor()" + +For example: + +# python -c "from guppy import hpy;hpy().monitor()" +<Monitor> +*** Connection 1 opened *** +<Monitor> lc +CID PID ARGV + 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid'] +<Monitor> sc 1 +Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN> +<Annex> int +Remote interactive console. To return to Annex, type '-'. +>>> hp.heap() +... + + +""" +import re +import Bcfg2.Server.Plugin + +class Guppy(Bcfg2.Server.Plugin.Plugin): + """Guppy is a debugging plugin to help trace memory leaks""" + name = 'Guppy' + __version__ = '$Id$' + __author__ = 'bcfg-dev@mcs.anl.gov' + + experimental = True + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + + self.Enable() + + def Enable(self): + """Enable remote debugging""" + try: + from guppy.heapy import Remote + Remote.on() + except: + self.logger.error("Failed to create Heapy context") + raise Bcfg2.Server.Plugin.PluginInitError + + def Disable(self): + """Disable remote debugging""" + try: + from guppy.heapy import Remote + Remote.off() + except: + self.logger.error("Failed to disable Heapy") + raise Bcfg2.Server.Plugin.PluginInitError + diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 3161a69e3..81fd3e173 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -39,14 +39,21 @@ class ClientMetadata(object): """Test to see if client is a member of group.""" return group in self.groups + def group_in_category(self, category): + for grp in self.query.all_groups_in_category(category): + if grp in self.groups: + return grp + return '' + class MetadataQuery(object): - def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups): + def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later self.by_name = by_name self.names_by_groups = by_groups self.names_by_profiles = by_profiles self.all_clients = get_clients self.all_groups = all_groups + self.all_groups_in_category = all_groups_in_category def by_groups(self, groups): return [self.by_name(name) for name in self.names_by_groups(groups)] @@ -105,7 +112,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, lambda:self.clients.keys(), self.get_client_names_by_groups, self.get_client_names_by_profiles, - self.get_all_group_names) + self.get_all_group_names, + self.get_all_groups_in_category) @classmethod def init_repo(cls, repo, groups, os_selection, clients): @@ -592,6 +600,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, [all_groups.update(g[1]) for g in self.groups.values()] return all_groups + def get_all_groups_in_category(self, category): + all_groups = set() + [all_groups.add(g) for g in self.categories \ + if self.categories[g] == category] + return all_groups + def get_client_names_by_profiles(self, profiles): return [client for client, profile in self.clients.iteritems() \ if profile in profiles] diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index b83b7444f..194330723 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -1,6 +1,7 @@ import cPickle import copy import gzip +import tarfile import glob import logging import lxml.etree @@ -467,6 +468,114 @@ class APTSource(Source): pkg not in self.blacklist and \ (len(self.whitelist) == 0 or pkg in self.whitelist) +class PACSource(Source): + basegroups = ['arch', 'parabola'] + ptype = 'pacman' + + def __init__(self, basepath, url, version, arches, components, groups, + rawurl, blacklist, whitelist, recommended): + Source.__init__(self, basepath, url, version, arches, components, groups, + rawurl, blacklist, whitelist, recommended) + self.pkgnames = set() + + self.url_map = [{'rawurl': self.rawurl, 'url': self.url, 'version': self.version, \ + 'components': self.components, 'arches': self.arches, 'groups': self.groups}] + + def save_state(self): + cache = file(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides), + cache, 2) + cache.close() + + def load_state(self): + data = file(self.cachefile) + self.pkgnames, self.deps, self.provides = cPickle.load(data) + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + if not self.rawurl: + return ["%s/%s/os/%s/%s.db.tar.gz" % \ + (self.url, part, arch, part) for part in self.components \ + for arch in self.arches] + else: + raise Exception("PACSource : RAWUrl not supported (yet)") + urls = property(get_urls) + + + def read_files(self): + bdeps = dict() + bprov = dict() + + if self.recommended: + depfnames = ['Depends', 'Pre-Depends', 'Recommends'] + else: + depfnames = ['Depends', 'Pre-Depends'] + + for fname in self.files: + if not self.rawurl: + barch = [x for x in fname.split('@') if x in self.arches][0] + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + + if barch not in bdeps: + bdeps[barch] = dict() + bprov[barch] = dict() + try: + print "try to read : " + fname + tar = tarfile.open(fname, "r") + reader = gzip.GzipFile(fname) + except: + print("Failed to read file %s" % fname) + raise + + for tarinfo in tar: + if tarinfo.isdir(): + self.pkgnames.add(tarinfo.name.rsplit("-",2)[0]) + print "added : " + tarinfo.name.rsplit("-",2)[0] + tar.close() + + self.deps['global'] = dict() + self.provides['global'] = dict() + for barch in bdeps: + self.deps[barch] = dict() + self.provides[barch] = dict() + for pkgname in self.pkgnames: + pset = set() + for barch in bdeps: + if pkgname not in bdeps[barch]: + bdeps[barch][pkgname] = [] + pset.add(tuple(bdeps[barch][pkgname])) + if len(pset) == 1: + self.deps['global'][pkgname] = pset.pop() + else: + for barch in bdeps: + self.deps[barch][pkgname] = bdeps[barch][pkgname] + provided = set() + for bprovided in bprov.values(): + provided.update(set(bprovided)) + for prov in provided: + prset = set() + for barch in bprov: + if prov not in bprov[barch]: + continue + prset.add(tuple(bprov[barch].get(prov, ()))) + if len(prset) == 1: + self.provides['global'][prov] = prset.pop() + else: + for barch in bprov: + self.provides[barch][prov] = bprov[barch].get(prov, ()) + self.save_state() + + def is_package(self, _, pkg): + return pkg in self.pkgnames and \ + pkg not in self.blacklist and \ + (len(self.whitelist) == 0 or pkg in self.whitelist) + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -742,6 +851,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.sources.append(APTSource(self.cachepath, **source_from_xml(s))) for s in xdata.findall('.//YUMSource'): self.sources.append(YUMSource(self.cachepath, **source_from_xml(s))) + for s in xdata.findall('.//PACSource'): + self.sources.append(PACSource(self.cachepath, **source_from_xml(s))) + cachefiles = [] for source in self.sources: cachefiles.append(source.cachefile) diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index 81220c0e3..9efe38552 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -49,7 +49,8 @@ if DATABASE_ENGINE == 'sqlite3' and DATABASE_NAME == '': try: TIME_ZONE = c.get('statistics', 'time_zone') except: - TIME_ZONE = 'America/Chicago' + if django.VERSION[0] == 1 and django.VERSION[1] > 2: + TIME_ZONE = None # Language code for this installation. All choices can be found here: # http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py index 6d9b5e952..f8fca1f90 100644 --- a/src/lib/Server/Reports/updatefix.py +++ b/src/lib/Server/Reports/updatefix.py @@ -139,8 +139,12 @@ def dosync(): fresh = True # ensure database connection are close, so that the management can do it's job right - cursor.close() - connection.close() + try: + cursor.close() + connection.close() + except: + # ignore any errors from missing/invalid dbs + pass # Do the syncdb according to the django version if "call_command" in dir(django.core.management): # this is available since django 1.0 alpha. diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 3407a1c53..10262e4a9 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -49,6 +49,7 @@ class Client: 'dryrun': Bcfg2.Options.CLIENT_DRYRUN, 'paranoid': Bcfg2.Options.CLIENT_PARANOID, 'bundle': Bcfg2.Options.CLIENT_BUNDLE, + 'bundle-quick': Bcfg2.Options.CLIENT_BUNDLEQUICK, 'indep': Bcfg2.Options.CLIENT_INDEP, 'file': Bcfg2.Options.CLIENT_FILE, 'interactive': Bcfg2.Options.INTERACTIVE, @@ -62,7 +63,6 @@ class Client: 'password': Bcfg2.Options.SERVER_PASSWORD, 'retries': Bcfg2.Options.CLIENT_RETRIES, 'kevlar': Bcfg2.Options.CLIENT_KEVLAR, - 'key': Bcfg2.Options.SERVER_KEY, 'decision-list': DECISION_LIST, 'encoding': Bcfg2.Options.ENCODING, 'omit-lock-check': Bcfg2.Options.OMIT_LOCK_CHECK, @@ -93,6 +93,13 @@ class Client: to_file=self.setup['filelog']) self.logger = logging.getLogger('bcfg2') self.logger.debug(self.setup) + if self.setup['bundle-quick']: + if self.setup['bundle'] == []: + self.logger.error("-Q option requires -b") + raise SystemExit(1) + elif self.setup['remove'] != False: + self.logger.error("-Q option incompatible with -r") + raise SystemExit(1) if 'drivers' in self.setup and self.setup['drivers'] == 'help': self.logger.info("The following drivers are available:") self.logger.info(Bcfg2.Client.Tools.drivers) @@ -251,6 +258,12 @@ class Client: self.fatal_error("Server error: %s" % (self.config.text)) return(1) + if self.setup['bundle-quick']: + newconfig = Bcfg2.Client.XML.XML('<Configuration/>') + [newconfig.append(bundle) for bundle in self.config.getchildren() if \ + bundle.tag == 'Bundle' and bundle.get('name') in self.setup['bundle']] + self.config = newconfig + self.tools = Bcfg2.Client.Frame.Frame(self.config, self.setup, times, self.setup['drivers'], @@ -281,7 +294,7 @@ class Client: except OSError: self.logger.error("Failed to unlock lockfile %s" % lockfile.name) - if not self.setup['file']: + if not self.setup['file'] and not self.setup['bundle-quick']: # upload statistics feedback = self.tools.GenerateStats() diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index d3a9bf8be..497c39174 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -303,7 +303,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): def do_showentries(self, args): """Show abstract configuration entries for a given host.""" arglen = len(args.split()) - if arglen not in [2, 3]: + if arglen not in [1, 2]: print("Usage: showentries <hostname> <type>") return client = args.split()[0] |