diff options
Diffstat (limited to 'pym')
-rw-r--r-- | pym/_emerge/main.py | 27 | ||||
-rw-r--r-- | pym/portage/emaint/__init__.py | 7 | ||||
-rw-r--r-- | pym/portage/emaint/defaults.py | 18 | ||||
-rw-r--r-- | pym/portage/emaint/main.py | 218 | ||||
-rw-r--r-- | pym/portage/emaint/module.py | 194 | ||||
-rw-r--r-- | pym/portage/emaint/modules/__init__.py | 7 | ||||
-rw-r--r-- | pym/portage/emaint/modules/binhost/__init__.py | 22 | ||||
-rw-r--r-- | pym/portage/emaint/modules/binhost/binhost.py | 167 | ||||
-rw-r--r-- | pym/portage/emaint/modules/config/__init__.py | 22 | ||||
-rw-r--r-- | pym/portage/emaint/modules/config/config.py | 101 | ||||
-rw-r--r-- | pym/portage/emaint/modules/logs/__init__.py | 51 | ||||
-rw-r--r-- | pym/portage/emaint/modules/logs/logs.py | 114 | ||||
-rw-r--r-- | pym/portage/emaint/modules/move/__init__.py | 33 | ||||
-rw-r--r-- | pym/portage/emaint/modules/move/move.py | 162 | ||||
-rw-r--r-- | pym/portage/emaint/modules/resume/__init__.py | 22 | ||||
-rw-r--r-- | pym/portage/emaint/modules/resume/resume.py | 58 | ||||
-rw-r--r-- | pym/portage/emaint/modules/world/__init__.py | 22 | ||||
-rw-r--r-- | pym/portage/emaint/modules/world/world.py | 89 | ||||
-rw-r--r-- | pym/portage/emaint/progress.py | 61 |
19 files changed, 1376 insertions, 19 deletions
diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 640f320fd..f19994c46 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -13,6 +13,7 @@ import platform import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.news:count_unread_news,display_news_notifications', + 'portage.emaint.modules.logs.logs:CleanLogs', ) from portage import os from portage import _encodings @@ -1351,29 +1352,17 @@ def clean_logs(settings): if "clean-logs" not in settings.features: return - clean_cmd = settings.get("PORT_LOGDIR_CLEAN") - if clean_cmd: - clean_cmd = shlex_split(clean_cmd) - if not clean_cmd: - return - logdir = settings.get("PORT_LOGDIR") if logdir is None or not os.path.isdir(logdir): return - variables = {"PORT_LOGDIR" : logdir} - cmd = [varexpand(x, mydict=variables) for x in clean_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - rval = 127 - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for " - "PORT_LOGDIR_CLEAN usage instructions.") + options = { + 'eerror': portage.output.EOutput().eerror, + # uncomment next line to output a succeeded message + #'einfo': portage.output.EOutput().einfo + } + cleanlogs = CleanLogs() + cleanlogs.clean(settings=settings, options=options) def setconfig_fallback(root_config): setconfig = root_config.setconfig diff --git a/pym/portage/emaint/__init__.py b/pym/portage/emaint/__init__.py new file mode 100644 index 000000000..5e0ae700a --- /dev/null +++ b/pym/portage/emaint/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program provides checks and maintenance +on a gentoo system. +""" + diff --git a/pym/portage/emaint/defaults.py b/pym/portage/emaint/defaults.py new file mode 100644 index 000000000..d9d83ffbb --- /dev/null +++ b/pym/portage/emaint/defaults.py @@ -0,0 +1,18 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# parser option data +CHECK = {"short": "-c", "long": "--check", + "help": "Check for problems (a default option for most modules)", + 'status': "Checking %s for problems", + 'func': 'check' + } + +FIX = {"short": "-f", "long": "--fix", + "help": "Attempt to fix problems (a default option for most modules)", + 'status': "Attempting to fix %s", + 'func': 'fix' + } + +# parser options +DEFAULT_OPTIONS = {'check': CHECK, 'fix': FIX} diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py new file mode 100644 index 000000000..dbc5f18cc --- /dev/null +++ b/pym/portage/emaint/main.py @@ -0,0 +1,218 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + + +import sys +import textwrap +from optparse import OptionParser, OptionValueError + + +import portage +from portage import os +from portage.emaint.module import Modules +from portage.emaint.progress import ProgressBar +from portage.emaint.defaults import DEFAULT_OPTIONS + +class OptionItem(object): + """class to hold module OptionParser options data + """ + + def __init__(self, opt, parser): + """ + @type opt: dictionary + @param opt: options parser options + """ + self.parser = parser + self.short = opt['short'] + self.long = opt['long'] + self.help = opt['help'] + self.status = opt['status'] + self.func = opt['func'] + self.action = opt.get('action', "callback") + self.type = opt.get('type', None) + self.dest = opt.get('dest', None) + self.callback = opt.get('callback', self._exclusive) + self.callback_kwargs = opt.get('callback_kwargs', {"var":"action"}) + + + def _exclusive(self, option, *args, **kw): + """Generic check for the 2 default options + """ + var = kw.get("var", None) + if var is None: + raise ValueError("var not specified to exclusive()") + if getattr(self.parser, var, ""): + raise OptionValueError("%s and %s are exclusive options" + % (getattr(self.parser, var), option)) + setattr(self.parser, var, str(option)) + + def check_action(self, action): + """Checks if 'action' is the same as this option + + @type action: string + @param action: the action to compare + @rtype: boolean + """ + if action == self.action: + return True + elif action == '/'.join([self.short, self.long]): + return True + return False + + +def usage(module_controller): + _usage = "usage: emaint [options] COMMAND" + + desc = "The emaint program provides an interface to system health " + \ + "checks and maintenance. See the emaint(1) man page " + \ + "for additional information about the following commands:" + + _usage += "\n\n" + for line in textwrap.wrap(desc, 65): + _usage += "%s\n" % line + _usage += "\nCommands:\n" + _usage += " %s" % "all".ljust(15) + \ + "Perform all supported commands\n" + textwrap.subsequent_indent = ' '.ljust(17) + for mod in module_controller.module_names: + desc = textwrap.wrap(module_controller.get_description(mod), 65) + _usage += " %s%s\n" % (mod.ljust(15), desc[0]) + for d in desc[1:]: + _usage += " %s%s\n" % (' '.ljust(15), d) + return _usage + + +def module_opts(module_controller, module): + _usage = " %s module options:\n" % module + opts = module_controller.get_func_descriptions(module) + if opts == {}: + opts = DEFAULT_OPTIONS + for opt in sorted(opts): + optd = opts[opt] + opto = " %s, %s" %(optd['short'], optd['long']) + _usage += '%s %s\n' % (opto.ljust(15),optd['help']) + _usage += '\n' + return _usage + + +class TaskHandler(object): + """Handles the running of the tasks it is given + """ + + def __init__(self, show_progress_bar=True, verbose=True, callback=None): + self.show_progress_bar = show_progress_bar + self.verbose = verbose + self.callback = callback + self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) + + + def run_tasks(self, tasks, func, status=None, verbose=True, options=None): + """Runs the module tasks""" + if tasks is None or func is None: + return + for task in tasks: + inst = task() + show_progress = self.show_progress_bar + # check if the function is capable of progressbar + # and possibly override it off + if show_progress and hasattr(inst, 'can_progressbar'): + show_progress = inst.can_progressbar(func) + if show_progress: + self.progress_bar.reset() + self.progress_bar.set_label(func + " " + inst.name()) + onProgress = self.progress_bar.start() + else: + onProgress = None + kwargs = { + 'onProgress': onProgress, + # pass in a copy of the options so a module can not pollute or change + # them for other tasks if there is more to do. + 'options': options.copy() + } + result = getattr(inst, func)(**kwargs) + if self.isatty and show_progress: + # make sure the final progress is displayed + self.progress_bar.display() + print() + self.progress_bar.stop() + if self.callback: + self.callback(result) + + +def print_results(results): + if results: + print() + print("\n".join(results)) + print("\n") + + +def emaint_main(myargv): + + # Similar to emerge, emaint needs a default umask so that created + # files (such as the world file) have sane permissions. + os.umask(0o22) + + module_controller = Modules(namepath="portage.emaint.modules") + module_names = module_controller.module_names[:] + module_names.insert(0, "all") + + + parser = OptionParser(usage=usage(module_controller), version=portage.VERSION) + # add default options + parser_options = [] + for opt in DEFAULT_OPTIONS: + parser_options.append(OptionItem(DEFAULT_OPTIONS[opt], parser)) + for mod in module_names[1:]: + desc = module_controller.get_func_descriptions(mod) + if desc: + for opt in desc: + parser_options.append(OptionItem(desc[opt], parser)) + for opt in parser_options: + parser.add_option(opt.short, opt.long, help=opt.help, action=opt.action, + type=opt.type, dest=opt.dest, + callback=opt.callback, callback_kwargs=opt.callback_kwargs) + + parser.action = None + + (options, args) = parser.parse_args(args=myargv) + #print('options', options, '\nargs', args, '\naction', parser.action) + if len(args) != 1: + parser.error("Incorrect number of arguments") + if args[0] not in module_names: + parser.error("%s target is not a known target" % args[0]) + + if parser.action: + action = parser.action + else: + action = "-c/--check" + long_action = action.split('/')[1].lstrip('-') + #print("DEBUG: action = ", action, long_action) + + if args[0] == "all": + tasks = [] + for m in module_names[1:]: + #print("DEBUG: module: %s, functions: " %(m, str(module_controller.get_functions(m)))) + if long_action in module_controller.get_functions(m): + tasks.append(module_controller.get_class(m)) + elif long_action in module_controller.get_functions(args[0]): + tasks = [module_controller.get_class(args[0] )] + else: + print("\nERROR: module '%s' does not have option '%s'\n" %(args[0], action)) + print(module_opts(module_controller, args[0])) + sys.exit(1) + func = status = None + for opt in parser_options: + if opt.check_action(action): + status = opt.status + func = opt.func + break + + # need to pass the parser options dict to the modules + # so they are available if needed. + task_opts = options.__dict__ + taskmaster = TaskHandler(callback=print_results) + taskmaster.run_tasks(tasks, func, status, options=task_opts) + diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py new file mode 100644 index 000000000..64b0c64b5 --- /dev/null +++ b/pym/portage/emaint/module.py @@ -0,0 +1,194 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + +from portage import os +from portage.exception import PortageException +from portage.cache.mappings import ProtectedDict + + +class InvalidModuleName(PortageException): + """An invalid or unknown module name.""" + + +class Module(object): + """Class to define and hold our plug-in module + + @type name: string + @param name: the module name + @type path: the path to the new module + """ + + def __init__(self, name, namepath): + """Some variables initialization""" + self.name = name + self._namepath = namepath + self.kids_names = [] + self.kids = {} + self.initialized = self._initialize() + + def _initialize(self): + """Initialize the plug-in module + + @rtype: boolean + """ + self.valid = False + try: + mod_name = ".".join([self._namepath, self.name]) + self._module = __import__(mod_name, [],[], ["not empty"]) + self.valid = True + except ImportError as e: + print("MODULE; failed import", mod_name, " error was:",e) + return False + self.module_spec = self._module.module_spec + for submodule in self.module_spec['provides']: + kid = self.module_spec['provides'][submodule] + kidname = kid['name'] + kid['module_name'] = '.'.join([mod_name, self.name]) + kid['is_imported'] = False + self.kids[kidname] = kid + self.kids_names.append(kidname) + return True + + def get_class(self, name): + if not name or name not in self.kids_names: + raise InvalidModuleName("Module name '%s' was invalid or not" + %name + "part of the module '%s'" %self.name) + kid = self.kids[name] + if kid['is_imported']: + module = kid['instance'] + else: + try: + module = __import__(kid['module_name'], [],[], ["not empty"]) + kid['instance'] = module + kid['is_imported'] = True + except ImportError: + raise + mod_class = getattr(module, kid['class']) + return mod_class + + +class Modules(object): + """Dynamic modules system for loading and retrieving any of the + installed emaint modules and/or provided class's + + @param path: Optional path to the "modules" directory or + defaults to the directory of this file + '/modules' + @param namepath: Optional python import path to the "modules" directory or + defaults to the directory name of this file + '.modules' + """ + + def __init__(self, path=None, namepath=None): + if path: + self._module_path = path + else: + self._module_path = os.path.join(( + os.path.dirname(os.path.realpath(__file__))), "modules") + if namepath: + self._namepath = namepath + else: + self._namepath = '.'.join(os.path.dirname( + os.path.realpath(__file__)), "modules") + self._modules = self._get_all_modules() + self.modules = ProtectedDict(self._modules) + self.module_names = sorted(self._modules) + #self.modules = {} + #for mod in self.module_names: + #self.module[mod] = LazyLoad( + + def _get_all_modules(self): + """scans the emaint modules dir for loadable modules + + @rtype: dictionary of module_plugins + """ + module_dir = self._module_path + importables = [] + names = os.listdir(module_dir) + for entry in names: + # skip any __init__ or __pycache__ files or directories + if entry.startswith('__'): + continue + try: + # test for statinfo to ensure it should a real module + # it will bail if it errors + os.lstat(os.path.join(module_dir, entry, '__init__.py')) + importables.append(entry) + except EnvironmentError: + pass + kids = {} + for entry in importables: + new_module = Module(entry, self._namepath) + for module_name in new_module.kids: + kid = new_module.kids[module_name] + kid['parent'] = new_module + kids[kid['name']] = kid + return kids + + def get_module_names(self): + """Convienence function to return the list of installed modules + available + + @rtype: list + @return: the installed module names available + """ + return self.module_names + + def get_class(self, modname): + """Retrieves a module class desired + + @type modname: string + @param modname: the module class name + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['parent'].get_class(modname) + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_description(self, modname): + """Retrieves the module class decription + + @type modname: string + @param modname: the module class name + @type string + @return: the modules class decription + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['description'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_functions(self, modname): + """Retrieves the module class exported function names + + @type modname: string + @param modname: the module class name + @type list + @return: the modules class exported function names + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['functions'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_func_descriptions(self, modname): + """Retrieves the module class exported functions descriptions + + @type modname: string + @param modname: the module class name + @type dictionary + @return: the modules class exported functions descriptions + """ + if modname and modname in self.module_names: + desc = self._modules[modname]['func_desc'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return desc diff --git a/pym/portage/emaint/modules/__init__.py b/pym/portage/emaint/modules/__init__.py new file mode 100644 index 000000000..35674e342 --- /dev/null +++ b/pym/portage/emaint/modules/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program plug-in module provides an automatic method +of adding/removing modules to perform checks and maintenance +on a gentoo system. +""" diff --git a/pym/portage/emaint/modules/binhost/__init__.py b/pym/portage/emaint/modules/binhost/__init__.py new file mode 100644 index 000000000..1a61af42b --- /dev/null +++ b/pym/portage/emaint/modules/binhost/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program module provides checks and maintenancefor: + Scanning, checking and fixing problems in the world file. +""" + + +module_spec = { + 'name': 'binhost', + 'description': "Provides functions to scan, check and " + \ + "Generate a metadata index for binary packages", + 'provides':{ + 'module1': { + 'name': "binhost", + 'class': "BinhostHandler", + 'description': "Generate a metadata index for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/binhost/binhost.py b/pym/portage/emaint/modules/binhost/binhost.py new file mode 100644 index 000000000..b540d7686 --- /dev/null +++ b/pym/portage/emaint/modules/binhost/binhost.py @@ -0,0 +1,167 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import stat + +import portage +from portage import os +from portage.util import writemsg + +import sys +if sys.hexversion >= 0x3000000: + long = int + +class BinhostHandler(object): + + short_desc = "Generate a metadata index for binary packages" + + def name(): + return "binhost" + name = staticmethod(name) + + def __init__(self): + eroot = portage.settings['EROOT'] + self._bintree = portage.db[eroot]["bintree"] + self._bintree.populate() + self._pkgindex_file = self._bintree._pkgindex_file + self._pkgindex = self._bintree._load_pkgindex() + + def _need_update(self, cpv, data): + + if "MD5" not in data: + return True + + size = data.get("SIZE") + if size is None: + return True + + mtime = data.get("MTIME") + if mtime is None: + return True + + pkg_path = self._bintree.getname(cpv) + try: + s = os.lstat(pkg_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + # We can't update the index for this one because + # it disappeared. + return False + + try: + if long(mtime) != s[stat.ST_MTIME]: + return True + if long(size) != long(s.st_size): + return True + except ValueError: + return True + + return False + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + missing = [] + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + if onProgress: + onProgress(maxval, i+1) + errors = ["'%s' is not in Packages" % cpv for cpv in missing] + stale = set(metadata).difference(cpv_all) + for cpv in stale: + errors.append("'%s' is not in the repository" % cpv) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + bintree = self._bintree + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + missing = [] + maxval = 0 + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + stale = set(metadata).difference(cpv_all) + if missing or stale: + from portage import locks + pkgindex_lock = locks.lockfile( + self._pkgindex_file, wantnewlockfile=1) + try: + # Repopulate with lock held. + bintree._populate() + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + + pkgindex = bintree._load_pkgindex() + self._pkgindex = pkgindex + + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + # Recount missing packages, with lock held. + del missing[:] + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + maxval = len(missing) + for i, cpv in enumerate(missing): + try: + metadata[cpv] = bintree._pkgindex_entry(cpv) + except portage.exception.InvalidDependString: + writemsg("!!! Invalid binary package: '%s'\n" % \ + bintree.getname(cpv), noiselevel=-1) + + if onProgress: + onProgress(maxval, i+1) + + for cpv in set(metadata).difference( + self._bintree.dbapi.cpv_all()): + del metadata[cpv] + + # We've updated the pkgindex, so set it to + # repopulate when necessary. + bintree.populated = False + + del pkgindex.packages[:] + pkgindex.packages.extend(metadata.values()) + from portage.util import atomic_ofstream + f = atomic_ofstream(self._pkgindex_file) + try: + self._pkgindex.write(f) + finally: + f.close() + finally: + locks.unlockfile(pkgindex_lock) + + if onProgress: + if maxval == 0: + maxval = 1 + onProgress(maxval, maxval) + return None diff --git a/pym/portage/emaint/modules/config/__init__.py b/pym/portage/emaint/modules/config/__init__.py new file mode 100644 index 000000000..22abb07b1 --- /dev/null +++ b/pym/portage/emaint/modules/config/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the emerge config tracker list +""" + + +module_spec = { + 'name': 'config', + 'description': "Provides functions to scan, check for and fix no " +\ + "longer installed config files in emerge's tracker file", + 'provides':{ + 'module1': { + 'name': "cleanconfmem", + 'class': "CleanConfig", + 'description': "Discard no longer installed config tracker entries", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/config/config.py b/pym/portage/emaint/modules/config/config.py new file mode 100644 index 000000000..a80d87d29 --- /dev/null +++ b/pym/portage/emaint/modules/config/config.py @@ -0,0 +1,101 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.const import PRIVATE_PATH +from portage.checksum import perform_md5 + + +class CleanConfig(object): + + short_desc = "Discard any no longer installed configs from emerge's tracker list" + + def __init__(self): + self.target = os.path.join(portage.settings["EROOT"], PRIVATE_PATH, 'config') + + def name(): + return "cleanconfmem" + name = staticmethod(name) + + def load_configlist(self): + + configs = {} + with open(self.target, 'r') as configfile: + lines = configfile.readlines() + for line in lines: + ls = line.split() + configs[ls[0]] = ls[1] + return configs + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + else: + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + return self._format_output(messages, chksums) + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + configs.pop(config) + else: + configs.pop(config) + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + lines = [] + keys = sorted(configs) + for key in keys: + line = ' '.join([key, configs[key]]) + lines.append(line) + lines.append('') + with open(self.target, 'w') as configfile: + configfile.write('\n'.join(lines)) + return self._format_output(messages, chksums, True) + + def _format_output(self, messages=[], chksums=[], cleaned=False): + output = [] + if messages: + output.append('Not Installed:') + output += messages + tot = '------------------------------------\n Total %i Not installed' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(messages)) + if chksums: + output.append('\nChecksums did not match:') + output += chksums + tot = '------------------------------------\n Total %i Checksums did not match' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(chksums)) + return output diff --git a/pym/portage/emaint/modules/logs/__init__.py b/pym/portage/emaint/modules/logs/__init__.py new file mode 100644 index 000000000..005b608a6 --- /dev/null +++ b/pym/portage/emaint/modules/logs/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the PORT_LOGDIR logs +""" + + +module_spec = { + 'name': 'logs', + 'description': "Provides functions to scan, check and clean old logs " +\ + "in the PORT_LOGDIR", + 'provides':{ + 'module1': { + 'name': "logs", + 'class': "CleanLogs", + 'description': "Clean out old logs from the PORT_LOGDIR", + 'functions': ['check','clean'], + 'func_desc': { + 'clean': { + "short": "-C", "long": "--clean", + "help": "Cleans out logs more than 7 days old (cleanlogs only)" + \ + " modulke-options: -t, -p", + 'status': "Cleaning %s", + 'func': 'clean' + }, + 'time': { + "short": "-t", "long": "--time", + "help": "(cleanlogs only): -t, --time Delete logs older than NUM of days", + 'status': "", + 'action': 'store', + 'type': 'int', + 'dest': 'NUM', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + }, + 'pretend': { + "short": "-p", "long": "--pretend", + "help": "(cleanlogs only): -p, --pretend Output logs that would be deleted", + 'status': "", + 'action': 'store_true', + 'dest': 'pretend', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + } + } + } + } + } diff --git a/pym/portage/emaint/modules/logs/logs.py b/pym/portage/emaint/modules/logs/logs.py new file mode 100644 index 000000000..32c8508f7 --- /dev/null +++ b/pym/portage/emaint/modules/logs/logs.py @@ -0,0 +1,114 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.util import shlex_split, varexpand + +## default clean command from make.globals +## PORT_LOGDIR_CLEAN = 'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +7 -delete' + +class CleanLogs(object): + + short_desc = "Clean PORT_LOGDIR logs" + + def name(): + return "logs" + name = staticmethod(name) + + + def can_progressbar(self, func): + return False + + + def check(self, **kwargs): + if kwargs: + options = kwargs.get('options', None) + if options: + options['pretend'] = True + return self.clean(**kwargs) + + + def clean(self, **kwargs): + """Log directory cleaning function + + @param **kwargs: optional dictionary of values used in this function are: + settings: portage settings instance: defaults to portage.settings + "PORT_LOGDIR": directory to clean + "PORT_LOGDIR_CLEAN": command for cleaning the logs. + options: dict: + 'NUM': int: number of days + 'pretend': boolean + 'eerror': defaults to None, optional output module to output errors. + 'einfo': defaults to None, optional output module to output info msgs. + """ + messages = [] + num_of_days = None + if kwargs: + # convuluted, I know, but portage.settings does not exist in + # kwargs.get() when called from _emerge.main.clean_logs() + settings = kwargs.get('settings', None) + if not settings: + settings = portage.settings + options = kwargs.get('options', None) + if options: + num_of_days = options.get('NUM', None) + pretend = options.get('pretend', False) + eerror = options.get('eerror', None) + einfo = options.get('einfo', None) + + clean_cmd = settings.get("PORT_LOGDIR_CLEAN") + if clean_cmd: + clean_cmd = shlex_split(clean_cmd) + if '-mtime' in clean_cmd and num_of_days is not None: + if num_of_days == 0: + i = clean_cmd.index('-mtime') + clean_cmd.remove('-mtime') + clean_cmd.pop(i) + else: + clean_cmd[clean_cmd.index('-mtime') +1] = \ + '+%s' % str(num_of_days) + if pretend: + if "-delete" in clean_cmd: + clean_cmd.remove("-delete") + + if not clean_cmd: + return [] + rval = self._clean_logs(clean_cmd, settings) + messages += self._convert_errors(rval, eerror, einfo) + return messages + + + @staticmethod + def _clean_logs(clean_cmd, settings): + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + variables = {"PORT_LOGDIR" : logdir} + cmd = [varexpand(x, mydict=variables) for x in clean_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + rval = 127 + return rval + + + @staticmethod + def _convert_errors(rval, eerror=None, einfo=None): + msg = [] + if rval != os.EX_OK: + msg.append("PORT_LOGDIR_CLEAN command returned %s" + % ("%d" % rval if rval else "None")) + msg.append("See the make.conf(5) man page for " + "PORT_LOGDIR_CLEAN usage instructions.") + if eerror: + for m in msg: + eerror(m) + else: + msg.append("PORT_LOGDIR_CLEAN command succeeded") + if einfo: + for m in msg: + einfo(m) + return msg diff --git a/pym/portage/emaint/modules/move/__init__.py b/pym/portage/emaint/modules/move/__init__.py new file mode 100644 index 000000000..5399440ce --- /dev/null +++ b/pym/portage/emaint/modules/move/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: + 1) "Performing package move updates for installed packages", + 2)"Perform package move updates for binary packages" +""" + + +module_spec = { + 'name': 'move', + 'description': "Provides functions to check for and move packages " +\ + "either installed or binary packages stored on this system", + 'provides':{ + 'module1': { + 'name': "moveinst", + 'class': "MoveInstalled", + 'description': "Perform package move updates for installed packages", + 'options': ['check', 'fix'], + 'functions': ['check', 'fix'], + 'func_desc': { + } + }, + 'module2':{ + 'name': "movebin", + 'class': "MoveBinary", + 'description': "Perform package move updates for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': { + } + } + } + } diff --git a/pym/portage/emaint/modules/move/move.py b/pym/portage/emaint/modules/move/move.py new file mode 100644 index 000000000..018e6cac1 --- /dev/null +++ b/pym/portage/emaint/modules/move/move.py @@ -0,0 +1,162 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class MoveHandler(object): + + def __init__(self, tree, porttree): + self._tree = tree + self._portdb = porttree.dbapi + self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] + self._master_repo = \ + self._portdb.getRepositoryName(self._portdb.porttree_root) + + def _grab_global_updates(self): + from portage.update import grab_updates, parse_updates + retupdates = {} + errors = [] + + for repo_name in self._portdb.getRepositories(): + repo = self._portdb.getRepositoryPath(repo_name) + updpath = os.path.join(repo, "profiles", "updates") + if not os.path.isdir(updpath): + continue + + try: + rawupdates = grab_updates(updpath) + except portage.exception.DirectoryNotFound: + rawupdates = [] + upd_commands = [] + for mykey, mystat, mycontent in rawupdates: + commands, errors = parse_updates(mycontent) + upd_commands.extend(commands) + errors.extend(errors) + retupdates[repo_name] = upd_commands + + if self._master_repo in retupdates: + retupdates['DEFAULT'] = retupdates[self._master_repo] + + return retupdates, errors + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + match = self._tree.dbapi.match + aux_get = self._tree.dbapi.aux_get + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + origcp, newcp = update_cmd[1:] + for cpv in match(origcp): + if repo_match(aux_get(cpv, ["repository"])[0]): + errors.append("'%s' moved to '%s'" % (cpv, newcp)) + elif update_cmd[0] == "slotmove": + pkg, origslot, newslot = update_cmd[1:] + for cpv in match(pkg): + slot, prepo = aux_get(cpv, ["SLOT", "repository"]) + if slot == origslot and repo_match(prepo): + errors.append("'%s' slot moved from '%s' to '%s'" % \ + (cpv, origslot, newslot)) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + cpv_all = self._tree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + meta_keys = self._update_keys + ['repository', 'EAPI'] + if onProgress: + onProgress(maxval, 0) + for i, cpv in enumerate(cpv_all): + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + eapi = metadata.pop('EAPI') + repository = metadata.pop('repository') + try: + updates = allupdates[repository] + except KeyError: + try: + updates = allupdates['DEFAULT'] + except KeyError: + continue + if not updates: + continue + metadata_updates = \ + portage.update_dbentries(updates, metadata, eapi=eapi) + if metadata_updates: + errors.append("'%s' has outdated metadata" % cpv) + if onProgress: + onProgress(maxval, i+1) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + move = self._tree.dbapi.move_ent + slotmove = self._tree.dbapi.move_slot_ent + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + move(update_cmd, repo_match=repo_match) + elif update_cmd[0] == "slotmove": + slotmove(update_cmd, repo_match=repo_match) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) + return errors + +class MoveInstalled(MoveHandler): + + short_desc = "Perform package move updates for installed packages" + + def name(): + return "moveinst" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) + +class MoveBinary(MoveHandler): + + short_desc = "Perform package move updates for binary packages" + + def name(): + return "movebin" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) diff --git a/pym/portage/emaint/modules/resume/__init__.py b/pym/portage/emaint/modules/resume/__init__.py new file mode 100644 index 000000000..60cffe9db --- /dev/null +++ b/pym/portage/emaint/modules/resume/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the "emerge --resume" lists +""" + + +module_spec = { + 'name': 'resume', + 'description': "Provides functions to scan, check and fix problems " +\ + "in the resume and/or resume_backup files", + 'provides':{ + 'module1': { + 'name': "cleanresume", + 'class': "CleanResume", + 'description': "Discard emerge --resume merge lists", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/resume/resume.py b/pym/portage/emaint/modules/resume/resume.py new file mode 100644 index 000000000..1bada5288 --- /dev/null +++ b/pym/portage/emaint/modules/resume/resume.py @@ -0,0 +1,58 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage + + +class CleanResume(object): + + short_desc = "Discard emerge --resume merge lists" + + def name(): + return "cleanresume" + name = staticmethod(name) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + messages = [] + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + d = mtimedb.get(k) + if d is None: + continue + if not isinstance(d, dict): + messages.append("unrecognized resume list: '%s'" % k) + continue + mergelist = d.get("mergelist") + if mergelist is None or not hasattr(mergelist, "__len__"): + messages.append("unrecognized resume list: '%s'" % k) + continue + messages.append("resume list '%s' contains %d packages" % \ + (k, len(mergelist))) + finally: + if onProgress: + onProgress(maxval, i+1) + return messages + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + delete_count = 0 + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + if mtimedb.pop(k, None) is not None: + delete_count += 1 + finally: + if onProgress: + onProgress(maxval, i+1) + if delete_count: + mtimedb.commit() diff --git a/pym/portage/emaint/modules/world/__init__.py b/pym/portage/emaint/modules/world/__init__.py new file mode 100644 index 000000000..103b5c5ba --- /dev/null +++ b/pym/portage/emaint/modules/world/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Fixing problems with the "world" file. +""" + + +module_spec = { + 'name': 'world', + 'description': "Provides functions to scan, " + + "check and fix problems in the world file", + 'provides':{ + 'module1':{ + 'name': "world", + 'class': "WorldHandler", + 'description': "Fix problems in the world file", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/world/world.py b/pym/portage/emaint/modules/world/world.py new file mode 100644 index 000000000..2c9dbffeb --- /dev/null +++ b/pym/portage/emaint/modules/world/world.py @@ -0,0 +1,89 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class WorldHandler(object): + + short_desc = "Fix problems in the world file" + + def name(): + return "world" + name = staticmethod(name) + + def __init__(self): + self.invalid = [] + self.not_installed = [] + self.okay = [] + from portage._sets import load_default_config + setconfig = load_default_config(portage.settings, + portage.db[portage.settings['EROOT']]) + self._sets = setconfig.getSets() + + def _check_world(self, onProgress): + eroot = portage.settings['EROOT'] + self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) + self.found = os.access(self.world_file, os.R_OK) + vardb = portage.db[eroot]["vartree"].dbapi + + from portage._sets import SETPREFIX + sets = self._sets + world_atoms = list(sets["selected"]) + maxval = len(world_atoms) + if onProgress: + onProgress(maxval, 0) + for i, atom in enumerate(world_atoms): + if not isinstance(atom, portage.dep.Atom): + if atom.startswith(SETPREFIX): + s = atom[len(SETPREFIX):] + if s in sets: + self.okay.append(atom) + else: + self.not_installed.append(atom) + else: + self.invalid.append(atom) + if onProgress: + onProgress(maxval, i+1) + continue + okay = True + if not vardb.match(atom): + self.not_installed.append(atom) + okay = False + if okay: + self.okay.append(atom) + if onProgress: + onProgress(maxval, i+1) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + self._check_world(onProgress) + errors = [] + if self.found: + errors += ["'%s' is not a valid atom" % x for x in self.invalid] + errors += ["'%s' is not installed" % x for x in self.not_installed] + else: + errors.append(self.world_file + " could not be opened for reading") + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + world_set = self._sets["selected"] + world_set.lock() + try: + world_set.load() # maybe it's changed on disk + before = set(world_set) + self._check_world(onProgress) + after = set(self.okay) + errors = [] + if before != after: + try: + world_set.replace(self.okay) + except portage.exception.PortageException: + errors.append("%s could not be opened for writing" % \ + self.world_file) + return errors + finally: + world_set.unlock() + diff --git a/pym/portage/emaint/progress.py b/pym/portage/emaint/progress.py new file mode 100644 index 000000000..e43c2afbd --- /dev/null +++ b/pym/portage/emaint/progress.py @@ -0,0 +1,61 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import time +import signal + +import portage + + +class ProgressHandler(object): + def __init__(self): + self.reset() + + def reset(self): + self.curval = 0 + self.maxval = 0 + self.last_update = 0 + self.min_display_latency = 0.2 + + def onProgress(self, maxval, curval): + self.maxval = maxval + self.curval = curval + cur_time = time.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + + def display(self): + raise NotImplementedError(self) + + +class ProgressBar(ProgressHandler): + """Class to set up and return a Progress Bar""" + + def __init__(self, isatty, **kwargs): + self.isatty = isatty + self.kwargs = kwargs + ProgressHandler.__init__(self) + self.progressBar = None + + def start(self): + if self.isatty: + self.progressBar = portage.output.TermProgressBar(**self.kwargs) + signal.signal(signal.SIGWINCH, self.sigwinch_handler) + else: + self.onProgress = None + return self.onProgress + + def set_label(self, _label): + self.kwargs['label'] = _label + + def display(self): + self.progressBar.set(self.curval, self.maxval) + + def sigwinch_handler(self, signum, frame): + lines, self.progressBar.term_columns = \ + portage.output.get_term_size() + + def stop(self): + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + |