diff options
21 files changed, 1429 insertions, 653 deletions
diff --git a/bin/emaint b/bin/emaint index 21604511b..bee46c40d 100755 --- a/bin/emaint +++ b/bin/emaint @@ -2,16 +2,29 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +"""'The emaint program provides an interface to system health + checks and maintenance. +""" + from __future__ import print_function -import errno -import re -import signal -import stat import sys -import textwrap -import time -from optparse import OptionParser, OptionValueError +import errno +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +except KeyboardInterrupt: + sys.exit(1) try: import portage @@ -20,629 +33,13 @@ except ImportError: sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage -from portage import os -from portage.util import writemsg - -if sys.hexversion >= 0x3000000: - long = int - -class WorldHandler(object): - - short_desc = "Fix problems in the world file" - - def name(): - return "world" - name = staticmethod(name) - - def __init__(self): - self.invalid = [] - self.not_installed = [] - self.okay = [] - from portage._sets import load_default_config - setconfig = load_default_config(portage.settings, - portage.db[portage.settings['EROOT']]) - self._sets = setconfig.getSets() - - def _check_world(self, onProgress): - eroot = portage.settings['EROOT'] - self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) - self.found = os.access(self.world_file, os.R_OK) - vardb = portage.db[eroot]["vartree"].dbapi - - from portage._sets import SETPREFIX - sets = self._sets - world_atoms = list(sets["selected"]) - maxval = len(world_atoms) - if onProgress: - onProgress(maxval, 0) - for i, atom in enumerate(world_atoms): - if not isinstance(atom, portage.dep.Atom): - if atom.startswith(SETPREFIX): - s = atom[len(SETPREFIX):] - if s in sets: - self.okay.append(atom) - else: - self.not_installed.append(atom) - else: - self.invalid.append(atom) - if onProgress: - onProgress(maxval, i+1) - continue - okay = True - if not vardb.match(atom): - self.not_installed.append(atom) - okay = False - if okay: - self.okay.append(atom) - if onProgress: - onProgress(maxval, i+1) - - def check(self, onProgress=None): - self._check_world(onProgress) - errors = [] - if self.found: - errors += ["'%s' is not a valid atom" % x for x in self.invalid] - errors += ["'%s' is not installed" % x for x in self.not_installed] - else: - errors.append(self.world_file + " could not be opened for reading") - return errors - - def fix(self, onProgress=None): - world_set = self._sets["selected"] - world_set.lock() - try: - world_set.load() # maybe it's changed on disk - before = set(world_set) - self._check_world(onProgress) - after = set(self.okay) - errors = [] - if before != after: - try: - world_set.replace(self.okay) - except portage.exception.PortageException: - errors.append("%s could not be opened for writing" % \ - self.world_file) - return errors - finally: - world_set.unlock() - -class BinhostHandler(object): - - short_desc = "Generate a metadata index for binary packages" - - def name(): - return "binhost" - name = staticmethod(name) - - def __init__(self): - eroot = portage.settings['EROOT'] - self._bintree = portage.db[eroot]["bintree"] - self._bintree.populate() - self._pkgindex_file = self._bintree._pkgindex_file - self._pkgindex = self._bintree._load_pkgindex() - - def _need_update(self, cpv, data): - - if "MD5" not in data: - return True - - size = data.get("SIZE") - if size is None: - return True - - mtime = data.get("MTIME") - if mtime is None: - return True - - pkg_path = self._bintree.getname(cpv) - try: - s = os.lstat(pkg_path) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - # We can't update the index for this one because - # it disappeared. - return False - - try: - if long(mtime) != s[stat.ST_MTIME]: - return True - if long(size) != long(s.st_size): - return True - except ValueError: - return True - - return False - - def check(self, onProgress=None): - missing = [] - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - if onProgress: - onProgress(maxval, i+1) - errors = ["'%s' is not in Packages" % cpv for cpv in missing] - stale = set(metadata).difference(cpv_all) - for cpv in stale: - errors.append("'%s' is not in the repository" % cpv) - return errors - - def fix(self, onProgress=None): - bintree = self._bintree - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - missing = [] - maxval = 0 - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - stale = set(metadata).difference(cpv_all) - if missing or stale: - from portage import locks - pkgindex_lock = locks.lockfile( - self._pkgindex_file, wantnewlockfile=1) - try: - # Repopulate with lock held. - bintree._populate() - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - - pkgindex = bintree._load_pkgindex() - self._pkgindex = pkgindex - - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - # Recount missing packages, with lock held. - del missing[:] - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - maxval = len(missing) - for i, cpv in enumerate(missing): - try: - metadata[cpv] = bintree._pkgindex_entry(cpv) - except portage.exception.InvalidDependString: - writemsg("!!! Invalid binary package: '%s'\n" % \ - bintree.getname(cpv), noiselevel=-1) - - if onProgress: - onProgress(maxval, i+1) - - for cpv in set(metadata).difference( - self._bintree.dbapi.cpv_all()): - del metadata[cpv] - - # We've updated the pkgindex, so set it to - # repopulate when necessary. - bintree.populated = False - - del pkgindex.packages[:] - pkgindex.packages.extend(metadata.values()) - from portage.util import atomic_ofstream - f = atomic_ofstream(self._pkgindex_file) - try: - self._pkgindex.write(f) - finally: - f.close() - finally: - locks.unlockfile(pkgindex_lock) - - if onProgress: - if maxval == 0: - maxval = 1 - onProgress(maxval, maxval) - return None - -class MoveHandler(object): - - def __init__(self, tree, porttree): - self._tree = tree - self._portdb = porttree.dbapi - self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] - self._master_repo = \ - self._portdb.getRepositoryName(self._portdb.porttree_root) - - def _grab_global_updates(self): - from portage.update import grab_updates, parse_updates - retupdates = {} - errors = [] - - for repo_name in self._portdb.getRepositories(): - repo = self._portdb.getRepositoryPath(repo_name) - updpath = os.path.join(repo, "profiles", "updates") - if not os.path.isdir(updpath): - continue - - try: - rawupdates = grab_updates(updpath) - except portage.exception.DirectoryNotFound: - rawupdates = [] - upd_commands = [] - for mykey, mystat, mycontent in rawupdates: - commands, errors = parse_updates(mycontent) - upd_commands.extend(commands) - errors.extend(errors) - retupdates[repo_name] = upd_commands - - if self._master_repo in retupdates: - retupdates['DEFAULT'] = retupdates[self._master_repo] - - return retupdates, errors +from portage.emaint.main import emaint_main - def check(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - match = self._tree.dbapi.match - aux_get = self._tree.dbapi.aux_get - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue - - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) - - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - origcp, newcp = update_cmd[1:] - for cpv in match(origcp): - if repo_match(aux_get(cpv, ["repository"])[0]): - errors.append("'%s' moved to '%s'" % (cpv, newcp)) - elif update_cmd[0] == "slotmove": - pkg, origslot, newslot = update_cmd[1:] - for cpv in match(pkg): - slot, prepo = aux_get(cpv, ["SLOT", "repository"]) - if slot == origslot and repo_match(prepo): - errors.append("'%s' slot moved from '%s' to '%s'" % \ - (cpv, origslot, newslot)) - if onProgress: - onProgress(0, 0) - - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - cpv_all = self._tree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - aux_update = self._tree.dbapi.aux_update - meta_keys = self._update_keys + ['repository', 'EAPI'] - if onProgress: - onProgress(maxval, 0) - for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - eapi = metadata.pop('EAPI') - repository = metadata.pop('repository') - try: - updates = allupdates[repository] - except KeyError: - try: - updates = allupdates['DEFAULT'] - except KeyError: - continue - if not updates: - continue - metadata_updates = \ - portage.update_dbentries(updates, metadata, eapi=eapi) - if metadata_updates: - errors.append("'%s' has outdated metadata" % cpv) - if onProgress: - onProgress(maxval, i+1) - return errors - - def fix(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - move = self._tree.dbapi.move_ent - slotmove = self._tree.dbapi.move_slot_ent - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue - - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) - - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - move(update_cmd, repo_match=repo_match) - elif update_cmd[0] == "slotmove": - slotmove(update_cmd, repo_match=repo_match) - if onProgress: - onProgress(0, 0) - - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) - return errors - -class MoveInstalled(MoveHandler): - - short_desc = "Perform package move updates for installed packages" - - def name(): - return "moveinst" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) - -class MoveBinary(MoveHandler): - - short_desc = "Perform package move updates for binary packages" - - def name(): - return "movebin" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) - -class VdbKeyHandler(object): - def name(): - return "vdbkeys" - name = staticmethod(name) - - def __init__(self): - self.list = portage.db[portage.settings["EROOT"]]["vartree"].dbapi.cpv_all() - self.missing = [] - self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"] - - for p in self.list: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - ismissing = True - for k in self.keys: - if os.path.exists(mydir+k): - ismissing = False - break - if ismissing: - self.missing.append(p) - - def check(self): - return ["%s has missing keys" % x for x in self.missing] - - def fix(self): - - errors = [] - - for p in self.missing: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - if not os.access(mydir+"environment.bz2", os.R_OK): - errors.append("Can't access %s" % (mydir+"environment.bz2")) - elif not os.access(mydir, os.W_OK): - errors.append("Can't create files in %s" % mydir) - else: - env = os.popen("bzip2 -dcq "+mydir+"environment.bz2", "r") - envlines = env.read().split("\n") - env.close() - for k in self.keys: - s = [l for l in envlines if l.startswith(k+"=")] - if len(s) > 1: - errors.append("multiple matches for %s found in %senvironment.bz2" % (k, mydir)) - elif len(s) == 0: - s = "" - else: - s = s[0].split("=",1)[1] - s = s.lstrip("$").strip("\'\"") - s = re.sub("(\\\\[nrt])+", " ", s) - s = " ".join(s.split()).strip() - if s != "": - try: - keyfile = open(mydir+os.sep+k, "w") - keyfile.write(s+"\n") - keyfile.close() - except (IOError, OSError) as e: - errors.append("Could not write %s, reason was: %s" % (mydir+k, e)) - - return errors - -class ProgressHandler(object): - def __init__(self): - self.curval = 0 - self.maxval = 0 - self.last_update = 0 - self.min_display_latency = 0.2 - - def onProgress(self, maxval, curval): - self.maxval = maxval - self.curval = curval - cur_time = time.time() - if cur_time - self.last_update >= self.min_display_latency: - self.last_update = cur_time - self.display() - - def display(self): - raise NotImplementedError(self) - -class CleanResume(object): - - short_desc = "Discard emerge --resume merge lists" - - def name(): - return "cleanresume" - name = staticmethod(name) - - def check(self, onProgress=None): - messages = [] - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - d = mtimedb.get(k) - if d is None: - continue - if not isinstance(d, dict): - messages.append("unrecognized resume list: '%s'" % k) - continue - mergelist = d.get("mergelist") - if mergelist is None or not hasattr(mergelist, "__len__"): - messages.append("unrecognized resume list: '%s'" % k) - continue - messages.append("resume list '%s' contains %d packages" % \ - (k, len(mergelist))) - finally: - if onProgress: - onProgress(maxval, i+1) - return messages - - def fix(self, onProgress=None): - delete_count = 0 - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - if mtimedb.pop(k, None) is not None: - delete_count += 1 - finally: - if onProgress: - onProgress(maxval, i+1) - if delete_count: - mtimedb.commit() - -def emaint_main(myargv): - - # Similar to emerge, emaint needs a default umask so that created - # files (such as the world file) have sane permissions. - os.umask(0o22) - - # TODO: Create a system that allows external modules to be added without - # the need for hard coding. - modules = { - "world" : WorldHandler, - "binhost":BinhostHandler, - "moveinst":MoveInstalled, - "movebin":MoveBinary, - "cleanresume":CleanResume - } - - module_names = list(modules) - module_names.sort() - module_names.insert(0, "all") - - def exclusive(option, *args, **kw): - var = kw.get("var", None) - if var is None: - raise ValueError("var not specified to exclusive()") - if getattr(parser, var, ""): - raise OptionValueError("%s and %s are exclusive options" % (getattr(parser, var), option)) - setattr(parser, var, str(option)) - - - usage = "usage: emaint [options] COMMAND" - - desc = "The emaint program provides an interface to system health " + \ - "checks and maintenance. See the emaint(1) man page " + \ - "for additional information about the following commands:" - - usage += "\n\n" - for line in textwrap.wrap(desc, 65): - usage += "%s\n" % line - usage += "\n" - usage += " %s" % "all".ljust(15) + \ - "Perform all supported commands\n" - for m in module_names[1:]: - usage += " %s%s\n" % (m.ljust(15), modules[m].short_desc) - - parser = OptionParser(usage=usage, version=portage.VERSION) - parser.add_option("-c", "--check", help="check for problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.add_option("-f", "--fix", help="attempt to fix problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.action = None - - - (options, args) = parser.parse_args(args=myargv) - if len(args) != 1: - parser.error("Incorrect number of arguments") - if args[0] not in module_names: - parser.error("%s target is not a known target" % args[0]) - - if parser.action: - action = parser.action - else: - print("Defaulting to --check") - action = "-c/--check" - - if args[0] == "all": - tasks = modules.values() - else: - tasks = [modules[args[0]]] - - - if action == "-c/--check": - func = "check" - else: - func = "fix" - - isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() - for task in tasks: - inst = task() - onProgress = None - if isatty: - progressBar = portage.output.TermProgressBar(title="Emaint", max_desc_length=26) - progressHandler = ProgressHandler() - onProgress = progressHandler.onProgress - def display(): - progressBar.set(progressHandler.curval, progressHandler.maxval) - progressHandler.display = display - def sigwinch_handler(signum, frame): - lines, progressBar.term_columns = \ - portage.output.get_term_size() - signal.signal(signal.SIGWINCH, sigwinch_handler) - progressBar.label(func + " " + inst.name()) - result = getattr(inst, func)(onProgress=onProgress) - if isatty: - # make sure the final progress is displayed - progressHandler.display() - print() - signal.signal(signal.SIGWINCH, signal.SIG_DFL) - if result: - print() - print("\n".join(result)) - print("\n") - - print("Finished") - -if __name__ == "__main__": +try: emaint_main(sys.argv[1:]) +except IOError as e: + if e.errno == errno.EACCES: + print("\nemaint: Need superuser access") + sys.exit(1) + else: + raise diff --git a/man/emaint.1 b/man/emaint.1 index dff6fddd9..c588a0bfe 100644 --- a/man/emaint.1 +++ b/man/emaint.1 @@ -19,9 +19,17 @@ Generate a metadata index for binary packages located in \fBPKGDIR\fR (for download by remote clients). See the \fBPORTAGE_BINHOST\fR documentation in the \fBmake.conf\fR(5) man page for additional information. .TP +.BR cleanconfig +Discard no longer installed config tracker entries. +.TP .BR cleanresume Discard merge lists saved for the \fBemerge\fR(1) \fB--resume\fR action. .TP +.BR logs +Clean out old logs from the \fBPORT_LOGDIR\fR using the command \fBPORT_LOGDIR_CLEAN\fR +See the \fBmake.conf\fR(5) man page for additional information as well as enabling the +\fB'clean-logs'\fR feature in emerge to do this automatically. +.TP .BR movebin Perform package move updates for binary packages located in \fBPKGDIR\fR. .TP @@ -30,23 +38,37 @@ Perform package move updates for installed packages. .TP .BR world Fix problems in the \fIworld\fR file. -.SH OPTIONS +.SH DEFAULT OPTIONS .TP .B \-c, \-\-check -Check for any problems that may exist. +Check for any problems that may exist. (all commands) .TP .B \-f, \-\-fix -Fix any problems that may exist. +Fix any problems that may exist. (not all commands) +.SH OPTIONS +.TP +.B \-C, \-\-clean +Cleans the logs from \fBPORT_LOGDIR\fR (logs command only) +.TP +.B \-p, \-\-pretend +Sets pretend mode (same as \-c, \-\-check) for use with the \-C, \-\-clean OPTION (logs command only) +.TP +.B \-t NUM, \-\-time NUM +Changes the minimum age \fBNUM\fR (in days) of the logs to be listed or deleted. (logs command only) .SH "REPORTING BUGS" Please report bugs via http://bugs.gentoo.org/ .SH AUTHORS .nf Mike Frysinger <vapier@gentoo.org> +Brian Dolbec <dolsen@gentoo.org> .fi .SH "FILES" .TP .B /var/lib/portage/world Contains a list of all user\-specified packages. +.TP +.B /var/lib/portage/config +Contains the paths and md5sums of all the config files being tracked. .SH "SEE ALSO" .BR emerge (1), .BR portage (5) diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 640f320fd..f19994c46 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -13,6 +13,7 @@ import platform import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.news:count_unread_news,display_news_notifications', + 'portage.emaint.modules.logs.logs:CleanLogs', ) from portage import os from portage import _encodings @@ -1351,29 +1352,17 @@ def clean_logs(settings): if "clean-logs" not in settings.features: return - clean_cmd = settings.get("PORT_LOGDIR_CLEAN") - if clean_cmd: - clean_cmd = shlex_split(clean_cmd) - if not clean_cmd: - return - logdir = settings.get("PORT_LOGDIR") if logdir is None or not os.path.isdir(logdir): return - variables = {"PORT_LOGDIR" : logdir} - cmd = [varexpand(x, mydict=variables) for x in clean_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - rval = 127 - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for " - "PORT_LOGDIR_CLEAN usage instructions.") + options = { + 'eerror': portage.output.EOutput().eerror, + # uncomment next line to output a succeeded message + #'einfo': portage.output.EOutput().einfo + } + cleanlogs = CleanLogs() + cleanlogs.clean(settings=settings, options=options) def setconfig_fallback(root_config): setconfig = root_config.setconfig diff --git a/pym/portage/emaint/__init__.py b/pym/portage/emaint/__init__.py new file mode 100644 index 000000000..5e0ae700a --- /dev/null +++ b/pym/portage/emaint/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program provides checks and maintenance +on a gentoo system. +""" + diff --git a/pym/portage/emaint/defaults.py b/pym/portage/emaint/defaults.py new file mode 100644 index 000000000..d9d83ffbb --- /dev/null +++ b/pym/portage/emaint/defaults.py @@ -0,0 +1,18 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# parser option data +CHECK = {"short": "-c", "long": "--check", + "help": "Check for problems (a default option for most modules)", + 'status': "Checking %s for problems", + 'func': 'check' + } + +FIX = {"short": "-f", "long": "--fix", + "help": "Attempt to fix problems (a default option for most modules)", + 'status': "Attempting to fix %s", + 'func': 'fix' + } + +# parser options +DEFAULT_OPTIONS = {'check': CHECK, 'fix': FIX} diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py new file mode 100644 index 000000000..dbc5f18cc --- /dev/null +++ b/pym/portage/emaint/main.py @@ -0,0 +1,218 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + + +import sys +import textwrap +from optparse import OptionParser, OptionValueError + + +import portage +from portage import os +from portage.emaint.module import Modules +from portage.emaint.progress import ProgressBar +from portage.emaint.defaults import DEFAULT_OPTIONS + +class OptionItem(object): + """class to hold module OptionParser options data + """ + + def __init__(self, opt, parser): + """ + @type opt: dictionary + @param opt: options parser options + """ + self.parser = parser + self.short = opt['short'] + self.long = opt['long'] + self.help = opt['help'] + self.status = opt['status'] + self.func = opt['func'] + self.action = opt.get('action', "callback") + self.type = opt.get('type', None) + self.dest = opt.get('dest', None) + self.callback = opt.get('callback', self._exclusive) + self.callback_kwargs = opt.get('callback_kwargs', {"var":"action"}) + + + def _exclusive(self, option, *args, **kw): + """Generic check for the 2 default options + """ + var = kw.get("var", None) + if var is None: + raise ValueError("var not specified to exclusive()") + if getattr(self.parser, var, ""): + raise OptionValueError("%s and %s are exclusive options" + % (getattr(self.parser, var), option)) + setattr(self.parser, var, str(option)) + + def check_action(self, action): + """Checks if 'action' is the same as this option + + @type action: string + @param action: the action to compare + @rtype: boolean + """ + if action == self.action: + return True + elif action == '/'.join([self.short, self.long]): + return True + return False + + +def usage(module_controller): + _usage = "usage: emaint [options] COMMAND" + + desc = "The emaint program provides an interface to system health " + \ + "checks and maintenance. See the emaint(1) man page " + \ + "for additional information about the following commands:" + + _usage += "\n\n" + for line in textwrap.wrap(desc, 65): + _usage += "%s\n" % line + _usage += "\nCommands:\n" + _usage += " %s" % "all".ljust(15) + \ + "Perform all supported commands\n" + textwrap.subsequent_indent = ' '.ljust(17) + for mod in module_controller.module_names: + desc = textwrap.wrap(module_controller.get_description(mod), 65) + _usage += " %s%s\n" % (mod.ljust(15), desc[0]) + for d in desc[1:]: + _usage += " %s%s\n" % (' '.ljust(15), d) + return _usage + + +def module_opts(module_controller, module): + _usage = " %s module options:\n" % module + opts = module_controller.get_func_descriptions(module) + if opts == {}: + opts = DEFAULT_OPTIONS + for opt in sorted(opts): + optd = opts[opt] + opto = " %s, %s" %(optd['short'], optd['long']) + _usage += '%s %s\n' % (opto.ljust(15),optd['help']) + _usage += '\n' + return _usage + + +class TaskHandler(object): + """Handles the running of the tasks it is given + """ + + def __init__(self, show_progress_bar=True, verbose=True, callback=None): + self.show_progress_bar = show_progress_bar + self.verbose = verbose + self.callback = callback + self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) + + + def run_tasks(self, tasks, func, status=None, verbose=True, options=None): + """Runs the module tasks""" + if tasks is None or func is None: + return + for task in tasks: + inst = task() + show_progress = self.show_progress_bar + # check if the function is capable of progressbar + # and possibly override it off + if show_progress and hasattr(inst, 'can_progressbar'): + show_progress = inst.can_progressbar(func) + if show_progress: + self.progress_bar.reset() + self.progress_bar.set_label(func + " " + inst.name()) + onProgress = self.progress_bar.start() + else: + onProgress = None + kwargs = { + 'onProgress': onProgress, + # pass in a copy of the options so a module can not pollute or change + # them for other tasks if there is more to do. + 'options': options.copy() + } + result = getattr(inst, func)(**kwargs) + if self.isatty and show_progress: + # make sure the final progress is displayed + self.progress_bar.display() + print() + self.progress_bar.stop() + if self.callback: + self.callback(result) + + +def print_results(results): + if results: + print() + print("\n".join(results)) + print("\n") + + +def emaint_main(myargv): + + # Similar to emerge, emaint needs a default umask so that created + # files (such as the world file) have sane permissions. + os.umask(0o22) + + module_controller = Modules(namepath="portage.emaint.modules") + module_names = module_controller.module_names[:] + module_names.insert(0, "all") + + + parser = OptionParser(usage=usage(module_controller), version=portage.VERSION) + # add default options + parser_options = [] + for opt in DEFAULT_OPTIONS: + parser_options.append(OptionItem(DEFAULT_OPTIONS[opt], parser)) + for mod in module_names[1:]: + desc = module_controller.get_func_descriptions(mod) + if desc: + for opt in desc: + parser_options.append(OptionItem(desc[opt], parser)) + for opt in parser_options: + parser.add_option(opt.short, opt.long, help=opt.help, action=opt.action, + type=opt.type, dest=opt.dest, + callback=opt.callback, callback_kwargs=opt.callback_kwargs) + + parser.action = None + + (options, args) = parser.parse_args(args=myargv) + #print('options', options, '\nargs', args, '\naction', parser.action) + if len(args) != 1: + parser.error("Incorrect number of arguments") + if args[0] not in module_names: + parser.error("%s target is not a known target" % args[0]) + + if parser.action: + action = parser.action + else: + action = "-c/--check" + long_action = action.split('/')[1].lstrip('-') + #print("DEBUG: action = ", action, long_action) + + if args[0] == "all": + tasks = [] + for m in module_names[1:]: + #print("DEBUG: module: %s, functions: " %(m, str(module_controller.get_functions(m)))) + if long_action in module_controller.get_functions(m): + tasks.append(module_controller.get_class(m)) + elif long_action in module_controller.get_functions(args[0]): + tasks = [module_controller.get_class(args[0] )] + else: + print("\nERROR: module '%s' does not have option '%s'\n" %(args[0], action)) + print(module_opts(module_controller, args[0])) + sys.exit(1) + func = status = None + for opt in parser_options: + if opt.check_action(action): + status = opt.status + func = opt.func + break + + # need to pass the parser options dict to the modules + # so they are available if needed. + task_opts = options.__dict__ + taskmaster = TaskHandler(callback=print_results) + taskmaster.run_tasks(tasks, func, status, options=task_opts) + diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py new file mode 100644 index 000000000..64b0c64b5 --- /dev/null +++ b/pym/portage/emaint/module.py @@ -0,0 +1,194 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + +from portage import os +from portage.exception import PortageException +from portage.cache.mappings import ProtectedDict + + +class InvalidModuleName(PortageException): + """An invalid or unknown module name.""" + + +class Module(object): + """Class to define and hold our plug-in module + + @type name: string + @param name: the module name + @type path: the path to the new module + """ + + def __init__(self, name, namepath): + """Some variables initialization""" + self.name = name + self._namepath = namepath + self.kids_names = [] + self.kids = {} + self.initialized = self._initialize() + + def _initialize(self): + """Initialize the plug-in module + + @rtype: boolean + """ + self.valid = False + try: + mod_name = ".".join([self._namepath, self.name]) + self._module = __import__(mod_name, [],[], ["not empty"]) + self.valid = True + except ImportError as e: + print("MODULE; failed import", mod_name, " error was:",e) + return False + self.module_spec = self._module.module_spec + for submodule in self.module_spec['provides']: + kid = self.module_spec['provides'][submodule] + kidname = kid['name'] + kid['module_name'] = '.'.join([mod_name, self.name]) + kid['is_imported'] = False + self.kids[kidname] = kid + self.kids_names.append(kidname) + return True + + def get_class(self, name): + if not name or name not in self.kids_names: + raise InvalidModuleName("Module name '%s' was invalid or not" + %name + "part of the module '%s'" %self.name) + kid = self.kids[name] + if kid['is_imported']: + module = kid['instance'] + else: + try: + module = __import__(kid['module_name'], [],[], ["not empty"]) + kid['instance'] = module + kid['is_imported'] = True + except ImportError: + raise + mod_class = getattr(module, kid['class']) + return mod_class + + +class Modules(object): + """Dynamic modules system for loading and retrieving any of the + installed emaint modules and/or provided class's + + @param path: Optional path to the "modules" directory or + defaults to the directory of this file + '/modules' + @param namepath: Optional python import path to the "modules" directory or + defaults to the directory name of this file + '.modules' + """ + + def __init__(self, path=None, namepath=None): + if path: + self._module_path = path + else: + self._module_path = os.path.join(( + os.path.dirname(os.path.realpath(__file__))), "modules") + if namepath: + self._namepath = namepath + else: + self._namepath = '.'.join(os.path.dirname( + os.path.realpath(__file__)), "modules") + self._modules = self._get_all_modules() + self.modules = ProtectedDict(self._modules) + self.module_names = sorted(self._modules) + #self.modules = {} + #for mod in self.module_names: + #self.module[mod] = LazyLoad( + + def _get_all_modules(self): + """scans the emaint modules dir for loadable modules + + @rtype: dictionary of module_plugins + """ + module_dir = self._module_path + importables = [] + names = os.listdir(module_dir) + for entry in names: + # skip any __init__ or __pycache__ files or directories + if entry.startswith('__'): + continue + try: + # test for statinfo to ensure it should a real module + # it will bail if it errors + os.lstat(os.path.join(module_dir, entry, '__init__.py')) + importables.append(entry) + except EnvironmentError: + pass + kids = {} + for entry in importables: + new_module = Module(entry, self._namepath) + for module_name in new_module.kids: + kid = new_module.kids[module_name] + kid['parent'] = new_module + kids[kid['name']] = kid + return kids + + def get_module_names(self): + """Convienence function to return the list of installed modules + available + + @rtype: list + @return: the installed module names available + """ + return self.module_names + + def get_class(self, modname): + """Retrieves a module class desired + + @type modname: string + @param modname: the module class name + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['parent'].get_class(modname) + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_description(self, modname): + """Retrieves the module class decription + + @type modname: string + @param modname: the module class name + @type string + @return: the modules class decription + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['description'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_functions(self, modname): + """Retrieves the module class exported function names + + @type modname: string + @param modname: the module class name + @type list + @return: the modules class exported function names + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['functions'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_func_descriptions(self, modname): + """Retrieves the module class exported functions descriptions + + @type modname: string + @param modname: the module class name + @type dictionary + @return: the modules class exported functions descriptions + """ + if modname and modname in self.module_names: + desc = self._modules[modname]['func_desc'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return desc diff --git a/pym/portage/emaint/modules/__init__.py b/pym/portage/emaint/modules/__init__.py new file mode 100644 index 000000000..35674e342 --- /dev/null +++ b/pym/portage/emaint/modules/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program plug-in module provides an automatic method +of adding/removing modules to perform checks and maintenance +on a gentoo system. +""" diff --git a/pym/portage/emaint/modules/binhost/__init__.py b/pym/portage/emaint/modules/binhost/__init__.py new file mode 100644 index 000000000..1a61af42b --- /dev/null +++ b/pym/portage/emaint/modules/binhost/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program module provides checks and maintenancefor: + Scanning, checking and fixing problems in the world file. +""" + + +module_spec = { + 'name': 'binhost', + 'description': "Provides functions to scan, check and " + \ + "Generate a metadata index for binary packages", + 'provides':{ + 'module1': { + 'name': "binhost", + 'class': "BinhostHandler", + 'description': "Generate a metadata index for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/binhost/binhost.py b/pym/portage/emaint/modules/binhost/binhost.py new file mode 100644 index 000000000..b540d7686 --- /dev/null +++ b/pym/portage/emaint/modules/binhost/binhost.py @@ -0,0 +1,167 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import stat + +import portage +from portage import os +from portage.util import writemsg + +import sys +if sys.hexversion >= 0x3000000: + long = int + +class BinhostHandler(object): + + short_desc = "Generate a metadata index for binary packages" + + def name(): + return "binhost" + name = staticmethod(name) + + def __init__(self): + eroot = portage.settings['EROOT'] + self._bintree = portage.db[eroot]["bintree"] + self._bintree.populate() + self._pkgindex_file = self._bintree._pkgindex_file + self._pkgindex = self._bintree._load_pkgindex() + + def _need_update(self, cpv, data): + + if "MD5" not in data: + return True + + size = data.get("SIZE") + if size is None: + return True + + mtime = data.get("MTIME") + if mtime is None: + return True + + pkg_path = self._bintree.getname(cpv) + try: + s = os.lstat(pkg_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + # We can't update the index for this one because + # it disappeared. + return False + + try: + if long(mtime) != s[stat.ST_MTIME]: + return True + if long(size) != long(s.st_size): + return True + except ValueError: + return True + + return False + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + missing = [] + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + if onProgress: + onProgress(maxval, i+1) + errors = ["'%s' is not in Packages" % cpv for cpv in missing] + stale = set(metadata).difference(cpv_all) + for cpv in stale: + errors.append("'%s' is not in the repository" % cpv) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + bintree = self._bintree + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + missing = [] + maxval = 0 + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + stale = set(metadata).difference(cpv_all) + if missing or stale: + from portage import locks + pkgindex_lock = locks.lockfile( + self._pkgindex_file, wantnewlockfile=1) + try: + # Repopulate with lock held. + bintree._populate() + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + + pkgindex = bintree._load_pkgindex() + self._pkgindex = pkgindex + + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + # Recount missing packages, with lock held. + del missing[:] + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + maxval = len(missing) + for i, cpv in enumerate(missing): + try: + metadata[cpv] = bintree._pkgindex_entry(cpv) + except portage.exception.InvalidDependString: + writemsg("!!! Invalid binary package: '%s'\n" % \ + bintree.getname(cpv), noiselevel=-1) + + if onProgress: + onProgress(maxval, i+1) + + for cpv in set(metadata).difference( + self._bintree.dbapi.cpv_all()): + del metadata[cpv] + + # We've updated the pkgindex, so set it to + # repopulate when necessary. + bintree.populated = False + + del pkgindex.packages[:] + pkgindex.packages.extend(metadata.values()) + from portage.util import atomic_ofstream + f = atomic_ofstream(self._pkgindex_file) + try: + self._pkgindex.write(f) + finally: + f.close() + finally: + locks.unlockfile(pkgindex_lock) + + if onProgress: + if maxval == 0: + maxval = 1 + onProgress(maxval, maxval) + return None diff --git a/pym/portage/emaint/modules/config/__init__.py b/pym/portage/emaint/modules/config/__init__.py new file mode 100644 index 000000000..22abb07b1 --- /dev/null +++ b/pym/portage/emaint/modules/config/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the emerge config tracker list +""" + + +module_spec = { + 'name': 'config', + 'description': "Provides functions to scan, check for and fix no " +\ + "longer installed config files in emerge's tracker file", + 'provides':{ + 'module1': { + 'name': "cleanconfmem", + 'class': "CleanConfig", + 'description': "Discard no longer installed config tracker entries", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/config/config.py b/pym/portage/emaint/modules/config/config.py new file mode 100644 index 000000000..a80d87d29 --- /dev/null +++ b/pym/portage/emaint/modules/config/config.py @@ -0,0 +1,101 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.const import PRIVATE_PATH +from portage.checksum import perform_md5 + + +class CleanConfig(object): + + short_desc = "Discard any no longer installed configs from emerge's tracker list" + + def __init__(self): + self.target = os.path.join(portage.settings["EROOT"], PRIVATE_PATH, 'config') + + def name(): + return "cleanconfmem" + name = staticmethod(name) + + def load_configlist(self): + + configs = {} + with open(self.target, 'r') as configfile: + lines = configfile.readlines() + for line in lines: + ls = line.split() + configs[ls[0]] = ls[1] + return configs + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + else: + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + return self._format_output(messages, chksums) + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + configs.pop(config) + else: + configs.pop(config) + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + lines = [] + keys = sorted(configs) + for key in keys: + line = ' '.join([key, configs[key]]) + lines.append(line) + lines.append('') + with open(self.target, 'w') as configfile: + configfile.write('\n'.join(lines)) + return self._format_output(messages, chksums, True) + + def _format_output(self, messages=[], chksums=[], cleaned=False): + output = [] + if messages: + output.append('Not Installed:') + output += messages + tot = '------------------------------------\n Total %i Not installed' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(messages)) + if chksums: + output.append('\nChecksums did not match:') + output += chksums + tot = '------------------------------------\n Total %i Checksums did not match' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(chksums)) + return output diff --git a/pym/portage/emaint/modules/logs/__init__.py b/pym/portage/emaint/modules/logs/__init__.py new file mode 100644 index 000000000..005b608a6 --- /dev/null +++ b/pym/portage/emaint/modules/logs/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the PORT_LOGDIR logs +""" + + +module_spec = { + 'name': 'logs', + 'description': "Provides functions to scan, check and clean old logs " +\ + "in the PORT_LOGDIR", + 'provides':{ + 'module1': { + 'name': "logs", + 'class': "CleanLogs", + 'description': "Clean out old logs from the PORT_LOGDIR", + 'functions': ['check','clean'], + 'func_desc': { + 'clean': { + "short": "-C", "long": "--clean", + "help": "Cleans out logs more than 7 days old (cleanlogs only)" + \ + " modulke-options: -t, -p", + 'status': "Cleaning %s", + 'func': 'clean' + }, + 'time': { + "short": "-t", "long": "--time", + "help": "(cleanlogs only): -t, --time Delete logs older than NUM of days", + 'status': "", + 'action': 'store', + 'type': 'int', + 'dest': 'NUM', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + }, + 'pretend': { + "short": "-p", "long": "--pretend", + "help": "(cleanlogs only): -p, --pretend Output logs that would be deleted", + 'status': "", + 'action': 'store_true', + 'dest': 'pretend', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + } + } + } + } + } diff --git a/pym/portage/emaint/modules/logs/logs.py b/pym/portage/emaint/modules/logs/logs.py new file mode 100644 index 000000000..32c8508f7 --- /dev/null +++ b/pym/portage/emaint/modules/logs/logs.py @@ -0,0 +1,114 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.util import shlex_split, varexpand + +## default clean command from make.globals +## PORT_LOGDIR_CLEAN = 'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +7 -delete' + +class CleanLogs(object): + + short_desc = "Clean PORT_LOGDIR logs" + + def name(): + return "logs" + name = staticmethod(name) + + + def can_progressbar(self, func): + return False + + + def check(self, **kwargs): + if kwargs: + options = kwargs.get('options', None) + if options: + options['pretend'] = True + return self.clean(**kwargs) + + + def clean(self, **kwargs): + """Log directory cleaning function + + @param **kwargs: optional dictionary of values used in this function are: + settings: portage settings instance: defaults to portage.settings + "PORT_LOGDIR": directory to clean + "PORT_LOGDIR_CLEAN": command for cleaning the logs. + options: dict: + 'NUM': int: number of days + 'pretend': boolean + 'eerror': defaults to None, optional output module to output errors. + 'einfo': defaults to None, optional output module to output info msgs. + """ + messages = [] + num_of_days = None + if kwargs: + # convuluted, I know, but portage.settings does not exist in + # kwargs.get() when called from _emerge.main.clean_logs() + settings = kwargs.get('settings', None) + if not settings: + settings = portage.settings + options = kwargs.get('options', None) + if options: + num_of_days = options.get('NUM', None) + pretend = options.get('pretend', False) + eerror = options.get('eerror', None) + einfo = options.get('einfo', None) + + clean_cmd = settings.get("PORT_LOGDIR_CLEAN") + if clean_cmd: + clean_cmd = shlex_split(clean_cmd) + if '-mtime' in clean_cmd and num_of_days is not None: + if num_of_days == 0: + i = clean_cmd.index('-mtime') + clean_cmd.remove('-mtime') + clean_cmd.pop(i) + else: + clean_cmd[clean_cmd.index('-mtime') +1] = \ + '+%s' % str(num_of_days) + if pretend: + if "-delete" in clean_cmd: + clean_cmd.remove("-delete") + + if not clean_cmd: + return [] + rval = self._clean_logs(clean_cmd, settings) + messages += self._convert_errors(rval, eerror, einfo) + return messages + + + @staticmethod + def _clean_logs(clean_cmd, settings): + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + variables = {"PORT_LOGDIR" : logdir} + cmd = [varexpand(x, mydict=variables) for x in clean_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + rval = 127 + return rval + + + @staticmethod + def _convert_errors(rval, eerror=None, einfo=None): + msg = [] + if rval != os.EX_OK: + msg.append("PORT_LOGDIR_CLEAN command returned %s" + % ("%d" % rval if rval else "None")) + msg.append("See the make.conf(5) man page for " + "PORT_LOGDIR_CLEAN usage instructions.") + if eerror: + for m in msg: + eerror(m) + else: + msg.append("PORT_LOGDIR_CLEAN command succeeded") + if einfo: + for m in msg: + einfo(m) + return msg diff --git a/pym/portage/emaint/modules/move/__init__.py b/pym/portage/emaint/modules/move/__init__.py new file mode 100644 index 000000000..5399440ce --- /dev/null +++ b/pym/portage/emaint/modules/move/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: + 1) "Performing package move updates for installed packages", + 2)"Perform package move updates for binary packages" +""" + + +module_spec = { + 'name': 'move', + 'description': "Provides functions to check for and move packages " +\ + "either installed or binary packages stored on this system", + 'provides':{ + 'module1': { + 'name': "moveinst", + 'class': "MoveInstalled", + 'description': "Perform package move updates for installed packages", + 'options': ['check', 'fix'], + 'functions': ['check', 'fix'], + 'func_desc': { + } + }, + 'module2':{ + 'name': "movebin", + 'class': "MoveBinary", + 'description': "Perform package move updates for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': { + } + } + } + } diff --git a/pym/portage/emaint/modules/move/move.py b/pym/portage/emaint/modules/move/move.py new file mode 100644 index 000000000..018e6cac1 --- /dev/null +++ b/pym/portage/emaint/modules/move/move.py @@ -0,0 +1,162 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class MoveHandler(object): + + def __init__(self, tree, porttree): + self._tree = tree + self._portdb = porttree.dbapi + self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] + self._master_repo = \ + self._portdb.getRepositoryName(self._portdb.porttree_root) + + def _grab_global_updates(self): + from portage.update import grab_updates, parse_updates + retupdates = {} + errors = [] + + for repo_name in self._portdb.getRepositories(): + repo = self._portdb.getRepositoryPath(repo_name) + updpath = os.path.join(repo, "profiles", "updates") + if not os.path.isdir(updpath): + continue + + try: + rawupdates = grab_updates(updpath) + except portage.exception.DirectoryNotFound: + rawupdates = [] + upd_commands = [] + for mykey, mystat, mycontent in rawupdates: + commands, errors = parse_updates(mycontent) + upd_commands.extend(commands) + errors.extend(errors) + retupdates[repo_name] = upd_commands + + if self._master_repo in retupdates: + retupdates['DEFAULT'] = retupdates[self._master_repo] + + return retupdates, errors + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + match = self._tree.dbapi.match + aux_get = self._tree.dbapi.aux_get + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + origcp, newcp = update_cmd[1:] + for cpv in match(origcp): + if repo_match(aux_get(cpv, ["repository"])[0]): + errors.append("'%s' moved to '%s'" % (cpv, newcp)) + elif update_cmd[0] == "slotmove": + pkg, origslot, newslot = update_cmd[1:] + for cpv in match(pkg): + slot, prepo = aux_get(cpv, ["SLOT", "repository"]) + if slot == origslot and repo_match(prepo): + errors.append("'%s' slot moved from '%s' to '%s'" % \ + (cpv, origslot, newslot)) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + cpv_all = self._tree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + meta_keys = self._update_keys + ['repository', 'EAPI'] + if onProgress: + onProgress(maxval, 0) + for i, cpv in enumerate(cpv_all): + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + eapi = metadata.pop('EAPI') + repository = metadata.pop('repository') + try: + updates = allupdates[repository] + except KeyError: + try: + updates = allupdates['DEFAULT'] + except KeyError: + continue + if not updates: + continue + metadata_updates = \ + portage.update_dbentries(updates, metadata, eapi=eapi) + if metadata_updates: + errors.append("'%s' has outdated metadata" % cpv) + if onProgress: + onProgress(maxval, i+1) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + move = self._tree.dbapi.move_ent + slotmove = self._tree.dbapi.move_slot_ent + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + move(update_cmd, repo_match=repo_match) + elif update_cmd[0] == "slotmove": + slotmove(update_cmd, repo_match=repo_match) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) + return errors + +class MoveInstalled(MoveHandler): + + short_desc = "Perform package move updates for installed packages" + + def name(): + return "moveinst" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) + +class MoveBinary(MoveHandler): + + short_desc = "Perform package move updates for binary packages" + + def name(): + return "movebin" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) diff --git a/pym/portage/emaint/modules/resume/__init__.py b/pym/portage/emaint/modules/resume/__init__.py new file mode 100644 index 000000000..60cffe9db --- /dev/null +++ b/pym/portage/emaint/modules/resume/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the "emerge --resume" lists +""" + + +module_spec = { + 'name': 'resume', + 'description': "Provides functions to scan, check and fix problems " +\ + "in the resume and/or resume_backup files", + 'provides':{ + 'module1': { + 'name': "cleanresume", + 'class': "CleanResume", + 'description': "Discard emerge --resume merge lists", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/resume/resume.py b/pym/portage/emaint/modules/resume/resume.py new file mode 100644 index 000000000..1bada5288 --- /dev/null +++ b/pym/portage/emaint/modules/resume/resume.py @@ -0,0 +1,58 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage + + +class CleanResume(object): + + short_desc = "Discard emerge --resume merge lists" + + def name(): + return "cleanresume" + name = staticmethod(name) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + messages = [] + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + d = mtimedb.get(k) + if d is None: + continue + if not isinstance(d, dict): + messages.append("unrecognized resume list: '%s'" % k) + continue + mergelist = d.get("mergelist") + if mergelist is None or not hasattr(mergelist, "__len__"): + messages.append("unrecognized resume list: '%s'" % k) + continue + messages.append("resume list '%s' contains %d packages" % \ + (k, len(mergelist))) + finally: + if onProgress: + onProgress(maxval, i+1) + return messages + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + delete_count = 0 + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + if mtimedb.pop(k, None) is not None: + delete_count += 1 + finally: + if onProgress: + onProgress(maxval, i+1) + if delete_count: + mtimedb.commit() diff --git a/pym/portage/emaint/modules/world/__init__.py b/pym/portage/emaint/modules/world/__init__.py new file mode 100644 index 000000000..103b5c5ba --- /dev/null +++ b/pym/portage/emaint/modules/world/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Fixing problems with the "world" file. +""" + + +module_spec = { + 'name': 'world', + 'description': "Provides functions to scan, " + + "check and fix problems in the world file", + 'provides':{ + 'module1':{ + 'name': "world", + 'class': "WorldHandler", + 'description': "Fix problems in the world file", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/world/world.py b/pym/portage/emaint/modules/world/world.py new file mode 100644 index 000000000..2c9dbffeb --- /dev/null +++ b/pym/portage/emaint/modules/world/world.py @@ -0,0 +1,89 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class WorldHandler(object): + + short_desc = "Fix problems in the world file" + + def name(): + return "world" + name = staticmethod(name) + + def __init__(self): + self.invalid = [] + self.not_installed = [] + self.okay = [] + from portage._sets import load_default_config + setconfig = load_default_config(portage.settings, + portage.db[portage.settings['EROOT']]) + self._sets = setconfig.getSets() + + def _check_world(self, onProgress): + eroot = portage.settings['EROOT'] + self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) + self.found = os.access(self.world_file, os.R_OK) + vardb = portage.db[eroot]["vartree"].dbapi + + from portage._sets import SETPREFIX + sets = self._sets + world_atoms = list(sets["selected"]) + maxval = len(world_atoms) + if onProgress: + onProgress(maxval, 0) + for i, atom in enumerate(world_atoms): + if not isinstance(atom, portage.dep.Atom): + if atom.startswith(SETPREFIX): + s = atom[len(SETPREFIX):] + if s in sets: + self.okay.append(atom) + else: + self.not_installed.append(atom) + else: + self.invalid.append(atom) + if onProgress: + onProgress(maxval, i+1) + continue + okay = True + if not vardb.match(atom): + self.not_installed.append(atom) + okay = False + if okay: + self.okay.append(atom) + if onProgress: + onProgress(maxval, i+1) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + self._check_world(onProgress) + errors = [] + if self.found: + errors += ["'%s' is not a valid atom" % x for x in self.invalid] + errors += ["'%s' is not installed" % x for x in self.not_installed] + else: + errors.append(self.world_file + " could not be opened for reading") + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + world_set = self._sets["selected"] + world_set.lock() + try: + world_set.load() # maybe it's changed on disk + before = set(world_set) + self._check_world(onProgress) + after = set(self.okay) + errors = [] + if before != after: + try: + world_set.replace(self.okay) + except portage.exception.PortageException: + errors.append("%s could not be opened for writing" % \ + self.world_file) + return errors + finally: + world_set.unlock() + diff --git a/pym/portage/emaint/progress.py b/pym/portage/emaint/progress.py new file mode 100644 index 000000000..e43c2afbd --- /dev/null +++ b/pym/portage/emaint/progress.py @@ -0,0 +1,61 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import time +import signal + +import portage + + +class ProgressHandler(object): + def __init__(self): + self.reset() + + def reset(self): + self.curval = 0 + self.maxval = 0 + self.last_update = 0 + self.min_display_latency = 0.2 + + def onProgress(self, maxval, curval): + self.maxval = maxval + self.curval = curval + cur_time = time.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + + def display(self): + raise NotImplementedError(self) + + +class ProgressBar(ProgressHandler): + """Class to set up and return a Progress Bar""" + + def __init__(self, isatty, **kwargs): + self.isatty = isatty + self.kwargs = kwargs + ProgressHandler.__init__(self) + self.progressBar = None + + def start(self): + if self.isatty: + self.progressBar = portage.output.TermProgressBar(**self.kwargs) + signal.signal(signal.SIGWINCH, self.sigwinch_handler) + else: + self.onProgress = None + return self.onProgress + + def set_label(self, _label): + self.kwargs['label'] = _label + + def display(self): + self.progressBar.set(self.curval, self.maxval) + + def sigwinch_handler(self, signum, frame): + lines, self.progressBar.term_columns = \ + portage.output.get_term_size() + + def stop(self): + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + |