diff options
author | Zac Medico <zmedico@gentoo.org> | 2010-08-21 11:18:40 -0700 |
---|---|---|
committer | Zac Medico <zmedico@gentoo.org> | 2010-08-21 11:18:40 -0700 |
commit | 42cab46428e69cf7423053b10940a275c38c04ff (patch) | |
tree | 3d136311af222afb8d0d9bda838c1d5674b6cf4f /pym/_emerge | |
parent | 99832a79bff002a7494c4fb31701451c13a5a435 (diff) | |
download | portage-42cab46428e69cf7423053b10940a275c38c04ff.tar.gz portage-42cab46428e69cf7423053b10940a275c38c04ff.tar.bz2 portage-42cab46428e69cf7423053b10940a275c38c04ff.zip |
Bug #324191 - Add support for FEATURES=compress-build-logs. The causes
all build logs to be compressed while they are being written. Log file
names have an extension that is appropriate for the compression type.
Currently, only gzip(1) compression is supported, so build logs will
have a '.gz' extension when this feature is enabled.
Diffstat (limited to 'pym/_emerge')
-rw-r--r-- | pym/_emerge/AbstractEbuildProcess.py | 14 | ||||
-rw-r--r-- | pym/_emerge/Binpkg.py | 20 | ||||
-rw-r--r-- | pym/_emerge/BinpkgVerifier.py | 34 | ||||
-rw-r--r-- | pym/_emerge/EbuildBuild.py | 15 | ||||
-rw-r--r-- | pym/_emerge/EbuildFetcher.py | 11 | ||||
-rw-r--r-- | pym/_emerge/EbuildPhase.py | 34 | ||||
-rw-r--r-- | pym/_emerge/MetadataRegen.py | 6 | ||||
-rw-r--r-- | pym/_emerge/PackageUninstall.py | 13 | ||||
-rw-r--r-- | pym/_emerge/PollScheduler.py | 38 | ||||
-rw-r--r-- | pym/_emerge/QueueScheduler.py | 6 | ||||
-rw-r--r-- | pym/_emerge/Scheduler.py | 40 | ||||
-rw-r--r-- | pym/_emerge/SpawnProcess.py | 16 |
12 files changed, 92 insertions, 155 deletions
diff --git a/pym/_emerge/AbstractEbuildProcess.py b/pym/_emerge/AbstractEbuildProcess.py index aca254191..8c87812f5 100644 --- a/pym/_emerge/AbstractEbuildProcess.py +++ b/pym/_emerge/AbstractEbuildProcess.py @@ -183,21 +183,11 @@ class AbstractEbuildProcess(SpawnProcess): phase = self.phase for line in lines: eerror(line, phase=phase, key=self.settings.mycpv, out=out) - logfile = self.logfile - if logfile is None: - logfile = self.settings.get("PORTAGE_LOG_FILE") msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: - if not self.background: - writemsg_stdout(msg, noiselevel=-1) - if logfile is not None: - log_file = codecs.open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], - errors='backslashreplace') - log_file.write(msg) - log_file.close() + self.scheduler.output(msg, + log_path=self.settings.get("PORTAGE_LOG_FILE")) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py index 66b7bd424..2201decaf 100644 --- a/pym/_emerge/Binpkg.py +++ b/pym/_emerge/Binpkg.py @@ -27,20 +27,8 @@ class Binpkg(CompositeTask): "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): - - if not self.background: - portage.util.writemsg_level(msg, - level=level, noiselevel=noiselevel) - - log_path = self.settings.get("PORTAGE_LOG_FILE") - if log_path is not None: - f = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - try: - f.write(msg) - finally: - f.close() + self.scheduler.output(msg, level=level, noiselevel=noiselevel, + log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): @@ -146,9 +134,7 @@ class Binpkg(CompositeTask): verifier = None if self._verify: - logfile = None - if self.background: - logfile = self.settings.get("PORTAGE_LOG_FILE") + logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg) self._start_task(verifier, self._verifier_exit) diff --git a/pym/_emerge/BinpkgVerifier.py b/pym/_emerge/BinpkgVerifier.py index 83a02c76c..02a942c94 100644 --- a/pym/_emerge/BinpkgVerifier.py +++ b/pym/_emerge/BinpkgVerifier.py @@ -7,7 +7,9 @@ import sys import portage from portage import os from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode +from portage import StringIO from portage.package.ebuild.fetch import _checksum_failure_temp_file import codecs @@ -27,27 +29,10 @@ class BinpkgVerifier(AsynchronousTask): rval = os.EX_OK stdout_orig = sys.stdout stderr_orig = sys.stderr - log_file = None - if self.background and self.logfile is not None: - if sys.hexversion >= 0x3000000: - # Since we are replacing the sys.std* streams, - # we need to use the normal open() function - # so that we get the right class (otherwise our - # code that expects the 'buffer' attribute - # will break). - log_file = open(_unicode_encode(self.logfile, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], - errors='backslashreplace') - else: - # For python2, sys.std* are expected to be binary streams. - log_file = open(_unicode_encode(self.logfile, - encoding=_encodings['fs'], errors='strict'), - mode='ab') + out = portage.StringIO() try: - if log_file is not None: - sys.stdout = log_file - sys.stderr = log_file + sys.stdout = out + sys.stderr = out try: bintree.digestCheck(pkg) except portage.exception.FileNotFound: @@ -69,7 +54,7 @@ class BinpkgVerifier(AsynchronousTask): if rval == os.EX_OK: # If this was successful, discard the log here since otherwise # we'll get multiple logs for the same package. - if log_file is not None: + if self.logfile is not None: try: os.unlink(self.logfile) except OSError: @@ -83,8 +68,11 @@ class BinpkgVerifier(AsynchronousTask): finally: sys.stdout = stdout_orig sys.stderr = stderr_orig - if log_file is not None: - log_file.close() + + msg = _unicode_decode(out.getvalue(), + encoding=_encodings['content'], errors='replace') + if msg: + self.scheduler.output(msg, log_path=self.logfile) self.returncode = rval self.wait() diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index b8694df96..5c108fee4 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -218,19 +218,8 @@ class EbuildBuild(CompositeTask): if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" - - log_path = self.settings.get("PORTAGE_LOG_FILE") - if log_path is not None: - log_file = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - try: - log_file.write(msg) - finally: - log_file.close() - - if not self.background: - portage.writemsg_stdout(msg, noiselevel=-1) + self.scheduler.output(msg, + log_path=self.settings.get("PORTAGE_LOG_FILE")) packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index 7f5bc6df4..63e423771 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -158,19 +158,10 @@ class EbuildFetcher(SpawnProcess): out = portage.StringIO() for line in lines: eerror(line, phase="unpack", key=self.pkg.cpv, out=out) - logfile = self.logfile msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: - if not self.background: - writemsg_stdout(msg, noiselevel=-1) - if logfile is not None: - log_file = codecs.open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], - errors='backslashreplace') - log_file.write(msg) - log_file.close() + self.scheduler.output(msg, log_path=self.logfile) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index 7fbc66849..d2d4dce3a 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -75,24 +75,11 @@ class EbuildPhase(CompositeTask): if self.phase == "install": out = portage.StringIO() - log_path = self.settings.get("PORTAGE_LOG_FILE") - log_file = None - if log_path is not None: - log_file = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - try: - _check_build_log(self.settings, out=out) - msg = _unicode_decode(out.getvalue(), - encoding=_encodings['content'], errors='replace') - if msg: - if not self.background: - writemsg_stdout(msg, noiselevel=-1) - if log_file is not None: - log_file.write(msg) - finally: - if log_file is not None: - log_file.close() + _check_build_log(self.settings, out=out) + msg = _unicode_decode(out.getvalue(), + encoding=_encodings['content'], errors='replace') + self.scheduler.output(msg, + log_path=self.settings.get("PORTAGE_LOG_FILE")) if fail: self._die_hooks() @@ -108,15 +95,8 @@ class EbuildPhase(CompositeTask): msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: - if not self.background: - writemsg_stdout(msg, noiselevel=-1) - log_path = self.settings.get("PORTAGE_LOG_FILE") - if log_path is not None: - log_file = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - log_file.write(msg) - log_file.close() + self.scheduler.output(msg, + log_path=self.settings.get("PORTAGE_LOG_FILE")) post_phase_cmds = _post_phase_cmds.get(self.phase) if post_phase_cmds is not None: diff --git a/pym/_emerge/MetadataRegen.py b/pym/_emerge/MetadataRegen.py index f819570d9..6c6dd6b90 100644 --- a/pym/_emerge/MetadataRegen.py +++ b/pym/_emerge/MetadataRegen.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2010 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -26,10 +26,6 @@ class MetadataRegen(PollScheduler): self._max_jobs = max_jobs self._max_load = max_load - self._sched_iface = self._sched_iface_class( - register=self._register, - schedule=self._schedule_wait, - unregister=self._unregister) self._valid_pkgs = set() self._cp_set = set() diff --git a/pym/_emerge/PackageUninstall.py b/pym/_emerge/PackageUninstall.py index 6f528663a..e806bc154 100644 --- a/pym/_emerge/PackageUninstall.py +++ b/pym/_emerge/PackageUninstall.py @@ -42,15 +42,4 @@ class PackageUninstall(AsynchronousTask): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: - if not background: - portage.util.writemsg_level(msg, - level=level, noiselevel=noiselevel) - - f = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - try: - f.write(msg) - finally: - f.close() - + self.scheduler.output(msg, level=level, noiselevel=noiselevel) diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py index 2a77f869e..2c9eb2a8c 100644 --- a/pym/_emerge/PollScheduler.py +++ b/pym/_emerge/PollScheduler.py @@ -1,10 +1,13 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2010 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import gzip import logging import select import time +from portage import _encodings +from portage import _unicode_encode from portage import os from portage.util import writemsg_level @@ -16,7 +19,7 @@ from _emerge.PollSelectAdapter import PollSelectAdapter class PollScheduler(object): class _sched_iface_class(SlotObject): - __slots__ = ("register", "schedule", "unregister") + __slots__ = ("output", "register", "schedule", "unregister") def __init__(self): self._max_jobs = 1 @@ -29,6 +32,12 @@ class PollScheduler(object): self._event_handler_id = 0 self._poll_obj = create_poll_instance() self._scheduling = False + self._background = False + self.sched_iface = self._sched_iface_class( + output=self._task_output, + register=self._register, + schedule=self._schedule_wait, + unregister=self._unregister) def _schedule(self): """ @@ -228,6 +237,31 @@ class PollScheduler(object): return event_handled + def _task_output(self, msg, log_path=None, level=0, noiselevel=-1): + """ + Output msg to stdout if not self._background. If log_path + is not None then append msg to the log (appends with + compression if the filename extension of log_path + corresponds to a supported compression type). + """ + + if not self._background: + writemsg_level(msg, level=level, noiselevel=noiselevel) + + if log_path is not None: + f = open(_unicode_encode(log_path, + encoding=_encodings['fs'], errors='strict'), + mode='ab') + + if log_path.endswith('.gz'): + # NOTE: The empty filename argument prevents us from triggering + # a bug in python3 which causes GzipFile to raise AttributeError + # if fileobj.name is bytes instead of unicode. + f = gzip.GzipFile(filename='', mode='ab', fileobj=f) + + f.write(_unicode_encode(msg)) + f.close() + _can_poll_device = None def can_poll_device(): diff --git a/pym/_emerge/QueueScheduler.py b/pym/_emerge/QueueScheduler.py index 8e1837c03..0e39d6ad3 100644 --- a/pym/_emerge/QueueScheduler.py +++ b/pym/_emerge/QueueScheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2010 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.PollScheduler import PollScheduler @@ -18,10 +18,6 @@ class QueueScheduler(PollScheduler): self._max_jobs = max_jobs self._max_load = max_load - self.sched_iface = self._sched_iface_class( - register=self._register, - schedule=self._schedule_wait, - unregister=self._unregister) self._queues = [] self._schedule_listeners = [] diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index 1c732c309..aa0a40290 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -4,6 +4,7 @@ from __future__ import print_function import codecs +import gzip import logging import shutil import sys @@ -78,7 +79,8 @@ class Scheduler(PollScheduler): class _iface_class(SlotObject): __slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge", - "dblinkElog", "dblinkEmergeLog", "fetch", "register", "schedule", + "dblinkElog", "dblinkEmergeLog", "fetch", + "output", "register", "schedule", "scheduleSetup", "scheduleUnpack", "scheduleYield", "unregister") @@ -209,7 +211,8 @@ class Scheduler(PollScheduler): dblinkDisplayMerge=self._dblink_display_merge, dblinkElog=self._dblink_elog, dblinkEmergeLog=self._dblink_emerge_log, - fetch=fetch_iface, register=self._register, + fetch=fetch_iface, output=self._task_output, + register=self._register, schedule=self._schedule_wait, scheduleSetup=self._schedule_setup, scheduleUnpack=self._schedule_unpack, @@ -562,22 +565,9 @@ class Scheduler(PollScheduler): installed = type_name == "installed" return self._pkg(cpv, type_name, root_config, installed=installed) - def _append_to_log_path(self, log_path, msg): - - f = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], - errors='backslashreplace') - try: - f.write(_unicode_decode(msg, - encoding=_encodings['content'], errors='replace')) - finally: - f.close() - def _dblink_elog(self, pkg_dblink, phase, func, msgs): log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE") - background = self._background out = StringIO() for msg in msgs: @@ -585,11 +575,7 @@ class Scheduler(PollScheduler): out_str = out.getvalue() - if not background: - portage.util.writemsg_stdout(out_str, noiselevel=-1) - - if log_path is not None: - self._append_to_log_path(log_path, out_str) + self._task_output(out_str, log_path=log_path) def _dblink_emerge_log(self, msg): self._logger.log(msg) @@ -603,10 +589,7 @@ class Scheduler(PollScheduler): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: - if not background: - portage.util.writemsg_level(msg, - level=level, noiselevel=noiselevel) - self._append_to_log_path(log_path, msg) + self._task_output(msg, log_path=log_path) def _dblink_ebuild_phase(self, pkg_dblink, pkg_dbapi, ebuild_path, phase): @@ -1100,11 +1083,14 @@ class Scheduler(PollScheduler): log_path = self._locate_failure_log(failed_pkg) if log_path is not None: try: - log_file = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') + log_file = open(_unicode_encode(log_path, + encoding=_encodings['fs'], errors='strict'), mode='rb') except IOError: pass + else: + if log_path.endswith('.gz'): + log_file = gzip.GzipFile(filename='', + mode='rb', fileobj=log_file) if log_file is not None: try: diff --git a/pym/_emerge/SpawnProcess.py b/pym/_emerge/SpawnProcess.py index 6e0586815..aeb206a06 100644 --- a/pym/_emerge/SpawnProcess.py +++ b/pym/_emerge/SpawnProcess.py @@ -6,10 +6,13 @@ from _emerge.PollConstants import PollConstants import sys from portage.cache.mappings import slot_dict_class import portage +from portage import _encodings +from portage import _unicode_encode from portage import os import fcntl import errno import array +import gzip class SpawnProcess(SubProcess): @@ -77,7 +80,12 @@ class SpawnProcess(SubProcess): fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd - files.log = open(logfile, mode='ab') + files.log = open(_unicode_encode(logfile, + encoding=_encodings['fs'], errors='strict'), mode='ab') + if logfile.endswith('.gz'): + files.log = gzip.GzipFile(filename='', mode='ab', + fileobj=files.log) + portage.util.apply_secpass_permissions(logfile, uid=portage.portage_uid, gid=portage.portage_gid, mode=0o660) @@ -188,7 +196,11 @@ class SpawnProcess(SubProcess): fcntl.fcntl(files.stdout.fileno(), fcntl.F_GETFL) ^ os.O_NONBLOCK) - buf.tofile(files.log) + try: + buf.tofile(files.log) + except TypeError: + # array.tofile() doesn't work with GzipFile + files.log.write(buf.tostring()) files.log.flush() else: self._unregister() |