diff options
| -rw-r--r-- | cnf/make.globals | 2 | ||||
| -rw-r--r-- | man/make.conf.5 | 4 | ||||
| -rw-r--r-- | pym/portage/const.py | 2 | ||||
| -rw-r--r-- | pym/portage/package/ebuild/doebuild.py | 47 | ||||
| -rw-r--r-- | pym/portage/tests/lafilefixer/__init__.py | 0 | ||||
| -rw-r--r-- | pym/portage/tests/lafilefixer/__test__ | 0 | ||||
| -rw-r--r-- | pym/portage/tests/lafilefixer/test_lafilefixer.py | 134 | ||||
| -rw-r--r-- | pym/portage/util/lafilefixer.py | 176 |
8 files changed, 363 insertions, 2 deletions
diff --git a/cnf/make.globals b/cnf/make.globals index 78088e006..bb11327c5 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -44,7 +44,7 @@ FETCHCOMMAND_RSYNC="rsync -avP \"\${URI}\" \"\${DISTDIR}/\${FILE}\"" RESUMECOMMAND_RSYNC="rsync -avP \"\${URI}\" \"\${DISTDIR}/\${FILE}\"" # Default user options -FEATURES="assume-digests distlocks fixpackages news parallel-fetch protect-owned +FEATURES="assume-digests distlocks fixpackages lafilefixing news parallel-fetch protect-owned sandbox sfperms strict unknown-features-warn unmerge-logs unmerge-orphans userfetch" diff --git a/man/make.conf.5 b/man/make.conf.5 index 71840f95a..4a5db0dae 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -272,6 +272,10 @@ prior to each build. Due to lack of proper cleanup, this feature can interfere with normal emerge operation and therefore it should not be left enabled for more than a short period of time. .TP +.B lafilefixing +Prevents modifiying .la files to not include other .la files and some other +fixes (order of flags, duplicated entries, ...) +.TP .B lmirror When \fImirror\fR is enabled in \fBFEATURES\fR, fetch files even when \fImirror\fR is also in the \fBebuild\fR(5) \fBRESTRICT\fR variable. diff --git a/pym/portage/const.py b/pym/portage/const.py index dc8155102..6bb465629 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -86,7 +86,7 @@ SUPPORTED_FEATURES = frozenset([ "assume-digests", "buildpkg", "buildsyspkg", "ccache", "collision-protect", "digest", "distcc", "distlocks", "fakeroot", "fail-clean", "fixpackages", "getbinpkg", - "installsources", "keeptemp", "keepwork", "lmirror", + "installsources", "keeptemp", "keepwork", "lafilefixing", "lmirror", "metadata-transfer", "mirror", "multilib-strict", "news", "noauto", "noclean", "nodoc", "noinfo", "noman", "nostrip" "notitles", "parallel-fetch", "parse-eapi-ebuild-head", diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index b6eedd416..d3abeeae0 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -52,6 +52,7 @@ from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ writemsg, writemsg_stdout, write_atomic from portage.util._pty import _create_pty_or_pipe +from portage.util.lafilefixer import rewrite_lafile from portage.versions import _pkgsplit def doebuild_environment(myebuild, mydo, myroot, mysettings, @@ -1556,6 +1557,8 @@ def _post_src_install_uid_fix(mysettings, out=None): (_shell_quote(mysettings["D"]),)) destdir = mysettings["D"] + logfile = mysettings.get("PORTAGE_LOG_FILE") + qa_out = StringIO() unicode_errors = [] while True: @@ -1563,6 +1566,7 @@ def _post_src_install_uid_fix(mysettings, out=None): unicode_error = False size = 0 counted_inodes = set() + lafilefixing_announced = False for parent, dirs, files in os.walk(destdir): try: @@ -1602,6 +1606,35 @@ def _post_src_install_uid_fix(mysettings, out=None): else: fpath = os.path.join(parent, fname) + if "lafilefixing" in mysettings["FEATURES"] and \ + fname.endswith(".la") and os.path.isfile(fpath): + f = codecs.open(_unicode_encode(os.path.realpath(fpath), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') + contents = f.read() + f.close() + try: + needs_update, new_contents = rewrite_lafile(contents) + except portage.exception.InvalidData as e: + needs_update = False + if not lafilefixing_announced: + lafilefixing_announced = True + writemsg("Fixing .la files\n") + msg = " %s is not a valid libtool archive, skipping\n" % fpath[len(destdir):] + qa_msg = "QA Notice: invalid .la file found: %s, %s" % (fpath[len(destdir):], e) + writemsg(msg) + eqawarn(qa_msg, key=mysettings.mycpv, out=qa_out) + if needs_update: + if not lafilefixing_announced: + lafilefixing_announced = True + writemsg("Fixing .la files\n") + writemsg(" %s\n" % fpath[len(destdir):]) + f = codecs.open(_unicode_encode(fpath, + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['content'], errors='replace') + f.write(new_contents) + f.close() + mystat = os.lstat(fpath) if stat.S_ISREG(mystat.st_mode) and \ mystat.st_ino not in counted_inodes: @@ -1632,6 +1665,20 @@ def _post_src_install_uid_fix(mysettings, out=None): for l in _merge_unicode_error(unicode_errors): eerror(l, phase='install', key=mysettings.mycpv, out=out) + msg = _unicode_decode(qa_out.getvalue(), + encoding=_encodings['content'], errors='replace') + if msg and logfile: + try: + f = codecs.open(_unicode_encode(logfile, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], + errors='replace') + except EnvironmentError: + pass + else: + f.write(msg) + f.close() + build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], 'build-info') diff --git a/pym/portage/tests/lafilefixer/__init__.py b/pym/portage/tests/lafilefixer/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pym/portage/tests/lafilefixer/__init__.py diff --git a/pym/portage/tests/lafilefixer/__test__ b/pym/portage/tests/lafilefixer/__test__ new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pym/portage/tests/lafilefixer/__test__ diff --git a/pym/portage/tests/lafilefixer/test_lafilefixer.py b/pym/portage/tests/lafilefixer/test_lafilefixer.py new file mode 100644 index 000000000..5e4e840f3 --- /dev/null +++ b/pym/portage/tests/lafilefixer/test_lafilefixer.py @@ -0,0 +1,134 @@ +# test_lafilefixer.py -- Portage Unit Testing Functionality +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.exception import InvalidData + +class test_lafilefixer(TestCase): + + def get_test_cases_clean(self): + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -lm'\n" + \ + "current=6\n" + \ + "age=0\n" + \ + "revision=2\n" + \ + "installed=yes\n" + \ + "dlopen=''\n" + \ + "dlpreopen=''\n" + \ + "libdir='/usr/lib64'\n" + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -lm'\n" + \ + "current=6\n" + \ + "age=0\n" + \ + "revision=2\n" + \ + "installed=yes\n" + \ + "dlopen=''\n" + \ + "dlpreopen=''\n" + \ + "libdir='/usr/lib64'\n" + yield "dependency_libs=' liba.la /usr/lib64/bar.la -lc'\n" + + def get_test_cases_update(self): + #.la -> -l* + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc'\n", \ + "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -L/usr/lib64 -la -lb -lc'\n" + #move stuff into inherited_linker_flags + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la -pthread /usr/lib64/libb.la -lc'\n" + \ + "inherited_linker_flags=''\n", \ + "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -L/usr/lib64 -la -lb -lc'\n" + \ + "inherited_linker_flags=' -pthread'\n" + #reorder + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la -R/usr/lib64 /usr/lib64/libb.la -lc'\n", \ + "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -R/usr/lib64 -L/usr/lib64 -la -lb -lc'\n" + #remove duplicates from dependency_libs (the original version didn't do it for inherited_linker_flags) + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libc.la -pthread -mt" + \ + " -L/usr/lib -R/usr/lib64 -lc /usr/lib64/libb.la -lc'\n" +\ + "inherited_linker_flags=' -pthread -pthread'\n", \ + "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' -R/usr/lib64 -L/usr/lib64 -L/usr/lib -la -lc -lb'\n" +\ + "inherited_linker_flags=' -pthread -pthread -mt'\n" + #-L rewriting + yield "dependency_libs=' -L/usr/X11R6/lib'\n", \ + "dependency_libs=' -L/usr/lib'\n" + yield "dependency_libs=' -L/usr/local/lib'\n", \ + "dependency_libs=' -L/usr/lib'\n" + yield "dependency_libs=' -L/usr/lib64/pkgconfig/../..'\n", \ + "dependency_libs=' -L/usr'\n" + yield "dependency_libs=' -L/usr/lib/pkgconfig/..'\n", \ + "dependency_libs=' -L/usr/lib'\n" + yield "dependency_libs=' -L/usr/lib/pkgconfig/../.. -L/usr/lib/pkgconfig/..'\n", \ + "dependency_libs=' -L/usr -L/usr/lib'\n" + + def get_test_cases_broken(self): + yield "" + #no dependency_libs + yield "dlname='libfoo.so.1'\n" + \ + "current=6\n" + \ + "age=0\n" + \ + "revision=2\n" + #borken dependency_libs + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc' \n" + #borken dependency_libs + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc\n" + #crap in dependency_libs + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" + #dependency_libs twice + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" +\ + "dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" + #inherited_linker_flags twice + yield "dlname='libfoo.so.1'\n" + \ + "library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ + "old_library='libpdf.a'\n" + \ + "inherited_linker_flags=''\n" +\ + "inherited_linker_flags=''\n" + + def testlafilefixer(self): + from portage.util.lafilefixer import _parse_lafile_contents, rewrite_lafile + + for clean_contents in self.get_test_cases_clean(): + self.assertEqual(rewrite_lafile(clean_contents), (False, None)) + + for original_contents, fixed_contents in self.get_test_cases_update(): + self.assertEqual(rewrite_lafile(original_contents), (True, fixed_contents)) + + for broken_contents in self.get_test_cases_broken(): + self.assertRaises(InvalidData, rewrite_lafile, broken_contents) diff --git a/pym/portage/util/lafilefixer.py b/pym/portage/util/lafilefixer.py new file mode 100644 index 000000000..90cc43541 --- /dev/null +++ b/pym/portage/util/lafilefixer.py @@ -0,0 +1,176 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import re + +from portage import os +from portage.exception import InvalidData + +######################################################### +# This an re-implementaion of dev-util/lafilefixer-0.5. +# rewrite_lafile() takes the contents of an lafile as a string +# It then parses the dependency_libs and inherited_linker_flags +# entries. +# We insist on dependency_libs being present. inherited_linker_flags +# is optional. +# There are strict rules about the syntax imposed by libtool's libltdl. +# See 'parse_dotla_file' and 'trim' functions in libltdl/ltdl.c. +# Note that duplicated entries of dependency_libs and inherited_linker_flags +# are ignored by libtool (last one wins), but we treat it as error (like +# lafilefixer does). +# What it does: +# * Replaces all .la files with absolut paths in dependency_libs with +# corresponding -l* and -L* entries +# (/usr/lib64/libfoo.la -> -L/usr/lib64 -lfoo) +# * Moves various flags (see flag_re below) to inherited_linker_flags, +# if such an entry was present. +# * Reorders dependency_libs such that all -R* entries precede -L* entries +# and these precede all other entries. +# * Remove duplicated entries from dependency_libs +# * Takes care that no entry to inherited_linker_flags is added that is +# already there. +######################################################### + +#These regexes are used to parse the interesting entries in the la file +dep_libs_re = re.compile("dependency_libs='(?P<value>[^']*)'$") +inh_link_flags_re = re.compile("inherited_linker_flags='(?P<value>[^']*)'$") + +#regexes for replacing stuff in -L entries. +#replace 'X11R6/lib' and 'local/lib' with 'lib', no idea what's this about. +X11_local_sub = re.compile("X11R6/lib|local/lib") +#get rid of the '..' +pkgconfig_sub1 = re.compile("usr/lib[^/]*/pkgconfig/\.\./\.\.") +pkgconfig_sub2 = re.compile("(?P<usrlib>usr/lib[^/]*)/pkgconfig/\.\.") + +#detect flags that should go into inherited_linker_flags instead of dependency_libs +flag_re = re.compile("-mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe|-threads") + +def _parse_lafile_contents(contents): + """ + Parses 'dependency_libs' and 'inherited_linker_flags' lines. + """ + + dep_libs = None + inh_link_flags = None + + for line in contents.split("\n"): + m = dep_libs_re.match(line) + if m: + if dep_libs is not None: + raise InvalidData("duplicated dependency_libs entry") + dep_libs = m.group("value") + continue + + m = inh_link_flags_re.match(line) + if m: + if inh_link_flags is not None: + raise InvalidData("duplicated inherited_linker_flags entry") + inh_link_flags = m.group("value") + continue + + return dep_libs, inh_link_flags + +def rewrite_lafile(contents): + """ + Given the contents of an .la file, parse and fix it. + Returns a bool, string tuple. + The bool indicates if the file needs fixing. + The string contains the fixed contents if the file needs fixing. + Raises 'InvalidData' if the .la file is invalid. + """ + #Parse the 'dependency_libs' and 'inherited_linker_flags' lines. + dep_libs, inh_link_flags = \ + _parse_lafile_contents(contents) + + if dep_libs is None: + raise InvalidData("missing or invalid dependency_libs") + + new_dep_libs = [] + new_inh_link_flags = [] + librpath = [] + libladir = [] + + if inh_link_flags is not None: + new_inh_link_flags = inh_link_flags.split() + + #Check entries in 'dependency_libs'. + for dep_libs_entry in dep_libs.split(): + if dep_libs_entry.startswith("-l"): + #-lfoo, keep it + if dep_libs_entry not in new_dep_libs: + new_dep_libs.append(dep_libs_entry) + + elif dep_libs_entry.endswith(".la"): + #Two cases: + #1) /usr/lib64/libfoo.la, turn it into -lfoo and append -L/usr/lib64 to libladir + #2) libfoo.la, keep it + dir, file = os.path.split(dep_libs_entry) + + if not dir or not file.startswith("lib"): + if dep_libs_entry not in new_dep_libs: + new_dep_libs.append(dep_libs_entry) + else: + #/usr/lib64/libfoo.la -> -lfoo + lib = "-l" + file[3:-3] + if lib not in new_dep_libs: + new_dep_libs.append(lib) + #/usr/lib64/libfoo.la -> -L/usr/lib64 + ladir = "-L" + dir + if ladir not in libladir: + libladir.append(ladir) + + elif dep_libs_entry.startswith("-L"): + #Do some replacement magic and store them in 'libladir'. + #This allows us to place all -L entries at the beginning + #of 'dependency_libs'. + ladir = dep_libs_entry + + ladir = X11_local_sub.sub("lib", ladir) + ladir = pkgconfig_sub1.sub("usr", ladir) + ladir = pkgconfig_sub2.sub("\g<usrlib>", ladir) + + if ladir not in libladir: + libladir.append(ladir) + + elif dep_libs_entry.startswith("-R"): + if dep_libs_entry not in librpath: + librpath.append(dep_libs_entry) + + elif flag_re.match(dep_libs_entry): + #All this stuff goes into inh_link_flags, if the la file has such an entry. + #If it doesn't, they stay in 'dependency_libs'. + if inh_link_flags is not None: + if dep_libs_entry not in new_inh_link_flags: + new_inh_link_flags.append(dep_libs_entry) + else: + if dep_libs_entry not in new_dep_libs: + new_dep_libs.append(dep_libs_entry) + + else: + raise InvalidData("Error: Unexpected entry '%s' in 'dependency_libs'" \ + % dep_libs_entry) + + #What should 'dependency_libs' and 'inherited_linker_flags' look like? + expected_dep_libs = "" + for x in (librpath, libladir, new_dep_libs): + if x: + expected_dep_libs += " " + " ".join(x) + + expected_inh_link_flags = "" + if new_inh_link_flags: + expected_inh_link_flags += " " + " ".join(new_inh_link_flags) + + #Don't touch the file if we don't need to, otherwise put the expected values into + #'contents' and write it into the la file. + if dep_libs == expected_dep_libs and \ + (inh_link_flags is None or expected_inh_link_flags == inh_link_flags): + return False, None + + contents = re.sub("dependency_libs='" + dep_libs + "'", \ + "dependency_libs='" + expected_dep_libs + "'" , contents) + + if inh_link_flags is not None: + contents = re.sub("inherited_linker_flags='" + inh_link_flags + "'", \ + "inherited_linker_flags='" + expected_inh_link_flags + "'" , contents) + + return True, contents |