@@ -, +, @@ --- lib/portage/dbapi/vartree.py | 56 +++++++++++++++++-- lib/portage/util/compare_files.py | 91 +++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 lib/portage/util/compare_files.py --- a/lib/portage/dbapi/vartree.py +++ a/lib/portage/dbapi/vartree.py @@ -30,6 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', + 'portage.util.compare_files:compare_files', 'portage.util.digraph:digraph', 'portage.util.env_update:env_update', 'portage.util.install_mask:install_mask_dir,InstallMask', @@ -3418,6 +3419,8 @@ class dblink(object): os = _os_merge + real_relative_paths = {} + collision_ignore = [] for x in portage.util.shlex_split( self.settings.get("COLLISION_IGNORE", "")): @@ -3469,8 +3472,12 @@ class dblink(object): previous = current progress_shown = True - dest_path = normalize_path( - os.path.join(destroot, f.lstrip(os.path.sep))) + dest_path = normalize_path(os.path.join(destroot, f.lstrip(os.path.sep))) + + # Relative path with symbolic links resolved only in parent directories + real_relative_path = os.path.join(os.path.realpath(os.path.dirname(dest_path)), os.path.basename(dest_path))[len(destroot):] + + real_relative_paths.setdefault(real_relative_path, []).append(f.lstrip(os.path.sep)) parent = os.path.dirname(dest_path) if parent not in dirs: @@ -3556,9 +3563,27 @@ class dblink(object): break if stopmerge: collisions.append(f) + + internal_collisions = {} + for real_relative_path, files in real_relative_paths.items(): + # Detect internal collisions between non-identical files. + if len(files) >= 2: + files.sort() + for i in range(len(files) - 1): + file1 = normalize_path(os.path.join(srcroot, files[i])) + file2 = normalize_path(os.path.join(srcroot, files[i+1])) + differences = compare_files(file1, file2) + # Ignore differences in times. + for time_type in ("atime", "mtime", "ctime"): + if time_type in differences: + differences.remove(time_type) + if differences: + internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] = differences + if progress_shown: showMessage(_("100% done\n")) - return collisions, dirs_ro, symlink_collisions, plib_collisions + + return collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions def _lstat_inode_map(self, path_iter): """ @@ -4081,7 +4106,7 @@ class dblink(object): if blocker.exists(): blockers.append(blocker) - collisions, dirs_ro, symlink_collisions, plib_collisions = \ + collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions = \ self._collision_protect(srcroot, destroot, others_in_slot + blockers, filelist, linklist) @@ -4109,6 +4134,29 @@ class dblink(object): eerror(msg) return 1 + if internal_collisions: + msg = _("Package '%s' has internal collisions (between files " + "in separate directories in the installation image (${D}) " + "corresponding to merged directories in the target " + "filesystem (${ROOT})):") % self.settings.mycpv + msg = textwrap.wrap(msg, 70) + msg.append("") + for k, v in sorted(internal_collisions.items()): + msg.append("\t%s" % os.path.join(destroot, k.lstrip(os.path.sep))) + for (file1, file2), differences in sorted(v.items()): + msg.append("\t\t%s" % os.path.join(destroot, file1.lstrip(os.path.sep))) + msg.append("\t\t%s" % os.path.join(destroot, file2.lstrip(os.path.sep))) + msg.append("\t\t\tDifferences: %s" % ", ".join(differences)) + msg.append("") + self._elog("eerror", "preinst", msg) + + msg = _("Package '%s' NOT merged due to internal collisions.") % \ + self.settings.mycpv + msg += _(" If necessary, refer to your elog messages for the whole " + "content of the above message.") + eerror(textwrap.wrap(msg, 70)) + return 1 + if symlink_collisions: # Symlink collisions need to be distinguished from other types # of collisions, in order to avoid confusion (see bug #409359). --- a/lib/portage/util/compare_files.py +++ a/lib/portage/util/compare_files.py @@ -0,0 +1,91 @@ +# Copyright 2019 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ["compare_files"] + +import io +import os +import stat +import sys + +from portage import _encodings +from portage import _unicode_encode +import portage.util._xattr + +def compare_files(file1, file2): + """ + Compare metadata and contents of two files. + + @param file1: File 1 + @type file1: str + @param file2: File 2 + @type file2: str + @rtype: list + @return: List of strings specifying types of differences + """ + + file1_stat = os.lstat(file1) + file2_stat = os.lstat(file2) + + differences = [] + + if (file1_stat.st_dev, file1_stat.st_ino) == (file2_stat.st_dev, file2_stat.st_ino): + return differences + + if stat.S_IFMT(file1_stat.st_mode) != stat.S_IFMT(file2_stat.st_mode): + differences.append("type") + if stat.S_IMODE(file1_stat.st_mode) != stat.S_IMODE(file2_stat.st_mode): + differences.append("mode") + if file1_stat.st_uid != file2_stat.st_uid: + differences.append("owner") + if file1_stat.st_gid != file2_stat.st_gid: + differences.append("group") + if file1_stat.st_rdev != file2_stat.st_rdev: + differences.append("device_number") + + if sorted(portage.util._xattr.xattr.get_all(file1, nofollow=True)) != sorted(portage.util._xattr.xattr.get_all(file2, nofollow=True)): + differences.append("xattr") + + if sys.hexversion >= 0x3030000: + if file1_stat.st_atime_ns != file2_stat.st_atime_ns: + differences.append("atime") + if file1_stat.st_mtime_ns != file2_stat.st_mtime_ns: + differences.append("mtime") + if file1_stat.st_ctime_ns != file2_stat.st_ctime_ns: + differences.append("ctime") + else: + if file1_stat.st_atime != file2_stat.st_atime: + differences.append("atime") + if file1_stat.st_mtime != file2_stat.st_mtime: + differences.append("mtime") + if file1_stat.st_ctime != file2_stat.st_ctime: + differences.append("ctime") + + if file1_stat.st_size != file2_stat.st_size: + differences.append("size") + differences.append("content") + else: + if stat.S_ISLNK(file1_stat.st_mode): + file1_stream = io.BytesIO(os.readlink(_unicode_encode(file1, + encoding=_encodings["fs"], + errors="strict"))) + else: + file1_stream = open(file1, "rb") + if stat.S_ISLNK(file2_stat.st_mode): + file2_stream = io.BytesIO(os.readlink(_unicode_encode(file2, + encoding=_encodings["fs"], + errors="strict"))) + else: + file2_stream = open(file2, "rb") + while True: + file1_content = file1_stream.read(4096) + file2_content = file2_stream.read(4096) + if file1_content == b"" or file2_content == b"": + break + if file1_content != file2_content: + differences.append("content") + break + file1_stream.close() + file2_stream.close() + + return differences --