Go to:
Gentoo Home
Documentation
Forums
Lists
Bugs
Planet
Store
Wiki
Get Gentoo!
Gentoo's Bugzilla – Attachment 586520 Details for
Bug 690484
Detection of internal collisions (between files in separate directories in the installation image (${D}) corresponding to merged directories in the target filesystem (${ROOT}))
Home
|
New
–
[Ex]
|
Browse
|
Search
|
Privacy Policy
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch
portage-internal_collisions_detection.patch (text/plain), 9.10 KB, created by
Arfrever Frehtes Taifersar Arahesis
on 2019-08-11 00:59:18 UTC
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Arfrever Frehtes Taifersar Arahesis
Created:
2019-08-11 00:59:18 UTC
Size:
9.10 KB
patch
obsolete
>From e71d9ada6ed417df6829a544d5063c52ff3ffaf7 Mon Sep 17 00:00:00 2001 >From: Arfrever Frehtes Taifersar Arahesis <Arfrever@Apache.Org> >Date: Wed, 7 Aug 2019 19:06:11 +0200 >Subject: [PATCH] dblink._collision_protect: Detect internal collisions. > >Implement detection of internal collisions (between files of the same package, >located in separate directories in the installation image (${D}) corresponding >to merged directories in the target filesystem (${ROOT})). > >This provides protection against overwriting some files when performing merging >of files from ${D} to ${ROOT} in some filesystem layouts (such as /-merged layout >or /usr-merged layout). > >Internal collisions between identical files are silently ignored. > >Bug: https://bugs.gentoo.org/690484 >Signed-off-by: Arfrever Frehtes Taifersar Arahesis <Arfrever@Apache.Org> >--- > lib/portage/dbapi/vartree.py | 53 +++++++++++++-- > lib/portage/util/_compare_files.py | 101 +++++++++++++++++++++++++++++ > 2 files changed, 150 insertions(+), 4 deletions(-) > create mode 100644 lib/portage/util/_compare_files.py > >diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py >index e2fce7736..30f7b0159 100644 >--- a/lib/portage/dbapi/vartree.py >+++ b/lib/portage/dbapi/vartree.py >@@ -30,6 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(), > 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ > 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ > 'grabdict,normalize_path,new_protect_filename', >+ 'portage.util._compare_files:compare_files', > 'portage.util.digraph:digraph', > 'portage.util.env_update:env_update', > 'portage.util.install_mask:install_mask_dir,InstallMask', >@@ -3418,6 +3419,8 @@ class dblink(object): > > os = _os_merge > >+ real_relative_paths = {} >+ > collision_ignore = [] > for x in portage.util.shlex_split( > self.settings.get("COLLISION_IGNORE", "")): >@@ -3469,8 +3472,12 @@ class dblink(object): > previous = current > progress_shown = True > >- dest_path = normalize_path( >- os.path.join(destroot, f.lstrip(os.path.sep))) >+ dest_path = normalize_path(os.path.join(destroot, f.lstrip(os.path.sep))) >+ >+ # Relative path with symbolic links resolved only in parent directories >+ real_relative_path = os.path.join(os.path.realpath(os.path.dirname(dest_path)), os.path.basename(dest_path))[len(destroot):] >+ >+ real_relative_paths.setdefault(real_relative_path, []).append(f.lstrip(os.path.sep)) > > parent = os.path.dirname(dest_path) > if parent not in dirs: >@@ -3556,9 +3563,24 @@ class dblink(object): > break > if stopmerge: > collisions.append(f) >+ >+ internal_collisions = {} >+ for real_relative_path, files in real_relative_paths.items(): >+ # Detect internal collisions between non-identical files. >+ if len(files) >= 2: >+ files.sort() >+ for i in range(len(files) - 1): >+ file1 = normalize_path(os.path.join(srcroot, files[i])) >+ file2 = normalize_path(os.path.join(srcroot, files[i+1])) >+ # Compare files, ignoring differences in times. >+ differences = compare_files(file1, file2, skipped_types=("atime", "mtime", "ctime")) >+ if differences: >+ internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] = differences >+ > if progress_shown: > showMessage(_("100% done\n")) >- return collisions, dirs_ro, symlink_collisions, plib_collisions >+ >+ return collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions > > def _lstat_inode_map(self, path_iter): > """ >@@ -4081,7 +4103,7 @@ class dblink(object): > if blocker.exists(): > blockers.append(blocker) > >- collisions, dirs_ro, symlink_collisions, plib_collisions = \ >+ collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions = \ > self._collision_protect(srcroot, destroot, > others_in_slot + blockers, filelist, linklist) > >@@ -4109,6 +4131,29 @@ class dblink(object): > eerror(msg) > return 1 > >+ if internal_collisions: >+ msg = _("Package '%s' has internal collisions between non-identical files " >+ "(located in separate directories in the installation image (${D}) " >+ "corresponding to merged directories in the target " >+ "filesystem (${ROOT})):") % self.settings.mycpv >+ msg = textwrap.wrap(msg, 70) >+ msg.append("") >+ for k, v in sorted(internal_collisions.items()): >+ msg.append("\t%s" % os.path.join(destroot, k.lstrip(os.path.sep))) >+ for (file1, file2), differences in sorted(v.items()): >+ msg.append("\t\t%s" % os.path.join(destroot, file1.lstrip(os.path.sep))) >+ msg.append("\t\t%s" % os.path.join(destroot, file2.lstrip(os.path.sep))) >+ msg.append("\t\t\tDifferences: %s" % ", ".join(differences)) >+ msg.append("") >+ self._elog("eerror", "preinst", msg) >+ >+ msg = _("Package '%s' NOT merged due to internal collisions " >+ "between non-identical files.") % self.settings.mycpv >+ msg += _(" If necessary, refer to your elog messages for the whole " >+ "content of the above message.") >+ eerror(textwrap.wrap(msg, 70)) >+ return 1 >+ > if symlink_collisions: > # Symlink collisions need to be distinguished from other types > # of collisions, in order to avoid confusion (see bug #409359). >diff --git a/lib/portage/util/_compare_files.py b/lib/portage/util/_compare_files.py >new file mode 100644 >index 000000000..73effe7da >--- /dev/null >+++ b/lib/portage/util/_compare_files.py >@@ -0,0 +1,101 @@ >+# Copyright 2019 Gentoo Authors >+# Distributed under the terms of the GNU General Public License v2 >+ >+__all__ = ["compare_files"] >+ >+import io >+import os >+import stat >+import sys >+ >+from portage import _encodings >+from portage import _unicode_encode >+from portage.util._xattr import xattr >+ >+def compare_files(file1, file2, skipped_types=()): >+ """ >+ Compare metadata and contents of two files. >+ >+ @param file1: File 1 >+ @type file1: str >+ @param file2: File 2 >+ @type file2: str >+ @param skipped_types: Tuple of strings specifying types of properties excluded from comparison. >+ Supported strings: type, mode, owner, group, device_number, xattr, atime, mtime, ctime, size, content >+ @type skipped_types: tuple of str >+ @rtype: tuple of str >+ @return: Tuple of strings specifying types of properties different between compared files >+ """ >+ >+ file1_stat = os.lstat(_unicode_encode(file1, encoding=_encodings["fs"], errors="strict")) >+ file2_stat = os.lstat(_unicode_encode(file2, encoding=_encodings["fs"], errors="strict")) >+ >+ differences = [] >+ >+ if (file1_stat.st_dev, file1_stat.st_ino) == (file2_stat.st_dev, file2_stat.st_ino): >+ return () >+ >+ if "type" not in skipped_types and stat.S_IFMT(file1_stat.st_mode) != stat.S_IFMT(file2_stat.st_mode): >+ differences.append("type") >+ if "mode" not in skipped_types and stat.S_IMODE(file1_stat.st_mode) != stat.S_IMODE(file2_stat.st_mode): >+ differences.append("mode") >+ if "owner" not in skipped_types and file1_stat.st_uid != file2_stat.st_uid: >+ differences.append("owner") >+ if "group" not in skipped_types and file1_stat.st_gid != file2_stat.st_gid: >+ differences.append("group") >+ if "device_number" not in skipped_types and file1_stat.st_rdev != file2_stat.st_rdev: >+ differences.append("device_number") >+ >+ if "xattr" not in skipped_types and sorted(xattr.get_all(file1, nofollow=True)) != sorted(xattr.get_all(file2, nofollow=True)): >+ differences.append("xattr") >+ >+ if sys.hexversion >= 0x3030000: >+ if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns: >+ differences.append("atime") >+ if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns: >+ differences.append("mtime") >+ if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns: >+ differences.append("ctime") >+ else: >+ if "atime" not in skipped_types and file1_stat.st_atime != file2_stat.st_atime: >+ differences.append("atime") >+ if "mtime" not in skipped_types and file1_stat.st_mtime != file2_stat.st_mtime: >+ differences.append("mtime") >+ if "ctime" not in skipped_types and file1_stat.st_ctime != file2_stat.st_ctime: >+ differences.append("ctime") >+ >+ if file1_stat.st_size != file2_stat.st_size: >+ if "size" not in skipped_types: >+ differences.append("size") >+ if "content" not in skipped_types: >+ differences.append("content") >+ else: >+ if "content" not in skipped_types: >+ if stat.S_ISLNK(file1_stat.st_mode): >+ file1_stream = io.BytesIO(os.readlink(_unicode_encode(file1, >+ encoding=_encodings["fs"], >+ errors="strict"))) >+ else: >+ file1_stream = open(_unicode_encode(file1, >+ encoding=_encodings["fs"], >+ errors="strict"), "rb") >+ if stat.S_ISLNK(file2_stat.st_mode): >+ file2_stream = io.BytesIO(os.readlink(_unicode_encode(file2, >+ encoding=_encodings["fs"], >+ errors="strict"))) >+ else: >+ file2_stream = open(_unicode_encode(file2, >+ encoding=_encodings["fs"], >+ errors="strict"), "rb") >+ while True: >+ file1_content = file1_stream.read(4096) >+ file2_content = file2_stream.read(4096) >+ if file1_content == b"" or file2_content == b"": >+ break >+ if file1_content != file2_content: >+ differences.append("content") >+ break >+ file1_stream.close() >+ file2_stream.close() >+ >+ return tuple(differences) >-- >2.22.0 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
Attachments on
bug 690484
:
584146
|
584148
|
584152
|
586062
|
586490
| 586520