Attachment #586520 for bug #690484

Lines 30-35 portage.proxy.lazyimport.lazyimport(globals(), Link Here

(-)a/lib/portage/dbapi/vartree.py (-4 / +49 lines)
30	'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \	30	'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \
31	'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \	31	'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \
32	'grabdict,normalize_path,new_protect_filename',	32	'grabdict,normalize_path,new_protect_filename',
		33	'portage.util._compare_files:compare_files',
33	'portage.util.digraph:digraph',	34	'portage.util.digraph:digraph',
34	'portage.util.env_update:env_update',	35	'portage.util.env_update:env_update',
35	'portage.util.install_mask:install_mask_dir,InstallMask',	36	'portage.util.install_mask:install_mask_dir,InstallMask',
Lines 3418-3423 class dblink(object): Link Here
3418		3419
3419	os = _os_merge	3420	os = _os_merge
3420		3421
		3422	real_relative_paths = {}
		3423
3421	collision_ignore = []	3424	collision_ignore = []
3422	for x in portage.util.shlex_split(	3425	for x in portage.util.shlex_split(
3423	self.settings.get("COLLISION_IGNORE", "")):	3426	self.settings.get("COLLISION_IGNORE", "")):
Lines 3469-3476 class dblink(object): Link Here
3469	previous = current	3472	previous = current
3470	progress_shown = True	3473	progress_shown = True
3471		3474
3472	dest_path = normalize_path(	3475	dest_path = normalize_path(os.path.join(destroot, f.lstrip(os.path.sep)))
3473	os.path.join(destroot, f.lstrip(os.path.sep)))	3476
		3477	# Relative path with symbolic links resolved only in parent directories
		3478	real_relative_path = os.path.join(os.path.realpath(os.path.dirname(dest_path)), os.path.basename(dest_path))[len(destroot):]
		3479
		3480	real_relative_paths.setdefault(real_relative_path, []).append(f.lstrip(os.path.sep))
3474		3481
3475	parent = os.path.dirname(dest_path)	3482	parent = os.path.dirname(dest_path)
3476	if parent not in dirs:	3483	if parent not in dirs:
Lines 3556-3564 class dblink(object): Link Here
3556	break	3563	break
3557	if stopmerge:	3564	if stopmerge:
3558	collisions.append(f)	3565	collisions.append(f)
		3566
		3567	internal_collisions = {}
		3568	for real_relative_path, files in real_relative_paths.items():
		3569	# Detect internal collisions between non-identical files.
		3570	if len(files) >= 2:
		3571	files.sort()
		3572	for i in range(len(files) - 1):
		3573	file1 = normalize_path(os.path.join(srcroot, files[i]))
		3574	file2 = normalize_path(os.path.join(srcroot, files[i+1]))
		3575	# Compare files, ignoring differences in times.
		3576	differences = compare_files(file1, file2, skipped_types=("atime", "mtime", "ctime"))
		3577	if differences:
		3578	internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] = differences
		3579
3559	if progress_shown:	3580	if progress_shown:
3560	showMessage(_("100% done\n"))	3581	showMessage(_("100% done\n"))
3561	return collisions, dirs_ro, symlink_collisions, plib_collisions	3582
		3583	return collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions
3562		3584
3563	def _lstat_inode_map(self, path_iter):	3585	def _lstat_inode_map(self, path_iter):
3564	"""	3586	"""
Lines 4081-4087 class dblink(object): Link Here
4081	if blocker.exists():	4103	if blocker.exists():
4082	blockers.append(blocker)	4104	blockers.append(blocker)
4083		4105
4084	collisions, dirs_ro, symlink_collisions, plib_collisions = \	4106	collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions = \
4085	self._collision_protect(srcroot, destroot,	4107	self._collision_protect(srcroot, destroot,
4086	others_in_slot + blockers, filelist, linklist)	4108	others_in_slot + blockers, filelist, linklist)
4087		4109
Lines 4109-4114 class dblink(object): Link Here
4109	eerror(msg)	4131	eerror(msg)
4110	return 1	4132	return 1
4111		4133
		4134	if internal_collisions:
		4135	msg = _("Package '%s' has internal collisions between non-identical files "
		4136	"(located in separate directories in the installation image (${D}) "
		4137	"corresponding to merged directories in the target "
		4138	"filesystem (${ROOT})):") % self.settings.mycpv
		4139	msg = textwrap.wrap(msg, 70)
		4140	msg.append("")
		4141	for k, v in sorted(internal_collisions.items()):
		4142	msg.append("\t%s" % os.path.join(destroot, k.lstrip(os.path.sep)))
		4143	for (file1, file2), differences in sorted(v.items()):
		4144	msg.append("\t\t%s" % os.path.join(destroot, file1.lstrip(os.path.sep)))
		4145	msg.append("\t\t%s" % os.path.join(destroot, file2.lstrip(os.path.sep)))
		4146	msg.append("\t\t\tDifferences: %s" % ", ".join(differences))
		4147	msg.append("")
		4148	self._elog("eerror", "preinst", msg)
		4149
		4150	msg = _("Package '%s' NOT merged due to internal collisions "
		4151	"between non-identical files.") % self.settings.mycpv
		4152	msg += _(" If necessary, refer to your elog messages for the whole "
		4153	"content of the above message.")
		4154	eerror(textwrap.wrap(msg, 70))
		4155	return 1
		4156
4112	if symlink_collisions:	4157	if symlink_collisions:
4113	# Symlink collisions need to be distinguished from other types	4158	# Symlink collisions need to be distinguished from other types
4114	# of collisions, in order to avoid confusion (see bug #409359).	4159	# of collisions, in order to avoid confusion (see bug #409359).




# Copyright 2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

__all__ = ["compare_files"]

import io
import os
import stat
import sys

from portage import _encodings
from portage import _unicode_encode
from portage.util._xattr import xattr

def compare_files(file1, file2, skipped_types=()):
	"""
	Compare metadata and contents of two files.

	@param file1: File 1
	@type file1: str
	@param file2: File 2
	@type file2: str
	@param skipped_types: Tuple of strings specifying types of properties excluded from comparison.
		Supported strings: type, mode, owner, group, device_number, xattr, atime, mtime, ctime, size, content
	@type skipped_types: tuple of str
	@rtype: tuple of str
	@return: Tuple of strings specifying types of properties different between compared files
	"""

	file1_stat = os.lstat(_unicode_encode(file1, encoding=_encodings["fs"], errors="strict"))
	file2_stat = os.lstat(_unicode_encode(file2, encoding=_encodings["fs"], errors="strict"))

	differences = []

	if (file1_stat.st_dev, file1_stat.st_ino) == (file2_stat.st_dev, file2_stat.st_ino):
		return ()

	if "type" not in skipped_types and stat.S_IFMT(file1_stat.st_mode) != stat.S_IFMT(file2_stat.st_mode):
		differences.append("type")
	if "mode" not in skipped_types and stat.S_IMODE(file1_stat.st_mode) != stat.S_IMODE(file2_stat.st_mode):
		differences.append("mode")
	if "owner" not in skipped_types and file1_stat.st_uid != file2_stat.st_uid:
		differences.append("owner")
	if "group" not in skipped_types and file1_stat.st_gid != file2_stat.st_gid:
		differences.append("group")
	if "device_number" not in skipped_types and file1_stat.st_rdev != file2_stat.st_rdev:
		differences.append("device_number")

	if "xattr" not in skipped_types and sorted(xattr.get_all(file1, nofollow=True)) != sorted(xattr.get_all(file2, nofollow=True)):
		differences.append("xattr")

	if sys.hexversion >= 0x3030000:
		if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns:
			differences.append("atime")
		if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns:
			differences.append("mtime")
		if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns:
			differences.append("ctime")
	else:
		if "atime" not in skipped_types and file1_stat.st_atime != file2_stat.st_atime:
			differences.append("atime")
		if "mtime" not in skipped_types and file1_stat.st_mtime != file2_stat.st_mtime:
			differences.append("mtime")
		if "ctime" not in skipped_types and file1_stat.st_ctime != file2_stat.st_ctime:
			differences.append("ctime")

	if file1_stat.st_size != file2_stat.st_size:
		if "size" not in skipped_types:
			differences.append("size")
		if "content" not in skipped_types:
			differences.append("content")
	else:
		if "content" not in skipped_types:
			if stat.S_ISLNK(file1_stat.st_mode):
				file1_stream = io.BytesIO(os.readlink(_unicode_encode(file1,
									encoding=_encodings["fs"],
									errors="strict")))
			else:
				file1_stream = open(_unicode_encode(file1,
							encoding=_encodings["fs"],
							errors="strict"), "rb")
			if stat.S_ISLNK(file2_stat.st_mode):
				file2_stream = io.BytesIO(os.readlink(_unicode_encode(file2,
									encoding=_encodings["fs"],
									errors="strict")))
			else:
				file2_stream = open(_unicode_encode(file2,
							encoding=_encodings["fs"],
							errors="strict"), "rb")
			while True:
				file1_content = file1_stream.read(4096)
				file2_content = file2_stream.read(4096)
				if file1_content == b"" or file2_content == b"":
					break
				if file1_content != file2_content:
					differences.append("content")
					break
			file1_stream.close()
			file2_stream.close()

	return tuple(differences)

Return to bug 690484

Line 0 Link Here

(-)a/lib/portage/util/_compare_files.py (-1 / +101 lines)
0	-	1	# Copyright 2019 Gentoo Authors
		2	# Distributed under the terms of the GNU General Public License v2
		3
		4	__all__ = ["compare_files"]
		5
		6	import io
		7	import os
		8	import stat
		9	import sys
		10
		11	from portage import _encodings
		12	from portage import _unicode_encode
		13	from portage.util._xattr import xattr
		14
		15	def compare_files(file1, file2, skipped_types=()):
		16	"""
		17	Compare metadata and contents of two files.
		18
		19	@param file1: File 1
		20	@type file1: str
		21	@param file2: File 2
		22	@type file2: str
		23	@param skipped_types: Tuple of strings specifying types of properties excluded from comparison.
		24	Supported strings: type, mode, owner, group, device_number, xattr, atime, mtime, ctime, size, content
		25	@type skipped_types: tuple of str
		26	@rtype: tuple of str
		27	@return: Tuple of strings specifying types of properties different between compared files
		28	"""
		29
		30	file1_stat = os.lstat(_unicode_encode(file1, encoding=_encodings["fs"], errors="strict"))
		31	file2_stat = os.lstat(_unicode_encode(file2, encoding=_encodings["fs"], errors="strict"))
		32
		33	differences = []
		34
		35	if (file1_stat.st_dev, file1_stat.st_ino) == (file2_stat.st_dev, file2_stat.st_ino):
		36	return ()
		37
		38	if "type" not in skipped_types and stat.S_IFMT(file1_stat.st_mode) != stat.S_IFMT(file2_stat.st_mode):
		39	differences.append("type")
		40	if "mode" not in skipped_types and stat.S_IMODE(file1_stat.st_mode) != stat.S_IMODE(file2_stat.st_mode):
		41	differences.append("mode")
		42	if "owner" not in skipped_types and file1_stat.st_uid != file2_stat.st_uid:
		43	differences.append("owner")
		44	if "group" not in skipped_types and file1_stat.st_gid != file2_stat.st_gid:
		45	differences.append("group")
		46	if "device_number" not in skipped_types and file1_stat.st_rdev != file2_stat.st_rdev:
		47	differences.append("device_number")
		48
		49	if "xattr" not in skipped_types and sorted(xattr.get_all(file1, nofollow=True)) != sorted(xattr.get_all(file2, nofollow=True)):
		50	differences.append("xattr")
		51
		52	if sys.hexversion >= 0x3030000:
		53	if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns:
		54	differences.append("atime")
		55	if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns:
		56	differences.append("mtime")
		57	if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns:
		58	differences.append("ctime")
		59	else:
		60	if "atime" not in skipped_types and file1_stat.st_atime != file2_stat.st_atime:
		61	differences.append("atime")
		62	if "mtime" not in skipped_types and file1_stat.st_mtime != file2_stat.st_mtime:
		63	differences.append("mtime")
		64	if "ctime" not in skipped_types and file1_stat.st_ctime != file2_stat.st_ctime:
		65	differences.append("ctime")
66
67	if file1_stat.st_size != file2_stat.st_size:
68	if "size" not in skipped_types:
69	differences.append("size")
70	if "content" not in skipped_types:
71	differences.append("content")
72	else:
73	if "content" not in skipped_types:
74	if stat.S_ISLNK(file1_stat.st_mode):
75	file1_stream = io.BytesIO(os.readlink(_unicode_encode(file1,
76	encoding=_encodings["fs"],
77	errors="strict")))
78	else:
79	file1_stream = open(_unicode_encode(file1,
80	encoding=_encodings["fs"],
81	errors="strict"), "rb")
82	if stat.S_ISLNK(file2_stat.st_mode):
83	file2_stream = io.BytesIO(os.readlink(_unicode_encode(file2,
84	encoding=_encodings["fs"],
85	errors="strict")))
86	else:
87	file2_stream = open(_unicode_encode(file2,
88	encoding=_encodings["fs"],
89	errors="strict"), "rb")
90	while True:
91	file1_content = file1_stream.read(4096)
92	file2_content = file2_stream.read(4096)
93	if file1_content == b"" or file2_content == b"":
94	break
95	if file1_content != file2_content:
96	differences.append("content")
97	break
98	file1_stream.close()
99	file2_stream.close()
100
101	return tuple(differences)