[patch] Add write ahead log

integrity-w-write-ahead.patch (text/plain), 5.75 KB, created by Sam on 2018-12-09 22:51:03 UTC

--- vartree.py
+++ vartree.py
@@ -1081,6 +1081,7 @@
 		new_contents = pkg.getcontents().copy()
 		removed = 0
 
+		self.startContentsRemoval(pkg.dbdir)
 		for filename in paths:
 			filename = _unicode_decode(filename,
 				encoding=_encodings['content'], errors='strict')
@@ -1109,6 +1110,7 @@
 				self.removeFromContentsMeta(pkg.dbdir, index, "ATTRS_CAPS")
 				removed += 1
 
+		self.stopContentsRemoval(pkg.dbdir)
 		if removed:
 			# Also remove corresponding NEEDED lines, so that they do
 			# no corrupt LinkageMap data for preserve-libs.
@@ -1142,6 +1144,162 @@
 
 			self.writeContentsToContentsFile(pkg, new_contents, new_needed=new_needed)
 
+	def startContentsRemoval(self, vdbdir):
+		contents_dir = os.path.join(vdbdir, "contents.d")
+		transaction_dir = os.path.join(vdbdir, "contents.d~")
+		manifest_file = os.path.join(contents_dir, "Manifest")
+		manifest_lines = ""
+
+		# Clean previously unfinished transaction @TODO also: either roll-back or roll-forward
+		if os.path.isdir(transaction_dir):
+			shutil.rmtree(transaction_dir)
+		if os.path.isdir(contents_dir):
+			shutil.rmtree(contents_dir)
+
+		# Set up transaction
+		os.mkdir(transaction_dir, 0o644)
+		files = [
+				"CONTENTS_DIGESTS_SHA1",
+				"CONTENTS_DIGESTS_SHA256",
+				"CONTENTS_DIGESTS_SHA512",
+				"CONTENTS_MODES",
+				"CONTENTS_ATTRS_PAX",
+				"CONTENTS_ATTRS_CAPS"
+			]
+		for f in files:
+			fname_src = os.path.join(vdbdir, f)
+			fname_dest = os.path.join(transaction_dir, f)
+
+			# Gracefully handle non-existent files
+			if os.path.isfile(fname_src):
+				shutil.copy2(fname_src, fname_dest)
+				manifest_lines += f + "\n"
+				manifest_lines += portage.checksum.perform_checksum(fname_src, "SHA1", 0)[0] + "\n"
+
+		# Write Manifest-file of transaction
+		os.mkdir(contents_dir, 0o644)
+		with open(manifest_file,"w") as f:
+			f.write(manifest_lines)
+
+	def stopContentsRemoval(self, vdbdir):
+		contents_dir = os.path.join(vdbdir, "contents.d")
+		transaction_dir = os.path.join(vdbdir, "contents.d~")
+		digests = []
+		transaction_files = []
+		all_files = [
+				"CONTENTS_DIGESTS_SHA1",
+				"CONTENTS_DIGESTS_SHA256",
+				"CONTENTS_DIGESTS_SHA512",
+				"CONTENTS_MODES",
+				"CONTENTS_ATTRS_PAX",
+				"CONTENTS_ATTRS_CAPS"
+			]
+
+		if not os.path.isdir(transaction_dir):
+			print("Failed creating transaction dir")
+			sys.exit(1)
+
+		# Read Manifest-file of contents
+		manifest_file = os.path.join(contents_dir, "Manifest")
+		if os.path.isfile(manifest_file):
+			with open(manifest_file,"r") as f:
+				lines = f.read().splitlines()
+
+			for i, line in enumerate(lines):
+				if (i%2) == 0:
+					transaction_files.append(line)
+				else:
+					digests.append(line)
+
+		# Check transactiondir against Manifest
+		for f in transaction_files:
+			file = os.path.join(transaction_dir, f)
+			if not os.path.isfile(file):
+				print("Manifest contains non-existing file '"+file+"'")
+				sys.exit(1)
+
+		# Setup contents_dir with links of vdbdir files
+		for i, f in enumerate(transaction_files):
+			fname_src = os.path.join(vdbdir, f)
+			fname_dest = os.path.join(contents_dir, f)
+
+			# Gracefully handle non-existent files
+			if os.path.isfile(fname_src):
+				if portage.checksum.perform_checksum(fname_src, "SHA1", 0)[0] != digests[i]:
+					print("According to Manifest, file in vdbdir was modified '" + fname_src + "'")
+					sys.exit(1)
+				else:
+					os.link(fname_src, fname_dest)
+			else:
+				print("File in Manifest no longer found in vdbdir '"+f+"'")
+				sys.exit(1)
+
+		# Sync contents_dir and transaction_dir to disk
+		if platform.system() == "Linux":
+			paths = []
+			for f in os.listdir(transaction_dir):
+				paths.append(os.path.join(transaction_dir, f))
+			for f in os.listdir(contents_dir):
+				paths.append(os.path.join(contents_dir, f))
+			paths = tuple(paths)
+
+			proc = SyncfsProcess(paths=paths,
+				scheduler=(
+						SchedulerInterface(portage._internal_caller and
+						global_event_loop() or EventLoop(main=False))
+					))
+
+			proc.start()
+			returncode = proc.wait()
+
+		# Link from transaction_dir
+		for f in transaction_files:
+			fname_src = os.path.join(transaction_dir, f)
+			fname_dest = os.path.join(vdbdir, f+"~")
+
+			# Gracefully handle non-existent files
+			if os.path.isfile(fname_src):
+				os.link(fname_src, fname_dest)
+			else:
+				print("Manifest contains file that no longer exists '"+f+"'")
+				sys.exit(1)
+
+		# Sync contents_dir and transaction_dir to disk
+		if platform.system() == "Linux":
+			paths = []
+			for f in transaction_files:
+				# Gracefully handle non-existent files
+				if os.path.isfile(os.path.join(vdbdir, f+"~")):
+					paths.append(os.path.join(vdbdir, f+"~"))
+				else:
+					print("Manifest contains file that no longer exists '"+f+"'")
+					sys.exit(1)
+			paths = tuple(paths)
+
+			proc = SyncfsProcess(paths=paths,
+				scheduler=(
+						SchedulerInterface(portage._internal_caller and
+						global_event_loop() or EventLoop(main=False))
+					))
+
+			proc.start()
+			returncode = proc.wait()
+
+		# Rename
+		for f in transaction_files:
+			fname_src = os.path.join(vdbdir, f+"~")
+			fname_dest = os.path.join(vdbdir, f)
+
+			# Gracefully handle non-existent files
+			if os.path.isfile(fname_src):
+				os.rename(fname_src, fname_dest) #atomic rename, doesn't require sync
+			else:
+				print("Manifest contains file that no longer exists '"+f+"'")
+				sys.exit(1)
+
+		shutil.rmtree(transaction_dir)
+		shutil.rmtree(contents_dir)
+
 	def removeFromContentsMeta(self, vdbdir, index, type):
 		contents_file = ""
 		if (type in
@@ -1151,7 +1309,7 @@
 			"MODES",
 			"ATTRS_PAX",
 			"ATTRS_CAPS"}):
-			contents_file = os.path.join(vdbdir, "CONTENTS_"+type)
+			contents_file = os.path.join(os.path.join(vdbdir, "contents.d~"),"CONTENTS_"+type)
 		else:
 			print("ERROR removeFromContentsMeta() got passed unexpected type "+type)