[patch] 0001-repoman-add-include-profiles-PROFILES.patch

0001-repoman-add-include-profiles-PROFILES.patch (text/plain), 6.14 KB, created by Sergei Trofimovich (RETIRED) on 2019-11-19 00:21:56 UTC

From ad5376684536df3a3b11be863e63674b8cf767c9 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <[e-mail]>
Date: Tue, 19 Nov 2019 00:09:53 +0000
Subject: [PATCH] repoman: add --include-profiles=PROFILES

repoman slows down ~linearly with amount of profiles being scanned.
In case of amd64 we have 28 stable profiles.

To speed up processing and fit into time budged of various CIs we can
split the work across different processes that handle different profiles.

Example benchmark on ::haskell overlay:
    $ ./repoman full --include-arches=amd64
    ~65 minutes
    $ ./repoman full --include-profiles=default/linux/amd64/17.0
    ~4 minutes
This allows for a crude sharding of work across processes and allows for
cheap tree-wide scans for early failures.

Bug: https://bugs.gentoo.org/700456
Signed-off-by: Sergei Trofimovich <[e-mail]>
---
 repoman/lib/repoman/actions.py                      | 4 ++++
 repoman/lib/repoman/argparser.py                    | 7 +++++++
 repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
 repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
 repoman/lib/repoman/scanner.py                      | 5 +++++
 repoman/man/repoman.1                               | 4 ++++
 6 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
index 1c9989a72..92d4d4e94 100644
--- a/repoman/lib/repoman/actions.py
+++ b/repoman/lib/repoman/actions.py
@@ -412,6 +412,10 @@ the whole commit message to abort.
 			report_options.append(
 				"--include-arches=\"%s\"" %
 				" ".join(sorted(self.scanner.include_arches)))
+		if self.scanner.include_profiles is not None:
+			report_options.append(
+				"--include-profiles=\"%s\"" %
+				" ".join(sorted(self.scanner.include_profiles)))
 
 		if portage_version is None:
 			sys.stderr.write("Failed to insert portage version in message!\n")
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index fa0e6ff90..670a0e91d 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
 			'A space separated list of arches used to '
 			'filter the selection of profiles for dependency checks'))
 
+	parser.add_argument(
+		'--include-profiles',
+		dest='include_profiles', metavar='PROFILES', action='append',
+		help=(
+			'A space separated list of profiles used to '
+			'define the selection of profiles for dependency checks'))
+
 	parser.add_argument(
 		'-d', '--include-dev', dest='include_dev', action='store_true',
 		default=False,
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
index c3cc0ddeb..9068760bb 100644
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
@@ -19,7 +19,8 @@ module_spec = {
 			'func_desc': {
 			},
 			'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
-				'repo_metadata', 'repo_settings', 'include_arches', 'caches',
+				'repo_metadata', 'repo_settings', 'include_arches',
+				'include_profiles', 'caches',
 				'repoman_incrementals', 'env', 'have', 'dev_keywords'
 			],
 			'func_kwargs': {
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
index d980f4eca..0b1d74483 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
 		@param options: cli options
 		@param repo_settings: repository settings instance
 		@param include_arches: set
+		@param include_profiles: set
 		@param caches: dictionary of our caches
 		@param repoman_incrementals: tuple
 		@param env: the environment
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
 		self.options = kwargs.get('options')
 		self.repo_settings = kwargs.get('repo_settings')
 		self.include_arches = kwargs.get('include_arches')
+		self.include_profiles = kwargs.get('include_profiles')
 		self.caches = kwargs.get('caches')
 		self.repoman_incrementals = kwargs.get('repoman_incrementals')
 		self.env = kwargs.get('env')
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
 				if arch not in self.include_arches:
 					continue
 
-			relevant_profiles.extend(
-				(keyword, groups, prof) for prof in self.profiles[arch])
+			for prof in self.profiles[arch]:
+				if self.include_profiles is not None:
+					if prof not in self.include_profiles:
+						continue
+				relevant_profiles.append((keyword, groups, prof))
 
 		relevant_profiles.sort(key=sort_key)
 
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
index 1b3242a51..06234b0ad 100644
--- a/repoman/lib/repoman/scanner.py
+++ b/repoman/lib/repoman/scanner.py
@@ -164,6 +164,10 @@ class Scanner(object):
 		if self.options.include_arches:
 			self.include_arches = set()
 			self.include_arches.update(*[x.split() for x in self.options.include_arches])
+		self.include_profiles = None
+		if self.options.include_profiles:
+			self.include_profiles = set()
+			self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
 
 		# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
 		# running `svn status` in every package dir will be too expensive.
@@ -190,6 +194,7 @@ class Scanner(object):
 			"repo_metadata": self.repo_metadata,
 			"profiles": profiles,
 			"include_arches": self.include_arches,
+			"include_profiles": self.include_profiles,
 			"caches": self.caches,
 			"repoman_incrementals": self.repoman_incrementals,
 			"env": self.env,
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
index 766146f57..f33fb6098 100644
--- a/repoman/man/repoman.1
+++ b/repoman/man/repoman.1
@@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
 A space separated list of arches used to filter the selection of
 profiles for dependency checks.
 .TP
+.BR "\-\-include\-profiles " PROFILES
+A space separated list of profiles used to
+define the selection of profiles for dependency checks.
+.TP
 \fB\-d\fR, \fB\-\-include\-dev\fR
 Include dev profiles in dependency checks.
 .TP
-- 
2.24.0