Created attachment 884761 [details] emerge --info # rm -rf /var/db/repos/qt/metadata/md5-cache/ && egencache --ignore-default-opts --repo=qt --update --jobs=12 Exception in callback EbuildMetadataPhase._async_start_done(<Task finishe...Error('EAPI')>) handle: <Handle EbuildMetadataPhase._async_start_done(<Task finishe...Error('EAPI')>)> Traceback (most recent call last): File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run self._context.run(self._callback, *self._args) File "/usr/lib/python3.12/site-packages/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done future.cancelled() or future.result() ^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/site-packages/_emerge/EbuildMetadataPhase.py", line 130, in _async_start retval = portage.doebuild( ^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/site-packages/portage/package/ebuild/doebuild.py", line 1030, in doebuild doebuild_environment( File "/usr/lib/python3.12/site-packages/portage/package/ebuild/doebuild.py", line 519, in doebuild_environment eapi = mysettings.configdict["pkg"]["EAPI"] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ File "/usr/lib/python3.12/site-packages/portage/util/__init__.py", line 1684, in __getitem__ return UserDict.__getitem__(self, item_key) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/site-packages/portage/cache/mappings.py", line 175, in __getitem__ return self.data[key] ~~~~~~~~~^^^^^ KeyError: 'EAPI' Terminated
This chnage to EbuildMetadataPhase made its use of self.settings non-atomic with respect to other EbuildMetadataPhase instances, so I need to handle that somehow: --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -44,6 +46,12 @@ class EbuildMetadataPhase(SubProcess): _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): + asyncio.ensure_future( + self._async_start(), loop=self.scheduler + ).add_done_callback(self._async_start_done) + + async def _async_start(self): + ebuild_path = self.ebuild_hash.location with open( @@ -75,6 +83,9 @@ class EbuildMetadataPhase(SubProcess):
I tested https://github.com/gentoo/portage/pull/1263 by running egencache --jobs=8 after a git pull and it worked fine.
Tried the pull request and it works fine for me as well.
https://github.com/gentoo/portage/pull/1263 has this reproducer that causes multiple ResolverPlayground based tests to fail: diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py index 2d26012873..75c86b615c 100644 --- a/lib/portage/tests/resolver/ResolverPlayground.py +++ b/lib/portage/tests/resolver/ResolverPlayground.py @@ -5,0 +6 @@ import fnmatch +import subprocess @@ -21,2 +21,0 @@ from portage.dbapi.bintree import binarytree -from portage.package.ebuild.config import config -from portage.package.ebuild.digestgen import digestgen @@ -326,16 +325,19 @@ class ResolverPlayground: - tmpsettings = config(clone=self.settings) - tmpsettings["PORTAGE_QUIET"] = "1" - for cpv in ebuilds: - a = Atom("=" + cpv, allow_repo=True) - repo = a.repo - if repo is None: - repo = "test_repo" - - repo_dir = self._get_repo_dir(repo) - ebuild_dir = os.path.join(repo_dir, a.cp) - ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild") - - portdb = self.trees[self.eroot]["porttree"].dbapi - tmpsettings["O"] = ebuild_dir - if not digestgen(mysettings=tmpsettings, myportdb=portdb): - raise AssertionError(f"digest creation failed for {ebuild_path}") + for repo_name in self._repositories: + if repo_name == "DEFAULT": + continue + egencache_cmd = [ + "egencache", + f"--repo={repo_name}", + "--update", + "--update-manifests", + "--sign-manifests=n", + "--strict-manifests=n", + f"--repositories-configuration={self.settings['PORTAGE_REPOSITORIES']}", + f"--jobs={portage.util.cpuinfo.get_cpu_count()}", + ] + result = subprocess.run( + egencache_cmd, + env=self.settings.environ(), + ) + if result.returncode != os.EX_OK: + raise AssertionError(f"command failed: {egencache_cmd}")
There are significant changes in https://github.com/gentoo/portage/pull/1263, so I'll go ahead and revert 7106dcd0a52e43b9cb493fc31ecf93af0a806280 in https://github.com/gentoo/portage/pull/1266 so that we can merge it later with its dependencies.
The bug has been referenced in the following commit(s): https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c528b1cf44f30d80a3ca5620a810e4fe2bd66f1 commit 5c528b1cf44f30d80a3ca5620a810e4fe2bd66f1 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 04:47:53 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-13 05:02:14 +0000 Revert "EbuildPhase: async_check_locale" This reverts commit c95fc64abf9698263090b3ffd4a056e989dd2be1 since we had assumed EbuildMetadataPhase._start would serialize access to the portdbapi doebuild_settings attribute, and that assumption broke when _async_start was introduced in order to call async_check_locale. Bug: https://bugs.gentoo.org/923841 Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/_emerge/EbuildMetadataPhase.py | 21 -------------------- lib/_emerge/EbuildPhase.py | 28 +-------------------------- lib/portage/package/ebuild/config.py | 26 ++++++++++++++----------- lib/portage/util/futures/_asyncio/__init__.py | 9 --------- lib/portage/util/locale.py | 28 +++++++++------------------ 5 files changed, 25 insertions(+), 87 deletions(-)
The bug has been referenced in the following commit(s): https://gitweb.gentoo.org/proj/portage.git/commit/?id=389bb304abf5705b9f0e9e75982a682f193af985 commit 389bb304abf5705b9f0e9e75982a682f193af985 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 04:35:02 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:31 +0000 async_aux_get: Use EbuildMetadataPhase deallocate_config future For the portdbapi async_aux_get method, there is not a very good place to store a config pool, so instead use asyncio.Lock to manage access to the portdbapi doebuild_settings attribute when using the main event loop in the main thread. For other threads, clone a config instance since we do not have a thread-safe config pool. This cloning is expensive, but since portage internals do not trigger this case, it suffices for now (an AssertionError ensures that internals do not trigger it). For the main event loop running in the main thread, performance with the asyncio.Lock should not be significantly different to performance prior to commit c95fc64abf96, since check_locale results are typically cached and before there was only a single shared doebuild_settings instance with access serialized via the EbuildMetadataPhase _start method. Update async_aux_get callers to use asyncio.ensure_future on the returned coroutine when needed, since it used to return a future instead of a coroutine, and sometimes a future is needed for add_done_callback usage. In the portdbapi async_fetch_map method, fix a broken reference to "future" which should have been "aux_get_future", an error discovered while testing this patch. Bug: https://bugs.gentoo.org/924319 Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale") Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/portage/_emirrordist/FetchIterator.py | 10 ++- lib/portage/dbapi/porttree.py | 129 ++++++++++++++++++++---------- lib/portage/tests/update/test_move_ent.py | 3 + 3 files changed, 97 insertions(+), 45 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=a42c2164ada634262ae1f791ad60298fe3468a94 commit a42c2164ada634262ae1f791ad60298fe3468a94 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 03:39:35 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:31 +0000 asyncio: Wrap asyncio.Lock for python 3.9 compat Wrap asyncio.Lock for compatibility with python 3.9 where the deprecated loop parameter is required in order to avoid "got Future <Future pending> attached to a different loop" errors. The pordbapi async_aux_get method can use asyncio.Lock to serialize access to its doebuild_settings attribute in order to prevent issues like bug 924319. Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dedea99ac13e0e75a83a78890ed73bced1b950b commit 0dedea99ac13e0e75a83a78890ed73bced1b950b Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 04:21:26 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:31 +0000 ResolverPlayground: Use egencache to create manifests Make the ResolverPlayground _create_ebuild_manifests method call egencache --jobs, which reliably triggers the KeyError from bug 924319 for multiple tests: lib/portage/tests/bin/test_doins.py::DoIns::testDoInsFallback Exception in callback EbuildMetadataPhase._async_start_done(<Task finishe...Error('EAPI')>) handle: <Handle EbuildMetadataPhase._async_start_done(<Task finishe...Error('EAPI')>)> Traceback (most recent call last): File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run self._context.run(self._callback, *self._args) File "lib/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done future.cancelled() or future.result() ^^^^^^^^^^^^^^^ File "lib/_emerge/EbuildMetadataPhase.py", line 130, in _async_start retval = portage.doebuild( ^^^^^^^^^^^^^^^^^ File "lib/portage/package/ebuild/doebuild.py", line 1030, in doebuild doebuild_environment( File "lib/portage/package/ebuild/doebuild.py", line 519, in doebuild_environment eapi = mysettings.configdict["pkg"]["EAPI"] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ File "lib/portage/util/__init__.py", line 1684, in __getitem__ return UserDict.__getitem__(self, item_key) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "lib/portage/cache/mappings.py", line 175, in __getitem__ return self.data[key] ~~~~~~~~~^^^^^ KeyError: 'EAPI' Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/portage/tests/dbapi/test_portdb_cache.py | 4 ++- lib/portage/tests/resolver/ResolverPlayground.py | 38 +++++++++++++----------- 2 files changed, 23 insertions(+), 19 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=f9ea958018c02b3ce07761c2d965260498add2af commit f9ea958018c02b3ce07761c2d965260498add2af Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 04:04:34 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:30 +0000 MetadataRegen: Use EbuildMetadataPhase deallocate_config For EbuildMetadataPhase consumers like MetadataRegen and depgraph, store a pool of config instances in a config_pool list, and return instaces to the list when the deallocate_config future is done. Bug: https://bugs.gentoo.org/924319 Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale") Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/_emerge/MetadataRegen.py | 16 ++++++++++++++-- lib/_emerge/depgraph.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=69fbd8f9f76df32d9bc72510e5d9348eb8f059bc commit 69fbd8f9f76df32d9bc72510e5d9348eb8f059bc Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 04:04:53 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:30 +0000 EbuildMetadataPhase: Add deallocate_config future Use a deallocate_config future to release self.settings when it is no longer needed. It's necessary to manage concurrency since commit c95fc64abf96 because mutation of self.settings is no longer limited to the EbuildMetadataPhase _start method, where exclusive access was guaranteed within the main thread. Add support to the isAlive() method to detect when the EbuildMetadataPhase has started but the pid is not yet available (due to async_check_locale usage from commit c95fc64abf96). This can be used to check if an EbuildMetadataPhase instance has been successfully started so that it can be relied upon to set the result of the deallocate_config future. Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/_emerge/EbuildMetadataPhase.py | 34 ++++++++++++++++++++++++++++++++++ lib/_emerge/SubProcess.py | 5 ++++- 2 files changed, 38 insertions(+), 1 deletion(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=414234a218bc79564ab17312d5cc247a4c8091d7 commit 414234a218bc79564ab17312d5cc247a4c8091d7 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2024-02-13 03:30:00 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2024-02-21 15:27:30 +0000 anydbm: Pickle support for multiprocessing spawn The egencache usage in ResolverPlayground that was used to trigger bug 924319 triggered a pickling error for AuxdbTestCase.test_anydbm with the multiprocessing spawn start method, so fix the anydbm cache module to omit the unpicklable database object from pickled state, and regenerate it after unpickling. Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico <zmedico@gentoo.org> lib/portage/cache/anydbm.py | 17 ++++++++++++++++- lib/portage/tests/dbapi/test_auxdb.py | 4 +--- 2 files changed, 17 insertions(+), 4 deletions(-)
