Event loop recursion occurs in EbuildFetcher._get_uri_map() when it calls the portdbapi.getFetchMap() method: > Traceback (most recent call last): > File "bin/emerge", line 51, in <module> > retval = emerge_main() > File "pym/_emerge/main.py", line 1282, in emerge_main > return run_action(emerge_config) > File "pym/_emerge/actions.py", line 3316, in run_action > retval = action_build(emerge_config, spinner=spinner) > File "pym/_emerge/actions.py", line 541, in action_build > retval = mergetask.merge() > File "pym/_emerge/Scheduler.py", line 1004, in merge > rval = self._merge() > File "pym/_emerge/Scheduler.py", line 1409, in _merge > self._main_loop() > File "pym/_emerge/Scheduler.py", line 1387, in _main_loop > self._event_loop.iteration() > File "pym/portage/util/_eventloop/EventLoop.py", line 281, in iteration > return self._iteration(*args) > File "pym/portage/util/_eventloop/EventLoop.py", line 302, in _iteration > if self._run_timeouts(): > File "pym/portage/util/_eventloop/EventLoop.py", line 597, in _run_timeouts > if self._run_idle_callbacks(): > File "pym/portage/util/_eventloop/EventLoop.py", line 556, in _run_idle_callbacks > if x._callback(*x._args): > File "pym/_emerge/AsynchronousTask.py", line 74, in _async_wait_cb > self.wait() > File "pym/_emerge/AsynchronousTask.py", line 57, in wait > self._wait_hook() > File "pym/_emerge/AsynchronousTask.py", line 175, in _wait_hook > self._exit_listener_stack.pop()(self) > File "pym/_emerge/EbuildPhase.py", line 204, in _ebuild_exit > self._ebuild_exit_unlocked(ebuild_process) > File "pym/_emerge/EbuildPhase.py", line 291, in _ebuild_exit_unlocked > self.wait() > File "pym/_emerge/AsynchronousTask.py", line 57, in wait > self._wait_hook() > File "pym/_emerge/AsynchronousTask.py", line 175, in _wait_hook > self._exit_listener_stack.pop()(self) > File "pym/_emerge/EbuildBuild.py", line 211, in _pre_clean_exit > already_fetched = fetcher.already_fetched(self.settings) > File "pym/_emerge/EbuildFetcher.py", line 36, in already_fetched > uri_map = self._get_uri_map() > File "pym/_emerge/EbuildFetcher.py", line 210, in _get_uri_map > useflags=use, mytree=mytree) > File "pym/portage/dbapi/porttree.py", line 734, in getFetchMap > ["EAPI", "SRC_URI"], mytree=mytree) > File "pym/portage/dbapi/porttree.py", line 587, in aux_get > myrepo=myrepo, loop=loop)) > File "pym/portage/util/_eventloop/EventLoop.py", line 827, in run_until_complete > self.iteration()
I've created a patch series to fix this, and posted it for review: https://archives.gentoo.org/gentoo-portage-dev/message/89ab52881a8cb44f78f0eedd2dc03532 https://github.com/gentoo/portage/pull/311
With the fixes for this bug, all unit tests pass with event loop recursion disabled!
The bug has been referenced in the following commit(s): https://gitweb.gentoo.org/proj/portage.git/commit/?id=0a5692bd4f4ab4a54daf2ce09112617cbc21c9ad commit 0a5692bd4f4ab4a54daf2ce09112617cbc21c9ad Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2018-04-22 22:22:20 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2018-04-23 18:39:47 +0000 EbuildFetcher: add async_already_fetched method (bug 653810) Add an async_already_fetched method to replace the synchronous already_fetched method, and use it to prevent event loop recursion. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> pym/_emerge/EbuildBuild.py | 8 +++++++- pym/_emerge/EbuildFetcher.py | 30 ++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=e512275c2997a2069049bbb5acdb40e155cf19c7 commit e512275c2997a2069049bbb5acdb40e155cf19c7 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2018-04-22 21:47:00 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2018-04-23 18:39:42 +0000 EbuildFetcher: use _async_uri_map in _start (bug 653810) Use _async_uri_map to avoid event loop recursion in _start. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> pym/_emerge/EbuildFetcher.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=9596fc68bf13b5e8f44b522d2b2e9c303615462f commit 9596fc68bf13b5e8f44b522d2b2e9c303615462f Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2018-04-22 21:18:53 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2018-04-23 18:39:37 +0000 EbuildFetcher: add _async_uri_map method (bug 653810) Add an _async_uri_map method to replace the synchronous _get_uri_map method. This will be used to prevent event loop recursion. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> pym/_emerge/EbuildFetcher.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=a0e96e92ae040052f9f11e1731e221346886aa3b commit a0e96e92ae040052f9f11e1731e221346886aa3b Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2018-04-22 20:20:45 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2018-04-23 18:39:31 +0000 EbuildFetcher: inherit CompositeTask (bug 653810) Make EbuildFetcher inherit CompositeTask, and move remaining code to a _EbuildFetcherProcess class that still inherits ForkProcess. The CompositeTask framework will be used to split EbuildFetcher into subtasks, in order to prevent event loop recursion. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> pym/_emerge/EbuildFetcher.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) https://gitweb.gentoo.org/proj/portage.git/commit/?id=8002d9343c385075ece19a131b0f584ec255a5b5 commit 8002d9343c385075ece19a131b0f584ec255a5b5 Author: Zac Medico <zmedico@gentoo.org> AuthorDate: 2018-04-22 19:57:09 +0000 Commit: Zac Medico <zmedico@gentoo.org> CommitDate: 2018-04-23 18:39:16 +0000 portdbapi: add async_fetch_map method (bug 653810) Add a portdbapi.async_fetch_map method which is identical to the existing portdbapi.getFetchMap method, but returns a future. This will be used by EbuildFetcher in order to avoid event loop recursion. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> pym/portage/dbapi/porttree.py | 75 +++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 17 deletions(-)}
Fixed in portage-2.3.40-r1.