[patch] Use concurrent.futures.ProcessPoolExecutor for python3.5+

0001-Use-ProcessPoolExecutor-bug-647964.patch (text/plain), 2.37 KB, created by Zac Medico on 2018-04-09 16:10:59 UTC

From 3cc21290a97ea0b734179054b354be3ef9696313 Mon Sep 17 00:00:00 2001
From: Zac Medico <[e-mail]>
Date: Mon, 9 Apr 2018 09:07:04 -0700
Subject: [PATCH] Use ProcessPoolExecutor (bug 647964)

Use ProcessPoolExecutor as a possible workaround for imap_unordered
deadlock.

Bug: https://bugs.gentoo.org/647964
---
 gemato/util.py | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/gemato/util.py b/gemato/util.py
index eb052b6..7563ec4 100644
--- a/gemato/util.py
+++ b/gemato/util.py
@@ -6,6 +6,11 @@
 import multiprocessing
 import sys
 
+try:
+	from concurrent.futures import ProcessPoolExecutor
+except ImportError:
+	ProcessPoolExecutor = None
+
 
 class MultiprocessingPoolWrapper(object):
     """
@@ -16,16 +21,25 @@ class MultiprocessingPoolWrapper(object):
     __slots__ = ['pool']
 
     def __init__(self, processes):
-        self.pool = multiprocessing.Pool(processes=processes)
+        # ProcessPoolExecutor.map supports chunksize in python3.5+,
+        # use it as a possible workaround for imap_unordered deadlock
+        # reported in https://bugs.gentoo.org/647964.
+        if sys.version_info >= (3, 5):
+            self.pool = ProcessPoolExecutor(max_workers=processes)
+        else:
+            self.pool = multiprocessing.Pool(processes=processes)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, exc_cb):
-        if exc_type is None:
-            self.pool.close()
-            self.pool.join()
-        self.pool.terminate()
+        if hasattr(self.pool, 'shutdown'):
+            self.pool.shutdown()
+        else:
+            if exc_type is None:
+                self.pool.close()
+                self.pool.join()
+            self.pool.terminate()
 
     def map(self, *args, **kwargs):
         return self.pool.map(*args, **kwargs)
@@ -35,12 +49,7 @@ class MultiprocessingPoolWrapper(object):
         Use imap_unordered() if available and safe to use. Fall back
         to regular map() otherwise.
         """
-        if sys.hexversion >= 0x03050400:
-            return self.pool.imap_unordered(*args, **kwargs)
-        else:
-            # in py<3.5.4 imap() swallows exceptions, so fall back
-            # to regular map()
-            return self.pool.map(*args, **kwargs)
+        return self.pool.map(*args, **kwargs)
 
 
 def path_starts_with(path, prefix):
-- 
2.13.6