Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions Lib/multiprocessing/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import sys
import socket
import struct
import threading
import time


Expand Down Expand Up @@ -283,22 +284,33 @@ class PipeConnection(_ConnectionBase):
with FILE_FLAG_OVERLAPPED.
"""
_got_empty_message = False
_send_ov = None

def __init__(self, handle, readable=True, writable=True):
super().__init__(handle, readable, writable)
# Several threads may write to the same connection. Keep track of
# all pending overlapped operations so that close() can cancel
# them.
self._send_ov = []
self._send_ov_lock = threading.Lock()
self._closing = False

def _close(self, _CloseHandle=_winapi.CloseHandle):
ov = self._send_ov
if ov is not None:
with self._send_ov_lock:
# Prevent new writes from being submitted after we have
# taken the snapshot of pending operations.
self._closing = True
overlapped = list(self._send_ov)
for ov in overlapped:
# Interrupt WaitForMultipleObjects() in _send_bytes()
ov.cancel()
_CloseHandle(self._handle)

def _send_bytes(self, buf):
if self._send_ov is not None:
# A connection should only be used by a single thread
raise ValueError("concurrent send_bytes() calls "
"are not supported")
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
self._send_ov = ov
with self._send_ov_lock:
if self._closing:
raise OSError(errno.EPIPE, "handle is closed")
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
self._send_ov.append(ov)
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
Expand All @@ -308,7 +320,8 @@ def _send_bytes(self, buf):
ov.cancel()
raise
finally:
self._send_ov = None
with self._send_ov_lock:
self._send_ov.remove(ov)
nwritten, err = ov.GetOverlappedResult(True)
if err == _winapi.ERROR_OPERATION_ABORTED:
# close() was called by another thread while
Expand Down
62 changes: 62 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -4083,6 +4083,68 @@ def test_duplex_false(self):
self.assertRaises(OSError, writer.recv)
self.assertRaises(OSError, writer.poll)

@unittest.skipUnless(sys.platform == 'win32',
'test Windows PipeConnection')
def test_concurrent_send_bytes(self):
if self.TYPE != 'processes':
self.skipTest('test requires a real PipeConnection')
# gh-162304: Several Pool threads can write to the same pipe. Fake the
# Win32 I/O calls to keep one write pending while a second thread
# submits another one, so the old single _send_ov slot is detected
# deterministically.
import _winapi

reader, writer = self.Pipe(duplex=False)
entered = threading.Event()
release = threading.Event()
errors = []

class FakeOverlapped:
def __init__(self, size):
self.event = object()
self.size = size

def cancel(self):
pass

def GetOverlappedResult(self, wait):
return self.size, 0

def fake_write(handle, buf, overlapped=False):
return FakeOverlapped(len(buf)), _winapi.ERROR_IO_PENDING

def fake_wait(handles, waitall, timeout):
entered.set()
release.wait(support.SHORT_TIMEOUT)
return _winapi.WAIT_OBJECT_0

def send():
try:
writer.send_bytes(b'x')
except BaseException as exc:
errors.append(exc)

with unittest.mock.patch.object(_winapi, 'WriteFile', fake_write), \
unittest.mock.patch.object(_winapi, 'WaitForMultipleObjects',
fake_wait):
t1 = threading.Thread(target=send)
t2 = threading.Thread(target=send)
try:
t1.start()
# t1 is now blocked inside fake_wait() with its write
# registered. A second write must be allowed concurrently.
self.assertTrue(entered.wait(support.SHORT_TIMEOUT))
t2.start()
t2.join(0.5)
finally:
release.set()
t1.join(support.SHORT_TIMEOUT)
t2.join(support.SHORT_TIMEOUT)

self.assertEqual(errors, [])
reader.close()
writer.close()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
def test_spawn_close(self):
# We test that a pipe connection can be closed by parent
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix a :exc:`ValueError` in :mod:`multiprocessing` on Windows when several
threads write to the same pipe simultaneously.
Loading