Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions Lib/asyncio/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,22 +157,31 @@ def get_all_awaited_by(pid):
return unwinder.get_all_awaited_by()


def build_async_tree(result, task_emoji="(T)", cor_emoji=""):
def build_async_tree(result, task_emoji="(T)", cor_emoji="", fut_emoji="(F)"):
"""
Build a list of strings for pretty-print an async call tree.
The call tree is produced by `get_all_async_stacks()`, prefixing tasks
with `task_emoji` and coroutine frames with `cor_emoji`.
with `task_emoji`, futures with `fut_emoji`, and coroutine frames with
`cor_emoji`.
"""
id2name, awaits, task_stacks = _index(result)
g = _task_graph(awaits)
cycles = _find_cycles(g)
if cycles:
raise CycleFoundException(cycles, id2name)
labels, children = _build_tree(id2name, awaits, task_stacks)
future_ids = {
node.task_id
for thread in result
for node in thread.awaited_by
if node.is_future
}

def pretty(node):
flag = task_emoji if node[0] == NodeType.TASK else cor_emoji
if node[0] == NodeType.TASK and node[1] in future_ids:
flag = fut_emoji
return f"{flag} {labels[node]}"

def render(root):
Expand Down
98 changes: 98 additions & 0 deletions Lib/test/test_external_inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,73 @@ async def main():
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
self.assertEqual(stacks["anext"], ["agen"])

@skip_if_not_supported
def test_future_between_tasks(self):
# gh-161502
from asyncio.tools import build_async_tree

async def worker(method):
return getattr(RemoteUnwinder(os.getpid()), method)()

async def main(method):
parent = asyncio.current_task()
parent.set_name("main")
future = asyncio.Future()
task = asyncio.create_task(worker(method), name="worker")
asyncio.future_add_to_awaited_by(task, future)
task.add_done_callback(future.set_result)
try:
await future
return task.result(), {
id(parent): [],
id(future): [id(parent)],
id(task): [id(future)],
}
finally:
asyncio.future_discard_from_awaited_by(task, future)

for method in ("get_all_awaited_by", "get_async_stack_trace"):
with self.subTest(method=method):
snapshot, expected = asyncio.run(main(method))
tree, = build_async_tree(snapshot)
rendered = "\n".join(tree)
self.assertIn("(T) main", rendered)
self.assertIn("(F) Future-", rendered)
self.assertIn("(T) worker", rendered)
relationships = {
node.task_id: [edge.task_name for edge in node.awaited_by]
for thread in snapshot for node in thread.awaited_by
}
self.assertEqual(relationships, expected)

@skip_if_not_supported
def test_shared_future_waiter(self):
async def main():
layers = [[asyncio.current_task()]]
for _ in range(14):
waiters = [asyncio.Future(), asyncio.Future()]
for child in layers[-1]:
for waiter in waiters:
asyncio.future_add_to_awaited_by(child, waiter)
self.addCleanup(asyncio.future_discard_from_awaited_by,
child, waiter)
layers.append(waiters)

leaf, root = layers[-1][0], layers[0][0]
asyncio.future_add_to_awaited_by(leaf, root)
self.addCleanup(asyncio.future_discard_from_awaited_by, leaf, root)

expected_ids = [id(node) for layer in layers for node in layer]
unwinder = RemoteUnwinder(os.getpid())
for method in (unwinder.get_all_awaited_by,
unwinder.get_async_stack_trace):
with self.subTest(method=method.__name__):
ids = [node.task_id for thread in method()
for node in thread.awaited_by]
self.assertCountEqual(ids, expected_ids)

asyncio.run(main())

@skip_if_not_supported
def test_async_generator_running_task_stack(self):
# gh-159620
Expand Down Expand Up @@ -4787,6 +4854,37 @@ def _assert_unwinder_limit_error(self, unwind, expected_substring):
f"{last_error!r}"
)

@skip_if_not_supported
def test_get_all_awaited_by_at_waiter_limit(self):
# gh-161502
async def main():
task = asyncio.current_task()
limit = self.TASK_WAITER_WALK_TASKS - 1
for _ in range(limit):
future = asyncio.Future()
asyncio.future_add_to_awaited_by(task, future)
self.addCleanup(asyncio.future_discard_from_awaited_by, task, future)

release = asyncio.Event()
child = asyncio.create_task(release.wait())
waiter = asyncio.create_task(asyncio.wait([child]), eager_start=True)
unwinder = RemoteUnwinder(os.getpid())
snapshot = unwinder.get_all_awaited_by()
count = sum(node.is_future for thread in snapshot
for node in thread.awaited_by)
self.assertEqual(count, limit)

extra = asyncio.Future()
asyncio.future_add_to_awaited_by(task, extra)
self.addCleanup(asyncio.future_discard_from_awaited_by, task, extra)
with self.assertRaisesRegex(RuntimeError, "Too many task waiters"):
unwinder.get_all_awaited_by()

release.set()
await waiter

asyncio.run(main())

@skip_if_not_supported
@unittest.skipIf(
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix missing tasks in ``python -m asyncio pstree`` output when tasks are
connected through an intermediate :class:`asyncio.Future`. Patch by Timofei
Ivankov
74 changes: 58 additions & 16 deletions Modules/_remote_debugging/asyncio.c
Original file line number Diff line number Diff line change
Expand Up @@ -534,10 +534,11 @@ parse_task_awaited_by(
uintptr_t task_address,
PyObject *awaited_by
) {
// Read the entire TaskObj at once
char task_obj[SIZEOF_TASK_OBJ];
size_t size = unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set
+ sizeof(char);
if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address,
(size_t)unwinder->async_debug_offsets.asyncio_task_object.size,
size,
task_obj) < 0) {
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object");
return -1;
Expand Down Expand Up @@ -571,7 +572,16 @@ process_single_task_node(
PyObject *result_item = NULL;
PyObject *coroutine_stack = NULL;

tn = parse_task_name(unwinder, task_addr);
char is_task;
// task_is_task = 1 => task
// task_is_task = 0 => future
if (read_char(unwinder,
task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task,
&is_task) < 0) {
goto error;
}
tn = is_task ? parse_task_name(unwinder, task_addr)
: PyUnicode_FromFormat("Future-%p", (void *)task_addr);
if (tn == NULL) {
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node");
goto error;
Expand Down Expand Up @@ -612,6 +622,7 @@ process_single_task_node(
PyStructSequence_SetItem(result_item, 1, tn); // steals ref
PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref
PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref
PyStructSequence_SetItem(result_item, 4, PyBool_FromLong(!is_task));

// References transferred to tuple
task_id = NULL;
Expand Down Expand Up @@ -652,37 +663,68 @@ process_single_task_node(
static int
process_task_waiters(
RemoteUnwinderObject *unwinder,
PyObject *result
PyObject *result,
bool futures_only
) {
Py_ssize_t initial_count = futures_only ? PyList_GET_SIZE(result) : 0;
PyObject *visited = PySet_New(NULL);
if (visited == NULL) {
return -1;
}
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(result); i++) {
PyObject *task_info = PyList_GET_ITEM(result, i);
if (PySet_Add(visited, PyStructSequence_GET_ITEM(task_info, 0)) < 0) {
goto error;
}
PyObject *waiters = PyStructSequence_GET_ITEM(task_info, 3);
for (Py_ssize_t j = 0; j < PyList_GET_SIZE(waiters); j++) {
if (PyList_GET_SIZE(result) >= MAX_TASK_WAITER_WALK_TASKS) {
PyErr_SetString(PyExc_RuntimeError,
"Too many task waiters (possible infinite loop)");
set_exception_cause(unwinder, PyExc_RuntimeError,
"Task waiter walk size limit exceeded");
return -1;
}
PyObject *waiter = PyList_GET_ITEM(waiters, j);
// CoroInfo item 1 holds the waiter task address stored by parse_task().
PyObject *task_id = PyStructSequence_GET_ITEM(waiter, 1);
int seen = PySet_Contains(visited, task_id);
if (seen > 0) {
continue;
}
if (seen < 0 || PySet_Add(visited, task_id) < 0) {
goto error;
}
void *task_ptr = PyLong_AsVoidPtr(task_id);
if (task_ptr == NULL && PyErr_Occurred()) {
set_exception_cause(unwinder, PyExc_RuntimeError,
"Failed to parse waiter task ID");
return -1;
goto error;
}
if (futures_only) {
char is_task;
if (read_char(unwinder,
(uintptr_t)task_ptr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task,
&is_task) < 0) {
goto error;
}
if (is_task) {
continue;
}
}
if (PyList_GET_SIZE(result) - initial_count >= MAX_TASK_WAITER_WALK_TASKS) {
PyErr_SetString(PyExc_RuntimeError,
"Too many task waiters (possible infinite loop)");
set_exception_cause(unwinder, PyExc_RuntimeError,
"Task waiter walk size limit exceeded");
goto error;
}
if (process_single_task_node(
Comment thread
pablogsal marked this conversation as resolved.
Resolved
unwinder, (uintptr_t)task_ptr, NULL, result) < 0)
{
return -1;
goto error;
}
}
}

Py_DECREF(visited);
return 0;

error:
Py_DECREF(visited);
return -1;
}

/* ============================================================================
Expand Down Expand Up @@ -927,7 +969,7 @@ append_awaited_by(
return -1;
}

return 0;
return process_task_waiters(unwinder, task_list, true);
}

/* ============================================================================
Expand Down Expand Up @@ -991,7 +1033,7 @@ process_running_task_chain(
}

// Now find all tasks that are waiting for this task and process them
if (process_task_waiters(unwinder, result) < 0) {
if (process_task_waiters(unwinder, result, false) < 0) {
return -1;
}

Expand Down
19 changes: 13 additions & 6 deletions Modules/_remote_debugging/clinic/module.c.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading