diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 6bc53f4d7d3e5f..1ee4a471ba5bb0 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -157,12 +157,13 @@ def get_all_awaited_by(pid): return unwinder.get_all_awaited_by() -def build_async_tree(result, task_emoji="(T)", cor_emoji=""): +def build_async_tree(result, task_emoji="(T)", cor_emoji="", fut_emoji="(F)"): """ Build a list of strings for pretty-print an async call tree. The call tree is produced by `get_all_async_stacks()`, prefixing tasks - with `task_emoji` and coroutine frames with `cor_emoji`. + with `task_emoji`, futures with `fut_emoji`, and coroutine frames with + `cor_emoji`. """ id2name, awaits, task_stacks = _index(result) g = _task_graph(awaits) @@ -170,9 +171,17 @@ def build_async_tree(result, task_emoji="(T)", cor_emoji=""): if cycles: raise CycleFoundException(cycles, id2name) labels, children = _build_tree(id2name, awaits, task_stacks) + future_ids = { + node.task_id + for thread in result + for node in thread.awaited_by + if node.is_future + } def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + if node[0] == NodeType.TASK and node[1] in future_ids: + flag = fut_emoji return f"{flag} {labels[node]}" def render(root): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 5aebe1fb6b18e9..abee03621b1ae8 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -608,6 +608,73 @@ async def main(): stacks = self._extract_coroutine_funcnames(asyncio.run(main())) self.assertEqual(stacks["anext"], ["agen"]) + @skip_if_not_supported + def test_future_between_tasks(self): + # gh-161502 + from asyncio.tools import build_async_tree + + async def worker(method): + return getattr(RemoteUnwinder(os.getpid()), method)() + + async def main(method): + parent = asyncio.current_task() + parent.set_name("main") + future = asyncio.Future() + task = asyncio.create_task(worker(method), name="worker") + asyncio.future_add_to_awaited_by(task, future) + task.add_done_callback(future.set_result) + try: + await future + return task.result(), { + id(parent): [], + id(future): [id(parent)], + id(task): [id(future)], + } + finally: + asyncio.future_discard_from_awaited_by(task, future) + + for method in ("get_all_awaited_by", "get_async_stack_trace"): + with self.subTest(method=method): + snapshot, expected = asyncio.run(main(method)) + tree, = build_async_tree(snapshot) + rendered = "\n".join(tree) + self.assertIn("(T) main", rendered) + self.assertIn("(F) Future-", rendered) + self.assertIn("(T) worker", rendered) + relationships = { + node.task_id: [edge.task_name for edge in node.awaited_by] + for thread in snapshot for node in thread.awaited_by + } + self.assertEqual(relationships, expected) + + @skip_if_not_supported + def test_shared_future_waiter(self): + async def main(): + layers = [[asyncio.current_task()]] + for _ in range(14): + waiters = [asyncio.Future(), asyncio.Future()] + for child in layers[-1]: + for waiter in waiters: + asyncio.future_add_to_awaited_by(child, waiter) + self.addCleanup(asyncio.future_discard_from_awaited_by, + child, waiter) + layers.append(waiters) + + leaf, root = layers[-1][0], layers[0][0] + asyncio.future_add_to_awaited_by(leaf, root) + self.addCleanup(asyncio.future_discard_from_awaited_by, leaf, root) + + expected_ids = [id(node) for layer in layers for node in layer] + unwinder = RemoteUnwinder(os.getpid()) + for method in (unwinder.get_all_awaited_by, + unwinder.get_async_stack_trace): + with self.subTest(method=method.__name__): + ids = [node.task_id for thread in method() + for node in thread.awaited_by] + self.assertCountEqual(ids, expected_ids) + + asyncio.run(main()) + @skip_if_not_supported def test_async_generator_running_task_stack(self): # gh-159620 @@ -4787,6 +4854,37 @@ def _assert_unwinder_limit_error(self, unwind, expected_substring): f"{last_error!r}" ) + @skip_if_not_supported + def test_get_all_awaited_by_at_waiter_limit(self): + # gh-161502 + async def main(): + task = asyncio.current_task() + limit = self.TASK_WAITER_WALK_TASKS - 1 + for _ in range(limit): + future = asyncio.Future() + asyncio.future_add_to_awaited_by(task, future) + self.addCleanup(asyncio.future_discard_from_awaited_by, task, future) + + release = asyncio.Event() + child = asyncio.create_task(release.wait()) + waiter = asyncio.create_task(asyncio.wait([child]), eager_start=True) + unwinder = RemoteUnwinder(os.getpid()) + snapshot = unwinder.get_all_awaited_by() + count = sum(node.is_future for thread in snapshot + for node in thread.awaited_by) + self.assertEqual(count, limit) + + extra = asyncio.Future() + asyncio.future_add_to_awaited_by(task, extra) + self.addCleanup(asyncio.future_discard_from_awaited_by, task, extra) + with self.assertRaisesRegex(RuntimeError, "Too many task waiters"): + unwinder.get_all_awaited_by() + + release.set() + await waiter + + asyncio.run(main()) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst b/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst new file mode 100644 index 00000000000000..f284c37bd73894 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst @@ -0,0 +1,3 @@ +Fix missing tasks in ``python -m asyncio pstree`` output when tasks are +connected through an intermediate :class:`asyncio.Future`. Patch by Timofei +Ivankov diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 787f7921d7d74a..04c0e0ba0200f5 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -534,10 +534,11 @@ parse_task_awaited_by( uintptr_t task_address, PyObject *awaited_by ) { - // Read the entire TaskObj at once char task_obj[SIZEOF_TASK_OBJ]; + size_t size = unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set + + sizeof(char); if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; @@ -571,7 +572,16 @@ process_single_task_node( PyObject *result_item = NULL; PyObject *coroutine_stack = NULL; - tn = parse_task_name(unwinder, task_addr); + char is_task; + // task_is_task = 1 => task + // task_is_task = 0 => future + if (read_char(unwinder, + task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task) < 0) { + goto error; + } + tn = is_task ? parse_task_name(unwinder, task_addr) + : PyUnicode_FromFormat("Future-%p", (void *)task_addr); if (tn == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); goto error; @@ -612,6 +622,7 @@ process_single_task_node( PyStructSequence_SetItem(result_item, 1, tn); // steals ref PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref + PyStructSequence_SetItem(result_item, 4, PyBool_FromLong(!is_task)); // References transferred to tuple task_id = NULL; @@ -652,37 +663,68 @@ process_single_task_node( static int process_task_waiters( RemoteUnwinderObject *unwinder, - PyObject *result + PyObject *result, + bool futures_only ) { + Py_ssize_t initial_count = futures_only ? PyList_GET_SIZE(result) : 0; + PyObject *visited = PySet_New(NULL); + if (visited == NULL) { + return -1; + } for (Py_ssize_t i = 0; i < PyList_GET_SIZE(result); i++) { PyObject *task_info = PyList_GET_ITEM(result, i); + if (PySet_Add(visited, PyStructSequence_GET_ITEM(task_info, 0)) < 0) { + goto error; + } PyObject *waiters = PyStructSequence_GET_ITEM(task_info, 3); for (Py_ssize_t j = 0; j < PyList_GET_SIZE(waiters); j++) { - if (PyList_GET_SIZE(result) >= MAX_TASK_WAITER_WALK_TASKS) { - PyErr_SetString(PyExc_RuntimeError, - "Too many task waiters (possible infinite loop)"); - set_exception_cause(unwinder, PyExc_RuntimeError, - "Task waiter walk size limit exceeded"); - return -1; - } PyObject *waiter = PyList_GET_ITEM(waiters, j); - // CoroInfo item 1 holds the waiter task address stored by parse_task(). PyObject *task_id = PyStructSequence_GET_ITEM(waiter, 1); + int seen = PySet_Contains(visited, task_id); + if (seen > 0) { + continue; + } + if (seen < 0 || PySet_Add(visited, task_id) < 0) { + goto error; + } void *task_ptr = PyLong_AsVoidPtr(task_id); if (task_ptr == NULL && PyErr_Occurred()) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse waiter task ID"); - return -1; + goto error; + } + if (futures_only) { + char is_task; + if (read_char(unwinder, + (uintptr_t)task_ptr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task) < 0) { + goto error; + } + if (is_task) { + continue; + } + } + if (PyList_GET_SIZE(result) - initial_count >= MAX_TASK_WAITER_WALK_TASKS) { + PyErr_SetString(PyExc_RuntimeError, + "Too many task waiters (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Task waiter walk size limit exceeded"); + goto error; } if (process_single_task_node( unwinder, (uintptr_t)task_ptr, NULL, result) < 0) { - return -1; + goto error; } } } + Py_DECREF(visited); return 0; + +error: + Py_DECREF(visited); + return -1; } /* ============================================================================ @@ -927,7 +969,7 @@ append_awaited_by( return -1; } - return 0; + return process_task_waiters(unwinder, task_list, true); } /* ============================================================================ @@ -991,7 +1033,7 @@ process_running_task_chain( } // Now find all tasks that are waiting for this task and process them - if (process_task_waiters(unwinder, result) < 0) { + if (process_task_waiters(unwinder, result, false) < 0) { return -1; } diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 28e5a3ca32ac80..26e4a3c2c2e84b 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -295,14 +295,17 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, "\n" " - thread_id (int): Identifier of the thread, or 0 for tasks in the\n" " interpreter\'s fallback task list.\n" -" - awaited_by (list[TaskInfo]): Tasks registered with this thread.\n" +" - awaited_by (list[TaskInfo]): Tasks registered with this thread and\n" +" intermediate futures in their awaited_by chains.\n" "\n" "Each TaskInfo contains:\n" " - task_id (int): Identifier of the task.\n" -" - task_name (str): Name of the task.\n" +" - task_name (str): Name of the task, or a generated name for\n" +" a future.\n" " - coroutine_stack (list[CoroInfo]): Stack of coroutine frames.\n" " - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures\n" " awaiting this task.\n" +" - is_future (bool, named-only): True for a plain Future.\n" "\n" "Each CoroInfo contains:\n" " - call_stack (list[FrameInfo]): Call stack frames for the coroutine.\n" @@ -367,14 +370,16 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "\n" " - thread_id (int): Identifier of the thread.\n" " - awaited_by (list[TaskInfo]): The running task and all its\n" -" transitive waiters.\n" +" transitive waiters, including intermediate futures.\n" "\n" "Each TaskInfo contains:\n" " - task_id (int): Identifier of the task.\n" -" - task_name (str): Name of the task.\n" +" - task_name (str): Name of the task, or a generated name for\n" +" a future.\n" " - coroutine_stack (list[CoroInfo]): Stack of coroutine frames.\n" " - awaited_by (list[CoroInfo]): Coroutine information for\n" " tasks or futures awaiting this task.\n" +" - is_future (bool, named-only): True for a plain Future.\n" "\n" "Each CoroInfo contains:\n" " - call_stack (list[FrameInfo]): Frames for the coroutine.\n" @@ -408,7 +413,9 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, " coroutine_stack=[\n" " CoroInfo(call_stack=[...], task_name=1)\n" " ],\n" -" awaited_by=[CoroInfo(call_stack=[...], task_name=2)]\n" +" awaited_by=[\n" +" CoroInfo(call_stack=[...], task_name=2)\n" +" ]\n" " ),\n" " TaskInfo(\n" " task_id=2,\n" @@ -1740,4 +1747,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=ef0d1eaac8262c2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2be4cb3f27f6c8d8 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c77ff9ae0abd5b..2031fe0364f4eb 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -34,12 +34,13 @@ static PyStructSequence_Field TaskInfo_fields[] = { {"task_name", "Task name"}, {"coroutine_stack", "Coroutine call stack"}, {"awaited_by", "Tasks awaiting this task"}, + {"is_future", "Whether this node is a plain Future rather than a Task"}, {NULL} }; PyStructSequence_Desc TaskInfo_desc = { "_remote_debugging.TaskInfo", - "Information about an asyncio task", + "Information about an asyncio task or future", TaskInfo_fields, 4 }; @@ -887,14 +888,17 @@ Get awaited_by relationships for tasks in the remote process. - thread_id (int): Identifier of the thread, or 0 for tasks in the interpreter's fallback task list. - - awaited_by (list[TaskInfo]): Tasks registered with this thread. + - awaited_by (list[TaskInfo]): Tasks registered with this thread and + intermediate futures in their awaited_by chains. Each TaskInfo contains: - task_id (int): Identifier of the task. - - task_name (str): Name of the task. + - task_name (str): Name of the task, or a generated name for + a future. - coroutine_stack (list[CoroInfo]): Stack of coroutine frames. - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures awaiting this task. + - is_future (bool, named-only): True for a plain Future. Each CoroInfo contains: - call_stack (list[FrameInfo]): Call stack frames for the coroutine. @@ -925,7 +929,7 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6a49cd345e8aec53 input=79e556b3973d21a8]*/ +/*[clinic end generated code: output=6a49cd345e8aec53 input=17fcd3d7bb835624]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; @@ -988,14 +992,16 @@ running task, tasks waiting for those tasks, etc.). - thread_id (int): Identifier of the thread. - awaited_by (list[TaskInfo]): The running task and all its - transitive waiters. + transitive waiters, including intermediate futures. Each TaskInfo contains: - task_id (int): Identifier of the task. - - task_name (str): Name of the task. + - task_name (str): Name of the task, or a generated name for + a future. - coroutine_stack (list[CoroInfo]): Stack of coroutine frames. - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures awaiting this task. + - is_future (bool, named-only): True for a plain Future. Each CoroInfo contains: - call_stack (list[FrameInfo]): Frames for the coroutine. @@ -1029,7 +1035,9 @@ Example output: coroutine_stack=[ CoroInfo(call_stack=[...], task_name=1) ], - awaited_by=[CoroInfo(call_stack=[...], task_name=2)] + awaited_by=[ + CoroInfo(call_stack=[...], task_name=2) + ] ), TaskInfo( task_id=2, @@ -1046,7 +1054,7 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=0532031a7c495491]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=a95958816358476a]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL;