Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Include/internal/pycore_pylifecycle.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc);
extern void _Py_InitVersion(void);
extern PyStatus _PyFaulthandler_Init(int enable);
extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
extern int _PyBuiltin_InitPythonFunctions(PyObject *dict);
extern PyStatus _PySys_Create(
PyThreadState *tstate,
PyObject **sysmod_p);
Expand Down
42 changes: 42 additions & 0 deletions Lib/_pybuiltins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""Builtins implemented in Python.
Comment thread
kumaraditya303 marked this conversation as resolved.

This module is frozen into the interpreter and imported during startup,
before the import system exists. The names listed in ``__all__`` are
copied into the ``builtins`` module.
"""

__all__ = ['anext']

_NOT_GIVEN = sentinel("_NOT_GIVEN")


def anext(async_iterator, default=_NOT_GIVEN, /):
"""Return the next item from the async iterator.

If default is given and the async iterator is exhausted,
it is returned instead of raising StopAsyncIteration.
"""
cls = type(async_iterator)
try:
# Looked up on the type, like the C slot am_anext.
anext_method = cls.__anext__
except AttributeError:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Strictly speaking, there is now a slight divergence where accessing __anext__ could raise something other than an AttributeError and this would leak. Should we worry about those cases? (previoulsy we directly accessed the structs, so we were bypassing getattr).

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

am_next is the underlying slot for __anext__ and the method is being looked up on the type so there isn't much divergence (the instance getattr cannot be triggered) so apart from contrived hand-crafted cases this is safe.

raise TypeError(
f"'{cls.__name__}' object is not an async iterator"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
f"'{cls.__name__}' object is not an async iterator"
f"{cls.__name__!r} object is not an async iterator"

) from None
awaitable = anext_method(async_iterator)
if default is _NOT_GIVEN:
return awaitable
return _anext_with_default(awaitable, default)


async def _anext_with_default(awaitable, default):
try:
return await awaitable
except StopAsyncIteration:
return default


for _name in __all__:
globals()[_name].__module__ = 'builtins'
del _name
22 changes: 18 additions & 4 deletions Lib/test/test_asyncgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import types
import unittest
import contextlib
import warnings

from test.support.import_helper import import_module
from test.support import gc_collect, requires_working_socket
Expand Down Expand Up @@ -709,7 +710,16 @@ def test_send():
async def test_throw():
p = ait_class()
obj = anext(p, "completed")
self.assertRaises(SyntaxError, obj.throw, SyntaxError)
with warnings.catch_warnings():

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't this a change of behavior then? maybe document it as well?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is but not important, throwing or sending something to just created generator always errors so in practice no one does it. The important part is that it raises exception for that and it does.

# Throwing into the unstarted anext() coroutine leaves the
# inner __anext__() awaitable never awaited.
warnings.simplefilter("ignore", RuntimeWarning)
self.assertRaises(SyntaxError, obj.throw, SyntaxError)
if isinstance(p, types.AsyncGeneratorType):
# The never-run asend() already registered the async
# generator with the loop's finalizer; close it explicitly
# so no aclose() task is left pending at loop close.
await p.aclose()
return "completed"

result = self.loop.run_until_complete(test_throw())
Expand Down Expand Up @@ -1132,9 +1142,13 @@ async def agenfn():
yield 'aaa'

agen = agenfn()
with contextlib.closing(anext(agen, "default").__await__()) as g:
with self.assertRaises(MyError):
g.throw(MyError())
with warnings.catch_warnings():
# Throwing into the unstarted anext() coroutine leaves the
# inner asend() awaitable never awaited.
warnings.simplefilter("ignore", RuntimeWarning)
with contextlib.closing(anext(agen, "default").__await__()) as g:
with self.assertRaises(MyError):
g.throw(MyError())

def run_test(test):
with self.subTest('pure-Python anext()'):
Expand Down
40 changes: 40 additions & 0 deletions Lib/test/test_asyncio/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,46 @@ async def main():
'async generator CallStackTestBase.test_stack_async_gen.<locals>.gen()',
stack_for_gen_nested_call[1])

async def test_stack_anext_default(self):
# anext() with a default wraps the awaitable in a coroutine, so the
# call graph of a suspended task sees through it into __anext__().

loop = asyncio.get_running_loop()
blocker = loop.create_future()

async def inner():
await blocker

class AIter:
def __aiter__(self):
return self

async def __anext__(self):
await inner()
return 1

async def main():
await anext(AIter(), None)

task = asyncio.create_task(main(), name='anext task')
await asyncio.sleep(0)
try:
stack = capture_test_stack(fut=task)
finally:
blocker.set_result(None)
await task

self.assertEqual(stack[0], [
'T<anext task>',
[
'a inner',
'a __anext__',
'a _anext_with_default',
'a main',
],
[]
])

def test_ag_frame_used_for_async_generator(self):
# Regression test for gh-148736: the ag_await branch of
# _build_graph_for_future must read ag_frame, not cr_frame.
Expand Down
8 changes: 6 additions & 2 deletions Lib/test/test_coroutines.py
Original file line number Diff line number Diff line change
Expand Up @@ -1312,8 +1312,12 @@ async def __anext__(self):
def __aiter__(self):
return self

with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable:
self.assertRaises(TypeError, anext_awaitable.close, 1)
with warnings.catch_warnings():
# Closing the unstarted anext() coroutine leaves the inner
# __anext__() coroutine never awaited.
warnings.simplefilter("ignore", RuntimeWarning)
with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable:
self.assertRaises(TypeError, anext_awaitable.close, 1)

def test_with_1(self):
class Manager:
Expand Down
3 changes: 2 additions & 1 deletion Lib/test/test_importlib/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ def import_importlib(module_name):
fresh = ('importlib',) if '.' in module_name else ()
frozen = import_helper.import_fresh_module(module_name)
source = import_helper.import_fresh_module(module_name, fresh=fresh,
blocked=('_frozen_importlib', '_frozen_importlib_external'))
blocked=('_frozen_importlib', '_frozen_importlib_external',
'_pybuiltins'))
return {'Frozen': frozen, 'Source': source}


Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_inspect/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -6174,7 +6174,7 @@ def test_builtins_have_signatures(self):
"next", "vars"}
no_signature |= needs_groups
# These have unrepresentable parameter default values of NULL
unsupported_signature = {"anext", "aiter", "iter"}
unsupported_signature = {"aiter", "iter"}
# These need *args support in Argument Clinic
needs_varargs = {"min", "max", "__build_class__"}
no_signature |= needs_varargs
Expand Down
8 changes: 7 additions & 1 deletion Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1623,7 +1623,8 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS)
BOOTSTRAP_HEADERS = \
Python/frozen_modules/importlib._bootstrap.h \
Python/frozen_modules/importlib._bootstrap_external.h \
Python/frozen_modules/zipimport.h
Python/frozen_modules/zipimport.h \
Python/frozen_modules/_pybuiltins.h

Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)

Expand Down Expand Up @@ -1664,6 +1665,7 @@ FROZEN_FILES_IN = \
Lib/importlib/_bootstrap.py \
Lib/importlib/_bootstrap_external.py \
Lib/zipimport.py \
Lib/_pybuiltins.py \
Lib/abc.py \
Lib/codecs.py \
Lib/io.py \
Expand All @@ -1690,6 +1692,7 @@ FROZEN_FILES_OUT = \
Python/frozen_modules/importlib._bootstrap.h \
Python/frozen_modules/importlib._bootstrap_external.h \
Python/frozen_modules/zipimport.h \
Python/frozen_modules/_pybuiltins.h \
Python/frozen_modules/abc.h \
Python/frozen_modules/codecs.h \
Python/frozen_modules/io.h \
Expand Down Expand Up @@ -1735,6 +1738,9 @@ Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_
Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
$(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h

Python/frozen_modules/_pybuiltins.h: Lib/_pybuiltins.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
$(FREEZE_MODULE_BOOTSTRAP) _pybuiltins $(srcdir)/Lib/_pybuiltins.py Python/frozen_modules/_pybuiltins.h

Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Implement :func:`anext` in Python instead of C, in a frozen ``_pybuiltins``

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it's a built-in change I think you should also mention it in What's New 3.16 because it's kind of a breaking change for anyone having a _pybuiltins momdule in their project.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll do that after I do the other ones as well like aiter.

module. The awaitable returned by ``anext(it, default)`` is now a plain
coroutine, so introspection tools such as :func:`asyncio.print_call_graph`
can see through it into :meth:`~object.__anext__`.
Loading
Loading