You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
347 lines
12 KiB
347 lines
12 KiB
"""Bridges between the `asyncio` module and Tornado IOLoop.
|
|
|
|
.. versionadded:: 3.2
|
|
|
|
This module integrates Tornado with the ``asyncio`` module introduced
|
|
in Python 3.4. This makes it possible to combine the two libraries on
|
|
the same event loop.
|
|
|
|
.. deprecated:: 5.0
|
|
|
|
While the code in this module is still used, it is now enabled
|
|
automatically when `asyncio` is available, so applications should
|
|
no longer need to refer to this module directly.
|
|
|
|
.. note::
|
|
|
|
Tornado requires the `~asyncio.AbstractEventLoop.add_reader` family of
|
|
methods, so it is not compatible with the `~asyncio.ProactorEventLoop` on
|
|
Windows. Use the `~asyncio.SelectorEventLoop` instead.
|
|
"""
|
|
|
|
import concurrent.futures
|
|
import functools
|
|
import sys
|
|
|
|
from threading import get_ident
|
|
from tornado.gen import convert_yielded
|
|
from tornado.ioloop import IOLoop, _Selectable
|
|
|
|
import asyncio
|
|
|
|
import typing
|
|
from typing import Any, TypeVar, Awaitable, Callable, Union, Optional
|
|
|
|
if typing.TYPE_CHECKING:
|
|
from typing import Set, Dict, Tuple # noqa: F401
|
|
|
|
_T = TypeVar("_T")
|
|
|
|
|
|
class BaseAsyncIOLoop(IOLoop):
|
|
def initialize( # type: ignore
|
|
self, asyncio_loop: asyncio.AbstractEventLoop, **kwargs: Any
|
|
) -> None:
|
|
self.asyncio_loop = asyncio_loop
|
|
# Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler)
|
|
self.handlers = {} # type: Dict[int, Tuple[Union[int, _Selectable], Callable]]
|
|
# Set of fds listening for reads/writes
|
|
self.readers = set() # type: Set[int]
|
|
self.writers = set() # type: Set[int]
|
|
self.closing = False
|
|
# If an asyncio loop was closed through an asyncio interface
|
|
# instead of IOLoop.close(), we'd never hear about it and may
|
|
# have left a dangling reference in our map. In case an
|
|
# application (or, more likely, a test suite) creates and
|
|
# destroys a lot of event loops in this way, check here to
|
|
# ensure that we don't have a lot of dead loops building up in
|
|
# the map.
|
|
#
|
|
# TODO(bdarnell): consider making self.asyncio_loop a weakref
|
|
# for AsyncIOMainLoop and make _ioloop_for_asyncio a
|
|
# WeakKeyDictionary.
|
|
for loop in list(IOLoop._ioloop_for_asyncio):
|
|
if loop.is_closed():
|
|
del IOLoop._ioloop_for_asyncio[loop]
|
|
IOLoop._ioloop_for_asyncio[asyncio_loop] = self
|
|
|
|
self._thread_identity = 0
|
|
|
|
super(BaseAsyncIOLoop, self).initialize(**kwargs)
|
|
|
|
def assign_thread_identity() -> None:
|
|
self._thread_identity = get_ident()
|
|
|
|
self.add_callback(assign_thread_identity)
|
|
|
|
def close(self, all_fds: bool = False) -> None:
|
|
self.closing = True
|
|
for fd in list(self.handlers):
|
|
fileobj, handler_func = self.handlers[fd]
|
|
self.remove_handler(fd)
|
|
if all_fds:
|
|
self.close_fd(fileobj)
|
|
# Remove the mapping before closing the asyncio loop. If this
|
|
# happened in the other order, we could race against another
|
|
# initialize() call which would see the closed asyncio loop,
|
|
# assume it was closed from the asyncio side, and do this
|
|
# cleanup for us, leading to a KeyError.
|
|
del IOLoop._ioloop_for_asyncio[self.asyncio_loop]
|
|
self.asyncio_loop.close()
|
|
|
|
def add_handler(
|
|
self, fd: Union[int, _Selectable], handler: Callable[..., None], events: int
|
|
) -> None:
|
|
fd, fileobj = self.split_fd(fd)
|
|
if fd in self.handlers:
|
|
raise ValueError("fd %s added twice" % fd)
|
|
self.handlers[fd] = (fileobj, handler)
|
|
if events & IOLoop.READ:
|
|
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
|
|
self.readers.add(fd)
|
|
if events & IOLoop.WRITE:
|
|
self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
|
|
self.writers.add(fd)
|
|
|
|
def update_handler(self, fd: Union[int, _Selectable], events: int) -> None:
|
|
fd, fileobj = self.split_fd(fd)
|
|
if events & IOLoop.READ:
|
|
if fd not in self.readers:
|
|
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
|
|
self.readers.add(fd)
|
|
else:
|
|
if fd in self.readers:
|
|
self.asyncio_loop.remove_reader(fd)
|
|
self.readers.remove(fd)
|
|
if events & IOLoop.WRITE:
|
|
if fd not in self.writers:
|
|
self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
|
|
self.writers.add(fd)
|
|
else:
|
|
if fd in self.writers:
|
|
self.asyncio_loop.remove_writer(fd)
|
|
self.writers.remove(fd)
|
|
|
|
def remove_handler(self, fd: Union[int, _Selectable]) -> None:
|
|
fd, fileobj = self.split_fd(fd)
|
|
if fd not in self.handlers:
|
|
return
|
|
if fd in self.readers:
|
|
self.asyncio_loop.remove_reader(fd)
|
|
self.readers.remove(fd)
|
|
if fd in self.writers:
|
|
self.asyncio_loop.remove_writer(fd)
|
|
self.writers.remove(fd)
|
|
del self.handlers[fd]
|
|
|
|
def _handle_events(self, fd: int, events: int) -> None:
|
|
fileobj, handler_func = self.handlers[fd]
|
|
handler_func(fileobj, events)
|
|
|
|
def start(self) -> None:
|
|
try:
|
|
old_loop = asyncio.get_event_loop()
|
|
except (RuntimeError, AssertionError):
|
|
old_loop = None # type: ignore
|
|
try:
|
|
self._setup_logging()
|
|
asyncio.set_event_loop(self.asyncio_loop)
|
|
self.asyncio_loop.run_forever()
|
|
finally:
|
|
asyncio.set_event_loop(old_loop)
|
|
|
|
def stop(self) -> None:
|
|
self.asyncio_loop.stop()
|
|
|
|
def call_at(
|
|
self, when: float, callback: Callable[..., None], *args: Any, **kwargs: Any
|
|
) -> object:
|
|
# asyncio.call_at supports *args but not **kwargs, so bind them here.
|
|
# We do not synchronize self.time and asyncio_loop.time, so
|
|
# convert from absolute to relative.
|
|
return self.asyncio_loop.call_later(
|
|
max(0, when - self.time()),
|
|
self._run_callback,
|
|
functools.partial(callback, *args, **kwargs),
|
|
)
|
|
|
|
def remove_timeout(self, timeout: object) -> None:
|
|
timeout.cancel() # type: ignore
|
|
|
|
def add_callback(self, callback: Callable, *args: Any, **kwargs: Any) -> None:
|
|
if get_ident() == self._thread_identity:
|
|
call_soon = self.asyncio_loop.call_soon
|
|
else:
|
|
call_soon = self.asyncio_loop.call_soon_threadsafe
|
|
try:
|
|
call_soon(self._run_callback, functools.partial(callback, *args, **kwargs))
|
|
except RuntimeError:
|
|
# "Event loop is closed". Swallow the exception for
|
|
# consistency with PollIOLoop (and logical consistency
|
|
# with the fact that we can't guarantee that an
|
|
# add_callback that completes without error will
|
|
# eventually execute).
|
|
pass
|
|
|
|
def add_callback_from_signal(
|
|
self, callback: Callable, *args: Any, **kwargs: Any
|
|
) -> None:
|
|
try:
|
|
self.asyncio_loop.call_soon_threadsafe(
|
|
self._run_callback, functools.partial(callback, *args, **kwargs)
|
|
)
|
|
except RuntimeError:
|
|
pass
|
|
|
|
def run_in_executor(
|
|
self,
|
|
executor: Optional[concurrent.futures.Executor],
|
|
func: Callable[..., _T],
|
|
*args: Any
|
|
) -> Awaitable[_T]:
|
|
return self.asyncio_loop.run_in_executor(executor, func, *args)
|
|
|
|
def set_default_executor(self, executor: concurrent.futures.Executor) -> None:
|
|
return self.asyncio_loop.set_default_executor(executor)
|
|
|
|
|
|
class AsyncIOMainLoop(BaseAsyncIOLoop):
|
|
"""``AsyncIOMainLoop`` creates an `.IOLoop` that corresponds to the
|
|
current ``asyncio`` event loop (i.e. the one returned by
|
|
``asyncio.get_event_loop()``).
|
|
|
|
.. deprecated:: 5.0
|
|
|
|
Now used automatically when appropriate; it is no longer necessary
|
|
to refer to this class directly.
|
|
|
|
.. versionchanged:: 5.0
|
|
|
|
Closing an `AsyncIOMainLoop` now closes the underlying asyncio loop.
|
|
"""
|
|
|
|
def initialize(self, **kwargs: Any) -> None: # type: ignore
|
|
super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), **kwargs)
|
|
|
|
def make_current(self) -> None:
|
|
# AsyncIOMainLoop already refers to the current asyncio loop so
|
|
# nothing to do here.
|
|
pass
|
|
|
|
|
|
class AsyncIOLoop(BaseAsyncIOLoop):
|
|
"""``AsyncIOLoop`` is an `.IOLoop` that runs on an ``asyncio`` event loop.
|
|
This class follows the usual Tornado semantics for creating new
|
|
``IOLoops``; these loops are not necessarily related to the
|
|
``asyncio`` default event loop.
|
|
|
|
Each ``AsyncIOLoop`` creates a new ``asyncio.EventLoop``; this object
|
|
can be accessed with the ``asyncio_loop`` attribute.
|
|
|
|
.. versionchanged:: 5.0
|
|
|
|
When an ``AsyncIOLoop`` becomes the current `.IOLoop`, it also sets
|
|
the current `asyncio` event loop.
|
|
|
|
.. deprecated:: 5.0
|
|
|
|
Now used automatically when appropriate; it is no longer necessary
|
|
to refer to this class directly.
|
|
"""
|
|
|
|
def initialize(self, **kwargs: Any) -> None: # type: ignore
|
|
self.is_current = False
|
|
loop = asyncio.new_event_loop()
|
|
try:
|
|
super(AsyncIOLoop, self).initialize(loop, **kwargs)
|
|
except Exception:
|
|
# If initialize() does not succeed (taking ownership of the loop),
|
|
# we have to close it.
|
|
loop.close()
|
|
raise
|
|
|
|
def close(self, all_fds: bool = False) -> None:
|
|
if self.is_current:
|
|
self.clear_current()
|
|
super(AsyncIOLoop, self).close(all_fds=all_fds)
|
|
|
|
def make_current(self) -> None:
|
|
if not self.is_current:
|
|
try:
|
|
self.old_asyncio = asyncio.get_event_loop()
|
|
except (RuntimeError, AssertionError):
|
|
self.old_asyncio = None # type: ignore
|
|
self.is_current = True
|
|
asyncio.set_event_loop(self.asyncio_loop)
|
|
|
|
def _clear_current_hook(self) -> None:
|
|
if self.is_current:
|
|
asyncio.set_event_loop(self.old_asyncio)
|
|
self.is_current = False
|
|
|
|
|
|
def to_tornado_future(asyncio_future: asyncio.Future) -> asyncio.Future:
|
|
"""Convert an `asyncio.Future` to a `tornado.concurrent.Future`.
|
|
|
|
.. versionadded:: 4.1
|
|
|
|
.. deprecated:: 5.0
|
|
Tornado ``Futures`` have been merged with `asyncio.Future`,
|
|
so this method is now a no-op.
|
|
"""
|
|
return asyncio_future
|
|
|
|
|
|
def to_asyncio_future(tornado_future: asyncio.Future) -> asyncio.Future:
|
|
"""Convert a Tornado yieldable object to an `asyncio.Future`.
|
|
|
|
.. versionadded:: 4.1
|
|
|
|
.. versionchanged:: 4.3
|
|
Now accepts any yieldable object, not just
|
|
`tornado.concurrent.Future`.
|
|
|
|
.. deprecated:: 5.0
|
|
Tornado ``Futures`` have been merged with `asyncio.Future`,
|
|
so this method is now equivalent to `tornado.gen.convert_yielded`.
|
|
"""
|
|
return convert_yielded(tornado_future)
|
|
|
|
|
|
if sys.platform == "win32" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
|
|
# "Any thread" and "selector" should be orthogonal, but there's not a clean
|
|
# interface for composing policies so pick the right base.
|
|
_BasePolicy = asyncio.WindowsSelectorEventLoopPolicy # type: ignore
|
|
else:
|
|
_BasePolicy = asyncio.DefaultEventLoopPolicy
|
|
|
|
|
|
class AnyThreadEventLoopPolicy(_BasePolicy): # type: ignore
|
|
"""Event loop policy that allows loop creation on any thread.
|
|
|
|
The default `asyncio` event loop policy only automatically creates
|
|
event loops in the main threads. Other threads must create event
|
|
loops explicitly or `asyncio.get_event_loop` (and therefore
|
|
`.IOLoop.current`) will fail. Installing this policy allows event
|
|
loops to be created automatically on any thread, matching the
|
|
behavior of Tornado versions prior to 5.0 (or 5.0 on Python 2).
|
|
|
|
Usage::
|
|
|
|
asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
|
|
|
|
.. versionadded:: 5.0
|
|
|
|
"""
|
|
|
|
def get_event_loop(self) -> asyncio.AbstractEventLoop:
|
|
try:
|
|
return super().get_event_loop()
|
|
except (RuntimeError, AssertionError):
|
|
# This was an AssertionError in python 3.4.2 (which ships with debian jessie)
|
|
# and changed to a RuntimeError in 3.4.3.
|
|
# "There is no current event loop in thread %r"
|
|
loop = self.new_event_loop()
|
|
self.set_event_loop(loop)
|
|
return loop
|