1
"""Base implementation of event loop.
3
The event loop can be broken up into a multiplexer (the part
4
responsible for notifying us of I/O events) and the event loop proper,
5
which wraps a multiplexer with functionality for scheduling callbacks,
6
immediately or at a given time in the future.
8
Whenever a public API takes a callback, subsequent positional
9
arguments will be passed to the callback if/when it is called. This
10
avoids the proliferation of trivial lambdas implementing closures.
11
Keyword arguments for the callback are not supported; this is a
12
conscious design decision, leaving the door open for keyword arguments
13
to modify the meaning of the API call itself.
18
import collections.abc
19
import concurrent.futures
37
except ImportError: # pragma: no cover
40
from . import constants
41
from . import coroutines
43
from . import exceptions
45
from . import protocols
46
from . import sslproto
47
from . import staggered
49
from . import timeouts
50
from . import transports
52
from .log import logger
55
__all__ = 'BaseEventLoop','Server',
58
# Minimum number of _scheduled timer handles before cleanup of
59
# cancelled handles is performed.
60
_MIN_SCHEDULED_TIMER_HANDLES = 100
62
# Minimum fraction of _scheduled timer handles that are cancelled
63
# before cleanup of cancelled handles is performed.
64
_MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5
67
_HAS_IPv6 = hasattr(socket, 'AF_INET6')
69
# Maximum timeout passed to select to avoid OS limitations
70
MAXIMUM_SELECT_TIMEOUT = 24 * 3600
73
def _format_handle(handle):
75
if isinstance(getattr(cb, '__self__', None), tasks.Task):
77
return repr(cb.__self__)
83
if fd == subprocess.PIPE:
85
elif fd == subprocess.STDOUT:
91
def _set_reuseport(sock):
92
if not hasattr(socket, 'SO_REUSEPORT'):
93
raise ValueError('reuse_port not supported by socket module')
96
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
98
raise ValueError('reuse_port not supported by socket module, '
99
'SO_REUSEPORT defined but not implemented.')
102
def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
103
# Try to skip getaddrinfo if "host" is already an IP. Users might have
104
# handled name resolution in their own code and pass in resolved IPs.
105
if not hasattr(socket, 'inet_pton'):
108
if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \
112
if type == socket.SOCK_STREAM:
113
proto = socket.IPPROTO_TCP
114
elif type == socket.SOCK_DGRAM:
115
proto = socket.IPPROTO_UDP
121
elif isinstance(port, bytes) and port == b'':
123
elif isinstance(port, str) and port == '':
126
# If port's a service name like "http", don't skip getaddrinfo.
129
except (TypeError, ValueError):
132
if family == socket.AF_UNSPEC:
133
afs = [socket.AF_INET]
135
afs.append(socket.AF_INET6)
139
if isinstance(host, bytes):
140
host = host.decode('idna')
142
# Linux's inet_pton doesn't accept an IPv6 zone index after host,
148
socket.inet_pton(af, host)
149
# The host has already been resolved.
150
if _HAS_IPv6 and af == socket.AF_INET6:
151
return af, type, proto, '', (host, port, flowinfo, scopeid)
153
return af, type, proto, '', (host, port)
157
# "host" is not an IP address.
161
def _interleave_addrinfos(addrinfos, first_address_family_count=1):
162
"""Interleave list of addrinfo tuples by family."""
163
# Group addresses by family
164
addrinfos_by_family = collections.OrderedDict()
165
for addr in addrinfos:
167
if family not in addrinfos_by_family:
168
addrinfos_by_family[family] = []
169
addrinfos_by_family[family].append(addr)
170
addrinfos_lists = list(addrinfos_by_family.values())
173
if first_address_family_count > 1:
174
reordered.extend(addrinfos_lists[0][:first_address_family_count - 1])
175
del addrinfos_lists[0][:first_address_family_count - 1]
177
a for a in itertools.chain.from_iterable(
178
itertools.zip_longest(*addrinfos_lists)
183
def _run_until_complete_cb(fut):
184
if not fut.cancelled():
185
exc = fut.exception()
186
if isinstance(exc, (SystemExit, KeyboardInterrupt)):
187
# Issue #22429: run_forever() already finished, no need to
190
futures._get_loop(fut).stop()
193
if hasattr(socket, 'TCP_NODELAY'):
194
def _set_nodelay(sock):
195
if (sock.family in {socket.AF_INET, socket.AF_INET6} and
196
sock.type == socket.SOCK_STREAM and
197
sock.proto == socket.IPPROTO_TCP):
198
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
200
def _set_nodelay(sock):
204
def _check_ssl_socket(sock):
205
if ssl is not None and isinstance(sock, ssl.SSLSocket):
206
raise TypeError("Socket cannot be of type SSLSocket")
209
class _SendfileFallbackProtocol(protocols.Protocol):
210
def __init__(self, transp):
211
if not isinstance(transp, transports._FlowControlMixin):
212
raise TypeError("transport should be _FlowControlMixin instance")
213
self._transport = transp
214
self._proto = transp.get_protocol()
215
self._should_resume_reading = transp.is_reading()
216
self._should_resume_writing = transp._protocol_paused
217
transp.pause_reading()
218
transp.set_protocol(self)
219
if self._should_resume_writing:
220
self._write_ready_fut = self._transport._loop.create_future()
222
self._write_ready_fut = None
224
async def drain(self):
225
if self._transport.is_closing():
226
raise ConnectionError("Connection closed by peer")
227
fut = self._write_ready_fut
232
def connection_made(self, transport):
233
raise RuntimeError("Invalid state: "
234
"connection should have been established already.")
236
def connection_lost(self, exc):
237
if self._write_ready_fut is not None:
238
# Never happens if peer disconnects after sending the whole content
239
# Thus disconnection is always an exception from user perspective
241
self._write_ready_fut.set_exception(
242
ConnectionError("Connection is closed by peer"))
244
self._write_ready_fut.set_exception(exc)
245
self._proto.connection_lost(exc)
247
def pause_writing(self):
248
if self._write_ready_fut is not None:
250
self._write_ready_fut = self._transport._loop.create_future()
252
def resume_writing(self):
253
if self._write_ready_fut is None:
255
self._write_ready_fut.set_result(False)
256
self._write_ready_fut = None
258
def data_received(self, data):
259
raise RuntimeError("Invalid state: reading should be paused")
261
def eof_received(self):
262
raise RuntimeError("Invalid state: reading should be paused")
264
async def restore(self):
265
self._transport.set_protocol(self._proto)
266
if self._should_resume_reading:
267
self._transport.resume_reading()
268
if self._write_ready_fut is not None:
270
# Basically it has no effect because protocol is switched back,
271
# no code should wait for it anymore.
272
self._write_ready_fut.cancel()
273
if self._should_resume_writing:
274
self._proto.resume_writing()
277
class Server(events.AbstractServer):
279
def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
280
ssl_handshake_timeout, ssl_shutdown_timeout=None):
282
self._sockets = sockets
283
# Weak references so we don't break Transport's ability to
284
# detect abandoned transports
285
self._clients = weakref.WeakSet()
287
self._protocol_factory = protocol_factory
288
self._backlog = backlog
289
self._ssl_context = ssl_context
290
self._ssl_handshake_timeout = ssl_handshake_timeout
291
self._ssl_shutdown_timeout = ssl_shutdown_timeout
292
self._serving = False
293
self._serving_forever_fut = None
294
self._context = contextvars.copy_context()
297
return f'<{self.__class__.__name__} sockets={self.sockets!r}>'
299
def _attach(self, transport):
300
assert self._sockets is not None
301
self._clients.add(transport)
303
def _detach(self, transport):
304
self._clients.discard(transport)
305
if len(self._clients) == 0 and self._sockets is None:
309
waiters = self._waiters
311
for waiter in waiters:
312
if not waiter.done():
313
waiter.set_result(None)
315
def _start_serving(self):
319
for sock in self._sockets:
320
sock.listen(self._backlog)
321
self._loop._start_serving(
322
self._protocol_factory, sock, self._ssl_context,
323
self, self._backlog, self._ssl_handshake_timeout,
324
self._ssl_shutdown_timeout, context=self._context)
329
def is_serving(self):
334
if self._sockets is None:
336
return tuple(trsock.TransportSocket(s) for s in self._sockets)
339
sockets = self._sockets
345
self._loop._stop_serving(sock)
347
self._serving = False
349
if (self._serving_forever_fut is not None and
350
not self._serving_forever_fut.done()):
351
self._serving_forever_fut.cancel()
352
self._serving_forever_fut = None
354
if len(self._clients) == 0:
357
def close_clients(self):
358
for transport in self._clients.copy():
361
def abort_clients(self):
362
for transport in self._clients.copy():
365
async def start_serving(self):
366
self._start_serving()
367
# Skip one loop iteration so that all 'loop.add_reader'
371
async def serve_forever(self):
372
if self._serving_forever_fut is not None:
374
f'server {self!r} is already being awaited on serve_forever()')
375
if self._sockets is None:
376
raise RuntimeError(f'server {self!r} is closed')
378
self._start_serving()
379
self._serving_forever_fut = self._loop.create_future()
382
await self._serving_forever_fut
383
except exceptions.CancelledError:
387
await self.wait_closed()
391
self._serving_forever_fut = None
393
async def wait_closed(self):
394
"""Wait until server is closed and all connections are dropped.
396
- If the server is not closed, wait.
397
- If it is closed, but there are still active connections, wait.
399
Anyone waiting here will be unblocked once both conditions
400
(server is closed and all connections have been dropped)
401
have become true, in either order.
403
Historical note: In 3.11 and before, this was broken, returning
404
immediately if the server was already closed, even if there
405
were still active connections. An attempted fix in 3.12.0 was
406
still broken, returning immediately if the server was still
407
open and there were no active connections. Hopefully in 3.12.1
410
# Waiters are unblocked by self._wakeup(), which is called
411
# from two places: self.close() and self._detach(), but only
412
# when both conditions have become true. To signal that this
413
# has happened, self._wakeup() sets self._waiters to None.
414
if self._waiters is None:
416
waiter = self._loop.create_future()
417
self._waiters.append(waiter)
421
class BaseEventLoop(events.AbstractEventLoop):
424
self._timer_cancelled_count = 0
426
self._stopping = False
427
self._ready = collections.deque()
429
self._default_executor = None
430
self._internal_fds = 0
431
# Identifier of the thread running the event loop, or None if the
432
# event loop is not running
433
self._thread_id = None
434
self._clock_resolution = time.get_clock_info('monotonic').resolution
435
self._exception_handler = None
436
self.set_debug(coroutines._is_debug_mode())
437
# The preserved state of async generator hooks.
438
self._old_agen_hooks = None
439
# In debug mode, if the execution of a callback or a step of a task
440
# exceed this duration in seconds, the slow callback/task is logged.
441
self.slow_callback_duration = 0.1
442
self._current_handle = None
443
self._task_factory = None
444
self._coroutine_origin_tracking_enabled = False
445
self._coroutine_origin_tracking_saved_depth = None
447
# A weak set of all asynchronous generators that are
448
# being iterated by the loop.
449
self._asyncgens = weakref.WeakSet()
450
# Set to True when `loop.shutdown_asyncgens` is called.
451
self._asyncgens_shutdown_called = False
452
# Set to True when `loop.shutdown_default_executor` is called.
453
self._executor_shutdown_called = False
457
f'<{self.__class__.__name__} running={self.is_running()} '
458
f'closed={self.is_closed()} debug={self.get_debug()}>'
461
def create_future(self):
462
"""Create a Future object attached to the loop."""
463
1
1
return futures.Future(loop=self)
465
def create_task(self, coro, **kwargs):
466
"""Schedule or begin executing a coroutine object.
468
Return a task object.
471
if self._task_factory is not None:
472
return self._task_factory(self, coro, **kwargs)
474
2
2
task = tasks.Task(coro, loop=self, **kwargs)
475
if task._source_traceback:
476
del task._source_traceback[-1]
480
# gh-128552: prevent a refcycle of
481
# task.exception().__traceback__->BaseEventLoop.create_task->task
484
def set_task_factory(self, factory):
485
"""Set a task factory that will be used by loop.create_task().
487
If factory is None the default task factory will be set.
489
If factory is a callable, it should have a signature matching
490
'(loop, coro, **kwargs)', where 'loop' will be a reference to the
491
active event loop, 'coro' will be a coroutine object, and **kwargs
492
will be arbitrary keyword arguments that should be passed on to
493
Task. The callable must return a Task.
495
if factory is not None and not callable(factory):
496
raise TypeError('task factory must be a callable or None')
497
self._task_factory = factory
499
def get_task_factory(self):
500
"""Return a task factory, or None if the default one is in use."""
501
return self._task_factory
503
def _make_socket_transport(self, sock, protocol, waiter=None, *,
504
extra=None, server=None):
505
"""Create socket transport."""
506
raise NotImplementedError
508
def _make_ssl_transport(
509
self, rawsock, protocol, sslcontext, waiter=None,
510
*, server_side=False, server_hostname=None,
511
extra=None, server=None,
512
ssl_handshake_timeout=None,
513
ssl_shutdown_timeout=None,
514
call_connection_made=True,
516
"""Create SSL transport."""
517
raise NotImplementedError
519
def _make_datagram_transport(self, sock, protocol,
520
address=None, waiter=None, extra=None):
521
"""Create datagram transport."""
522
raise NotImplementedError
524
def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
526
"""Create read pipe transport."""
527
raise NotImplementedError
529
def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
531
"""Create write pipe transport."""
532
raise NotImplementedError
534
async def _make_subprocess_transport(self, protocol, args, shell,
535
stdin, stdout, stderr, bufsize,
536
extra=None, **kwargs):
537
"""Create subprocess transport."""
538
raise NotImplementedError
540
def _write_to_self(self):
541
"""Write a byte to self-pipe, to wake up the event loop.
543
This may be called from a different thread.
545
The subclass is responsible for implementing the self-pipe.
547
raise NotImplementedError
549
def _process_events(self, event_list):
550
"""Process selector events."""
551
raise NotImplementedError
553
def _check_closed(self):
555
raise RuntimeError('Event loop is closed')
557
def _check_default_executor(self):
558
if self._executor_shutdown_called:
559
raise RuntimeError('Executor shutdown has been called')
561
def _asyncgen_finalizer_hook(self, agen):
562
self._asyncgens.discard(agen)
563
if not self.is_closed():
564
self.call_soon_threadsafe(self.create_task, agen.aclose())
566
def _asyncgen_firstiter_hook(self, agen):
567
if self._asyncgens_shutdown_called:
569
f"asynchronous generator {agen!r} was scheduled after "
570
f"loop.shutdown_asyncgens() call",
571
ResourceWarning, source=self)
573
self._asyncgens.add(agen)
575
async def shutdown_asyncgens(self):
576
"""Shutdown all active asynchronous generators."""
577
self._asyncgens_shutdown_called = True
579
if not len(self._asyncgens):
580
# If Python version is <3.6 or we don't have any asynchronous
584
closing_agens = list(self._asyncgens)
585
self._asyncgens.clear()
587
results = await tasks.gather(
588
*[ag.aclose() for ag in closing_agens],
589
return_exceptions=True)
591
for result, agen in zip(results, closing_agens):
592
if isinstance(result, Exception):
593
self.call_exception_handler({
594
'message': f'an error occurred during closing of '
595
f'asynchronous generator {agen!r}',
600
async def shutdown_default_executor(self, timeout=None):
601
"""Schedule the shutdown of the default executor.
603
The timeout parameter specifies the amount of time the executor will
604
be given to finish joining. The default value is None, which means
605
that the executor will be given an unlimited amount of time.
607
self._executor_shutdown_called = True
608
if self._default_executor is None:
610
future = self.create_future()
611
thread = threading.Thread(target=self._do_shutdown, args=(future,))
614
async with timeouts.timeout(timeout):
617
warnings.warn("The executor did not finishing joining "
618
f"its threads within {timeout} seconds.",
619
RuntimeWarning, stacklevel=2)
620
self._default_executor.shutdown(wait=False)
624
def _do_shutdown(self, future):
626
self._default_executor.shutdown(wait=True)
627
if not self.is_closed():
628
self.call_soon_threadsafe(futures._set_result_unless_cancelled,
630
except Exception as ex:
631
if not self.is_closed() and not future.cancelled():
632
self.call_soon_threadsafe(future.set_exception, ex)
634
def _check_running(self):
635
if self.is_running():
636
raise RuntimeError('This event loop is already running')
637
if events._get_running_loop() is not None:
639
'Cannot run the event loop while another loop is running')
641
def _run_forever_setup(self):
642
"""Prepare the run loop to process events.
644
This method exists so that custom event loop subclasses (e.g., event loops
645
that integrate a GUI event loop with Python's event loop) have access to all the
649
self._check_running()
650
self._set_coroutine_origin_tracking(self._debug)
652
self._old_agen_hooks = sys.get_asyncgen_hooks()
653
self._thread_id = threading.get_ident()
654
sys.set_asyncgen_hooks(
655
firstiter=self._asyncgen_firstiter_hook,
656
finalizer=self._asyncgen_finalizer_hook
659
events._set_running_loop(self)
661
def _run_forever_cleanup(self):
662
"""Clean up after an event loop finishes the looping over events.
664
This method exists so that custom event loop subclasses (e.g., event loops
665
that integrate a GUI event loop with Python's event loop) have access to all the
668
self._stopping = False
669
self._thread_id = None
670
events._set_running_loop(None)
671
self._set_coroutine_origin_tracking(False)
672
# Restore any pre-existing async generator hooks.
673
if self._old_agen_hooks is not None:
674
sys.set_asyncgen_hooks(*self._old_agen_hooks)
675
self._old_agen_hooks = None
677
def run_forever(self):
678
"""Run until stop() is called."""
679
self._run_forever_setup()
682
2
750
self._run_once()
686
self._run_forever_cleanup()
688
def run_until_complete(self, future):
689
"""Run until the Future is done.
691
If the argument is a coroutine, it is wrapped in a Task.
693
WARNING: It would be disastrous to call run_until_complete()
694
with the same coroutine twice -- it would wrap it in two
695
different Tasks and that can't be good.
697
Return the Future's result, or raise its exception.
700
self._check_running()
702
new_task = not futures.isfuture(future)
703
future = tasks.ensure_future(future, loop=self)
705
# An exception is raised if the future didn't complete, so there
706
# is no need to log the "destroy pending task" message
707
future._log_destroy_pending = False
709
future.add_done_callback(_run_until_complete_cb)
711
750
self.run_forever()
713
if new_task and future.done() and not future.cancelled():
714
# The coroutine raised a BaseException. Consume the exception
715
# to not log a warning, the caller doesn't have access to the
720
future.remove_done_callback(_run_until_complete_cb)
721
if not future.done():
722
raise RuntimeError('Event loop stopped before Future completed.')
724
return future.result()
727
"""Stop running the event loop.
729
Every callback already scheduled will still run. This simply
730
informs run_forever to stop looping after a complete iteration.
732
self._stopping = True
735
"""Close the event loop.
737
This clears the queues and shuts down the executor,
738
but does not wait for the executor to finish.
740
The event loop must not be running.
742
if self.is_running():
743
raise RuntimeError("Cannot close a running event loop")
747
logger.debug("Close %r", self)
750
self._scheduled.clear()
751
self._executor_shutdown_called = True
752
executor = self._default_executor
753
if executor is not None:
754
self._default_executor = None
755
executor.shutdown(wait=False)
758
"""Returns True if the event loop was closed."""
761
def __del__(self, _warn=warnings.warn):
762
if not self.is_closed():
763
_warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
764
if not self.is_running():
767
def is_running(self):
768
"""Returns True if the event loop is running."""
769
return (self._thread_id is not None)
771
1
1
def time(self):
772
"""Return the time according to the event loop's clock.
774
This is a float expressed in seconds since an epoch, but the
775
epoch, precision, accuracy and drift are unspecified and may
776
differ per event loop.
778
11
11
return time.monotonic()
780
def call_later(self, delay, callback, *args, context=None):
781
"""Arrange for a callback to be called at a given time.
783
Return a Handle: an opaque object with a cancel() method that
784
can be used to cancel the call.
786
The delay can be an int or float, expressed in seconds. It is
787
always relative to the current time.
789
Each callback will be called exactly once. If two callbacks
790
are scheduled for exactly the same time, it is undefined which
791
will be called first.
793
Any positional arguments after the callback will be passed to
794
the callback when it is called.
797
raise TypeError('delay must not be None')
798
timer = self.call_at(self.time() + delay, callback, *args,
800
if timer._source_traceback:
801
del timer._source_traceback[-1]
804
def call_at(self, when, callback, *args, context=None):
805
"""Like call_later(), but uses an absolute time.
807
Absolute time corresponds to the event loop's time() method.
810
raise TypeError("when cannot be None")
814
self._check_callback(callback, 'call_at')
815
timer = events.TimerHandle(when, callback, args, self, context)
816
if timer._source_traceback:
817
del timer._source_traceback[-1]
818
heapq.heappush(self._scheduled, timer)
819
timer._scheduled = True
822
def call_soon(self, callback, *args, context=None):
823
"""Arrange for a callback to be called as soon as possible.
825
This operates as a FIFO queue: callbacks are called in the
826
order in which they are registered. Each callback will be
829
Any positional arguments after the callback will be passed to
830
the callback when it is called.
835
self._check_callback(callback, 'call_soon')
836
2
2
handle = self._call_soon(callback, args, context)
837
if handle._source_traceback:
838
del handle._source_traceback[-1]
841
def _check_callback(self, callback, method):
842
if (coroutines.iscoroutine(callback) or
843
coroutines._iscoroutinefunction(callback)):
845
f"coroutines cannot be used with {method}()")
846
if not callable(callback):
848
f'a callable object was expected by {method}(), '
851
def _call_soon(self, callback, args, context):
852
handle = events.Handle(callback, args, self, context)
853
if handle._source_traceback:
854
del handle._source_traceback[-1]
855
self._ready.append(handle)
858
def _check_thread(self):
859
"""Check that the current thread is the thread running the event loop.
861
Non-thread-safe methods of this class make this assumption and will
862
likely behave incorrectly when the assumption is violated.
864
Should only be called when (self._debug == True). The caller is
865
responsible for checking this condition for performance reasons.
867
if self._thread_id is None:
869
thread_id = threading.get_ident()
870
if thread_id != self._thread_id:
872
"Non-thread-safe operation invoked on an event loop other "
873
"than the current one")
875
def call_soon_threadsafe(self, callback, *args, context=None):
876
"""Like call_soon(), but thread-safe."""
879
self._check_callback(callback, 'call_soon_threadsafe')
880
1
2
handle = events._ThreadSafeHandle(callback, args, self, context)
881
self._ready.append(handle)
882
if handle._source_traceback:
883
del handle._source_traceback[-1]
884
if handle._source_traceback:
885
del handle._source_traceback[-1]
886
self._write_to_self()
889
def run_in_executor(self, executor, func, *args):
892
self._check_callback(func, 'run_in_executor')
894
executor = self._default_executor
895
# Only check when the default executor is being used
896
self._check_default_executor()
898
executor = concurrent.futures.ThreadPoolExecutor(
899
thread_name_prefix='asyncio'
901
self._default_executor = executor
902
return futures.wrap_future(
903
executor.submit(func, *args), loop=self)
905
def set_default_executor(self, executor):
906
if not isinstance(executor, concurrent.futures.ThreadPoolExecutor):
907
raise TypeError('executor must be ThreadPoolExecutor instance')
908
self._default_executor = executor
910
def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
911
msg = [f"{host}:{port!r}"]
913
msg.append(f'family={family!r}')
915
msg.append(f'type={type!r}')
917
msg.append(f'proto={proto!r}')
919
msg.append(f'flags={flags!r}')
921
logger.debug('Get address info %s', msg)
924
addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags)
925
dt = self.time() - t0
927
msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}'
928
if dt >= self.slow_callback_duration:
934
async def getaddrinfo(self, host, port, *,
935
family=0, type=0, proto=0, flags=0):
937
getaddr_func = self._getaddrinfo_debug
939
getaddr_func = socket.getaddrinfo
941
return await self.run_in_executor(
942
None, getaddr_func, host, port, family, type, proto, flags)
944
async def getnameinfo(self, sockaddr, flags=0):
945
return await self.run_in_executor(
946
None, socket.getnameinfo, sockaddr, flags)
948
async def sock_sendfile(self, sock, file, offset=0, count=None,
950
if self._debug and sock.gettimeout() != 0:
951
raise ValueError("the socket must be non-blocking")
952
_check_ssl_socket(sock)
953
self._check_sendfile_params(sock, file, offset, count)
955
return await self._sock_sendfile_native(sock, file,
957
except exceptions.SendfileNotAvailableError:
960
return await self._sock_sendfile_fallback(sock, file,
963
async def _sock_sendfile_native(self, sock, file, offset, count):
964
# NB: sendfile syscall is not supported for SSL sockets and
965
# non-mmap files even if sendfile is supported by OS
966
raise exceptions.SendfileNotAvailableError(
967
f"syscall sendfile is not available for socket {sock!r} "
968
f"and file {file!r} combination")
970
async def _sock_sendfile_fallback(self, sock, file, offset, count):
971
if hasattr(file, 'seek'):
974
min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
975
if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
977
buf = bytearray(blocksize)
982
blocksize = min(count - total_sent, blocksize)
985
view = memoryview(buf)[:blocksize]
986
read = await self.run_in_executor(None, file.readinto, view)
989
await self.sock_sendall(sock, view[:read])
993
if total_sent > 0 and hasattr(file, 'seek'):
994
file.seek(offset + total_sent)
996
def _check_sendfile_params(self, sock, file, offset, count):
997
if 'b' not in getattr(file, 'mode', 'b'):
998
raise ValueError("file should be opened in binary mode")
999
if not sock.type == socket.SOCK_STREAM:
1000
raise ValueError("only SOCK_STREAM type sockets are supported")
1001
if count is not None:
1002
if not isinstance(count, int):
1004
"count must be a positive integer (got {!r})".format(count))
1007
"count must be a positive integer (got {!r})".format(count))
1008
if not isinstance(offset, int):
1010
"offset must be a non-negative integer (got {!r})".format(
1014
"offset must be a non-negative integer (got {!r})".format(
1017
async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None):
1018
"""Create, bind and connect one socket."""
1020
exceptions.append(my_exceptions)
1021
family, type_, proto, _, address = addr_info
1025
sock = socket.socket(family=family, type=type_, proto=proto)
1026
sock.setblocking(False)
1027
if local_addr_infos is not None:
1028
for lfamily, _, _, _, laddr in local_addr_infos:
1029
# skip local addresses of different family
1030
if lfamily != family:
1035
except OSError as exc:
1037
f'error while attempting to bind on '
1038
f'address {laddr!r}: {str(exc).lower()}'
1040
exc = OSError(exc.errno, msg)
1041
my_exceptions.append(exc)
1042
else: # all bind attempts failed
1044
raise my_exceptions.pop()
1046
raise OSError(f"no matching local address with {family=} found")
1047
await self.sock_connect(sock, address)
1049
except OSError as exc:
1050
my_exceptions.append(exc)
1053
if sock is not None:
1057
# An error when closing a newly created socket is
1058
# not important, but it can overwrite more important
1059
# non-OSError error. So ignore it.
1063
exceptions = my_exceptions = None
1065
async def create_connection(
1066
self, protocol_factory, host=None, port=None,
1067
*, ssl=None, family=0,
1068
proto=0, flags=0, sock=None,
1069
local_addr=None, server_hostname=None,
1070
ssl_handshake_timeout=None,
1071
ssl_shutdown_timeout=None,
1072
happy_eyeballs_delay=None, interleave=None,
1074
"""Connect to a TCP server.
1076
Create a streaming transport connection to a given internet host and
1077
port: socket family AF_INET or socket.AF_INET6 depending on host (or
1078
family if specified), socket type SOCK_STREAM. protocol_factory must
1079
be a callable returning a protocol instance.
1081
This method is a coroutine which will try to establish the
1082
connection in the background. When successful, the coroutine
1083
returns a (transport, protocol) pair.
1085
if server_hostname is not None and not ssl:
1086
raise ValueError('server_hostname is only meaningful with ssl')
1088
if server_hostname is None and ssl:
1089
# Use host as default for server_hostname. It is an error
1090
# if host is empty or not set, e.g. when an
1091
# already-connected socket was passed or when only a port
1092
# is given. To avoid this error, you can pass
1093
# server_hostname='' -- this will bypass the hostname
1094
# check. (This also means that if host is a numeric
1095
# IP/IPv6 address, we will attempt to verify that exact
1096
# address; this will probably fail, but it is possible to
1097
# create a certificate for a specific IP address, so we
1098
# don't judge it here.)
1100
raise ValueError('You must set server_hostname '
1101
'when using ssl without a host')
1102
server_hostname = host
1104
if ssl_handshake_timeout is not None and not ssl:
1106
'ssl_handshake_timeout is only meaningful with ssl')
1108
if ssl_shutdown_timeout is not None and not ssl:
1110
'ssl_shutdown_timeout is only meaningful with ssl')
1112
if sock is not None:
1113
_check_ssl_socket(sock)
1115
if happy_eyeballs_delay is not None and interleave is None:
1116
# If using happy eyeballs, default to interleave addresses by family
1119
if host is not None or port is not None:
1120
if sock is not None:
1122
'host/port and sock can not be specified at the same time')
1124
infos = await self._ensure_resolved(
1125
(host, port), family=family,
1126
type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self)
1128
raise OSError('getaddrinfo() returned empty list')
1130
if local_addr is not None:
1131
laddr_infos = await self._ensure_resolved(
1132
local_addr, family=family,
1133
type=socket.SOCK_STREAM, proto=proto,
1134
flags=flags, loop=self)
1136
raise OSError('getaddrinfo() returned empty list')
1141
infos = _interleave_addrinfos(infos, interleave)
1144
if happy_eyeballs_delay is None:
1145
# not using happy eyeballs
1146
for addrinfo in infos:
1148
sock = await self._connect_sock(
1149
exceptions, addrinfo, laddr_infos)
1153
else: # using happy eyeballs
1154
sock = (await staggered.staggered_race(
1156
# can't use functools.partial as it keeps a reference
1158
lambda addrinfo=addrinfo: self._connect_sock(
1159
exceptions, addrinfo, laddr_infos
1161
for addrinfo in infos
1163
happy_eyeballs_delay,
1165
))[0] # can't use sock, _, _ as it keeks a reference to exceptions
1168
exceptions = [exc for sub in exceptions for exc in sub]
1171
raise ExceptionGroup("create_connection failed", exceptions)
1172
if len(exceptions) == 1:
1175
# If they all have the same str(), raise one.
1176
model = str(exceptions[0])
1177
if all(str(exc) == model for exc in exceptions):
1179
# Raise a combined exception so the user can see all
1180
# the various error messages.
1181
raise OSError('Multiple exceptions: {}'.format(
1182
', '.join(str(exc) for exc in exceptions)))
1184
# No exceptions were collected, raise a timeout error
1185
raise TimeoutError('create_connection failed')
1192
'host and port was not specified and no sock specified')
1193
if sock.type != socket.SOCK_STREAM:
1194
# We allow AF_INET, AF_INET6, AF_UNIX as long as they
1196
# We support passing AF_UNIX sockets even though we have
1197
# a dedicated API for that: create_unix_connection.
1198
# Disallowing AF_UNIX in this method, breaks backwards
1201
f'A Stream Socket was expected, got {sock!r}')
1203
transport, protocol = await self._create_connection_transport(
1204
sock, protocol_factory, ssl, server_hostname,
1205
ssl_handshake_timeout=ssl_handshake_timeout,
1206
ssl_shutdown_timeout=ssl_shutdown_timeout)
1208
# Get the socket from the transport because SSL transport closes
1209
# the old socket and creates a new SSL socket
1210
sock = transport.get_extra_info('socket')
1211
logger.debug("%r connected to %s:%r: (%r, %r)",
1212
sock, host, port, transport, protocol)
1213
return transport, protocol
1215
async def _create_connection_transport(
1216
self, sock, protocol_factory, ssl,
1217
server_hostname, server_side=False,
1218
ssl_handshake_timeout=None,
1219
ssl_shutdown_timeout=None, context=None):
1222
sock.setblocking(False)
1223
context = context if context is not None else contextvars.copy_context()
1225
protocol = protocol_factory()
1226
waiter = self.create_future()
1228
sslcontext = None if isinstance(ssl, bool) else ssl
1229
transport = self._make_ssl_transport(
1230
sock, protocol, sslcontext, waiter,
1231
server_side=server_side, server_hostname=server_hostname,
1232
ssl_handshake_timeout=ssl_handshake_timeout,
1233
ssl_shutdown_timeout=ssl_shutdown_timeout,
1236
transport = self._make_socket_transport(sock, protocol, waiter, context=context)
1238
# gh-153133: close the socket if the transport is never created.
1248
return transport, protocol
1250
async def sendfile(self, transport, file, offset=0, count=None,
1252
"""Send a file to transport.
1254
Return the total number of bytes which were sent.
1256
The method uses high-performance os.sendfile if available.
1258
file must be a regular file object opened in binary mode.
1260
offset tells from where to start reading the file. If specified,
1261
count is the total number of bytes to transmit as opposed to
1262
sending the file until EOF is reached. File position is updated on
1263
return or also in case of error in which case file.tell()
1264
can be used to figure out the number of bytes
1267
fallback set to True makes asyncio to manually read and send
1268
the file when the platform does not support the sendfile syscall
1269
(e.g. Windows or SSL socket on Unix).
1271
Raise SendfileNotAvailableError if the system does not support
1272
sendfile syscall and fallback is False.
1274
if transport.is_closing():
1275
raise RuntimeError("Transport is closing")
1276
mode = getattr(transport, '_sendfile_compatible',
1277
constants._SendfileMode.UNSUPPORTED)
1278
if mode is constants._SendfileMode.UNSUPPORTED:
1280
f"sendfile is not supported for transport {transport!r}")
1281
if mode is constants._SendfileMode.TRY_NATIVE:
1283
return await self._sendfile_native(transport, file,
1285
except exceptions.SendfileNotAvailableError:
1291
f"fallback is disabled and native sendfile is not "
1292
f"supported for transport {transport!r}")
1293
return await self._sendfile_fallback(transport, file,
1296
async def _sendfile_native(self, transp, file, offset, count):
1297
raise exceptions.SendfileNotAvailableError(
1298
"sendfile syscall is not supported")
1300
async def _sendfile_fallback(self, transp, file, offset, count):
1301
if hasattr(file, 'seek'):
1303
blocksize = min(count, 16384) if count else 16384
1304
buf = bytearray(blocksize)
1306
proto = _SendfileFallbackProtocol(transp)
1310
blocksize = min(count - total_sent, blocksize)
1313
view = memoryview(buf)[:blocksize]
1314
read = await self.run_in_executor(None, file.readinto, view)
1316
return total_sent # EOF
1317
transp.write(view[:read])
1321
if total_sent > 0 and hasattr(file, 'seek'):
1322
file.seek(offset + total_sent)
1323
await proto.restore()
1325
async def start_tls(self, transport, protocol, sslcontext, *,
1327
server_hostname=None,
1328
ssl_handshake_timeout=None,
1329
ssl_shutdown_timeout=None):
1330
"""Upgrade transport to TLS.
1332
Return a new transport that *protocol* should start using
1336
raise RuntimeError('Python ssl module is not available')
1338
if not isinstance(sslcontext, ssl.SSLContext):
1340
f'sslcontext is expected to be an instance of ssl.SSLContext, '
1341
f'got {sslcontext!r}')
1343
if not getattr(transport, '_start_tls_compatible', False):
1345
f'transport {transport!r} is not supported by start_tls()')
1347
waiter = self.create_future()
1348
ssl_protocol = sslproto.SSLProtocol(
1349
self, protocol, sslcontext, waiter,
1350
server_side, server_hostname,
1351
ssl_handshake_timeout=ssl_handshake_timeout,
1352
ssl_shutdown_timeout=ssl_shutdown_timeout,
1353
call_connection_made=False)
1355
# Pause early so that "ssl_protocol.data_received()" doesn't
1356
# have a chance to get called before "ssl_protocol.connection_made()".
1357
transport.pause_reading()
1359
# gh-142352: move buffered StreamReader data to SSLProtocol
1361
from .streams import StreamReaderProtocol
1362
if isinstance(protocol, StreamReaderProtocol):
1363
stream_reader = getattr(protocol, '_stream_reader', None)
1364
if stream_reader is not None:
1365
buffer = stream_reader._buffer
1367
ssl_protocol._incoming.write(buffer)
1370
transport.set_protocol(ssl_protocol)
1371
conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
1372
resume_cb = self.call_soon(transport.resume_reading)
1376
except BaseException:
1382
return ssl_protocol._app_transport
1384
async def create_datagram_endpoint(self, protocol_factory,
1385
local_addr=None, remote_addr=None, *,
1386
family=0, proto=0, flags=0,
1388
allow_broadcast=None, sock=None):
1389
"""Create datagram connection."""
1390
if sock is not None:
1391
if sock.type == socket.SOCK_STREAM:
1393
f'A datagram socket was expected, got {sock!r}')
1394
if (local_addr or remote_addr or
1395
family or proto or flags or
1396
reuse_port or allow_broadcast):
1397
# show the problematic kwargs in exception msg
1398
opts = dict(local_addr=local_addr, remote_addr=remote_addr,
1399
family=family, proto=proto, flags=flags,
1400
reuse_port=reuse_port,
1401
allow_broadcast=allow_broadcast)
1402
problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v)
1404
f'socket modifier keyword arguments can not be used '
1405
f'when sock is specified. ({problems})')
1406
sock.setblocking(False)
1409
if not (local_addr or remote_addr):
1411
raise ValueError('unexpected address family')
1412
addr_pairs_info = (((family, proto), (None, None)),)
1413
elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX:
1414
for addr in (local_addr, remote_addr):
1415
if addr is not None and not isinstance(addr, str):
1416
raise TypeError('string is expected')
1418
if local_addr and local_addr[0] not in (0, '\x00'):
1420
if stat.S_ISSOCK(os.stat(local_addr).st_mode):
1421
os.remove(local_addr)
1422
except FileNotFoundError:
1424
except OSError as err:
1425
# Directory may have permissions only to create socket.
1426
logger.error('Unable to check or remove stale UNIX '
1430
addr_pairs_info = (((family, proto),
1431
(local_addr, remote_addr)), )
1433
# join address by (family, protocol)
1434
addr_infos = {} # Using order preserving dict
1435
for idx, addr in ((0, local_addr), (1, remote_addr)):
1436
if addr is not None:
1437
if not (isinstance(addr, tuple) and len(addr) == 2):
1438
raise TypeError('2-tuple is expected')
1440
infos = await self._ensure_resolved(
1441
addr, family=family, type=socket.SOCK_DGRAM,
1442
proto=proto, flags=flags, loop=self)
1444
raise OSError('getaddrinfo() returned empty list')
1446
for fam, _, pro, _, address in infos:
1448
if key not in addr_infos:
1449
addr_infos[key] = [None, None]
1450
addr_infos[key][idx] = address
1452
# each addr has to have info for each (family, proto) pair
1454
(key, addr_pair) for key, addr_pair in addr_infos.items()
1455
if not ((local_addr and addr_pair[0] is None) or
1456
(remote_addr and addr_pair[1] is None))]
1458
if not addr_pairs_info:
1459
raise ValueError('can not get address information')
1463
for ((family, proto),
1464
(local_address, remote_address)) in addr_pairs_info:
1468
sock = socket.socket(
1469
family=family, type=socket.SOCK_DGRAM, proto=proto)
1471
_set_reuseport(sock)
1474
socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
1475
sock.setblocking(False)
1478
sock.bind(local_address)
1480
if not allow_broadcast:
1481
await self.sock_connect(sock, remote_address)
1482
r_addr = remote_address
1483
except OSError as exc:
1484
if sock is not None:
1486
exceptions.append(exc)
1488
if sock is not None:
1496
protocol = protocol_factory()
1497
waiter = self.create_future()
1498
transport = self._make_datagram_transport(
1499
sock, protocol, r_addr, waiter)
1502
logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
1503
"created: (%r, %r)",
1504
local_addr, remote_addr, transport, protocol)
1506
logger.debug("Datagram endpoint remote_addr=%r created: "
1508
remote_addr, transport, protocol)
1516
return transport, protocol
1518
async def _ensure_resolved(self, address, *,
1519
family=0, type=socket.SOCK_STREAM,
1520
proto=0, flags=0, loop):
1521
host, port = address[:2]
1522
info = _ipaddr_info(host, port, family, type, proto, *address[2:])
1523
if info is not None:
1524
# "host" is already a resolved IP.
1527
return await loop.getaddrinfo(host, port, family=family, type=type,
1528
proto=proto, flags=flags)
1530
async def _create_server_getaddrinfo(self, host, port, family, flags):
1531
infos = await self._ensure_resolved((host, port), family=family,
1532
type=socket.SOCK_STREAM,
1533
flags=flags, loop=self)
1535
raise OSError(f'getaddrinfo({host!r}) returned empty list')
1538
async def create_server(
1539
self, protocol_factory, host=None, port=None,
1541
family=socket.AF_UNSPEC,
1542
flags=socket.AI_PASSIVE,
1549
ssl_handshake_timeout=None,
1550
ssl_shutdown_timeout=None,
1551
start_serving=True):
1552
"""Create a TCP server.
1554
The host parameter can be a string, in that case the TCP server is
1555
bound to host and port.
1557
The host parameter can also be a sequence of strings and in that
1558
case the TCP server is bound to all hosts of the sequence. If
1559
a host appears multiple times (possibly indirectly e.g. when
1560
hostnames resolve to the same IP address), the server is only bound
1563
Return a Server object which can be used to stop the service.
1565
This method is a coroutine.
1567
if isinstance(ssl, bool):
1568
raise TypeError('ssl argument must be an SSLContext or None')
1570
if ssl_handshake_timeout is not None and ssl is None:
1572
'ssl_handshake_timeout is only meaningful with ssl')
1574
if ssl_shutdown_timeout is not None and ssl is None:
1576
'ssl_shutdown_timeout is only meaningful with ssl')
1578
if sock is not None:
1579
_check_ssl_socket(sock)
1581
if host is not None or port is not None:
1582
if sock is not None:
1584
'host/port and sock can not be specified at the same time')
1586
if reuse_address is None:
1587
reuse_address = os.name == "posix" and sys.platform != "cygwin"
1591
elif (isinstance(host, str) or
1592
not isinstance(host, collections.abc.Iterable)):
1597
fs = [self._create_server_getaddrinfo(host, port, family=family,
1600
infos = await tasks.gather(*fs)
1601
infos = set(itertools.chain.from_iterable(infos))
1606
af, socktype, proto, canonname, sa = res
1608
sock = socket.socket(af, socktype, proto)
1609
except socket.error:
1610
# Assume it's a bad family/type/protocol combination.
1612
logger.warning('create_server() failed to create '
1613
'socket.socket(%r, %r, %r)',
1614
af, socktype, proto, exc_info=True)
1616
sockets.append(sock)
1619
socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
1620
# Since Linux 6.12.9, SO_REUSEPORT is not allowed
1621
# on other address families than AF_INET/AF_INET6.
1622
if reuse_port and af in (socket.AF_INET, socket.AF_INET6):
1623
_set_reuseport(sock)
1626
socket.SOL_SOCKET, socket.SO_KEEPALIVE, True)
1627
# Disable IPv4/IPv6 dual stack support (enabled by
1628
# default on Linux) which makes a single socket
1629
# listen on both address families.
1631
af == socket.AF_INET6 and
1632
hasattr(socket, 'IPPROTO_IPV6')):
1633
sock.setsockopt(socket.IPPROTO_IPV6,
1638
except OSError as err:
1639
msg = ('error while attempting '
1640
'to bind on address %r: %s'
1641
% (sa, str(err).lower()))
1642
if err.errno == errno.EADDRNOTAVAIL:
1643
# Assume the family is not enabled (bpo-30945)
1649
raise OSError(err.errno, msg) from None
1652
raise OSError('could not bind on any address out of %r'
1653
% ([info[4] for info in infos],))
1658
for sock in sockets:
1662
raise ValueError('Neither host/port nor sock were specified')
1663
if sock.type != socket.SOCK_STREAM:
1664
raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1667
for sock in sockets:
1668
sock.setblocking(False)
1670
server = Server(self, sockets, protocol_factory,
1671
ssl, backlog, ssl_handshake_timeout,
1672
ssl_shutdown_timeout)
1674
server._start_serving()
1675
# Skip one loop iteration so that all 'loop.add_reader'
1677
await tasks.sleep(0)
1680
logger.info("%r is serving", server)
1683
async def connect_accepted_socket(
1684
self, protocol_factory, sock,
1686
ssl_handshake_timeout=None,
1687
ssl_shutdown_timeout=None):
1688
if sock.type != socket.SOCK_STREAM:
1689
raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1691
if ssl_handshake_timeout is not None and not ssl:
1693
'ssl_handshake_timeout is only meaningful with ssl')
1695
if ssl_shutdown_timeout is not None and not ssl:
1697
'ssl_shutdown_timeout is only meaningful with ssl')
1699
_check_ssl_socket(sock)
1701
transport, protocol = await self._create_connection_transport(
1702
sock, protocol_factory, ssl, '', server_side=True,
1703
ssl_handshake_timeout=ssl_handshake_timeout,
1704
ssl_shutdown_timeout=ssl_shutdown_timeout)
1706
# Get the socket from the transport because SSL transport closes
1707
# the old socket and creates a new SSL socket
1708
sock = transport.get_extra_info('socket')
1709
logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
1710
return transport, protocol
1712
async def connect_read_pipe(self, protocol_factory, pipe):
1713
protocol = protocol_factory()
1714
waiter = self.create_future()
1715
transport = self._make_read_pipe_transport(pipe, protocol, waiter)
1724
logger.debug('Read pipe %r connected: (%r, %r)',
1725
pipe.fileno(), transport, protocol)
1726
return transport, protocol
1728
async def connect_write_pipe(self, protocol_factory, pipe):
1729
protocol = protocol_factory()
1730
waiter = self.create_future()
1731
transport = self._make_write_pipe_transport(pipe, protocol, waiter)
1740
logger.debug('Write pipe %r connected: (%r, %r)',
1741
pipe.fileno(), transport, protocol)
1742
return transport, protocol
1744
def _log_subprocess(self, msg, stdin, stdout, stderr):
1746
if stdin is not None:
1747
info.append(f'stdin={_format_pipe(stdin)}')
1748
if stdout is not None and stderr == subprocess.STDOUT:
1749
info.append(f'stdout=stderr={_format_pipe(stdout)}')
1751
if stdout is not None:
1752
info.append(f'stdout={_format_pipe(stdout)}')
1753
if stderr is not None:
1754
info.append(f'stderr={_format_pipe(stderr)}')
1755
logger.debug(' '.join(info))
1757
async def subprocess_shell(self, protocol_factory, cmd, *,
1758
stdin=subprocess.PIPE,
1759
stdout=subprocess.PIPE,
1760
stderr=subprocess.PIPE,
1761
universal_newlines=False,
1762
shell=True, bufsize=0,
1763
encoding=None, errors=None, text=None,
1765
if not isinstance(cmd, (bytes, str)):
1766
raise ValueError("cmd must be a string")
1767
if universal_newlines:
1768
raise ValueError("universal_newlines must be False")
1770
raise ValueError("shell must be True")
1772
raise ValueError("bufsize must be 0")
1774
raise ValueError("text must be False")
1775
if encoding is not None:
1776
raise ValueError("encoding must be None")
1777
if errors is not None:
1778
raise ValueError("errors must be None")
1780
protocol = protocol_factory()
1783
# don't log parameters: they may contain sensitive information
1784
# (password) and may be too long
1785
debug_log = 'run shell command %r' % cmd
1786
self._log_subprocess(debug_log, stdin, stdout, stderr)
1787
transport = await self._make_subprocess_transport(
1788
protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
1789
if self._debug and debug_log is not None:
1790
logger.info('%s: %r', debug_log, transport)
1791
return transport, protocol
1793
async def subprocess_exec(self, protocol_factory, program, *args,
1794
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1795
stderr=subprocess.PIPE, universal_newlines=False,
1796
shell=False, bufsize=0,
1797
encoding=None, errors=None, text=None,
1799
if universal_newlines:
1800
raise ValueError("universal_newlines must be False")
1802
raise ValueError("shell must be False")
1804
raise ValueError("bufsize must be 0")
1806
raise ValueError("text must be False")
1807
if encoding is not None:
1808
raise ValueError("encoding must be None")
1809
if errors is not None:
1810
raise ValueError("errors must be None")
1812
popen_args = (program,) + args
1813
protocol = protocol_factory()
1816
# don't log parameters: they may contain sensitive information
1817
# (password) and may be too long
1818
debug_log = f'execute program {program!r}'
1819
self._log_subprocess(debug_log, stdin, stdout, stderr)
1820
transport = await self._make_subprocess_transport(
1821
protocol, popen_args, False, stdin, stdout, stderr,
1823
if self._debug and debug_log is not None:
1824
logger.info('%s: %r', debug_log, transport)
1825
return transport, protocol
1827
def get_exception_handler(self):
1828
"""Return an exception handler, or None if the default one is in use.
1830
return self._exception_handler
1832
def set_exception_handler(self, handler):
1833
"""Set handler as the new event loop exception handler.
1835
If handler is None, the default exception handler will
1838
If handler is a callable object, it should have a
1839
signature matching '(loop, context)', where 'loop'
1840
will be a reference to the active event loop, 'context'
1841
will be a dict object (see `call_exception_handler()`
1842
documentation for details about context).
1844
if handler is not None and not callable(handler):
1845
raise TypeError(f'A callable object or None is expected, '
1847
self._exception_handler = handler
1849
def default_exception_handler(self, context):
1850
"""Default exception handler.
1852
This is called when an exception occurs and no exception
1853
handler is set, and can be called by a custom exception
1854
handler that wants to defer to the default behavior.
1856
This default handler logs the error message and other
1857
context-dependent information. In debug mode, a truncated
1858
stack trace is also appended showing where the given object
1859
(e.g. a handle or future or task) was created, if any.
1861
The context parameter has the same meaning as in
1862
`call_exception_handler()`.
1864
message = context.get('message')
1866
message = 'Unhandled exception in event loop'
1868
exception = context.get('exception')
1869
if exception is not None:
1870
exc_info = (type(exception), exception, exception.__traceback__)
1874
if ('source_traceback' not in context and
1875
self._current_handle is not None and
1876
self._current_handle._source_traceback):
1877
context['handle_traceback'] = \
1878
self._current_handle._source_traceback
1880
log_lines = [message]
1881
for key in sorted(context):
1882
if key in {'message', 'exception'}:
1884
value = context[key]
1885
if key == 'source_traceback':
1886
tb = ''.join(traceback.format_list(value))
1887
value = 'Object created at (most recent call last):\n'
1888
value += tb.rstrip()
1889
elif key == 'handle_traceback':
1890
tb = ''.join(traceback.format_list(value))
1891
value = 'Handle created at (most recent call last):\n'
1892
value += tb.rstrip()
1895
log_lines.append(f'{key}: {value}')
1897
logger.error('\n'.join(log_lines), exc_info=exc_info)
1899
def call_exception_handler(self, context):
1900
"""Call the current event loop's exception handler.
1902
The context argument is a dict containing the following keys:
1904
- 'message': Error message;
1905
- 'exception' (optional): Exception object;
1906
- 'future' (optional): Future instance;
1907
- 'task' (optional): Task instance;
1908
- 'handle' (optional): Handle instance;
1909
- 'protocol' (optional): Protocol instance;
1910
- 'transport' (optional): Transport instance;
1911
- 'socket' (optional): Socket instance;
1912
- 'source_traceback' (optional): Traceback of the source;
1913
- 'handle_traceback' (optional): Traceback of the handle;
1914
- 'asyncgen' (optional): Asynchronous generator that caused
1917
New keys maybe introduced in the future.
1919
Note: do not overload this method in an event loop subclass.
1920
For custom exception handling, use the
1921
`set_exception_handler()` method.
1923
if self._exception_handler is None:
1925
self.default_exception_handler(context)
1926
except (SystemExit, KeyboardInterrupt):
1928
except BaseException:
1929
# Second protection layer for unexpected errors
1930
# in the default implementation, as well as for subclassed
1931
# event loops with overloaded "default_exception_handler".
1932
logger.error('Exception in default exception handler',
1937
thing = context.get("task")
1939
# Even though Futures don't have a context,
1940
# Task is a subclass of Future,
1941
# and sometimes the 'future' key holds a Task.
1942
thing = context.get("future")
1944
# Handles also have a context.
1945
thing = context.get("handle")
1946
if thing is not None and hasattr(thing, "get_context"):
1947
ctx = thing.get_context()
1948
if ctx is not None and hasattr(ctx, "run"):
1949
ctx.run(self._exception_handler, self, context)
1951
self._exception_handler(self, context)
1952
except (SystemExit, KeyboardInterrupt):
1954
except BaseException as exc:
1955
# Exception in the user set custom exception handler.
1957
# Let's try default handler.
1958
self.default_exception_handler({
1959
'message': 'Unhandled error in exception handler',
1963
except (SystemExit, KeyboardInterrupt):
1965
except BaseException:
1966
# Guard 'default_exception_handler' in case it is
1968
logger.error('Exception in default exception handler '
1969
'while handling an unexpected error '
1970
'in custom exception handler',
1973
def _add_callback(self, handle):
1974
"""Add a Handle to _ready."""
1975
if not handle._cancelled:
1976
self._ready.append(handle)
1978
def _add_callback_signalsafe(self, handle):
1979
"""Like _add_callback() but called from a signal handler."""
1980
self._add_callback(handle)
1981
self._write_to_self()
1983
def _timer_handle_cancelled(self, handle):
1984
"""Notification that a TimerHandle has been cancelled."""
1985
if handle._scheduled:
1986
self._timer_cancelled_count += 1
1988
def _run_once(self):
1989
"""Run one full iteration of the event loop.
1991
This calls all currently ready callbacks, polls for I/O,
1992
schedules the resulting callbacks, and finally schedules
1993
'call_later' callbacks.
1996
sched_count = len(self._scheduled)
1997
if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and
1998
self._timer_cancelled_count / sched_count >
1999
_MIN_CANCELLED_TIMER_HANDLES_FRACTION):
2000
# Remove delayed calls that were cancelled if their number
2003
for handle in self._scheduled:
2004
if handle._cancelled:
2005
handle._scheduled = False
2007
new_scheduled.append(handle)
2009
heapq.heapify(new_scheduled)
2010
self._scheduled = new_scheduled
2011
self._timer_cancelled_count = 0
2013
# Remove delayed calls that were cancelled from head of queue.
2014
while self._scheduled and self._scheduled[0]._cancelled:
2015
self._timer_cancelled_count -= 1
2016
handle = heapq.heappop(self._scheduled)
2017
handle._scheduled = False
2020
if self._ready or self._stopping:
2022
elif self._scheduled:
2023
# Compute the desired timeout.
2024
timeout = self._scheduled[0]._when - self.time()
2025
if timeout > MAXIMUM_SELECT_TIMEOUT:
2026
timeout = MAXIMUM_SELECT_TIMEOUT
2030
22
138
event_list = self._selector.select(timeout)
2031
5
self._process_events(event_list)
2032
# Needed to break cycles when an exception occurs.
2035
# Handle 'later' callbacks that are ready.
2036
12
now = self.time()
2037
# Ensure that `end_time` is strictly increasing
2038
# when the clock resolution is too small.
2039
end_time = now + max(self._clock_resolution, math.ulp(now))
2040
while self._scheduled:
2041
handle = self._scheduled[0]
2042
if handle._when >= end_time:
2044
handle = heapq.heappop(self._scheduled)
2045
handle._scheduled = False
2046
self._ready.append(handle)
2048
# This is the only place where callbacks are actually *called*.
2049
# All other places just add them to ready.
2050
# Note: We run all currently scheduled callbacks, but not any
2051
# callbacks scheduled by callbacks run this time around --
2052
# they will be run the next time (after another I/O poll).
2053
# Use an idiom that is thread-safe without using locks.
2054
ntodo = len(self._ready)
2055
for i in range(ntodo):
2056
handle = self._ready.popleft()
2057
if handle._cancelled:
2061
self._current_handle = handle
2064
dt = self.time() - t0
2065
if dt >= self.slow_callback_duration:
2066
logger.warning('Executing %s took %.3f seconds',
2067
_format_handle(handle), dt)
2069
self._current_handle = None
2071
36
593
handle._run()
2072
handle = None # Needed to break cycles when an exception occurs.
2074
def _set_coroutine_origin_tracking(self, enabled):
2075
if bool(enabled) == bool(self._coroutine_origin_tracking_enabled):
2079
self._coroutine_origin_tracking_saved_depth = (
2080
sys.get_coroutine_origin_tracking_depth())
2081
sys.set_coroutine_origin_tracking_depth(
2082
constants.DEBUG_STACK_DEPTH)
2084
sys.set_coroutine_origin_tracking_depth(
2085
self._coroutine_origin_tracking_saved_depth)
2087
self._coroutine_origin_tracking_enabled = enabled
2089
def get_debug(self):
2092
def set_debug(self, enabled):
2093
self._debug = enabled
2095
if self.is_running():
2096
self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled)