[pull] master from aio-libs:master

.github/workflows/ci-cd.yml

@@ -260,7 +260,7 @@ jobs:
     needs: gen_llhttp
 
     runs-on: ubuntu-latest
-    timeout-minutes: 7
+    timeout-minutes: 9
     steps:
     - name: Checkout project
       uses: actions/checkout@v4

CHANGES.rst

@@ -10,6 +10,34 @@
 
 .. towncrier release notes start
 
+3.11.18 (2025-04-20)
+====================
+
+Bug fixes
+---------
+
+- Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`.
+
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`7686`.
+
+
+
+- Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`.
+
+  The problem first appeared in 3.11.17
+
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`10764`.
+
+
+
+
+----
+
+
 3.11.17 (2025-04-19)
 ====================
 

CONTRIBUTORS.txt

@@ -250,6 +250,7 @@ Martin Sucha
 Mathias Fröjdman
 Mathieu Dugré
 Matt VanEseltine
+Matthew Go
 Matthias Marquardt
 Matthieu Hauglustaine
 Matthieu Rigal

aiohttp/_websocket/reader_py.py

@@ -458,8 +458,7 @@ def _feed_data(self, data: bytes) -> None:
                     self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos])
                     if self._has_mask:
                         assert self._frame_mask is not None
-                        payload_bytearray = bytearray()
-                        payload_bytearray.join(self._payload_fragments)
+                        payload_bytearray = bytearray(b"".join(self._payload_fragments))
                         websocket_mask(self._frame_mask, payload_bytearray)
                         payload = payload_bytearray
                     else:

aiohttp/connector.py

@@ -1150,7 +1150,13 @@ def _warn_about_tls_in_tls(
         if req.request_info.url.scheme != "https":
             return
 
-        asyncio_supports_tls_in_tls = getattr(
+        # Check if uvloop is being used, which supports TLS in TLS,
+        # otherwise assume that asyncio's native transport is being used.
+        if type(underlying_transport).__module__.startswith("uvloop"):
+            return
+
+        # Support in asyncio was added in Python 3.11 (bpo-44011)
+        asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr(
             underlying_transport,
             "_start_tls_compatible",
             False,

tests/conftest.py

@@ -32,6 +32,13 @@
 except ImportError:
     TRUSTME = False
 
+
+try:
+    import uvloop
+except ImportError:
+    uvloop = None  # type: ignore[assignment]
+
+
 pytest_plugins = ("aiohttp.pytest_plugin", "pytester")
 
 IS_HPUX = sys.platform.startswith("hp-ux")
@@ -234,6 +241,16 @@ def selector_loop() -> Iterator[asyncio.AbstractEventLoop]:
         yield _loop
 
 
+@pytest.fixture
+def uvloop_loop() -> Iterator[asyncio.AbstractEventLoop]:
+    policy = uvloop.EventLoopPolicy()
+    asyncio.set_event_loop_policy(policy)
+
+    with loop_context(policy.new_event_loop) as _loop:
+        asyncio.set_event_loop(_loop)
+        yield _loop
+
+
 @pytest.fixture
 def netrc_contents(
     tmp_path: Path,

tests/test_proxy_functional.py

@@ -1,6 +1,7 @@
 import asyncio
 import os
 import pathlib
+import platform
 import ssl
 import sys
 from re import match as match_regex
@@ -240,6 +241,32 @@ async def test_https_proxy_unsupported_tls_in_tls(
     await asyncio.sleep(0.1)
 
 
+@pytest.mark.usefixtures("uvloop_loop")
+@pytest.mark.skipif(
+    platform.system() == "Windows" or sys.implementation.name != "cpython",
+    reason="uvloop is not supported on Windows and non-CPython implementations",
+)
+@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*")
+# Filter out the warning from
+# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226
+# otherwise this test will fail because the proxy will die with an error.
+async def test_uvloop_secure_https_proxy(
+    client_ssl_ctx: ssl.SSLContext,
+    secure_proxy_url: URL,
+) -> None:
+    """Ensure HTTPS sites are accessible through a secure proxy without warning when using uvloop."""
+    conn = aiohttp.TCPConnector()
+    sess = aiohttp.ClientSession(connector=conn)
+    url = URL("https://example.com")
+
+    async with sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) as response:
+        assert response.status == 200
+
+    await sess.close()
+    await conn.close()
+    await asyncio.sleep(0.1)
+
+
 @pytest.fixture
 def proxy_test_server(
     aiohttp_raw_server: AiohttpRawServer,

tests/test_websocket_parser.py

@@ -1,13 +1,21 @@
 import asyncio
 import pickle
+import random
 import struct
 from typing import Optional, Union
 from unittest import mock
 
 import pytest
 
 from aiohttp._websocket import helpers as _websocket_helpers
-from aiohttp._websocket.helpers import PACK_CLOSE_CODE, PACK_LEN1, PACK_LEN2
+from aiohttp._websocket.helpers import (
+    PACK_CLOSE_CODE,
+    PACK_LEN1,
+    PACK_LEN2,
+    PACK_LEN3,
+    PACK_RANDBITS,
+    websocket_mask,
+)
 from aiohttp._websocket.models import WS_DEFLATE_TRAILING
 from aiohttp._websocket.reader import WebSocketDataQueue
 from aiohttp.base_protocol import BaseProtocol
@@ -52,6 +60,7 @@ def build_frame(
     noheader: bool = False,
     is_fin: bool = True,
     ZLibBackend: Optional[ZLibBackendWrapper] = None,
+    mask: bool = False,
 ) -> bytes:
     # Send a frame over the websocket with message as its payload.
     compress = False
@@ -72,11 +81,21 @@ def build_frame(
     if compress:
         header_first_byte |= 0x40
 
+    mask_bit = 0x80 if mask else 0
+
     if msg_length < 126:
-        header = PACK_LEN1(header_first_byte, msg_length)
+        header = PACK_LEN1(header_first_byte, msg_length | mask_bit)
+    elif msg_length < 65536:
+        header = PACK_LEN2(header_first_byte, 126 | mask_bit, msg_length)
     else:
-        assert msg_length < (1 << 16)
-        header = PACK_LEN2(header_first_byte, 126, msg_length)
+        header = PACK_LEN3(header_first_byte, 127 | mask_bit, msg_length)
+
+    if mask:
+        assert not noheader
+        mask_bytes = PACK_RANDBITS(random.getrandbits(32))
+        message_arr = bytearray(message)
+        websocket_mask(mask_bytes, message_arr)
+        return header + mask_bytes + message_arr
 
     if noheader:
         return message
@@ -352,6 +371,51 @@ def test_fragmentation_header(
     assert res == WSMessageText(data="a", size=1, extra="")
 
 
+def test_large_message(
+    out: WebSocketDataQueue, parser: PatchableWebSocketReader
+) -> None:
+    large_payload = b"b" * 131072
+    data = build_frame(large_payload, WSMsgType.BINARY)
+    parser._feed_data(data)
+
+    res = out._buffer[0]
+    assert res == WSMessageBinary(data=large_payload, size=131072, extra="")
+
+
+def test_large_masked_message(
+    out: WebSocketDataQueue, parser: PatchableWebSocketReader
+) -> None:
+    large_payload = b"b" * 131072
+    data = build_frame(large_payload, WSMsgType.BINARY, mask=True)
+    parser._feed_data(data)
+
+    res = out._buffer[0]
+    assert res == WSMessageBinary(data=large_payload, size=131072, extra="")
+
+
+def test_fragmented_masked_message(
+    out: WebSocketDataQueue, parser: PatchableWebSocketReader
+) -> None:
+    large_payload = b"b" * 100
+    data = build_frame(large_payload, WSMsgType.BINARY, mask=True)
+    for i in range(len(data)):
+        parser._feed_data(data[i : i + 1])
+
+    res = out._buffer[0]
+    assert res == WSMessageBinary(data=large_payload, size=100, extra="")
+
+
+def test_large_fragmented_masked_message(
+    out: WebSocketDataQueue, parser: PatchableWebSocketReader
+) -> None:
+    large_payload = b"b" * 131072
+    data = build_frame(large_payload, WSMsgType.BINARY, mask=True)
+    for i in range(0, len(data), 16384):
+        parser._feed_data(data[i : i + 16384])
+    res = out._buffer[0]
+    assert res == WSMessageBinary(data=large_payload, size=131072, extra="")
+
+
 def test_continuation(
     out: WebSocketDataQueue, parser: PatchableWebSocketReader
 ) -> None: