From ae0668899a144edb06c4d2ca9cec96b760a93e6a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 6 Oct 2023 08:14:39 +0400 Subject: [PATCH 001/131] Drop support for Python 3.7 (#3143) --- .github/workflows/ci.yml | 8 +------- .pre-commit-config.yaml | 4 ++-- changelog/3143.removal.rst | 1 + dev-requirements.txt | 15 ++++++--------- docs/contributing.rst | 9 +++++---- noxfile.py | 8 ++------ pyproject.toml | 3 +-- src/urllib3/_base_connection.py | 3 +-- src/urllib3/_collections.py | 4 ++-- src/urllib3/connection.py | 2 +- src/urllib3/connectionpool.py | 3 +-- src/urllib3/contrib/securetransport.py | 2 +- src/urllib3/contrib/socks.py | 23 ++++++++++------------- src/urllib3/poolmanager.py | 3 +-- src/urllib3/response.py | 2 +- src/urllib3/util/request.py | 2 +- src/urllib3/util/ssl_.py | 16 ++++++++-------- src/urllib3/util/ssltransport.py | 2 +- src/urllib3/util/timeout.py | 2 +- test/__init__.py | 3 +-- test/test_response.py | 4 ---- test/test_ssltransport.py | 2 +- test/test_util.py | 5 ++--- test/with_dummyserver/test_https.py | 2 -- 24 files changed, 51 insertions(+), 77 deletions(-) create mode 100644 changelog/3143.removal.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fa25a91ab..d0e56f7644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: - macos-11 - windows-latest @@ -43,10 +43,6 @@ jobs: nox-session: [''] include: - experimental: false - - python-version: "pypy-3.7" - os: ubuntu-latest - experimental: false - nox-session: test-pypy - python-version: "pypy-3.8" os: ubuntu-latest experimental: false @@ -73,8 +69,6 @@ jobs: exclude: # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it # https://github.com/python/cpython/issues/83001 - - python-version: "3.7" - os: ubuntu-22.04 - python-version: "3.8" os: ubuntu-22.04 # Testing with non-final CPython on macOS is too slow for CI. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc05ec6820..6ffe5459e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: ["--py37-plus"] + args: ["--py38-plus"] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black - args: ["--target-version", "py37"] + args: ["--target-version", "py38"] - repo: https://github.com/PyCQA/isort rev: 5.12.0 diff --git a/changelog/3143.removal.rst b/changelog/3143.removal.rst new file mode 100644 index 0000000000..21130f07cb --- /dev/null +++ b/changelog/3143.removal.rst @@ -0,0 +1 @@ +Removed support for Python 3.7. diff --git a/dev-requirements.txt b/dev-requirements.txt index 70f5097338..1679aef27f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,13 +1,10 @@ -coverage==7.0.4 -tornado==6.2 +coverage==7.3.2 +tornado==6.3.3 PySocks==1.7.1 -pytest==7.4.0 +pytest==7.4.2 pytest-timeout==2.1.0 -trustme==0.9.0 -# We have to install at most cryptography 39.0.2 for PyPy<7.3.10 -# versions of Python 3.7, 3.8, and 3.9. -cryptography==39.0.2;implementation_name=="pypy" and implementation_version<"7.3.10" -cryptography==41.0.4;implementation_name!="pypy" or implementation_version>="7.3.10" +trustme==1.1.0 +cryptography==41.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 -pytest-memray==1.4.0;python_version>="3.8" and python_version<"3.12" and sys_platform!="win32" and implementation_name=="cpython" +pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython" diff --git a/docs/contributing.rst b/docs/contributing.rst index a75e5f6901..f5e81082eb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -36,12 +36,12 @@ We use some external dependencies, multiple interpreters and code coverage analysis while running test suite. Our ``noxfile.py`` handles much of this for you:: - $ nox --reuse-existing-virtualenvs --sessions test-3.7 test-3.9 + $ nox --reuse-existing-virtualenvs --sessions test-3.8 test-3.9 [ Nox will create virtualenv if needed, install the specified dependencies, and run the commands in order.] - nox > Running session test-3.7 + nox > Running session test-3.8 ....... ....... - nox > Session test-3.7 was successful. + nox > Session test-3.8 was successful. ....... ....... nox > Running session test-3.9 @@ -63,10 +63,11 @@ suite:: [ Nox will create virtualenv if needed, install the specified dependencies, and run the commands in order.] ....... ....... - nox > Session test-3.7 was successful. nox > Session test-3.8 was successful. nox > Session test-3.9 was successful. nox > Session test-3.10 was successful. + nox > Session test-3.11 was successful. + nox > Session test-3.12 was successful. nox > Session test-pypy was successful. Our test suite `runs continuously on GitHub Actions diff --git a/noxfile.py b/noxfile.py index 4573370a4b..f0c7f1421b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -25,11 +25,7 @@ def tests_impl( session.run("python", "-m", "OpenSSL.debug") memray_supported = True - if ( - sys.implementation.name != "cpython" - or sys.version_info < (3, 8) - or sys.version_info.releaselevel != "final" - ): + if sys.implementation.name != "cpython" or sys.version_info.releaselevel != "final": memray_supported = False # pytest-memray requires CPython 3.8+ elif sys.platform == "win32": memray_supported = False @@ -58,7 +54,7 @@ def tests_impl( ) -@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12", "pypy"]) def test(session: nox.Session) -> None: tests_impl(session) diff --git a/pyproject.toml b/pyproject.toml index 2b33cc9c28..dcdc6d7e97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -35,7 +34,7 @@ classifiers = [ "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries", ] -requires-python = ">=3.7" +requires-python = ">=3.8" dynamic = ["version"] [project.optional-dependencies] diff --git a/src/urllib3/_base_connection.py b/src/urllib3/_base_connection.py index 25b633af25..bb349c744b 100644 --- a/src/urllib3/_base_connection.py +++ b/src/urllib3/_base_connection.py @@ -28,8 +28,7 @@ class _ResponseOptions(typing.NamedTuple): if typing.TYPE_CHECKING: import ssl - - from typing_extensions import Literal, Protocol + from typing import Literal, Protocol from .response import BaseHTTPResponse diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index 7f9dca7fa8..f1cb60654a 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -8,7 +8,7 @@ if typing.TYPE_CHECKING: # We can only import Protocol if TYPE_CHECKING because it's a development # dependency, and is not available at runtime. - from typing_extensions import Protocol + from typing import Protocol class HasGettableStringKeys(Protocol): def keys(self) -> typing.Iterator[str]: @@ -239,7 +239,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]): def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str): super().__init__() - self._container = {} # 'dict' is insert-ordered in Python 3.7+ + self._container = {} # 'dict' is insert-ordered if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 4a71225ce6..6eb3f0d28c 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -14,7 +14,7 @@ from socket import timeout as SocketTimeout if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal from .response import HTTPResponse from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 2479405bd5..113d264d79 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -52,8 +52,7 @@ if typing.TYPE_CHECKING: import ssl - - from typing_extensions import Literal + from typing import Literal from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py index 11beb3dfef..2bc374bc4e 100644 --- a/src/urllib3/contrib/securetransport.py +++ b/src/urllib3/contrib/securetransport.py @@ -92,7 +92,7 @@ ) if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal __all__ = ["inject_into_urllib3", "extract_from_urllib3"] diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py index 5e552ddaed..6c3bb764b2 100644 --- a/src/urllib3/contrib/socks.py +++ b/src/urllib3/contrib/socks.py @@ -71,19 +71,16 @@ except ImportError: ssl = None # type: ignore[assignment] -try: - from typing import TypedDict - - class _TYPE_SOCKS_OPTIONS(TypedDict): - socks_version: int - proxy_host: str | None - proxy_port: str | None - username: str | None - password: str | None - rdns: bool - -except ImportError: # Python 3.7 - _TYPE_SOCKS_OPTIONS = typing.Dict[str, typing.Any] # type: ignore[misc, assignment] +from typing import TypedDict + + +class _TYPE_SOCKS_OPTIONS(TypedDict): + socks_version: int + proxy_host: str | None + proxy_port: str | None + username: str | None + password: str | None + rdns: bool class SOCKSConnection(HTTPConnection): diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index 02b2f622a1..7a998ee896 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -26,8 +26,7 @@ if typing.TYPE_CHECKING: import ssl - - from typing_extensions import Literal + from typing import Literal __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 12097ea9c2..9e7aab41fb 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -58,7 +58,7 @@ from .util.retry import Retry if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal from .connectionpool import HTTPConnectionPool diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index 7d6866f3ad..e6905ffca4 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -9,7 +9,7 @@ from .util import to_bytes if typing.TYPE_CHECKING: - from typing_extensions import Final + from typing import Final # Pass as a value within ``headers`` to skip # emitting some HTTP headers that are added automatically. diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index e35e394030..e26227ab14 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -42,7 +42,7 @@ def _is_bpo_43522_fixed( """ if implementation_name == "pypy": # https://foss.heptapod.net/pypy/pypy/-/issues/3129 - return pypy_version_info >= (7, 3, 8) and version_info >= (3, 8) # type: ignore[operator] + return pypy_version_info >= (7, 3, 8) # type: ignore[operator] elif implementation_name == "cpython": major_minor = version_info[:2] micro = version_info[2] @@ -79,8 +79,7 @@ def _is_has_never_check_common_name_reliable( if typing.TYPE_CHECKING: from ssl import VerifyMode - - from typing_extensions import Literal, TypedDict + from typing import Literal, TypedDict from .ssltransport import SSLTransport as SSLTransportType @@ -322,12 +321,13 @@ def create_urllib3_context( # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is # necessary for conditional client cert authentication with TLS 1.3. # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older - # versions of Python. We only enable on Python 3.7.4+ or if certificate - # verification is enabled to work around Python issue #37428 + # versions of Python. We only enable if certificate verification is enabled to work + # around Python issue #37428 # See: https://bugs.python.org/issue37428 - if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( - context, "post_handshake_auth", None - ) is not None: + if ( + cert_reqs == ssl.CERT_REQUIRED + and getattr(context, "post_handshake_auth", None) is not None + ): context.post_handshake_auth = True # The order of the below lines setting verify_mode and check_hostname diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py index 5ec86473b4..fa9f2b37c5 100644 --- a/src/urllib3/util/ssltransport.py +++ b/src/urllib3/util/ssltransport.py @@ -8,7 +8,7 @@ from ..exceptions import ProxySchemeUnsupported if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT diff --git a/src/urllib3/util/timeout.py b/src/urllib3/util/timeout.py index ec090f69cc..f044625c35 100644 --- a/src/urllib3/util/timeout.py +++ b/src/urllib3/util/timeout.py @@ -8,7 +8,7 @@ from ..exceptions import TimeoutStateError if typing.TYPE_CHECKING: - from typing_extensions import Final + from typing import Final class _TYPE_DEFAULT(Enum): diff --git a/test/__init__.py b/test/__init__.py index a1629e7021..0c3c8e08b9 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -43,8 +43,7 @@ if typing.TYPE_CHECKING: import ssl - - from typing_extensions import Literal + from typing import Literal _RT = typing.TypeVar("_RT") # return type diff --git a/test/test_response.py b/test/test_response.py index c6d9d1528a..4403307752 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -4,7 +4,6 @@ import http.client as httplib import socket import ssl -import sys import typing import zlib from base64 import b64decode @@ -74,9 +73,6 @@ def test_multiple_chunks(self) -> None: assert buffer.get(4) == b"rbaz" assert len(buffer) == 0 - @pytest.mark.skipif( - sys.version_info < (3, 8), reason="pytest-memray requires Python 3.8+" - ) @pytest.mark.limit_memory("12.5 MB") # assert that we're not doubling memory usage def test_memory_usage(self) -> None: # Allocate 10 1MiB chunks diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index cace51db96..21d556fc93 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -15,7 +15,7 @@ from urllib3.util.ssltransport import SSLTransport if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal # consume_socket can iterate forever, we add timeouts to prevent halting. PER_TEST_TIMEOUT = 60 diff --git a/test/test_util.py b/test/test_util.py index 0c46aa1dd3..8ed92ee189 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -42,7 +42,7 @@ from . import clear_warnings if typing.TYPE_CHECKING: - from typing_extensions import Literal + from typing import Literal # This number represents a time in seconds, it doesn't mean anything in # isolation. Setting to a high-ish value to avoid conflicts with the smaller @@ -1075,8 +1075,7 @@ def test_ssl_wrap_socket_sni_none_no_warn(self) -> None: # Python OK -> reliable ("OpenSSL 1.1.1", 0x10101000, "cpython", (3, 9, 3), None, True), # PyPy: depends on the version - ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 6, 9), (7, 3, 7), False), - ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 7, 13), (7, 3, 9), False), + ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 9, 9), (7, 3, 7), False), ("OpenSSL 1.1.1", 0x101010CF, "pypy", (3, 8, 12), (7, 3, 8), True), # OpenSSL OK -> reliable ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 2), None, True), diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index ec37d92b02..014bb68887 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -5,7 +5,6 @@ import os.path import shutil import ssl -import sys import tempfile import warnings from pathlib import Path @@ -913,7 +912,6 @@ def test_tls_version_maximum_and_minimum(self) -> None: finally: conn.close() - @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python 3.8+") def test_sslkeylogfile( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 312a47b596859fe8ae1cea228e4525f2d0345a75 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 6 Oct 2023 20:20:21 +0400 Subject: [PATCH 002/131] Drop support for SecureTransport (#3146) --- changelog/2681.removal.rst | 1 + docs/reference/contrib/index.rst | 1 - docs/reference/contrib/securetransport.rst | 31 - pyproject.toml | 1 - src/urllib3/connection.py | 7 +- .../contrib/_securetransport/__init__.py | 0 .../contrib/_securetransport/bindings.py | 430 --------- .../contrib/_securetransport/low_level.py | 474 --------- src/urllib3/contrib/securetransport.py | 913 ------------------ src/urllib3/response.py | 10 +- src/urllib3/util/__init__.py | 2 - src/urllib3/util/ssl_.py | 1 - test/__init__.py | 34 - test/contrib/test_securetransport.py | 68 -- test/with_dummyserver/test_https.py | 15 +- .../test_proxy_poolmanager.py | 13 +- test/with_dummyserver/test_socketlevel.py | 23 +- 17 files changed, 12 insertions(+), 2012 deletions(-) create mode 100644 changelog/2681.removal.rst delete mode 100644 docs/reference/contrib/securetransport.rst delete mode 100644 src/urllib3/contrib/_securetransport/__init__.py delete mode 100644 src/urllib3/contrib/_securetransport/bindings.py delete mode 100644 src/urllib3/contrib/_securetransport/low_level.py delete mode 100644 src/urllib3/contrib/securetransport.py delete mode 100644 test/contrib/test_securetransport.py diff --git a/changelog/2681.removal.rst b/changelog/2681.removal.rst new file mode 100644 index 0000000000..75e44a5c83 --- /dev/null +++ b/changelog/2681.removal.rst @@ -0,0 +1 @@ +Removed support for the SecureTransport TLS implementation. diff --git a/docs/reference/contrib/index.rst b/docs/reference/contrib/index.rst index e233241616..7785541912 100644 --- a/docs/reference/contrib/index.rst +++ b/docs/reference/contrib/index.rst @@ -7,5 +7,4 @@ prime time or that require optional third-party dependencies. .. toctree:: pyopenssl - securetransport socks diff --git a/docs/reference/contrib/securetransport.rst b/docs/reference/contrib/securetransport.rst deleted file mode 100644 index d4af1b8ba0..0000000000 --- a/docs/reference/contrib/securetransport.rst +++ /dev/null @@ -1,31 +0,0 @@ -macOS SecureTransport -===================== -.. warning:: - DEPRECATED: This module is deprecated and will be removed in urllib3 v2.1.0. - Read more in this `issue `_. - -`SecureTranport `_ -support for urllib3 via ctypes. - -This makes platform-native TLS available to urllib3 users on macOS without the -use of a compiler. This is an important feature because the Python Package -Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL -that ships with macOS is not capable of doing TLSv1.2. The only way to resolve -this is to give macOS users an alternative solution to the problem, and that -solution is to use SecureTransport. - -We use ctypes here because this solution must not require a compiler. That's -because Pip is not allowed to require a compiler either. - -This code is a bastardised version of the code found in Will Bond's -`oscrypto `_ library. An enormous debt -is owed to him for blazing this trail for us. For that reason, this code -should be considered to be covered both by urllib3's license and by -`oscrypto's `_. - -To use this module, simply import and inject it: - -.. code-block:: python - - import urllib3.contrib.securetransport - urllib3.contrib.securetransport.inject_into_urllib3() diff --git a/pyproject.toml b/pyproject.toml index dcdc6d7e97..a4574d3869 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,6 @@ filterwarnings = [ "error", '''default:urllib3 v2.0 only supports OpenSSL 1.1.1+.*''', '''default:'urllib3\[secure\]' extra is deprecated and will be removed in urllib3 v2\.1\.0.*:DeprecationWarning''', - '''default:'urllib3\.contrib\.securetransport' module is deprecated and will be removed in urllib3 v2\.1\.0.*:DeprecationWarning''', '''default:No IPv6 support. Falling back to IPv4:urllib3.exceptions.HTTPWarning''', '''default:No IPv6 support. skipping:urllib3.exceptions.HTTPWarning''', '''default:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning''', diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 6eb3f0d28c..38a2fd6dfa 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -757,10 +757,9 @@ def _ssl_wrap_socket_and_match_hostname( ): context.check_hostname = False - # Try to load OS default certs if none are given. - # We need to do the hasattr() check for our custom - # pyOpenSSL and SecureTransport SSLContext objects - # because neither support load_default_certs(). + # Try to load OS default certs if none are given. We need to do the hasattr() check + # for custom pyOpenSSL SSLContext objects because they don't support + # load_default_certs(). if ( not ca_certs and not ca_cert_dir diff --git a/src/urllib3/contrib/_securetransport/__init__.py b/src/urllib3/contrib/_securetransport/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/urllib3/contrib/_securetransport/bindings.py b/src/urllib3/contrib/_securetransport/bindings.py deleted file mode 100644 index 3e4cd466ea..0000000000 --- a/src/urllib3/contrib/_securetransport/bindings.py +++ /dev/null @@ -1,430 +0,0 @@ -# type: ignore - -""" -This module uses ctypes to bind a whole bunch of functions and constants from -SecureTransport. The goal here is to provide the low-level API to -SecureTransport. These are essentially the C-level functions and constants, and -they're pretty gross to work with. - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - - Copyright (c) 2015-2016 Will Bond - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" - -from __future__ import annotations - -import platform -from ctypes import ( - CDLL, - CFUNCTYPE, - POINTER, - c_bool, - c_byte, - c_char_p, - c_int32, - c_long, - c_size_t, - c_uint32, - c_ulong, - c_void_p, -) -from ctypes.util import find_library - -if platform.system() != "Darwin": - raise ImportError("Only macOS is supported") - -version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split("."))) -if version_info < (10, 8): - raise OSError( - f"Only OS X 10.8 and newer are supported, not {version_info[0]}.{version_info[1]}" - ) - - -def load_cdll(name: str, macos10_16_path: str) -> CDLL: - """Loads a CDLL by name, falling back to known path on 10.16+""" - try: - # Big Sur is technically 11 but we use 10.16 due to the Big Sur - # beta being labeled as 10.16. - path: str | None - if version_info >= (10, 16): - path = macos10_16_path - else: - path = find_library(name) - if not path: - raise OSError # Caught and reraised as 'ImportError' - return CDLL(path, use_errno=True) - except OSError: - raise ImportError(f"The library {name} failed to load") from None - - -Security = load_cdll( - "Security", "/System/Library/Frameworks/Security.framework/Security" -) -CoreFoundation = load_cdll( - "CoreFoundation", - "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", -) - - -Boolean = c_bool -CFIndex = c_long -CFStringEncoding = c_uint32 -CFData = c_void_p -CFString = c_void_p -CFArray = c_void_p -CFMutableArray = c_void_p -CFDictionary = c_void_p -CFError = c_void_p -CFType = c_void_p -CFTypeID = c_ulong - -CFTypeRef = POINTER(CFType) -CFAllocatorRef = c_void_p - -OSStatus = c_int32 - -CFDataRef = POINTER(CFData) -CFStringRef = POINTER(CFString) -CFArrayRef = POINTER(CFArray) -CFMutableArrayRef = POINTER(CFMutableArray) -CFDictionaryRef = POINTER(CFDictionary) -CFArrayCallBacks = c_void_p -CFDictionaryKeyCallBacks = c_void_p -CFDictionaryValueCallBacks = c_void_p - -SecCertificateRef = POINTER(c_void_p) -SecExternalFormat = c_uint32 -SecExternalItemType = c_uint32 -SecIdentityRef = POINTER(c_void_p) -SecItemImportExportFlags = c_uint32 -SecItemImportExportKeyParameters = c_void_p -SecKeychainRef = POINTER(c_void_p) -SSLProtocol = c_uint32 -SSLCipherSuite = c_uint32 -SSLContextRef = POINTER(c_void_p) -SecTrustRef = POINTER(c_void_p) -SSLConnectionRef = c_uint32 -SecTrustResultType = c_uint32 -SecTrustOptionFlags = c_uint32 -SSLProtocolSide = c_uint32 -SSLConnectionType = c_uint32 -SSLSessionOption = c_uint32 - - -try: - Security.SecItemImport.argtypes = [ - CFDataRef, - CFStringRef, - POINTER(SecExternalFormat), - POINTER(SecExternalItemType), - SecItemImportExportFlags, - POINTER(SecItemImportExportKeyParameters), - SecKeychainRef, - POINTER(CFArrayRef), - ] - Security.SecItemImport.restype = OSStatus - - Security.SecCertificateGetTypeID.argtypes = [] - Security.SecCertificateGetTypeID.restype = CFTypeID - - Security.SecIdentityGetTypeID.argtypes = [] - Security.SecIdentityGetTypeID.restype = CFTypeID - - Security.SecKeyGetTypeID.argtypes = [] - Security.SecKeyGetTypeID.restype = CFTypeID - - Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] - Security.SecCertificateCreateWithData.restype = SecCertificateRef - - Security.SecCertificateCopyData.argtypes = [SecCertificateRef] - Security.SecCertificateCopyData.restype = CFDataRef - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SecIdentityCreateWithCertificate.argtypes = [ - CFTypeRef, - SecCertificateRef, - POINTER(SecIdentityRef), - ] - Security.SecIdentityCreateWithCertificate.restype = OSStatus - - Security.SecKeychainCreate.argtypes = [ - c_char_p, - c_uint32, - c_void_p, - Boolean, - c_void_p, - POINTER(SecKeychainRef), - ] - Security.SecKeychainCreate.restype = OSStatus - - Security.SecKeychainDelete.argtypes = [SecKeychainRef] - Security.SecKeychainDelete.restype = OSStatus - - Security.SecPKCS12Import.argtypes = [ - CFDataRef, - CFDictionaryRef, - POINTER(CFArrayRef), - ] - Security.SecPKCS12Import.restype = OSStatus - - SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE( - OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) - ) - - Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] - Security.SSLSetIOFuncs.restype = OSStatus - - Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerID.restype = OSStatus - - Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetCertificate.restype = OSStatus - - Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] - Security.SSLSetCertificateAuthorities.restype = OSStatus - - Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] - Security.SSLSetConnection.restype = OSStatus - - Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerDomainName.restype = OSStatus - - Security.SSLHandshake.argtypes = [SSLContextRef] - Security.SSLHandshake.restype = OSStatus - - Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLRead.restype = OSStatus - - Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLWrite.restype = OSStatus - - Security.SSLClose.argtypes = [SSLContextRef] - Security.SSLClose.restype = OSStatus - - Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberSupportedCiphers.restype = OSStatus - - Security.SSLGetSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetSupportedCiphers.restype = OSStatus - - Security.SSLSetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - c_size_t, - ] - Security.SSLSetEnabledCiphers.restype = OSStatus - - Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberEnabledCiphers.restype = OSStatus - - Security.SSLGetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetEnabledCiphers.restype = OSStatus - - Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] - Security.SSLGetNegotiatedCipher.restype = OSStatus - - Security.SSLGetNegotiatedProtocolVersion.argtypes = [ - SSLContextRef, - POINTER(SSLProtocol), - ] - Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - - Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] - Security.SSLCopyPeerTrust.restype = OSStatus - - Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] - Security.SecTrustSetAnchorCertificates.restype = OSStatus - - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] - Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - - Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] - Security.SecTrustEvaluate.restype = OSStatus - - Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] - Security.SecTrustGetCertificateCount.restype = CFIndex - - Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] - Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef - - Security.SSLCreateContext.argtypes = [ - CFAllocatorRef, - SSLProtocolSide, - SSLConnectionType, - ] - Security.SSLCreateContext.restype = SSLContextRef - - Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] - Security.SSLSetSessionOption.restype = OSStatus - - Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMin.restype = OSStatus - - Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMax.restype = OSStatus - - try: - Security.SSLSetALPNProtocols.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetALPNProtocols.restype = OSStatus - except AttributeError: - # Supported only in 10.12+ - pass - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SSLReadFunc = SSLReadFunc - Security.SSLWriteFunc = SSLWriteFunc - Security.SSLContextRef = SSLContextRef - Security.SSLProtocol = SSLProtocol - Security.SSLCipherSuite = SSLCipherSuite - Security.SecIdentityRef = SecIdentityRef - Security.SecKeychainRef = SecKeychainRef - Security.SecTrustRef = SecTrustRef - Security.SecTrustResultType = SecTrustResultType - Security.SecExternalFormat = SecExternalFormat - Security.OSStatus = OSStatus - - Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, "kSecImportExportPassphrase" - ) - Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, "kSecImportItemIdentity" - ) - - # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [CFTypeRef] - CoreFoundation.CFRetain.restype = CFTypeRef - - CoreFoundation.CFRelease.argtypes = [CFTypeRef] - CoreFoundation.CFRelease.restype = None - - CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] - CoreFoundation.CFGetTypeID.restype = CFTypeID - - CoreFoundation.CFStringCreateWithCString.argtypes = [ - CFAllocatorRef, - c_char_p, - CFStringEncoding, - ] - CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - - CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] - CoreFoundation.CFStringGetCStringPtr.restype = c_char_p - - CoreFoundation.CFStringGetCString.argtypes = [ - CFStringRef, - c_char_p, - CFIndex, - CFStringEncoding, - ] - CoreFoundation.CFStringGetCString.restype = c_bool - - CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] - CoreFoundation.CFDataCreate.restype = CFDataRef - - CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] - CoreFoundation.CFDataGetLength.restype = CFIndex - - CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] - CoreFoundation.CFDataGetBytePtr.restype = c_void_p - - CoreFoundation.CFDictionaryCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - POINTER(CFTypeRef), - CFIndex, - CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks, - ] - CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - - CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] - CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef - - CoreFoundation.CFArrayCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreate.restype = CFArrayRef - - CoreFoundation.CFArrayCreateMutable.argtypes = [ - CFAllocatorRef, - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - - CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] - CoreFoundation.CFArrayAppendValue.restype = None - - CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] - CoreFoundation.CFArrayGetCount.restype = CFIndex - - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] - CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p - - CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, "kCFAllocatorDefault" - ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeArrayCallBacks" - ) - CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryKeyCallBacks" - ) - CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryValueCallBacks" - ) - - CoreFoundation.CFTypeRef = CFTypeRef - CoreFoundation.CFArrayRef = CFArrayRef - CoreFoundation.CFStringRef = CFStringRef - CoreFoundation.CFDictionaryRef = CFDictionaryRef - -except AttributeError: - raise ImportError("Error initializing ctypes") from None - - -class CFConst: - """ - A class object that acts as essentially a namespace for CoreFoundation - constants. - """ - - kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) diff --git a/src/urllib3/contrib/_securetransport/low_level.py b/src/urllib3/contrib/_securetransport/low_level.py deleted file mode 100644 index e23569972c..0000000000 --- a/src/urllib3/contrib/_securetransport/low_level.py +++ /dev/null @@ -1,474 +0,0 @@ -""" -Low-level helpers for the SecureTransport bindings. - -These are Python functions that are not directly related to the high-level APIs -but are necessary to get them to work. They include a whole bunch of low-level -CoreFoundation messing about and memory management. The concerns in this module -are almost entirely about trying to avoid memory leaks and providing -appropriate and useful assistance to the higher-level code. -""" -from __future__ import annotations - -import base64 -import ctypes -import itertools -import os -import re -import ssl -import struct -import tempfile -import typing - -from .bindings import ( # type: ignore[attr-defined] - CFArray, - CFConst, - CFData, - CFDictionary, - CFMutableArray, - CFString, - CFTypeRef, - CoreFoundation, - SecKeychainRef, - Security, -) - -# This regular expression is used to grab PEM data out of a PEM bundle. -_PEM_CERTS_RE = re.compile( - b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL -) - - -def _cf_data_from_bytes(bytestring: bytes) -> CFData: - """ - Given a bytestring, create a CFData object from it. This CFData object must - be CFReleased by the caller. - """ - return CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring) - ) - - -def _cf_dictionary_from_tuples( - tuples: list[tuple[typing.Any, typing.Any]] -) -> CFDictionary: - """ - Given a list of Python tuples, create an associated CFDictionary. - """ - dictionary_size = len(tuples) - - # We need to get the dictionary keys and values out in the same order. - keys = (t[0] for t in tuples) - values = (t[1] for t in tuples) - cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys) - cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values) - - return CoreFoundation.CFDictionaryCreate( - CoreFoundation.kCFAllocatorDefault, - cf_keys, - cf_values, - dictionary_size, - CoreFoundation.kCFTypeDictionaryKeyCallBacks, - CoreFoundation.kCFTypeDictionaryValueCallBacks, - ) - - -def _cfstr(py_bstr: bytes) -> CFString: - """ - Given a Python binary data, create a CFString. - The string must be CFReleased by the caller. - """ - c_str = ctypes.c_char_p(py_bstr) - cf_str = CoreFoundation.CFStringCreateWithCString( - CoreFoundation.kCFAllocatorDefault, - c_str, - CFConst.kCFStringEncodingUTF8, - ) - return cf_str - - -def _create_cfstring_array(lst: list[bytes]) -> CFMutableArray: - """ - Given a list of Python binary data, create an associated CFMutableArray. - The array must be CFReleased by the caller. - - Raises an ssl.SSLError on failure. - """ - cf_arr = None - try: - cf_arr = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cf_arr: - raise MemoryError("Unable to allocate memory!") - for item in lst: - cf_str = _cfstr(item) - if not cf_str: - raise MemoryError("Unable to allocate memory!") - try: - CoreFoundation.CFArrayAppendValue(cf_arr, cf_str) - finally: - CoreFoundation.CFRelease(cf_str) - except BaseException as e: - if cf_arr: - CoreFoundation.CFRelease(cf_arr) - raise ssl.SSLError(f"Unable to allocate array: {e}") from None - return cf_arr - - -def _cf_string_to_unicode(value: CFString) -> str | None: - """ - Creates a Unicode string from a CFString object. Used entirely for error - reporting. - - Yes, it annoys me quite a lot that this function is this complex. - """ - value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) - - string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, CFConst.kCFStringEncodingUTF8 - ) - if string is None: - buffer = ctypes.create_string_buffer(1024) - result = CoreFoundation.CFStringGetCString( - value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 - ) - if not result: - raise OSError("Error copying C string from CFStringRef") - string = buffer.value - if string is not None: - string = string.decode("utf-8") - return string # type: ignore[no-any-return] - - -def _assert_no_error( - error: int, exception_class: type[BaseException] | None = None -) -> None: - """ - Checks the return code and throws an exception if there is an error to - report - """ - if error == 0: - return - - cf_error_string = Security.SecCopyErrorMessageString(error, None) - output = _cf_string_to_unicode(cf_error_string) - CoreFoundation.CFRelease(cf_error_string) - - if output is None or output == "": - output = f"OSStatus {error}" - - if exception_class is None: - exception_class = ssl.SSLError - - raise exception_class(output) - - -def _cert_array_from_pem(pem_bundle: bytes) -> CFArray: - """ - Given a bundle of certs in PEM format, turns them into a CFArray of certs - that can be used to validate a cert chain. - """ - # Normalize the PEM bundle's line endings. - pem_bundle = pem_bundle.replace(b"\r\n", b"\n") - - der_certs = [ - base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) - ] - if not der_certs: - raise ssl.SSLError("No root certificates specified") - - cert_array = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cert_array: - raise ssl.SSLError("Unable to allocate memory!") - - try: - for der_bytes in der_certs: - certdata = _cf_data_from_bytes(der_bytes) - if not certdata: - raise ssl.SSLError("Unable to allocate memory!") - cert = Security.SecCertificateCreateWithData( - CoreFoundation.kCFAllocatorDefault, certdata - ) - CoreFoundation.CFRelease(certdata) - if not cert: - raise ssl.SSLError("Unable to build cert object!") - - CoreFoundation.CFArrayAppendValue(cert_array, cert) - CoreFoundation.CFRelease(cert) - except Exception: - # We need to free the array before the exception bubbles further. - # We only want to do that if an error occurs: otherwise, the caller - # should free. - CoreFoundation.CFRelease(cert_array) - raise - - return cert_array - - -def _is_cert(item: CFTypeRef) -> bool: - """ - Returns True if a given CFTypeRef is a certificate. - """ - expected = Security.SecCertificateGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected # type: ignore[no-any-return] - - -def _is_identity(item: CFTypeRef) -> bool: - """ - Returns True if a given CFTypeRef is an identity. - """ - expected = Security.SecIdentityGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected # type: ignore[no-any-return] - - -def _temporary_keychain() -> tuple[SecKeychainRef, str]: - """ - This function creates a temporary Mac keychain that we can use to work with - credentials. This keychain uses a one-time password and a temporary file to - store the data. We expect to have one keychain per socket. The returned - SecKeychainRef must be freed by the caller, including calling - SecKeychainDelete. - - Returns a tuple of the SecKeychainRef and the path to the temporary - directory that contains it. - """ - # Unfortunately, SecKeychainCreate requires a path to a keychain. This - # means we cannot use mkstemp to use a generic temporary file. Instead, - # we're going to create a temporary directory and a filename to use there. - # This filename will be 8 random bytes expanded into base64. We also need - # some random bytes to password-protect the keychain we're creating, so we - # ask for 40 random bytes. - random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode("utf-8") - password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 - tempdirectory = tempfile.mkdtemp() - - keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") - - # We now want to create the keychain itself. - keychain = Security.SecKeychainRef() - status = Security.SecKeychainCreate( - keychain_path, len(password), password, False, None, ctypes.byref(keychain) - ) - _assert_no_error(status) - - # Having created the keychain, we want to pass it off to the caller. - return keychain, tempdirectory - - -def _load_items_from_file( - keychain: SecKeychainRef, path: str -) -> tuple[list[CFTypeRef], list[CFTypeRef]]: - """ - Given a single file, loads all the trust objects from it into arrays and - the keychain. - Returns a tuple of lists: the first list is a list of identities, the - second a list of certs. - """ - certificates = [] - identities = [] - result_array = None - - with open(path, "rb") as f: - raw_filedata = f.read() - - try: - filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) - ) - result_array = CoreFoundation.CFArrayRef() - result = Security.SecItemImport( - filedata, # cert data - None, # Filename, leaving it out for now - None, # What the type of the file is, we don't care - None, # what's in the file, we don't care - 0, # import flags - None, # key params, can include passphrase in the future - keychain, # The keychain to insert into - ctypes.byref(result_array), # Results - ) - _assert_no_error(result) - - # A CFArray is not very useful to us as an intermediary - # representation, so we are going to extract the objects we want - # and then free the array. We don't need to keep hold of keys: the - # keychain already has them! - result_count = CoreFoundation.CFArrayGetCount(result_array) - for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) - item = ctypes.cast(item, CoreFoundation.CFTypeRef) - - if _is_cert(item): - CoreFoundation.CFRetain(item) - certificates.append(item) - elif _is_identity(item): - CoreFoundation.CFRetain(item) - identities.append(item) - finally: - if result_array: - CoreFoundation.CFRelease(result_array) - - CoreFoundation.CFRelease(filedata) - - return (identities, certificates) - - -def _load_client_cert_chain(keychain: SecKeychainRef, *paths: str | None) -> CFArray: - """ - Load certificates and maybe keys from a number of files. Has the end goal - of returning a CFArray containing one SecIdentityRef, and then zero or more - SecCertificateRef objects, suitable for use as a client certificate trust - chain. - """ - # Ok, the strategy. - # - # This relies on knowing that macOS will not give you a SecIdentityRef - # unless you have imported a key into a keychain. This is a somewhat - # artificial limitation of macOS (for example, it doesn't necessarily - # affect iOS), but there is nothing inside Security.framework that lets you - # get a SecIdentityRef without having a key in a keychain. - # - # So the policy here is we take all the files and iterate them in order. - # Each one will use SecItemImport to have one or more objects loaded from - # it. We will also point at a keychain that macOS can use to work with the - # private key. - # - # Once we have all the objects, we'll check what we actually have. If we - # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise, - # we'll take the first certificate (which we assume to be our leaf) and - # ask the keychain to give us a SecIdentityRef with that cert's associated - # key. - # - # We'll then return a CFArray containing the trust chain: one - # SecIdentityRef and then zero-or-more SecCertificateRef objects. The - # responsibility for freeing this CFArray will be with the caller. This - # CFArray must remain alive for the entire connection, so in practice it - # will be stored with a single SSLSocket, along with the reference to the - # keychain. - certificates = [] - identities = [] - - # Filter out bad paths. - filtered_paths = (path for path in paths if path) - - try: - for file_path in filtered_paths: - new_identities, new_certs = _load_items_from_file(keychain, file_path) - identities.extend(new_identities) - certificates.extend(new_certs) - - # Ok, we have everything. The question is: do we have an identity? If - # not, we want to grab one from the first cert we have. - if not identities: - new_identity = Security.SecIdentityRef() - status = Security.SecIdentityCreateWithCertificate( - keychain, certificates[0], ctypes.byref(new_identity) - ) - _assert_no_error(status) - identities.append(new_identity) - - # We now want to release the original certificate, as we no longer - # need it. - CoreFoundation.CFRelease(certificates.pop(0)) - - # We now need to build a new CFArray that holds the trust chain. - trust_chain = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - for item in itertools.chain(identities, certificates): - # ArrayAppendValue does a CFRetain on the item. That's fine, - # because the finally block will release our other refs to them. - CoreFoundation.CFArrayAppendValue(trust_chain, item) - - return trust_chain - finally: - for obj in itertools.chain(identities, certificates): - CoreFoundation.CFRelease(obj) - - -TLS_PROTOCOL_VERSIONS = { - "SSLv2": (0, 2), - "SSLv3": (3, 0), - "TLSv1": (3, 1), - "TLSv1.1": (3, 2), - "TLSv1.2": (3, 3), -} - - -def _build_tls_unknown_ca_alert(version: str) -> bytes: - """ - Builds a TLS alert record for an unknown CA. - """ - ver_maj, ver_min = TLS_PROTOCOL_VERSIONS[version] - severity_fatal = 0x02 - description_unknown_ca = 0x30 - msg = struct.pack(">BB", severity_fatal, description_unknown_ca) - msg_len = len(msg) - record_type_alert = 0x15 - record = struct.pack(">BBBH", record_type_alert, ver_maj, ver_min, msg_len) + msg - return record - - -class SecurityConst: - """ - A class object that acts as essentially a namespace for Security constants. - """ - - kSSLSessionOptionBreakOnServerAuth = 0 - - kSSLProtocol2 = 1 - kSSLProtocol3 = 2 - kTLSProtocol1 = 4 - kTLSProtocol11 = 7 - kTLSProtocol12 = 8 - # SecureTransport does not support TLS 1.3 even if there's a constant for it - kTLSProtocol13 = 10 - kTLSProtocolMaxSupported = 999 - - kSSLClientSide = 1 - kSSLStreamType = 0 - - kSecFormatPEMSequence = 10 - - kSecTrustResultInvalid = 0 - kSecTrustResultProceed = 1 - # This gap is present on purpose: this was kSecTrustResultConfirm, which - # is deprecated. - kSecTrustResultDeny = 3 - kSecTrustResultUnspecified = 4 - kSecTrustResultRecoverableTrustFailure = 5 - kSecTrustResultFatalTrustFailure = 6 - kSecTrustResultOtherError = 7 - - errSSLProtocol = -9800 - errSSLWouldBlock = -9803 - errSSLClosedGraceful = -9805 - errSSLClosedNoNotify = -9816 - errSSLClosedAbort = -9806 - - errSSLXCertChainInvalid = -9807 - errSSLCrypto = -9809 - errSSLInternal = -9810 - errSSLCertExpired = -9814 - errSSLCertNotYetValid = -9815 - errSSLUnknownRootCert = -9812 - errSSLNoRootCert = -9813 - errSSLHostNameMismatch = -9843 - errSSLPeerHandshakeFail = -9824 - errSSLPeerUserCancelled = -9839 - errSSLWeakPeerEphemeralDHKey = -9850 - errSSLServerAuthCompleted = -9841 - errSSLRecordOverflow = -9847 - - errSecVerifyFailed = -67808 - errSecNoTrustSettings = -25263 - errSecItemNotFound = -25300 - errSecInvalidTrustSettings = -25262 diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py deleted file mode 100644 index 2bc374bc4e..0000000000 --- a/src/urllib3/contrib/securetransport.py +++ /dev/null @@ -1,913 +0,0 @@ -""" -SecureTranport support for urllib3 via ctypes. - -This makes platform-native TLS available to urllib3 users on macOS without the -use of a compiler. This is an important feature because the Python Package -Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL -that ships with macOS is not capable of doing TLSv1.2. The only way to resolve -this is to give macOS users an alternative solution to the problem, and that -solution is to use SecureTransport. - -We use ctypes here because this solution must not require a compiler. That's -because pip is not allowed to require a compiler either. - -This is not intended to be a seriously long-term solution to this problem. -The hope is that PEP 543 will eventually solve this issue for us, at which -point we can retire this contrib module. But in the short term, we need to -solve the impending tire fire that is Python on Mac without this kind of -contrib module. So...here we are. - -To use this module, simply import and inject it:: - - import urllib3.contrib.securetransport - urllib3.contrib.securetransport.inject_into_urllib3() - -Happy TLSing! - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - -.. code-block:: - - Copyright (c) 2015-2016 Will Bond - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" - -from __future__ import annotations - -import contextlib -import ctypes -import errno -import os.path -import shutil -import socket -import ssl -import struct -import threading -import typing -import warnings -import weakref -from socket import socket as socket_cls - -from .. import util -from ._securetransport.bindings import ( # type: ignore[attr-defined] - CoreFoundation, - Security, -) -from ._securetransport.low_level import ( - SecurityConst, - _assert_no_error, - _build_tls_unknown_ca_alert, - _cert_array_from_pem, - _create_cfstring_array, - _load_client_cert_chain, - _temporary_keychain, -) - -warnings.warn( - "'urllib3.contrib.securetransport' module is deprecated and will be removed " - "in urllib3 v2.1.0. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2681", - category=DeprecationWarning, - stacklevel=2, -) - -if typing.TYPE_CHECKING: - from typing import Literal - -__all__ = ["inject_into_urllib3", "extract_from_urllib3"] - -orig_util_SSLContext = util.ssl_.SSLContext - -# This dictionary is used by the read callback to obtain a handle to the -# calling wrapped socket. This is a pretty silly approach, but for now it'll -# do. I feel like I should be able to smuggle a handle to the wrapped socket -# directly in the SSLConnectionRef, but for now this approach will work I -# guess. -# -# We need to lock around this structure for inserts, but we don't do it for -# reads/writes in the callbacks. The reasoning here goes as follows: -# -# 1. It is not possible to call into the callbacks before the dictionary is -# populated, so once in the callback the id must be in the dictionary. -# 2. The callbacks don't mutate the dictionary, they only read from it, and -# so cannot conflict with any of the insertions. -# -# This is good: if we had to lock in the callbacks we'd drastically slow down -# the performance of this code. -_connection_refs: weakref.WeakValueDictionary[ - int, WrappedSocket -] = weakref.WeakValueDictionary() -_connection_ref_lock = threading.Lock() - -# Limit writes to 16kB. This is OpenSSL's limit, but we'll cargo-cult it over -# for no better reason than we need *a* limit, and this one is right there. -SSL_WRITE_BLOCKSIZE = 16384 - -# Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of -# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. -# TLSv1 to 1.2 are supported on macOS 10.8+ -_protocol_to_min_max = { - util.ssl_.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), # type: ignore[attr-defined] - util.ssl_.PROTOCOL_TLS_CLIENT: ( # type: ignore[attr-defined] - SecurityConst.kTLSProtocol1, - SecurityConst.kTLSProtocol12, - ), -} - -if hasattr(ssl, "PROTOCOL_SSLv2"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, - SecurityConst.kSSLProtocol2, - ) -if hasattr(ssl, "PROTOCOL_SSLv3"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, - SecurityConst.kSSLProtocol3, - ) -if hasattr(ssl, "PROTOCOL_TLSv1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, - SecurityConst.kTLSProtocol1, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, - SecurityConst.kTLSProtocol11, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_2"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, - SecurityConst.kTLSProtocol12, - ) - - -_tls_version_to_st: dict[int, int] = { - ssl.TLSVersion.MINIMUM_SUPPORTED: SecurityConst.kTLSProtocol1, - ssl.TLSVersion.TLSv1: SecurityConst.kTLSProtocol1, - ssl.TLSVersion.TLSv1_1: SecurityConst.kTLSProtocol11, - ssl.TLSVersion.TLSv1_2: SecurityConst.kTLSProtocol12, - ssl.TLSVersion.MAXIMUM_SUPPORTED: SecurityConst.kTLSProtocol12, -} - - -def inject_into_urllib3() -> None: - """ - Monkey-patch urllib3 with SecureTransport-backed SSL-support. - """ - util.SSLContext = SecureTransportContext # type: ignore[assignment] - util.ssl_.SSLContext = SecureTransportContext # type: ignore[assignment] - util.IS_SECURETRANSPORT = True - util.ssl_.IS_SECURETRANSPORT = True - - -def extract_from_urllib3() -> None: - """ - Undo monkey-patching by :func:`inject_into_urllib3`. - """ - util.SSLContext = orig_util_SSLContext - util.ssl_.SSLContext = orig_util_SSLContext - util.IS_SECURETRANSPORT = False - util.ssl_.IS_SECURETRANSPORT = False - - -def _read_callback( - connection_id: int, data_buffer: int, data_length_pointer: bytearray -) -> int: - """ - SecureTransport read callback. This is called by ST to request that data - be returned from the socket. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - requested_length = data_length_pointer[0] - - timeout = wrapped_socket.gettimeout() - error = None - read_count = 0 - - try: - while read_count < requested_length: - if timeout is None or timeout >= 0: - if not util.wait_for_read(base_socket, timeout): - raise OSError(errno.EAGAIN, "timed out") - - remaining = requested_length - read_count - buffer = (ctypes.c_char * remaining).from_address( - data_buffer + read_count - ) - chunk_size = base_socket.recv_into(buffer, remaining) - read_count += chunk_size - if not chunk_size: - if not read_count: - return SecurityConst.errSSLClosedGraceful - break - except OSError as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = read_count - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = read_count - - if read_count != requested_length: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -def _write_callback( - connection_id: int, data_buffer: int, data_length_pointer: bytearray -) -> int: - """ - SecureTransport write callback. This is called by ST to request that data - actually be sent on the network. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - bytes_to_write = data_length_pointer[0] - data = ctypes.string_at(data_buffer, bytes_to_write) - - timeout = wrapped_socket.gettimeout() - error = None - sent = 0 - - try: - while sent < bytes_to_write: - if timeout is None or timeout >= 0: - if not util.wait_for_write(base_socket, timeout): - raise OSError(errno.EAGAIN, "timed out") - chunk_sent = base_socket.send(data) - sent += chunk_sent - - # This has some needless copying here, but I'm not sure there's - # much value in optimising this data path. - data = data[chunk_sent:] - except OSError as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = sent - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = sent - - if sent != bytes_to_write: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -# We need to keep these two objects references alive: if they get GC'd while -# in use then SecureTransport could attempt to call a function that is in freed -# memory. That would be...uh...bad. Yeah, that's the word. Bad. -_read_callback_pointer = Security.SSLReadFunc(_read_callback) -_write_callback_pointer = Security.SSLWriteFunc(_write_callback) - - -class WrappedSocket: - """ - API-compatibility wrapper for Python's OpenSSL wrapped socket object. - """ - - def __init__(self, socket: socket_cls) -> None: - self.socket = socket - self.context = None - self._io_refs = 0 - self._closed = False - self._real_closed = False - self._exception: Exception | None = None - self._keychain = None - self._keychain_dir: str | None = None - self._client_cert_chain = None - - # We save off the previously-configured timeout and then set it to - # zero. This is done because we use select and friends to handle the - # timeouts, but if we leave the timeout set on the lower socket then - # Python will "kindly" call select on that socket again for us. Avoid - # that by forcing the timeout to zero. - self._timeout = self.socket.gettimeout() - self.socket.settimeout(0) - - @contextlib.contextmanager - def _raise_on_error(self) -> typing.Generator[None, None, None]: - """ - A context manager that can be used to wrap calls that do I/O from - SecureTransport. If any of the I/O callbacks hit an exception, this - context manager will correctly propagate the exception after the fact. - This avoids silently swallowing those exceptions. - - It also correctly forces the socket closed. - """ - self._exception = None - - # We explicitly don't catch around this yield because in the unlikely - # event that an exception was hit in the block we don't want to swallow - # it. - yield - if self._exception is not None: - exception, self._exception = self._exception, None - self._real_close() - raise exception - - def _set_alpn_protocols(self, protocols: list[bytes] | None) -> None: - """ - Sets up the ALPN protocols on the context. - """ - if not protocols: - return - protocols_arr = _create_cfstring_array(protocols) - try: - result = Security.SSLSetALPNProtocols(self.context, protocols_arr) - _assert_no_error(result) - finally: - CoreFoundation.CFRelease(protocols_arr) - - def _custom_validate(self, verify: bool, trust_bundle: bytes | None) -> None: - """ - Called when we have set custom validation. We do this in two cases: - first, when cert validation is entirely disabled; and second, when - using a custom trust DB. - Raises an SSLError if the connection is not trusted. - """ - # If we disabled cert validation, just say: cool. - if not verify or trust_bundle is None: - return - - successes = ( - SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed, - ) - try: - trust_result = self._evaluate_trust(trust_bundle) - if trust_result in successes: - return - reason = f"error code: {int(trust_result)}" - exc = None - except Exception as e: - # Do not trust on error - reason = f"exception: {e!r}" - exc = e - - # SecureTransport does not send an alert nor shuts down the connection. - rec = _build_tls_unknown_ca_alert(self.version()) - self.socket.sendall(rec) - # close the connection immediately - # l_onoff = 1, activate linger - # l_linger = 0, linger for 0 seoncds - opts = struct.pack("ii", 1, 0) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, opts) - self._real_close() - raise ssl.SSLError(f"certificate verify failed, {reason}") from exc - - def _evaluate_trust(self, trust_bundle: bytes) -> int: - # We want data in memory, so load it up. - if os.path.isfile(trust_bundle): - with open(trust_bundle, "rb") as f: - trust_bundle = f.read() - - cert_array = None - trust = Security.SecTrustRef() - - try: - # Get a CFArray that contains the certs we want. - cert_array = _cert_array_from_pem(trust_bundle) - - # Ok, now the hard part. We want to get the SecTrustRef that ST has - # created for this connection, shove our CAs into it, tell ST to - # ignore everything else it knows, and then ask if it can build a - # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - raise ssl.SSLError("Failed to copy trust reference") - - result = Security.SecTrustSetAnchorCertificates(trust, cert_array) - _assert_no_error(result) - - result = Security.SecTrustSetAnchorCertificatesOnly(trust, True) - _assert_no_error(result) - - trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) - _assert_no_error(result) - finally: - if trust: - CoreFoundation.CFRelease(trust) - - if cert_array is not None: - CoreFoundation.CFRelease(cert_array) - - return trust_result.value # type: ignore[no-any-return] - - def handshake( - self, - server_hostname: bytes | str | None, - verify: bool, - trust_bundle: bytes | None, - min_version: int, - max_version: int, - client_cert: str | None, - client_key: str | None, - client_key_passphrase: typing.Any, - alpn_protocols: list[bytes] | None, - ) -> None: - """ - Actually performs the TLS handshake. This is run automatically by - wrapped socket, and shouldn't be needed in user code. - """ - # First, we do the initial bits of connection setup. We need to create - # a context, set its I/O funcs, and set the connection reference. - self.context = Security.SSLCreateContext( - None, SecurityConst.kSSLClientSide, SecurityConst.kSSLStreamType - ) - result = Security.SSLSetIOFuncs( - self.context, _read_callback_pointer, _write_callback_pointer - ) - _assert_no_error(result) - - # Here we need to compute the handle to use. We do this by taking the - # id of self modulo 2**31 - 1. If this is already in the dictionary, we - # just keep incrementing by one until we find a free space. - with _connection_ref_lock: - handle = id(self) % 2147483647 - while handle in _connection_refs: - handle = (handle + 1) % 2147483647 - _connection_refs[handle] = self - - result = Security.SSLSetConnection(self.context, handle) - _assert_no_error(result) - - # If we have a server hostname, we should set that too. - # RFC6066 Section 3 tells us not to use SNI when the host is an IP, but we have - # to do it anyway to match server_hostname against the server certificate - if server_hostname: - if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode("utf-8") - - result = Security.SSLSetPeerDomainName( - self.context, server_hostname, len(server_hostname) - ) - _assert_no_error(result) - - # Setup the ALPN protocols. - self._set_alpn_protocols(alpn_protocols) - - # Set the minimum and maximum TLS versions. - result = Security.SSLSetProtocolVersionMin(self.context, min_version) - _assert_no_error(result) - - result = Security.SSLSetProtocolVersionMax(self.context, max_version) - _assert_no_error(result) - - # If there's a trust DB, we need to use it. We do that by telling - # SecureTransport to break on server auth. We also do that if we don't - # want to validate the certs at all: we just won't actually do any - # authing in that case. - if not verify or trust_bundle is not None: - result = Security.SSLSetSessionOption( - self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True - ) - _assert_no_error(result) - - # If there's a client cert, we need to use it. - if client_cert: - self._keychain, self._keychain_dir = _temporary_keychain() - self._client_cert_chain = _load_client_cert_chain( - self._keychain, client_cert, client_key - ) - result = Security.SSLSetCertificate(self.context, self._client_cert_chain) - _assert_no_error(result) - - while True: - with self._raise_on_error(): - result = Security.SSLHandshake(self.context) - - if result == SecurityConst.errSSLWouldBlock: - raise socket.timeout("handshake timed out") - elif result == SecurityConst.errSSLServerAuthCompleted: - self._custom_validate(verify, trust_bundle) - continue - else: - _assert_no_error(result) - break - - def fileno(self) -> int: - return self.socket.fileno() - - # Copy-pasted from Python 3.5 source code - def _decref_socketios(self) -> None: - if self._io_refs > 0: - self._io_refs -= 1 - if self._closed: - self.close() - - def recv(self, bufsiz: int) -> bytes: - buffer = ctypes.create_string_buffer(bufsiz) - bytes_read = self.recv_into(buffer, bufsiz) - data = buffer[:bytes_read] - return typing.cast(bytes, data) - - def recv_into( - self, buffer: ctypes.Array[ctypes.c_char], nbytes: int | None = None - ) -> int: - # Read short on EOF. - if self._real_closed: - return 0 - - if nbytes is None: - nbytes = len(buffer) - - buffer = (ctypes.c_char * nbytes).from_buffer(buffer) - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLRead( - self.context, buffer, nbytes, ctypes.byref(processed_bytes) - ) - - # There are some result codes that we want to treat as "not always - # errors". Specifically, those are errSSLWouldBlock, - # errSSLClosedGraceful, and errSSLClosedNoNotify. - if result == SecurityConst.errSSLWouldBlock: - # If we didn't process any bytes, then this was just a time out. - # However, we can get errSSLWouldBlock in situations when we *did* - # read some data, and in those cases we should just read "short" - # and return. - if processed_bytes.value == 0: - # Timed out, no data read. - raise socket.timeout("recv timed out") - elif result in ( - SecurityConst.errSSLClosedGraceful, - SecurityConst.errSSLClosedNoNotify, - ): - # The remote peer has closed this connection. We should do so as - # well. Note that we don't actually return here because in - # principle this could actually be fired along with return data. - # It's unlikely though. - self._real_close() - else: - _assert_no_error(result) - - # Ok, we read and probably succeeded. We should return whatever data - # was actually read. - return processed_bytes.value - - def settimeout(self, timeout: float) -> None: - self._timeout = timeout - - def gettimeout(self) -> float | None: - return self._timeout - - def send(self, data: bytes) -> int: - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLWrite( - self.context, data, len(data), ctypes.byref(processed_bytes) - ) - - if result == SecurityConst.errSSLWouldBlock and processed_bytes.value == 0: - # Timed out - raise socket.timeout("send timed out") - else: - _assert_no_error(result) - - # We sent, and probably succeeded. Tell them how much we sent. - return processed_bytes.value - - def sendall(self, data: bytes) -> None: - total_sent = 0 - while total_sent < len(data): - sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) - total_sent += sent - - def shutdown(self) -> None: - with self._raise_on_error(): - Security.SSLClose(self.context) - - def close(self) -> None: - self._closed = True - # TODO: should I do clean shutdown here? Do I have to? - if self._io_refs <= 0: - self._real_close() - - def _real_close(self) -> None: - self._real_closed = True - if self.context: - CoreFoundation.CFRelease(self.context) - self.context = None - if self._client_cert_chain: - CoreFoundation.CFRelease(self._client_cert_chain) - self._client_cert_chain = None - if self._keychain: - Security.SecKeychainDelete(self._keychain) - CoreFoundation.CFRelease(self._keychain) - shutil.rmtree(self._keychain_dir) - self._keychain = self._keychain_dir = None - return self.socket.close() - - def getpeercert(self, binary_form: bool = False) -> bytes | None: - # Urgh, annoying. - # - # Here's how we do this: - # - # 1. Call SSLCopyPeerTrust to get hold of the trust object for this - # connection. - # 2. Call SecTrustGetCertificateAtIndex for index 0 to get the leaf. - # 3. To get the CN, call SecCertificateCopyCommonName and process that - # string so that it's of the appropriate type. - # 4. To get the SAN, we need to do something a bit more complex: - # a. Call SecCertificateCopyValues to get the data, requesting - # kSecOIDSubjectAltName. - # b. Mess about with this dictionary to try to get the SANs out. - # - # This is gross. Really gross. It's going to be a few hundred LoC extra - # just to repeat something that SecureTransport can *already do*. So my - # operating assumption at this time is that what we want to do is - # instead to just flag to urllib3 that it shouldn't do its own hostname - # validation when using SecureTransport. - if not binary_form: - raise ValueError("SecureTransport only supports dumping binary certs") - trust = Security.SecTrustRef() - certdata = None - der_bytes = None - - try: - # Grab the trust store. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - # Probably we haven't done the handshake yet. No biggie. - return None - - cert_count = Security.SecTrustGetCertificateCount(trust) - if not cert_count: - # Also a case that might happen if we haven't handshaked. - # Handshook? Handshaken? - return None - - leaf = Security.SecTrustGetCertificateAtIndex(trust, 0) - assert leaf - - # Ok, now we want the DER bytes. - certdata = Security.SecCertificateCopyData(leaf) - assert certdata - - data_length = CoreFoundation.CFDataGetLength(certdata) - data_buffer = CoreFoundation.CFDataGetBytePtr(certdata) - der_bytes = ctypes.string_at(data_buffer, data_length) - finally: - if certdata: - CoreFoundation.CFRelease(certdata) - if trust: - CoreFoundation.CFRelease(trust) - - return der_bytes - - def version(self) -> str: - protocol = Security.SSLProtocol() - result = Security.SSLGetNegotiatedProtocolVersion( - self.context, ctypes.byref(protocol) - ) - _assert_no_error(result) - if protocol.value == SecurityConst.kTLSProtocol13: - raise ssl.SSLError("SecureTransport does not support TLS 1.3") - elif protocol.value == SecurityConst.kTLSProtocol12: - return "TLSv1.2" - elif protocol.value == SecurityConst.kTLSProtocol11: - return "TLSv1.1" - elif protocol.value == SecurityConst.kTLSProtocol1: - return "TLSv1" - elif protocol.value == SecurityConst.kSSLProtocol3: - return "SSLv3" - elif protocol.value == SecurityConst.kSSLProtocol2: - return "SSLv2" - else: - raise ssl.SSLError(f"Unknown TLS version: {protocol!r}") - - -def makefile( - self: socket_cls, - mode: ( - Literal["r"] | Literal["w"] | Literal["rw"] | Literal["wr"] | Literal[""] - ) = "r", - buffering: int | None = None, - *args: typing.Any, - **kwargs: typing.Any, -) -> typing.BinaryIO | typing.TextIO: - # We disable buffering with SecureTransport because it conflicts with - # the buffering that ST does internally (see issue #1153 for more). - buffering = 0 - return socket_cls.makefile(self, mode, buffering, *args, **kwargs) - - -WrappedSocket.makefile = makefile # type: ignore[attr-defined] - - -class SecureTransportContext: - """ - I am a wrapper class for the SecureTransport library, to translate the - interface of the standard library ``SSLContext`` object to calls into - SecureTransport. - """ - - def __init__(self, protocol: int) -> None: - self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED - self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED - if protocol not in (None, ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_CLIENT): - self._min_version, self._max_version = _protocol_to_min_max[protocol] - - self._options = 0 - self._verify = False - self._trust_bundle: bytes | None = None - self._client_cert: str | None = None - self._client_key: str | None = None - self._client_key_passphrase = None - self._alpn_protocols: list[bytes] | None = None - - @property - def check_hostname(self) -> Literal[True]: - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - return True - - @check_hostname.setter - def check_hostname(self, value: typing.Any) -> None: - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - - @property - def options(self) -> int: - # TODO: Well, crap. - # - # So this is the bit of the code that is the most likely to cause us - # trouble. Essentially we need to enumerate all of the SSL options that - # users might want to use and try to see if we can sensibly translate - # them, or whether we should just ignore them. - return self._options - - @options.setter - def options(self, value: int) -> None: - # TODO: Update in line with above. - self._options = value - - @property - def verify_mode(self) -> int: - return ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE - - @verify_mode.setter - def verify_mode(self, value: int) -> None: - self._verify = value == ssl.CERT_REQUIRED - - def set_default_verify_paths(self) -> None: - # So, this has to do something a bit weird. Specifically, what it does - # is nothing. - # - # This means that, if we had previously had load_verify_locations - # called, this does not undo that. We need to do that because it turns - # out that the rest of the urllib3 code will attempt to load the - # default verify paths if it hasn't been told about any paths, even if - # the context itself was sometime earlier. We resolve that by just - # ignoring it. - pass - - def load_default_certs(self) -> None: - return self.set_default_verify_paths() - - def set_ciphers(self, ciphers: typing.Any) -> None: - raise ValueError("SecureTransport doesn't support custom cipher strings") - - def load_verify_locations( - self, - cafile: str | None = None, - capath: str | None = None, - cadata: bytes | None = None, - ) -> None: - # OK, we only really support cadata and cafile. - if capath is not None: - raise ValueError("SecureTransport does not support cert directories") - - # Raise if cafile does not exist. - if cafile is not None: - with open(cafile): - pass - - self._trust_bundle = cafile or cadata # type: ignore[assignment] - - def load_cert_chain( - self, - certfile: str, - keyfile: str | None = None, - password: str | None = None, - ) -> None: - self._client_cert = certfile - self._client_key = keyfile - self._client_cert_passphrase = password - - def set_alpn_protocols(self, protocols: list[str | bytes]) -> None: - """ - Sets the ALPN protocols that will later be set on the context. - - Raises a NotImplementedError if ALPN is not supported. - """ - if not hasattr(Security, "SSLSetALPNProtocols"): - raise NotImplementedError( - "SecureTransport supports ALPN only in macOS 10.12+" - ) - self._alpn_protocols = [util.util.to_bytes(p, "ascii") for p in protocols] - - def wrap_socket( - self, - sock: socket_cls, - server_side: bool = False, - do_handshake_on_connect: bool = True, - suppress_ragged_eofs: bool = True, - server_hostname: bytes | str | None = None, - ) -> WrappedSocket: - # So, what do we do here? Firstly, we assert some properties. This is a - # stripped down shim, so there is some functionality we don't support. - # See PEP 543 for the real deal. - assert not server_side - assert do_handshake_on_connect - assert suppress_ragged_eofs - - # Ok, we're good to go. Now we want to create the wrapped socket object - # and store it in the appropriate place. - wrapped_socket = WrappedSocket(sock) - - # Now we can handshake - wrapped_socket.handshake( - server_hostname, - self._verify, - self._trust_bundle, - _tls_version_to_st[self._minimum_version], - _tls_version_to_st[self._maximum_version], - self._client_cert, - self._client_key, - self._client_key_passphrase, - self._alpn_protocols, - ) - return wrapped_socket - - @property - def minimum_version(self) -> int: - return self._minimum_version - - @minimum_version.setter - def minimum_version(self, minimum_version: int) -> None: - self._minimum_version = minimum_version - - @property - def maximum_version(self) -> int: - return self._maximum_version - - @maximum_version.setter - def maximum_version(self, maximum_version: int) -> None: - self._maximum_version = maximum_version diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 9e7aab41fb..367ccfe425 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -767,13 +767,9 @@ def _fp_read(self, amt: int | None = None) -> bytes: assert self._fp c_int_max = 2**31 - 1 if ( - ( - (amt and amt > c_int_max) - or (self.length_remaining and self.length_remaining > c_int_max) - ) - and not util.IS_SECURETRANSPORT - and (util.IS_PYOPENSSL or sys.version_info < (3, 10)) - ): + (amt and amt > c_int_max) + or (self.length_remaining and self.length_remaining > c_int_max) + ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)): buffer = io.BytesIO() # Besides `max_chunk_amt` being a maximum chunk size, it # affects memory overhead of reading a response by this diff --git a/src/urllib3/util/__init__.py b/src/urllib3/util/__init__.py index ff56c55bae..534126033c 100644 --- a/src/urllib3/util/__init__.py +++ b/src/urllib3/util/__init__.py @@ -8,7 +8,6 @@ from .ssl_ import ( ALPN_PROTOCOLS, IS_PYOPENSSL, - IS_SECURETRANSPORT, SSLContext, assert_fingerprint, create_urllib3_context, @@ -22,7 +21,6 @@ __all__ = ( "IS_PYOPENSSL", - "IS_SECURETRANSPORT", "SSLContext", "ALPN_PROTOCOLS", "Retry", diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index e26227ab14..e0a7c04a3c 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -16,7 +16,6 @@ SSLTransport = None HAS_NEVER_CHECK_COMMON_NAME = False IS_PYOPENSSL = False -IS_SECURETRANSPORT = False ALPN_PROTOCOLS = ["http/1.1"] _TYPE_VERSION_INFO = typing.Tuple[int, int, int, str, int] diff --git a/test/__init__.py b/test/__init__.py index 0c3c8e08b9..1bf153cfcb 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -156,31 +156,6 @@ def notZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]: ) -# Hack to make pytest evaluate a condition at test runtime instead of collection time. -def lazy_condition(condition: typing.Callable[[], bool]) -> bool: - class LazyCondition: - def __bool__(self) -> bool: - return condition() - - return typing.cast(bool, LazyCondition()) - - -def onlySecureTransport() -> typing.Callable[[_TestFuncT], _TestFuncT]: - """Runs this test when SecureTransport is in use.""" - return pytest.mark.skipif( - lazy_condition(lambda: not ssl_.IS_SECURETRANSPORT), - reason="Test only runs with SecureTransport", - ) - - -def notSecureTransport() -> typing.Callable[[_TestFuncT], _TestFuncT]: - """Skips this test when SecureTransport is in use.""" - return pytest.mark.skipif( - lazy_condition(lambda: ssl_.IS_SECURETRANSPORT), - reason="Test does not run with SecureTransport", - ) - - _requires_network_has_route = None @@ -217,15 +192,6 @@ def _has_route() -> bool: ) -def requires_ssl_context_keyfile_password() -> ( - typing.Callable[[_TestFuncT], _TestFuncT] -): - return pytest.mark.skipif( - lazy_condition(lambda: ssl_.IS_SECURETRANSPORT), - reason="Test requires password parameter for SSLContext.load_cert_chain()", - ) - - def resolvesLocalhostFQDN() -> typing.Callable[[_TestFuncT], _TestFuncT]: """Test requires successful resolving of 'localhost.'""" return pytest.mark.skipif( diff --git a/test/contrib/test_securetransport.py b/test/contrib/test_securetransport.py deleted file mode 100644 index ac41fe5caa..0000000000 --- a/test/contrib/test_securetransport.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import base64 -import contextlib -import socket -import ssl - -import pytest - -try: - from urllib3.contrib.securetransport import WrappedSocket -except ImportError: - pass - - -def setup_module() -> None: - try: - from urllib3.contrib.securetransport import inject_into_urllib3 - - inject_into_urllib3() - except ImportError as e: - pytest.skip(f"Could not import SecureTransport: {repr(e)}") - - -def teardown_module() -> None: - try: - from urllib3.contrib.securetransport import extract_from_urllib3 - - extract_from_urllib3() - except ImportError: - pass - - -from ..test_util import TestUtilSSL # noqa: E402, F401 - -# SecureTransport does not support TLSv1.3 -# https://github.com/urllib3/urllib3/issues/1674 -from ..with_dummyserver.test_https import ( # noqa: E402, F401 - TestHTTPS, - TestHTTPS_TLSv1, - TestHTTPS_TLSv1_1, - TestHTTPS_TLSv1_2, -) -from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401 - TestClientCerts, - TestSNI, - TestSocketClosing, - TestSSL, -) - - -def test_no_crash_with_empty_trust_bundle() -> None: - with contextlib.closing(socket.socket()) as s: - ws = WrappedSocket(s) - with pytest.raises(ssl.SSLError): - ws._custom_validate(True, b"") - - -def test_no_crash_with_invalid_trust_bundle() -> None: - invalid_cert = base64.b64encode(b"invalid-cert") - cert_bundle = ( - b"-----BEGIN CERTIFICATE-----\n" + invalid_cert + b"\n-----END CERTIFICATE-----" - ) - - with contextlib.closing(socket.socket()) as s: - ws = WrappedSocket(s) - with pytest.raises(ssl.SSLError): - ws._custom_validate(True, cert_bundle) diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 014bb68887..158df83b8f 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -12,9 +12,7 @@ LONG_TIMEOUT, SHORT_TIMEOUT, TARPIT_HOST, - notSecureTransport, requires_network, - requires_ssl_context_keyfile_password, resolvesLocalhostFQDN, ) from test.conftest import ServerConfig @@ -189,7 +187,6 @@ def test_client_no_intermediate(self) -> None: with pytest.raises((SSLError, ProtocolError)): https_pool.request("GET", "/certificate", retries=False) - @requires_ssl_context_keyfile_password() def test_client_key_password(self) -> None: with HTTPSConnectionPool( self.host, @@ -204,7 +201,6 @@ def test_client_key_password(self) -> None: subject = r.json() assert subject["organizationalUnitName"].startswith("Testing cert") - @requires_ssl_context_keyfile_password() def test_client_encrypted_key_requires_password(self) -> None: with HTTPSConnectionPool( self.host, @@ -265,7 +261,6 @@ def test_context_combines_with_ca_certs(self) -> None: assert r.status == 200 assert not warn.called, warn.call_args_list - @notSecureTransport() # SecureTransport does not support cert directories def test_ca_dir_verified(self, tmp_path: Path) -> None: # OpenSSL looks up certificates by the hash for their name, see c_rehash # TODO infer the bytes using `cryptography.x509.Name.public_bytes`. @@ -552,12 +547,7 @@ def test_verify_none_and_good_fingerprint(self) -> None: ) as https_pool: https_pool.request("GET", "/") - @notSecureTransport() def test_good_fingerprint_and_hostname_mismatch(self) -> None: - # This test doesn't run with SecureTransport because we don't turn off - # hostname validation without turning off all validation, which this - # test doesn't do (deliberately). We should revisit this if we make - # new decisions. with HTTPSConnectionPool( "127.0.0.1", self.port, @@ -975,13 +965,12 @@ def test_default_ssl_context_ssl_min_max_versions(self) -> None: ctx = urllib3.util.ssl_.create_urllib3_context() assert ctx.minimum_version == ssl.TLSVersion.TLSv1_2 # urllib3 sets a default maximum version only when it is - # injected with PyOpenSSL- or SecureTransport-backed - # SSL-support. + # injected with PyOpenSSL SSL-support. # Otherwise, the default maximum version is set by Python's # `ssl.SSLContext`. The value respects OpenSSL configuration and # can be different from `ssl.TLSVersion.MAXIMUM_SUPPORTED`. # https://github.com/urllib3/urllib3/issues/2477#issuecomment-1151452150 - if util.IS_PYOPENSSL or util.IS_SECURETRANSPORT: + if util.IS_PYOPENSSL: expected_maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED else: expected_maximum_version = ssl.SSLContext( diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index f4620643f5..9d5f90761e 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -9,7 +9,7 @@ import socket import ssl import tempfile -from test import LONG_TIMEOUT, SHORT_TIMEOUT, onlySecureTransport, withPyOpenSSL +from test import LONG_TIMEOUT, SHORT_TIMEOUT, withPyOpenSSL from test.conftest import ServerConfig import pytest @@ -103,17 +103,6 @@ def test_https_proxy_pyopenssl_not_supported(self) -> None: ): https.request("GET", f"{self.https_url}/") - @onlySecureTransport() - def test_https_proxy_securetransport_not_supported(self) -> None: - with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: - r = https.request("GET", f"{self.http_url}/") - assert r.status == 200 - - with pytest.raises( - ProxySchemeUnsupported, match="isn't available on non-native SSLContext" - ): - https.request("GET", f"{self.https_url}/") - def test_https_proxy_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index cae6b241c7..8635edc0c9 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -18,14 +18,7 @@ import zlib from collections import OrderedDict from pathlib import Path -from test import ( - LONG_TIMEOUT, - SHORT_TIMEOUT, - notSecureTransport, - notWindows, - requires_ssl_context_keyfile_password, - resolvesLocalhostFQDN, -) +from test import LONG_TIMEOUT, SHORT_TIMEOUT, notWindows, resolvesLocalhostFQDN from threading import Event from unittest import mock @@ -329,11 +322,9 @@ def socket_handler(listener: socket.socket) -> None: done_receiving.set() done_receiving.set() - @requires_ssl_context_keyfile_password() def test_client_cert_with_string_password(self) -> None: self.run_client_cert_with_password_test("letmein") - @requires_ssl_context_keyfile_password() def test_client_cert_with_bytes_password(self) -> None: self.run_client_cert_with_password_test(b"letmein") @@ -385,7 +376,6 @@ def socket_handler(listener: socket.socket) -> None: assert len(client_certs) == 1 - @requires_ssl_context_keyfile_password() def test_load_keyfile_with_invalid_password(self) -> None: assert ssl_.SSLContext is not None context = ssl_.SSLContext(ssl_.PROTOCOL_SSLv23) @@ -396,9 +386,6 @@ def test_load_keyfile_with_invalid_password(self) -> None: password=b"letmei", ) - # For SecureTransport, the validation that would raise an error in - # this case is deferred. - @notSecureTransport() def test_load_invalid_cert_file(self) -> None: assert ssl_.SSLContext is not None context = ssl_.SSLContext(ssl_.PROTOCOL_SSLv23) @@ -993,9 +980,7 @@ def consume_ssl_socket(listener: socket.socket) -> None: ) as f: ssl_sock.close() f.close() - # SecureTransport is supposed to raise OSError but raises - # ssl.SSLError when closed because ssl_sock.context is None - with pytest.raises((OSError, ssl.SSLError)): + with pytest.raises(OSError): ssl_sock.sendall(b"hello") assert ssl_sock.fileno() == -1 @@ -1316,7 +1301,6 @@ def socket_handler(listener: socket.socket) -> None: ): pool.request("GET", "/", retries=False) - @notSecureTransport() def test_ssl_read_timeout(self) -> None: timed_out = Event() @@ -1600,9 +1584,6 @@ def socket_handler(listener: socket.socket) -> None: pool.request("GET", "/", retries=False, timeout=LONG_TIMEOUT) assert server_closed.wait(LONG_TIMEOUT), "The socket was not terminated" - # SecureTransport can read only small pieces of data at the moment. - # https://github.com/urllib3/urllib3/pull/2674 - @notSecureTransport() @pytest.mark.skipif( os.environ.get("CI") == "true" and sys.implementation.name == "pypy", reason="too slow to run in CI", From 11f5f5e19bfaab9cdfa2ce9223e62b8f05dd2a99 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 6 Oct 2023 20:21:40 +0400 Subject: [PATCH 003/131] Drop support for urllib3[secure] extra (#3147) --- changelog/2680.removal.rst | 1 + dev-requirements.txt | 2 ++ noxfile.py | 6 +++--- pyproject.toml | 8 -------- src/urllib3/__init__.py | 17 ----------------- src/urllib3/contrib/pyopenssl.py | 6 +++--- 6 files changed, 9 insertions(+), 31 deletions(-) create mode 100644 changelog/2680.removal.rst diff --git a/changelog/2680.removal.rst b/changelog/2680.removal.rst new file mode 100644 index 0000000000..ad43b9bd3a --- /dev/null +++ b/changelog/2680.removal.rst @@ -0,0 +1 @@ +Removed support for the urllib3[secure] extra. diff --git a/dev-requirements.txt b/dev-requirements.txt index 1679aef27f..845fc98f8a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,6 +3,8 @@ tornado==6.3.3 PySocks==1.7.1 pytest==7.4.2 pytest-timeout==2.1.0 +pyOpenSSL==23.2.0 +idna==3.4 trustme==1.1.0 cryptography==41.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" diff --git a/noxfile.py b/noxfile.py index f0c7f1421b..9a4f576912 100644 --- a/noxfile.py +++ b/noxfile.py @@ -9,7 +9,7 @@ def tests_impl( session: nox.Session, - extras: str = "socks,secure,brotli,zstd", + extras: str = "socks,brotli,zstd", byte_string_comparisons: bool = True, ) -> None: # Install deps and the package itself. @@ -65,7 +65,7 @@ def test_brotlipy(session: nox.Session) -> None: 'brotlicffi' that we still don't blow up. """ session.install("brotlipy") - tests_impl(session, extras="socks,secure", byte_string_comparisons=False) + tests_impl(session, extras="socks", byte_string_comparisons=False) def git_clone(session: nox.Session, git_url: str) -> None: @@ -156,7 +156,7 @@ def mypy(session: nox.Session) -> None: @nox.session def docs(session: nox.Session) -> None: session.install("-r", "docs/requirements.txt") - session.install(".[socks,secure,brotli,zstd]") + session.install(".[socks,brotli,zstd]") session.chdir("docs") if os.path.exists("_build"): diff --git a/pyproject.toml b/pyproject.toml index a4574d3869..6aac7948ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,13 +45,6 @@ brotli = [ zstd = [ "zstandard>=0.18.0", ] -secure = [ - "pyOpenSSL>=17.1.0", - "cryptography>=1.9", - "idna>=2.0.0", - "certifi", - "urllib3-secure-extra", -] socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] @@ -85,7 +78,6 @@ log_level = "DEBUG" filterwarnings = [ "error", '''default:urllib3 v2.0 only supports OpenSSL 1.1.1+.*''', - '''default:'urllib3\[secure\]' extra is deprecated and will be removed in urllib3 v2\.1\.0.*:DeprecationWarning''', '''default:No IPv6 support. Falling back to IPv4:urllib3.exceptions.HTTPWarning''', '''default:No IPv6 support. skipping:urllib3.exceptions.HTTPWarning''', '''default:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning''', diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 32c1f0025f..26fc5770e4 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -44,23 +44,6 @@ "See: https://github.com/urllib3/urllib3/issues/2168" ) -# === NOTE TO REPACKAGERS AND VENDORS === -# Please delete this block, this logic is only -# for urllib3 being distributed via PyPI. -# See: https://github.com/urllib3/urllib3/issues/2680 -try: - import urllib3_secure_extra # type: ignore # noqa: F401 -except ModuleNotFoundError: - pass -else: - warnings.warn( - "'urllib3[secure]' extra is deprecated and will be removed " - "in urllib3 v2.1.0. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2680", - category=DeprecationWarning, - stacklevel=2, - ) - __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" __version__ = __version__ diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py index 74b35883bf..3987d6320d 100644 --- a/src/urllib3/contrib/pyopenssl.py +++ b/src/urllib3/contrib/pyopenssl.py @@ -8,10 +8,10 @@ * `pyOpenSSL`_ (tested with 16.0.0) * `cryptography`_ (minimum 1.3.4, from pyopenssl) -* `idna`_ (minimum 2.0, from cryptography) +* `idna`_ (minimum 2.0) -However, pyOpenSSL depends on cryptography, which depends on idna, so while we -use all three directly here we end up having relatively few packages required. +However, pyOpenSSL depends on cryptography, so while we use all three directly here we +end up having relatively few packages required. You can install them with the following command: From 431c07db7cabd01829631ec37473b4fbf4341175 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 9 Oct 2023 16:54:24 +0400 Subject: [PATCH 004/131] Add Python 3.12 macOS to CI (#3152) --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e56f7644..ddbc8a17a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,16 +64,11 @@ jobs: os: ubuntu-20.04 # CPython 3.9.2 is not available for ubuntu-22.04. experimental: false nox-session: test-3.9 - - python-version: "3.12" - experimental: true exclude: # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it # https://github.com/python/cpython/issues/83001 - python-version: "3.8" os: ubuntu-22.04 - # Testing with non-final CPython on macOS is too slow for CI. - - python-version: "3.12" - os: macos-11 runs-on: ${{ matrix.os }} name: ${{ fromJson('{"macos-11":"macOS","windows-latest":"Windows","ubuntu-latest":"Ubuntu","ubuntu-20.04":"Ubuntu 20.04 (OpenSSL 1.1.1)","ubuntu-22.04":"Ubuntu 22.04 (OpenSSL 3.0)"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session}} From 3c6e079d1395ae815484b4a8f0ed7657a4dc8d0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 20:39:25 +0000 Subject: [PATCH 005/131] Bump ossf/scorecard-action from 2.2.0 to 2.3.0 Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/08b4669551908b1024bb425080c797723083c031...483ef80eb98fb506c348f7d62e28055e49fe2398) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/scorecards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index fd29d3e25e..18891ce8c1 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: "Run Scorecard" - uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0 + uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 with: results_file: results.sarif results_format: sarif From 4e98d57809dacab1cbe625fddeec1a290c478ea9 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Tue, 17 Oct 2023 21:07:33 +0300 Subject: [PATCH 006/131] Bring 2.0.7 & 1.26.18 to main (#3161) * Merge pull request from GHSA-g4mx-q9vg-27p4 * Release 2.0.7 --- .readthedocs.yml | 2 +- CHANGES.rst | 10 ++++++++++ dummyserver/handlers.py | 6 ++++++ src/urllib3/_collections.py | 20 ++++++++++++++++++++ src/urllib3/_version.py | 2 +- src/urllib3/connectionpool.py | 5 +++++ src/urllib3/poolmanager.py | 7 +++++-- test/with_dummyserver/test_connectionpool.py | 11 +++++++++++ test/with_dummyserver/test_poolmanager.py | 14 ++++++++++++++ 9 files changed, 73 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 78bd2064f6..7c59df5b3a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3" + python: "3.11" python: install: diff --git a/CHANGES.rst b/CHANGES.rst index 27038fef47..6c37aeba10 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +2.0.7 (2023-10-17) +================== + +* Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. + 2.0.6 (2023-10-02) ================== @@ -167,6 +172,11 @@ Fixed * Fixed a socket leak if ``HTTPConnection.connect()`` fails (`#2571 `__). * Fixed ``urllib3.contrib.pyopenssl.WrappedSocket`` and ``urllib3.contrib.securetransport.WrappedSocket`` close methods (`#2970 `__) +1.26.18 (2023-10-17) +==================== + +* Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. + 1.26.17 (2023-10-02) ==================== diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 9fde80e041..86201a116f 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -281,6 +281,12 @@ def encodingrequest(self, request: httputil.HTTPServerRequest) -> Response: def headers(self, request: httputil.HTTPServerRequest) -> Response: return Response(json.dumps(dict(request.headers))) + def headers_and_params(self, request: httputil.HTTPServerRequest) -> Response: + params = request_params(request) + return Response( + json.dumps({"headers": dict(request.headers), "params": params}) + ) + def multi_headers(self, request: httputil.HTTPServerRequest) -> Response: return Response(json.dumps({"headers": list(request.headers.get_all())})) diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index f1cb60654a..55b0324797 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -10,6 +10,8 @@ # dependency, and is not available at runtime. from typing import Protocol + from typing_extensions import Self + class HasGettableStringKeys(Protocol): def keys(self) -> typing.Iterator[str]: ... @@ -391,6 +393,24 @@ def getlist( # meets our external interface requirement of `Union[List[str], _DT]`. return vals[1:] + def _prepare_for_method_change(self) -> Self: + """ + Remove content-specific header fields before changing the request + method to GET or HEAD according to RFC 9110, Section 15.4. + """ + content_specific_headers = [ + "Content-Encoding", + "Content-Language", + "Content-Location", + "Content-Type", + "Content-Length", + "Digest", + "Last-Modified", + ] + for header in content_specific_headers: + self.discard(header) + return self + # Backwards compatibility for httplib getheaders = getlist getallmatchingheaders = getlist diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index 2d0d430896..e2b88f1d68 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.0.6" +__version__ = "2.0.7" diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 113d264d79..70048b7aed 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -11,6 +11,7 @@ from types import TracebackType from ._base_connection import _TYPE_BODY +from ._collections import HTTPHeaderDict from ._request_methods import RequestMethods from .connection import ( BaseSSLError, @@ -892,7 +893,11 @@ def urlopen( # type: ignore[override] redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + body = None + headers = HTTPHeaderDict(headers)._prepare_for_method_change() try: retries = retries.increment(method, url, response=response, _pool=self) diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index 7a998ee896..8b8fac9e0d 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -7,7 +7,7 @@ from types import TracebackType from urllib.parse import urljoin -from ._collections import RecentlyUsedContainer +from ._collections import HTTPHeaderDict, RecentlyUsedContainer from ._request_methods import RequestMethods from .connection import ProxyConfig from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme @@ -448,9 +448,12 @@ def urlopen( # type: ignore[override] # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) - # RFC 7231, Section 6.4.4 if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + kw["body"] = None + kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() retries = kw.get("retries") if not isinstance(retries, Retry): diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index fdfb2c9aba..ebfaf3878f 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -480,6 +480,17 @@ def test_redirect(self) -> None: assert r.status == 200 assert r.data == b"Dummy server!" + def test_303_redirect_makes_request_lose_body(self) -> None: + with HTTPConnectionPool(self.host, self.port) as pool: + response = pool.request( + "POST", + "/redirect", + fields={"target": "/headers_and_params", "status": "303 See Other"}, + ) + data = response.json() + assert data["params"] == {} + assert "Content-Type" not in HTTPHeaderDict(data["headers"]) + def test_bad_connect(self) -> None: with HTTPConnectionPool("badhost.invalid", self.port) as pool: with pytest.raises(MaxRetryError) as e: diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index da802a38b3..ab0111e45b 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -244,6 +244,20 @@ def test_redirect_without_preload_releases_connection(self) -> None: assert r._pool.num_connections == 1 assert len(http.pools) == 1 + def test_303_redirect_makes_request_lose_body(self) -> None: + with PoolManager() as http: + response = http.request( + "POST", + f"{self.base_url}/redirect", + fields={ + "target": f"{self.base_url}/headers_and_params", + "status": "303 See Other", + }, + ) + data = response.json() + assert data["params"] == {} + assert "Content-Type" not in HTTPHeaderDict(data["headers"]) + def test_unknown_scheme(self) -> None: with PoolManager() as http: unknown_scheme = "unknown" From 0a3f8283101b736a9234aca0be72fb1709244e51 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 30 Oct 2023 15:58:54 -0700 Subject: [PATCH 007/131] Don't actually attempt to connect to evil.com in test_deprecated_no_scheme --- test/test_poolmanager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py index 821e218b18..ab5f20309b 100644 --- a/test/test_poolmanager.py +++ b/test/test_poolmanager.py @@ -259,11 +259,15 @@ def test_http_connection_from_context_case_insensitive(self) -> None: assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_deprecated_no_scheme(self) -> None: + @patch("urllib3.poolmanager.PoolManager.connection_from_host") + def test_deprecated_no_scheme(self, connection_from_host: mock.MagicMock) -> None: + # Don't actually make a network connection, just verify the DeprecationWarning + connection_from_host.side_effect = ConnectionError("Not attempting connection") p = PoolManager() with pytest.warns(DeprecationWarning) as records: - p.request(method="GET", url="evil.com://good.com") + with pytest.raises(ConnectionError): + p.request(method="GET", url="evil.com://good.com") msg = ( "URLs without a scheme (ie 'https://') are deprecated and will raise an error " From a711ed8266efdbf53805892a733370409d5f9fbf Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 30 Oct 2023 15:59:46 -0700 Subject: [PATCH 008/131] Use a mark for requires_network in test suite --- pyproject.toml | 5 ++++- test/__init__.py | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6aac7948ae..141b263a83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,10 @@ include = [ [tool.pytest.ini_options] xfail_strict = true python_classes = ["Test", "*TestCase"] -markers = ["limit_memory"] +markers = [ + "limit_memory: Limit memory with memray", + "requires_network: This test needs access to the Internet", +] log_level = "DEBUG" filterwarnings = [ "error", diff --git a/test/__init__.py b/test/__init__.py index 1bf153cfcb..062613614c 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -181,14 +181,26 @@ def _has_route() -> bool: else: raise + def _decorator_requires_internet( + decorator: typing.Callable[[_TestFuncT], _TestFuncT] + ) -> typing.Callable[[_TestFuncT], _TestFuncT]: + """Mark a decorator with the "requires_internet" mark""" + + def wrapper(f: _TestFuncT) -> typing.Any: + return pytest.mark.requires_network(decorator(f)) + + return wrapper + global _requires_network_has_route if _requires_network_has_route is None: _requires_network_has_route = _has_route() - return pytest.mark.skipif( - not _requires_network_has_route, - reason="Can't run the test because the network is unreachable", + return _decorator_requires_internet( + pytest.mark.skipif( + not _requires_network_has_route, + reason="Can't run the test because the network is unreachable", + ) ) From 7d0648b53cf4159e7f3ebb8353cb642942ec93e6 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 31 Oct 2023 13:51:29 +0400 Subject: [PATCH 009/131] Fix downstream requests test --- noxfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/noxfile.py b/noxfile.py index 9a4f576912..b7e80d4992 100644 --- a/noxfile.py +++ b/noxfile.py @@ -117,6 +117,9 @@ def downstream_requests(session: nox.Session) -> None: session.install(".[socks]", silent=False) session.install("-r", "requirements-dev.txt", silent=False) + # Workaround until https://github.com/psf/httpbin/pull/29 gets released + session.install("flask<3", "werkzeug<3", silent=False) + session.cd(root) session.install(".", silent=False) session.cd(f"{tmp_dir}/requests") From b99cc396b2b760362469796d086fae118aec7fb7 Mon Sep 17 00:00:00 2001 From: FineFindus <63370021+FineFindus@users.noreply.github.com> Date: Fri, 3 Nov 2023 20:57:22 +0100 Subject: [PATCH 010/131] Replace deprecated set-output in GitHub Actions --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4a516c281c..6fb36c29dc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,7 +37,7 @@ jobs: - name: "Generate hashes" id: hash run: | - cd dist && echo "::set-output name=hashes::$(sha256sum * | base64 -w0)" + cd dist && echo "hashes=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - name: "Upload dists" uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 From ff764a01499203a7c6fbe2e6c0a5a670cf26745c Mon Sep 17 00:00:00 2001 From: Ravi <86048085+ravi-kale@users.noreply.github.com> Date: Sat, 4 Nov 2023 03:08:35 +0530 Subject: [PATCH 011/131] Allow loading CA certificates from memory for proxies (#3150) Co-authored-by: Illia Volochii --- changelog/3065.bugfix.rst | 3 +++ src/urllib3/poolmanager.py | 2 ++ test/with_dummyserver/test_proxy_poolmanager.py | 10 ++++++++++ 3 files changed, 15 insertions(+) create mode 100644 changelog/3065.bugfix.rst diff --git a/changelog/3065.bugfix.rst b/changelog/3065.bugfix.rst new file mode 100644 index 0000000000..e9be364072 --- /dev/null +++ b/changelog/3065.bugfix.rst @@ -0,0 +1,3 @@ +Fixed an issue where it was not possible to pass the ca_cert_data keyword argument in a proxy context when making SSL +requests. Previously, attempting to pass ca_cert_data in a proxy context would result in an error. +This issue has been resolved, and users can now pass ca_cert_data when making SSL requests through a proxy context. diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index 8b8fac9e0d..32da0a00ab 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -38,6 +38,7 @@ "cert_file", "cert_reqs", "ca_certs", + "ca_cert_data", "ssl_version", "ssl_minimum_version", "ssl_maximum_version", @@ -73,6 +74,7 @@ class PoolKey(typing.NamedTuple): key_cert_file: str | None key_cert_reqs: str | None key_ca_certs: str | None + key_ca_cert_data: str | bytes | None key_ssl_version: int | str | None key_ssl_minimum_version: ssl.TLSVersion | None key_ssl_maximum_version: ssl.TLSVersion | None diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 9d5f90761e..09898c92e2 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -78,6 +78,16 @@ def test_https_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 + def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: + with open(DEFAULT_CA) as pem_file: + pem_file_data = pem_file.read() + with proxy_from_url(self.https_proxy_url, ca_cert_data=pem_file_data) as https: + r = https.request("GET", f"{self.https_url}/") + assert r.status == 200 + + r = https.request("GET", f"{self.http_url}/") + assert r.status == 200 + def test_https_proxy_with_proxy_ssl_context(self) -> None: proxy_ssl_context = create_urllib3_context() proxy_ssl_context.load_verify_locations(DEFAULT_CA) From 5fc48e711b33c08eea1c1ea8209870f45e8baf05 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Fri, 3 Nov 2023 16:24:18 -0400 Subject: [PATCH 012/131] Treat x-gzip content encoding as gzip According to RFC 9110, the "x-gzip" content coding should be treated as "gzip". Fixes #3174 --- changelog/3174.bugfix.rst | 1 + src/urllib3/response.py | 6 ++++-- test/test_response.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changelog/3174.bugfix.rst diff --git a/changelog/3174.bugfix.rst b/changelog/3174.bugfix.rst new file mode 100644 index 0000000000..cf3e458eb7 --- /dev/null +++ b/changelog/3174.bugfix.rst @@ -0,0 +1 @@ +Fixed decoding Gzip-encoded responses which specified ``x-gzip`` content-encoding. diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 367ccfe425..37936f9397 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -208,7 +208,9 @@ def _get_decoder(mode: str) -> ContentDecoder: if "," in mode: return MultiDecoder(mode) - if mode == "gzip": + # According to RFC 9110 section 8.4.1.3, recipients should + # consider x-gzip equivalent to gzip + if mode in ("gzip", "x-gzip"): return GzipDecoder() if brotli is not None and mode == "br": @@ -280,7 +282,7 @@ def get(self, n: int) -> bytes: class BaseHTTPResponse(io.IOBase): - CONTENT_DECODERS = ["gzip", "deflate"] + CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] if brotli is not None: CONTENT_DECODERS += ["br"] if zstd is not None: diff --git a/test/test_response.py b/test/test_response.py index 4403307752..98ae544eb4 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -229,14 +229,15 @@ def test_chunked_decoding_deflate2(self) -> None: assert r.read() == b"" assert r.read() == b"" - def test_chunked_decoding_gzip(self) -> None: + @pytest.mark.parametrize("content_encoding", ["gzip", "x-gzip"]) + def test_chunked_decoding_gzip(self, content_encoding: str) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() fp = BytesIO(data) r = HTTPResponse( - fp, headers={"content-encoding": "gzip"}, preload_content=False + fp, headers={"content-encoding": content_encoding}, preload_content=False ) assert r.read(1) == b"f" From 872768312273d67ca9b9481028c11acaf213da86 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 5 Nov 2023 15:31:31 +0400 Subject: [PATCH 013/131] Remove Sphinx version pin --- docs/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index dff30723d9..3edab4d772 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,5 @@ -r ../dev-requirements.txt -# https://github.com/sphinx-doc/sphinx/issues/11662#issuecomment-1713887182 -sphinx>3.0.0,<7.2.5 +sphinx>=7.2.6 requests furo sphinx-copybutton From 5fa8ea621579edf4eb94addc9fb1bc5873256381 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 31 Oct 2023 11:58:14 +0400 Subject: [PATCH 014/131] Fix lint on Python 3.12 GitHub Actions now defaults to Python 3.12, which made flake8 6.0 report E231 errors because it was looking at f-strings incorrectly. Upgrading to 6.1 fixed the issue, but introduced E721 that I fixed too. This commit also changes `nox -s lint` and `nox -s mypy` to use Python 3.12, allowing to remove one type ignore. --- .pre-commit-config.yaml | 2 +- noxfile.py | 4 +-- test/test_ssltransport.py | 6 ++--- test/tz_stub.py | 2 +- test/with_dummyserver/test_connectionpool.py | 4 +-- test/with_dummyserver/test_https.py | 2 +- .../test_proxy_poolmanager.py | 25 +++++++++---------- test/with_dummyserver/test_socketlevel.py | 2 +- 8 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ffe5459e2..04e65f8cd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: isort - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: [flake8-2020] diff --git a/noxfile.py b/noxfile.py index b7e80d4992..c0b0b9f0e6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -134,7 +134,7 @@ def format(session: nox.Session) -> None: lint(session) -@nox.session +@nox.session(python="3.12") def lint(session: nox.Session) -> None: session.install("pre-commit") session.run("pre-commit", "run", "--all-files") @@ -142,7 +142,7 @@ def lint(session: nox.Session) -> None: mypy(session) -@nox.session(python="3.8") +@nox.session(python="3.12") def mypy(session: nox.Session) -> None: """Run mypy.""" session.install("-r", "mypy-requirements.txt") diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 21d556fc93..a28c2e4e15 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -91,11 +91,11 @@ def validate_response( def validate_peercert(ssl_socket: SSLTransport) -> None: binary_cert = ssl_socket.getpeercert(binary_form=True) - assert type(binary_cert) == bytes + assert isinstance(binary_cert, bytes) assert len(binary_cert) > 0 cert = ssl_socket.getpeercert() - assert type(cert) == dict + assert isinstance(cert, dict) assert "serialNumber" in cert assert cert["serialNumber"] != "" @@ -222,7 +222,7 @@ def test_ssl_object_attributes(self) -> None: sock, self.client_context, server_hostname="localhost" ) as ssock: cipher = ssock.cipher() - assert type(cipher) == tuple + assert isinstance(cipher, tuple) # No chosen protocol through ALPN or NPN. assert ssock.selected_alpn_protocol() is None diff --git a/test/tz_stub.py b/test/tz_stub.py index 41b114bb2a..27f119e634 100644 --- a/test/tz_stub.py +++ b/test/tz_stub.py @@ -9,7 +9,7 @@ import pytest try: - import zoneinfo # type: ignore[import] + import zoneinfo except ImportError: # Python < 3.9 from backports import zoneinfo # type: ignore[no-redef] diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index ebfaf3878f..e8178a19f3 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -363,7 +363,7 @@ def test_connection_error_retries(self) -> None: with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) - assert type(e.value.reason) == NewConnectionError + assert isinstance(e.value.reason, NewConnectionError) def test_timeout_success(self) -> None: timeout = Timeout(connect=3, read=5, total=None) @@ -495,7 +495,7 @@ def test_bad_connect(self) -> None: with HTTPConnectionPool("badhost.invalid", self.port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=5) - assert type(e.value.reason) == NameResolutionError + assert isinstance(e.value.reason, NameResolutionError) def test_keepalive(self) -> None: with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) as pool: diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 158df83b8f..e532fda301 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -1099,7 +1099,7 @@ def test_hostname_checks_common_name_respected( # IP addresses should fail for commonName. else: assert err is not None - assert type(err.reason) == SSLError + assert isinstance(err.reason, SSLError) assert isinstance( err.reason.args[0], (ssl.SSLCertVerificationError, CertificateError) ) diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 09898c92e2..1f7365b55b 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -158,10 +158,9 @@ def test_proxy_conn_fail_from_dns( with pytest.raises(MaxRetryError) as e: http.request("GET", f"{target_url}/") - assert type(e.value.reason) == ProxyError - assert ( - type(e.value.reason.original_error) - == urllib3.exceptions.NameResolutionError + assert isinstance(e.value.reason, ProxyError) + assert isinstance( + e.value.reason.original_error, urllib3.exceptions.NameResolutionError ) def test_oldapi(self) -> None: @@ -477,7 +476,7 @@ def test_forwarding_proxy_request_timeout( # We sent the request to the proxy but didn't get any response # so we're not sure if that's being caused by the proxy or the # target so we put the blame on the target. - assert type(e.value.reason) == ReadTimeoutError + assert isinstance(e.value.reason, ReadTimeoutError) @requires_network() @pytest.mark.parametrize( @@ -497,7 +496,7 @@ def test_tunneling_proxy_request_timeout( timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT) proxy.request("GET", target_url, timeout=timeout) - assert type(e.value.reason) == ReadTimeoutError + assert isinstance(e.value.reason, ReadTimeoutError) @requires_network() @pytest.mark.parametrize( @@ -524,8 +523,8 @@ def test_forwarding_proxy_connect_timeout( with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == ProxyError - assert type(e.value.reason.original_error) == ConnectTimeoutError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, ConnectTimeoutError) @requires_network() @pytest.mark.parametrize( @@ -543,8 +542,8 @@ def test_tunneling_proxy_connect_timeout( with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == ProxyError - assert type(e.value.reason.original_error) == ConnectTimeoutError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, ConnectTimeoutError) @requires_network() @pytest.mark.parametrize( @@ -567,8 +566,8 @@ def test_https_proxy_tls_error( ) as proxy: with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == ProxyError - assert type(e.value.reason.original_error) == SSLError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, SSLError) @requires_network() @pytest.mark.parametrize( @@ -598,7 +597,7 @@ def test_proxy_https_target_tls_error( ) as proxy: with pytest.raises(MaxRetryError) as e: proxy.request("GET", self.https_url) - assert type(e.value.reason) == SSLError + assert isinstance(e.value.reason, SSLError) def test_scheme_host_case_insensitive(self) -> None: """Assert that upper-case schemes and hosts are normalized.""" diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 8635edc0c9..1773971053 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1260,7 +1260,7 @@ def http_socket_handler(listener: socket.socket) -> None: errored.set() # Avoid a ConnectionAbortedError on Windows. - assert type(e.value.reason) == ProxyError + assert isinstance(e.value.reason, ProxyError) assert "Your proxy appears to only use HTTP and not HTTPS" in str( e.value.reason ) From 6fc4260934b4a780c01a685f3f7982055e1c73e2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 5 Nov 2023 15:21:44 +0400 Subject: [PATCH 015/131] Use more precise type checks --- test/test_ssltransport.py | 10 +++++----- test/with_dummyserver/test_connectionpool.py | 6 +++--- test/with_dummyserver/test_https.py | 14 +++++++------- test/with_dummyserver/test_socketlevel.py | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index a28c2e4e15..2a7a0387b4 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -91,11 +91,11 @@ def validate_response( def validate_peercert(ssl_socket: SSLTransport) -> None: binary_cert = ssl_socket.getpeercert(binary_form=True) - assert isinstance(binary_cert, bytes) + assert type(binary_cert) is bytes assert len(binary_cert) > 0 cert = ssl_socket.getpeercert() - assert isinstance(cert, dict) + assert type(cert) is dict assert "serialNumber" in cert assert cert["serialNumber"] != "" @@ -222,7 +222,7 @@ def test_ssl_object_attributes(self) -> None: sock, self.client_context, server_hostname="localhost" ) as ssock: cipher = ssock.cipher() - assert isinstance(cipher, tuple) + assert type(cipher) is tuple # No chosen protocol through ALPN or NPN. assert ssock.selected_alpn_protocol() is None @@ -484,11 +484,11 @@ def test_tls_in_tls_makefile_rw_text(self) -> None: write.flush() response = read.read() - assert isinstance(response, str) + assert type(response) is str if "\r" not in response: # Carriage return will be removed when reading as a file on # some platforms. We add it before the comparison. - assert isinstance(response, str) + assert type(response) is str response = response.replace("\n", "\r\n") validate_response(response, binary=False) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index e8178a19f3..c0f4d9d5cf 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -363,7 +363,7 @@ def test_connection_error_retries(self) -> None: with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) - assert isinstance(e.value.reason, NewConnectionError) + assert type(e.value.reason) is NewConnectionError def test_timeout_success(self) -> None: timeout = Timeout(connect=3, read=5, total=None) @@ -495,7 +495,7 @@ def test_bad_connect(self) -> None: with HTTPConnectionPool("badhost.invalid", self.port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=5) - assert isinstance(e.value.reason, NameResolutionError) + assert type(e.value.reason) is NameResolutionError def test_keepalive(self) -> None: with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) as pool: @@ -1063,7 +1063,7 @@ def test_headers_not_modified_by_request( conn.request("GET", "/headers", chunked=chunked) assert pool.headers == {"key": "val"} - assert isinstance(pool.headers, header_type) + assert type(pool.headers) is header_type with HTTPConnectionPool(self.host, self.port) as pool: if pool_request: diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index e532fda301..da3f630cee 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -213,7 +213,7 @@ def test_client_encrypted_key_requires_password(self) -> None: with pytest.raises(MaxRetryError, match="password is required") as e: https_pool.request("GET", "/certificate") - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError def test_verified(self) -> None: with HTTPSConnectionPool( @@ -293,7 +293,7 @@ def test_invalid_common_name(self) -> None: ) as https_pool: with pytest.raises(MaxRetryError) as e: https_pool.request("GET", "/", retries=0) - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError assert "doesn't match" in str( e.value.reason ) or "certificate verify failed" in str(e.value.reason) @@ -308,7 +308,7 @@ def test_verified_with_bad_ca_certs(self) -> None: ) as https_pool: with pytest.raises(MaxRetryError) as e: https_pool.request("GET", "/") - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError assert ( "certificate verify failed" in str(e.value.reason) # PyPy is more specific @@ -342,7 +342,7 @@ def test_verified_without_ca_certs(self) -> None: ) as https_pool: with pytest.raises(MaxRetryError) as e: https_pool.request("GET", "/") - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError # there is a different error message depending on whether or # not pyopenssl is injected assert ( @@ -490,7 +490,7 @@ def test_assert_invalid_fingerprint(self) -> None: def _test_request(pool: HTTPSConnectionPool) -> SSLError: with pytest.raises(MaxRetryError) as cm: pool.request("GET", "/", retries=0) - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError return cm.value.reason with HTTPSConnectionPool( @@ -533,7 +533,7 @@ def test_verify_none_and_bad_fingerprint(self) -> None: ) as https_pool: with pytest.raises(MaxRetryError) as cm: https_pool.request("GET", "/", retries=0) - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError def test_verify_none_and_good_fingerprint(self) -> None: with HTTPSConnectionPool( @@ -1099,7 +1099,7 @@ def test_hostname_checks_common_name_respected( # IP addresses should fail for commonName. else: assert err is not None - assert isinstance(err.reason, SSLError) + assert type(err.reason) is SSLError assert isinstance( err.reason.args[0], (ssl.SSLCertVerificationError, CertificateError) ) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 1773971053..5935888330 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1260,7 +1260,7 @@ def http_socket_handler(listener: socket.socket) -> None: errored.set() # Avoid a ConnectionAbortedError on Windows. - assert isinstance(e.value.reason, ProxyError) + assert type(e.value.reason) is ProxyError assert "Your proxy appears to only use HTTP and not HTTPS" in str( e.value.reason ) @@ -1389,7 +1389,7 @@ def request() -> None: with pytest.raises(MaxRetryError) as cm: request() - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError # Should not hang, see https://github.com/urllib3/urllib3/issues/529 with pytest.raises(MaxRetryError): request() @@ -1883,7 +1883,7 @@ def _test_broken_header_parsing( for record in logs: if ( "Failed to parse headers" in record.msg - and isinstance(record.args, tuple) + and type(record.args) is tuple and _url_from_pool(pool, "/") == record.args[0] ): if ( From f7cd7f3f632cf5224f627536f02c2abf7e4146d1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 6 Nov 2023 15:56:59 +0400 Subject: [PATCH 016/131] Stop naming urllib3/requests tests "integration" (#3182) --- .github/PULL_REQUEST_TEMPLATE/release.md | 2 +- .github/workflows/{integration.yml => downstream.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{integration.yml => downstream.yml} (97%) diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md index 9d295b14dc..c8dfe983c3 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -1,4 +1,4 @@ -* [ ] See if all tests, including integration, pass +* [ ] See if all tests, including downstream, pass * [ ] Get the release pull request approved by a [CODEOWNER](https://github.com/urllib3/urllib3/blob/main/.github/CODEOWNERS) * [ ] Squash merge the release pull request with message "`Release `" * [ ] Tag with X.Y.Z, push tag on urllib3/urllib3 (not on your fork, update `` accordingly) diff --git a/.github/workflows/integration.yml b/.github/workflows/downstream.yml similarity index 97% rename from .github/workflows/integration.yml rename to .github/workflows/downstream.yml index 374d8efb1c..3cd2b2a068 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/downstream.yml @@ -5,7 +5,7 @@ on: [push, pull_request] permissions: "read-all" jobs: - integration: + downstream: strategy: fail-fast: false matrix: From e601a0e8826fc734712f298e92c1a87cee1a08a7 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 12 Nov 2023 01:56:21 -0800 Subject: [PATCH 017/131] Check _has_route *within* the test function (#3187) Test selection occurs after collection, so if we're in an environment where we don't want to make outbound network connections, defer the check to after selection. This was the aim of #3166, but it wasn't tested in an environment where ConnectionError is raised. --- test/__init__.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 062613614c..7eb50ef074 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -10,6 +10,7 @@ import typing import warnings from collections.abc import Sequence +from functools import wraps from importlib.abc import Loader, MetaPathFinder from importlib.machinery import ModuleSpec from types import ModuleType, TracebackType @@ -29,8 +30,6 @@ except ImportError: zstd = None -import functools - from urllib3 import util from urllib3.connectionpool import ConnectionPool from urllib3.exceptions import HTTPWarning @@ -181,6 +180,20 @@ def _has_route() -> bool: else: raise + def _skip_if_no_route(f: _TestFuncT) -> _TestFuncT: + """Skip test exuction if network is unreachable""" + + @wraps(f) + def wrapper(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + global _requires_network_has_route + if _requires_network_has_route is None: + _requires_network_has_route = _has_route() + if not _requires_network_has_route: + pytest.skip("Can't run the test because the network is unreachable") + return f(*args, **kwargs) + + return typing.cast(_TestFuncT, wrapper) + def _decorator_requires_internet( decorator: typing.Callable[[_TestFuncT], _TestFuncT] ) -> typing.Callable[[_TestFuncT], _TestFuncT]: @@ -191,17 +204,7 @@ def wrapper(f: _TestFuncT) -> typing.Any: return wrapper - global _requires_network_has_route - - if _requires_network_has_route is None: - _requires_network_has_route = _has_route() - - return _decorator_requires_internet( - pytest.mark.skipif( - not _requires_network_has_route, - reason="Can't run the test because the network is unreachable", - ) - ) + return _decorator_requires_internet(_skip_if_no_route) def resolvesLocalhostFQDN() -> typing.Callable[[_TestFuncT], _TestFuncT]: @@ -213,7 +216,7 @@ def resolvesLocalhostFQDN() -> typing.Callable[[_TestFuncT], _TestFuncT]: def withPyOpenSSL(test: typing.Callable[..., _RT]) -> typing.Callable[..., _RT]: - @functools.wraps(test) + @wraps(test) def wrapper(*args: typing.Any, **kwargs: typing.Any) -> _RT: if not pyopenssl: pytest.skip("pyopenssl not available, skipping test.") From 77f71d3fbc6b747849e0205929d0e519ba77457b Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sun, 12 Nov 2023 22:53:12 +0200 Subject: [PATCH 018/131] Mention myself in README --- README.md | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 27df7a1aa5..1d94fceb71 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ Tidelift will coordinate the fix and disclosure with maintainers. - [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson) - [@pquentin](https://github.com/pquentin) (Quentin Pradet) +- [@illia-v](https://github.com/illia-v) (Illia Volochii) - [@theacodes](https://github.com/theacodes) (Thea Flowers) - [@haikuginger](https://github.com/haikuginger) (Jess Shapiro) - [@lukasa](https://github.com/lukasa) (Cory Benfield) diff --git a/pyproject.toml b/pyproject.toml index 141b263a83..1d0d7ed279 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ authors = [ maintainers = [ {name = "Seth Michael Larson", email="sethmichaellarson@gmail.com"}, {name = "Quentin Pradet", email="quentin@pradet.me"}, + {name = "Illia Volochii", email = "illia.volochii@gmail.com"}, ] classifiers = [ "Environment :: Web Environment", From 69be2992f8a25a1f27e49f339e4d5b98dec07462 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 13 Nov 2023 16:20:01 +0400 Subject: [PATCH 019/131] Release 2.1.0 Co-authored-by: Illia Volochii --- CHANGES.rst | 20 ++++++++++++++++++++ changelog/2680.removal.rst | 1 - changelog/2681.removal.rst | 1 - changelog/3065.bugfix.rst | 3 --- changelog/3143.removal.rst | 1 - changelog/3174.bugfix.rst | 1 - pyproject.toml | 2 +- src/urllib3/__init__.py | 4 ++-- src/urllib3/_version.py | 2 +- 9 files changed, 24 insertions(+), 11 deletions(-) delete mode 100644 changelog/2680.removal.rst delete mode 100644 changelog/2681.removal.rst delete mode 100644 changelog/3065.bugfix.rst delete mode 100644 changelog/3143.removal.rst delete mode 100644 changelog/3174.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index 6c37aeba10..208073e203 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,23 @@ +2.1.0 (2023-11-13) +================== + +Read the `v2 migration guide `__ for help upgrading to the latest version of urllib3. + +Removals +-------- + +- Removed support for the deprecated urllib3[secure] extra. (`#2680 `__) +- Removed support for the deprecated SecureTransport TLS implementation. (`#2681 `__) +- Removed support for the end-of-life Python 3.7. (`#3143 `__) + + +Bugfixes +-------- + +- Allowed loading CA certificates from memory for proxies. (`#3065 `__) +- Fixed decoding Gzip-encoded responses which specified ``x-gzip`` content-encoding. (`#3174 `__) + + 2.0.7 (2023-10-17) ================== diff --git a/changelog/2680.removal.rst b/changelog/2680.removal.rst deleted file mode 100644 index ad43b9bd3a..0000000000 --- a/changelog/2680.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Removed support for the urllib3[secure] extra. diff --git a/changelog/2681.removal.rst b/changelog/2681.removal.rst deleted file mode 100644 index 75e44a5c83..0000000000 --- a/changelog/2681.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Removed support for the SecureTransport TLS implementation. diff --git a/changelog/3065.bugfix.rst b/changelog/3065.bugfix.rst deleted file mode 100644 index e9be364072..0000000000 --- a/changelog/3065.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an issue where it was not possible to pass the ca_cert_data keyword argument in a proxy context when making SSL -requests. Previously, attempting to pass ca_cert_data in a proxy context would result in an error. -This issue has been resolved, and users can now pass ca_cert_data when making SSL requests through a proxy context. diff --git a/changelog/3143.removal.rst b/changelog/3143.removal.rst deleted file mode 100644 index 21130f07cb..0000000000 --- a/changelog/3143.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Removed support for Python 3.7. diff --git a/changelog/3174.bugfix.rst b/changelog/3174.bugfix.rst deleted file mode 100644 index cf3e458eb7..0000000000 --- a/changelog/3174.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed decoding Gzip-encoded responses which specified ``x-gzip`` content-encoding. diff --git a/pyproject.toml b/pyproject.toml index 1d0d7ed279..bb11c6d0cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ markers = [ log_level = "DEBUG" filterwarnings = [ "error", - '''default:urllib3 v2.0 only supports OpenSSL 1.1.1+.*''', + '''default:urllib3 v2 only supports OpenSSL 1.1.1+.*''', '''default:No IPv6 support. Falling back to IPv4:urllib3.exceptions.HTTPWarning''', '''default:No IPv6 support. skipping:urllib3.exceptions.HTTPWarning''', '''default:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning''', diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 26fc5770e4..46c89762c2 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -32,14 +32,14 @@ else: if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive: warnings.warn( - "urllib3 v2.0 only supports OpenSSL 1.1.1+, currently " + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " "See: https://github.com/urllib3/urllib3/issues/3020", exceptions.NotOpenSSLWarning, ) elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive: raise ImportError( - "urllib3 v2.0 only supports OpenSSL 1.1.1+, currently " + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " "See: https://github.com/urllib3/urllib3/issues/2168" ) diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index e2b88f1d68..409ba3f53a 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.0.7" +__version__ = "2.1.0" From 4dd4a824e60ba3bb81f319cf2b8e320aa8a87dc6 Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 13 Nov 2023 16:09:30 +0000 Subject: [PATCH 020/131] Document the parameters for `urllib3.request()` Co-authored-by: Illia Volochii --- src/urllib3/__init__.py | 55 ++++++++++++++++++++++++++++++ src/urllib3/_request_methods.py | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 46c89762c2..1bd3010b3f 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -132,6 +132,61 @@ def request( Therefore, its side effects could be shared across dependencies relying on it. To avoid side effects create a new ``PoolManager`` instance and use it instead. The method does not accept low-level ``**urlopen_kw`` keyword arguments. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. + + :param bool preload_content: + If True, the response's body will be preloaded into memory. + + :param bool decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param redirect: + If True, automatically handle redirects (status codes 301, 302, + 303, 307, 308). Each redirect counts as a retry. Disabling retries + will disable redirect, too. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + + :param timeout: + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. """ return _DEFAULT_POOL.request( diff --git a/src/urllib3/_request_methods.py b/src/urllib3/_request_methods.py index 1d0f3465ad..3ce7603bda 100644 --- a/src/urllib3/_request_methods.py +++ b/src/urllib3/_request_methods.py @@ -85,6 +85,30 @@ def request( option to drop down to more specific methods when necessary, such as :meth:`request_encode_url`, :meth:`request_encode_body`, or even the lowest level :meth:`urlopen`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the request body. Values are processed + by :func:`urllib.parse.urlencode`. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. """ method = method.upper() @@ -130,6 +154,20 @@ def request_encode_url( """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. """ if headers is None: headers = self.headers @@ -186,6 +224,28 @@ def request_encode_body( be overwritten because it depends on the dynamic random boundary string which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param encode_multipart: + If True, encode the ``fields`` using the multipart/form-data MIME + format. + + :param multipart_boundary: + If not specified, then a random boundary will be generated using + :func:`urllib3.filepost.choose_boundary`. """ if headers is None: headers = self.headers From 052617c5da9b04bd8d14507f511a4bddba4caf55 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 15 Nov 2023 15:58:58 +0400 Subject: [PATCH 021/131] Run test_requesting_large_resources_via_ssl separately (#3181) --- .github/workflows/ci.yml | 17 +++++++++++++++ changelog/3181.feature.rst | 4 ++++ noxfile.py | 10 ++++++++- pyproject.toml | 1 + test/conftest.py | 26 +++++++++++++++++++++++ test/with_dummyserver/test_socketlevel.py | 6 +----- 6 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 changelog/3181.feature.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddbc8a17a9..3a58a92c1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,22 @@ jobs: nox-session: [''] include: - experimental: false + # integration + # 3.8 and 3.9 have a known issue with large SSL requests that we work around: + # https://github.com/urllib3/urllib3/pull/3181#issuecomment-1794830698 + - python-version: "3.8" + os: ubuntu-latest + experimental: false + nox-session: test_integration + - python-version: "3.9" + os: ubuntu-latest + experimental: false + nox-session: test_integration + - python-version: "3.12" + os: ubuntu-latest + experimental: false + nox-session: test_integration + # pypy - python-version: "pypy-3.8" os: ubuntu-latest experimental: false @@ -56,6 +72,7 @@ jobs: experimental: false nox-session: test-pypy - python-version: "3.x" + # brotli os: ubuntu-latest experimental: false nox-session: test_brotlipy diff --git a/changelog/3181.feature.rst b/changelog/3181.feature.rst new file mode 100644 index 0000000000..82d6af24ba --- /dev/null +++ b/changelog/3181.feature.rst @@ -0,0 +1,4 @@ +Note to redistributors: the urllib3 test suite has been separated in +two. To run integration tests, you now need to run the tests a second +time with the `--integration` pytest flag, as in this example: `nox +-rs test-3.12 -- --integration`. diff --git a/noxfile.py b/noxfile.py index c0b0b9f0e6..907d82cdd4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,6 +11,7 @@ def tests_impl( session: nox.Session, extras: str = "socks,brotli,zstd", byte_string_comparisons: bool = True, + integration: bool = False, ) -> None: # Install deps and the package itself. session.install("-r", "dev-requirements.txt") @@ -44,6 +45,7 @@ def tests_impl( *("--memray", "--hide-memray-summary") if memray_supported else (), "-v", "-ra", + *(("--integration",) if integration else ()), f"--color={'yes' if 'GITHUB_ACTIONS' in os.environ else 'auto'}", "--tb=native", "--durations=10", @@ -59,7 +61,13 @@ def test(session: nox.Session) -> None: tests_impl(session) -@nox.session(python=["3"]) +@nox.session(python="3") +def test_integration(session: nox.Session) -> None: + """Run integration tests""" + tests_impl(session, integration=True) + + +@nox.session(python="3") def test_brotlipy(session: nox.Session) -> None: """Check that if 'brotlipy' is installed instead of 'brotli' or 'brotlicffi' that we still don't blow up. diff --git a/pyproject.toml b/pyproject.toml index bb11c6d0cb..6b850412a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ python_classes = ["Test", "*TestCase"] markers = [ "limit_memory: Limit memory with memray", "requires_network: This test needs access to the Internet", + "integration: Slow integrations tests not run by default", ] log_level = "DEBUG" filterwarnings = [ diff --git a/test/conftest.py b/test/conftest.py index 9aafdacfac..f464f704c3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -20,6 +20,32 @@ from .tz_stub import stub_timezone_ctx +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--integration", + action="store_true", + default=False, + help="run integration tests only", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + integration_mode = bool(config.getoption("--integration")) + skip_integration = pytest.mark.skip( + reason="skipping, need --integration option to run" + ) + skip_normal = pytest.mark.skip( + reason="skipping non integration tests in --integration mode" + ) + for item in items: + if "integration" in item.keywords and not integration_mode: + item.add_marker(skip_integration) + elif integration_mode and "integration" not in item.keywords: + item.add_marker(skip_normal) + + class ServerConfig(typing.NamedTuple): scheme: str host: str diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 5935888330..c0701fe00a 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -11,7 +11,6 @@ import shutil import socket import ssl -import sys import tempfile import time import typing @@ -1584,10 +1583,7 @@ def socket_handler(listener: socket.socket) -> None: pool.request("GET", "/", retries=False, timeout=LONG_TIMEOUT) assert server_closed.wait(LONG_TIMEOUT), "The socket was not terminated" - @pytest.mark.skipif( - os.environ.get("CI") == "true" and sys.implementation.name == "pypy", - reason="too slow to run in CI", - ) + @pytest.mark.integration @pytest.mark.parametrize( "preload_content,read_amt", [(True, None), (False, None), (False, 2**31)] ) From 3b354498929a63f350bd5cdfc9bae8ecc5ff8af4 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 15 Nov 2023 19:59:24 +0400 Subject: [PATCH 022/131] Clarify Tornado involvement in dummyserver names --- dummyserver/https_proxy.py | 2 +- dummyserver/testcase.py | 8 ++++---- dummyserver/{server.py => tornadoserver.py} | 4 ++-- test/conftest.py | 10 +++++++--- test/contrib/test_socks.py | 2 +- test/test_connectionpool.py | 2 +- test/test_ssltransport.py | 2 +- test/with_dummyserver/test_connectionpool.py | 2 +- test/with_dummyserver/test_https.py | 4 ++-- test/with_dummyserver/test_poolmanager.py | 2 +- test/with_dummyserver/test_proxy_poolmanager.py | 2 +- test/with_dummyserver/test_socketlevel.py | 4 ++-- 12 files changed, 24 insertions(+), 20 deletions(-) rename dummyserver/{server.py => tornadoserver.py} (98%) diff --git a/dummyserver/https_proxy.py b/dummyserver/https_proxy.py index 79dae1cd03..87c6eceeab 100755 --- a/dummyserver/https_proxy.py +++ b/dummyserver/https_proxy.py @@ -10,7 +10,7 @@ import tornado.web from dummyserver.proxy import ProxyHandler -from dummyserver.server import DEFAULT_CERTS, ssl_options_to_context +from dummyserver.tornadoserver import DEFAULT_CERTS, ssl_options_to_context def run_proxy(port: int, certs: dict[str, typing.Any] = DEFAULT_CERTS) -> None: diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index a681106732..77d3d8ba95 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -12,12 +12,12 @@ from dummyserver.handlers import TestingApp from dummyserver.proxy import ProxyHandler -from dummyserver.server import ( +from dummyserver.tornadoserver import ( DEFAULT_CERTS, HAS_IPV6, SocketServerThread, - run_loop_in_thread, run_tornado_app, + run_tornado_loop_in_thread, ) from urllib3.connection import HTTPConnection from urllib3.util.ssltransport import SSLTransport @@ -172,7 +172,7 @@ class HTTPDummyServerTestCase: @classmethod def _start_server(cls) -> None: with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_loop_in_thread()) + io_loop = stack.enter_context(run_tornado_loop_in_thread()) async def run_app() -> None: app = web.Application([(r".*", TestingApp)]) @@ -240,7 +240,7 @@ class HTTPDummyProxyTestCase: @classmethod def setup_class(cls) -> None: with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_loop_in_thread()) + io_loop = stack.enter_context(run_tornado_loop_in_thread()) async def run_app() -> None: app = web.Application([(r".*", TestingApp)]) diff --git a/dummyserver/server.py b/dummyserver/tornadoserver.py similarity index 98% rename from dummyserver/server.py rename to dummyserver/tornadoserver.py index 2683c9ec62..64c31c7227 100755 --- a/dummyserver/server.py +++ b/dummyserver/tornadoserver.py @@ -262,7 +262,7 @@ async def inner_fn() -> R: @contextlib.contextmanager -def run_loop_in_thread() -> Generator[tornado.ioloop.IOLoop, None, None]: +def run_tornado_loop_in_thread() -> Generator[tornado.ioloop.IOLoop, None, None]: loop_started: concurrent.futures.Future[ tuple[tornado.ioloop.IOLoop, asyncio.Event] ] = concurrent.futures.Future() @@ -296,7 +296,7 @@ async def run() -> None: def main() -> int: - # For debugging dummyserver itself - python -m dummyserver.server + # For debugging dummyserver itself - PYTHONPATH=src python -m dummyserver.tornadoserver from .handlers import TestingApp host = "127.0.0.1" diff --git a/test/conftest.py b/test/conftest.py index f464f704c3..e4c6e8ee64 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -13,8 +13,12 @@ from dummyserver.handlers import TestingApp from dummyserver.proxy import ProxyHandler -from dummyserver.server import HAS_IPV6, run_loop_in_thread, run_tornado_app from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.tornadoserver import ( + HAS_IPV6, + run_tornado_app, + run_tornado_loop_in_thread, +) from urllib3.util import ssl_ from .tz_stub import stub_timezone_ctx @@ -79,7 +83,7 @@ def run_server_in_thread( ca.cert_pem.write_to_path(ca_cert_path) server_certs = _write_cert_to_dir(server_cert, tmpdir) - with run_loop_in_thread() as io_loop: + with run_tornado_loop_in_thread() as io_loop: async def run_app() -> int: app = web.Application([(r".*", TestingApp)]) @@ -107,7 +111,7 @@ def run_server_and_proxy_in_thread( server_certs = _write_cert_to_dir(server_cert, tmpdir) proxy_certs = _write_cert_to_dir(proxy_cert, tmpdir, "proxy") - with run_loop_in_thread() as io_loop: + with run_tornado_loop_in_thread() as io_loop: async def run_app() -> tuple[ServerConfig, ServerConfig]: app = web.Application([(r".*", TestingApp)]) diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py index 2878cc8d8b..5510f36214 100644 --- a/test/contrib/test_socks.py +++ b/test/contrib/test_socks.py @@ -11,8 +11,8 @@ import pytest import socks as py_socks # type: ignore[import] -from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import IPV4SocketDummyServerTestCase +from dummyserver.tornadoserver import DEFAULT_CA, DEFAULT_CERTS from urllib3.contrib import socks from urllib3.exceptions import ConnectTimeoutError, NewConnectionError diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index d81d33f7bd..dbf2208581 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -12,7 +12,7 @@ import pytest -from dummyserver.server import DEFAULT_CA +from dummyserver.tornadoserver import DEFAULT_CA from urllib3 import Retry from urllib3.connection import HTTPConnection from urllib3.connectionpool import ( diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 2a7a0387b4..4fa2d84cb6 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -9,8 +9,8 @@ import pytest -from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import SocketDummyServerTestCase, consume_socket +from dummyserver.tornadoserver import DEFAULT_CA, DEFAULT_CERTS from urllib3.util import ssl_ from urllib3.util.ssltransport import SSLTransport diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index c0f4d9d5cf..8b1eb171c0 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -12,8 +12,8 @@ import pytest -from dummyserver.server import HAS_IPV6_AND_DNS, NoIPv6Warning from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase +from dummyserver.tornadoserver import HAS_IPV6_AND_DNS, NoIPv6Warning from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict from urllib3.connection import _get_default_user_agent diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index da3f630cee..45f0a3b974 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -23,13 +23,13 @@ import urllib3.util as util import urllib3.util.ssl_ -from dummyserver.server import ( +from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.tornadoserver import ( DEFAULT_CA, DEFAULT_CA_KEY, DEFAULT_CERTS, encrypt_key_pem, ) -from dummyserver.testcase import HTTPSDummyServerTestCase from urllib3 import HTTPSConnectionPool from urllib3.connection import RECENT_DATE, HTTPSConnection, VerifiedHTTPSConnection from urllib3.exceptions import ( diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index ab0111e45b..9e80fad360 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -7,8 +7,8 @@ import pytest -from dummyserver.server import HAS_IPV6 from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase +from dummyserver.tornadoserver import HAS_IPV6 from urllib3 import HTTPHeaderDict, HTTPResponse, request from urllib3.connectionpool import port_by_scheme from urllib3.exceptions import MaxRetryError, URLSchemeUnknown diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 1f7365b55b..5c306cdccf 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -16,8 +16,8 @@ import trustme import urllib3.exceptions -from dummyserver.server import DEFAULT_CA, HAS_IPV6, get_unreachable_address from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase +from dummyserver.tornadoserver import DEFAULT_CA, HAS_IPV6, get_unreachable_address from urllib3 import HTTPResponse from urllib3._collections import HTTPHeaderDict from urllib3.connection import VerifiedHTTPSConnection diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index c0701fe00a..9422e5d885 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -24,13 +24,13 @@ import pytest import trustme -from dummyserver.server import ( +from dummyserver.testcase import SocketDummyServerTestCase, consume_socket +from dummyserver.tornadoserver import ( DEFAULT_CA, DEFAULT_CERTS, encrypt_key_pem, get_unreachable_address, ) -from dummyserver.testcase import SocketDummyServerTestCase, consume_socket from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager, util from urllib3._collections import HTTPHeaderDict from urllib3.connection import HTTPConnection, _get_default_user_agent From 402cba152839b246426c67dd8ed22b53ce6e8379 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 16 Nov 2023 09:30:54 +0400 Subject: [PATCH 023/131] Add Hypercorn server implementation to test infrastructure --- dev-requirements.txt | 3 + dummyserver/handlers.py | 9 +++ dummyserver/hypercornserver.py | 83 +++++++++++++++++++++++ dummyserver/testcase.py | 26 ++++++- mypy-requirements.txt | 3 + noxfile.py | 4 +- test/with_dummyserver/test_poolmanager.py | 19 +++++- 7 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 dummyserver/hypercornserver.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 845fc98f8a..266c1be646 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,3 +10,6 @@ cryptography==41.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython" +trio==0.23.1 +quart-trio==0.11.0 +hypercorn==0.15.0 \ No newline at end of file diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 86201a116f..69e0ad8fce 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -13,6 +13,7 @@ from io import BytesIO from urllib.parse import urlsplit +from quart_trio import QuartTrio from tornado import httputil from tornado.web import RequestHandler @@ -362,3 +363,11 @@ def redirect_after(self, request: httputil.HTTPServerRequest) -> Response: def shutdown(self, request: httputil.HTTPServerRequest) -> typing.NoReturn: sys.exit() + + +hypercorn_app = QuartTrio(__name__) + + +@hypercorn_app.route("/") +async def hello() -> str: + return "Dummy Hypercorn server!" diff --git a/dummyserver/hypercornserver.py b/dummyserver/hypercornserver.py new file mode 100644 index 0000000000..02afcd85b6 --- /dev/null +++ b/dummyserver/hypercornserver.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import concurrent.futures +import contextlib +import functools +import sys +import threading +from typing import Generator + +import hypercorn +import hypercorn.trio +import trio +from quart_trio import QuartTrio + + +# https://github.com/pgjones/hypercorn/blob/19dfb96411575a6a647cdea63fa581b48ebb9180/src/hypercorn/utils.py#L172-L178 +async def graceful_shutdown(shutdown_event: threading.Event) -> None: + while True: + if shutdown_event.is_set(): + return + await trio.sleep(0.1) + + +async def _start_server( + config: hypercorn.Config, + app: QuartTrio, + ready_event: threading.Event, + shutdown_event: threading.Event, +) -> None: + async with trio.open_nursery() as nursery: + config.bind = await nursery.start( + functools.partial( + hypercorn.trio.serve, + app, + config, + shutdown_trigger=functools.partial(graceful_shutdown, shutdown_event), + ) + ) + ready_event.set() + + +@contextlib.contextmanager +def run_hypercorn_in_thread( + config: hypercorn.Config, app: QuartTrio +) -> Generator[None, None, None]: + ready_event = threading.Event() + shutdown_event = threading.Event() + + with concurrent.futures.ThreadPoolExecutor( + 1, thread_name_prefix="hypercorn dummyserver" + ) as executor: + future = executor.submit( + trio.run, + _start_server, # type: ignore[arg-type] + config, + app, + ready_event, + shutdown_event, + ) + ready_event.wait(5) + if not ready_event.is_set(): + raise Exception("most likely failed to start server") + + yield + + shutdown_event.set() + future.result() + + +def main() -> int: + # For debugging dummyserver itself - PYTHONPATH=src python -m dummyserver.hypercornserver + from .handlers import hypercorn_app + + config = hypercorn.Config() + config.bind = ["localhost:0"] + ready_event = threading.Event() + shutdown_event = threading.Event() + trio.run(_start_server, config, hypercorn_app, ready_event, shutdown_event) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 77d3d8ba95..c3c8be7ebe 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -7,10 +7,12 @@ import threading import typing +import hypercorn import pytest from tornado import httpserver, ioloop, web -from dummyserver.handlers import TestingApp +from dummyserver.handlers import TestingApp, hypercorn_app +from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.proxy import ProxyHandler from dummyserver.tornadoserver import ( DEFAULT_CERTS, @@ -21,6 +23,7 @@ ) from urllib3.connection import HTTPConnection from urllib3.util.ssltransport import SSLTransport +from urllib3.util.url import parse_url def consume_socket( @@ -292,6 +295,27 @@ class IPv6HTTPDummyProxyTestCase(HTTPDummyProxyTestCase): proxy_host_alt = "127.0.0.1" +class HypercornDummyServerTestCase: + host = "localhost" + port: typing.ClassVar[int] + base_url: typing.ClassVar[str] + + _stack: typing.ClassVar[contextlib.ExitStack] + + @classmethod + def setup_class(cls) -> None: + with contextlib.ExitStack() as stack: + config = hypercorn.Config() + config.bind = [f"{cls.host}:0"] + stack.enter_context(run_hypercorn_in_thread(config, hypercorn_app)) + cls._stack = stack.pop_all() + cls.port = typing.cast(int, parse_url(config.bind[0]).port) + + @classmethod + def teardown_class(cls) -> None: + cls._stack.close() + + class ConnectionMarker: """ Marks an HTTP(S)Connection's socket after a request was made. diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 2bb46d737b..b8a3d34c34 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,6 +4,9 @@ cryptography>=1.3.4 tornado>=6.1 pytest>=6.2 trustme==0.9.0 +trio==0.23.1 +quart-trio==0.11.0 +hypercorn==0.15.0 types-backports types-requests nox diff --git a/noxfile.py b/noxfile.py index 907d82cdd4..c63c3de92f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,7 +10,9 @@ def tests_impl( session: nox.Session, extras: str = "socks,brotli,zstd", - byte_string_comparisons: bool = True, + # hypercorn dependency h2 compares bytes and strings + # https://github.com/python-hyper/h2/issues/1236 + byte_string_comparisons: bool = False, integration: bool = False, ) -> None: # Install deps and the package itself. diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 9e80fad360..e272bdaea9 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -7,7 +7,11 @@ import pytest -from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase +from dummyserver.testcase import ( + HTTPDummyServerTestCase, + HypercornDummyServerTestCase, + IPv6HTTPDummyServerTestCase, +) from dummyserver.tornadoserver import HAS_IPV6 from urllib3 import HTTPHeaderDict, HTTPResponse, request from urllib3.connectionpool import port_by_scheme @@ -16,6 +20,19 @@ from urllib3.util.retry import Retry +class TestHypercornPoolManager(HypercornDummyServerTestCase): + @classmethod + def setup_class(cls) -> None: + super().setup_class() + cls.base_url = f"http://{cls.host}:{cls.port}" + + def test_index(self) -> None: + with PoolManager() as http: + r = http.request("GET", self.base_url + "/") + assert r.status == 200 + assert r.data == b"Dummy Hypercorn server!" + + class TestPoolManager(HTTPDummyServerTestCase): @classmethod def setup_class(cls) -> None: From 3351be511c241073f0bc98cd02092a571fb8dbf7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 17 Nov 2023 20:12:04 +0400 Subject: [PATCH 024/131] Migrate TestPoolManager to Hypercorn --- dummyserver/app.py | 106 ++++++++++++++++++++++ dummyserver/handlers.py | 9 -- dummyserver/hypercornserver.py | 2 +- dummyserver/testcase.py | 5 +- test/__init__.py | 2 +- test/with_dummyserver/test_poolmanager.py | 26 ++---- 6 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 dummyserver/app.py diff --git a/dummyserver/app.py b/dummyserver/app.py new file mode 100644 index 0000000000..f1dc58c517 --- /dev/null +++ b/dummyserver/app.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import contextlib +import gzip +import zlib +from io import BytesIO + +from quart import make_response, request + +# TODO switch to Response if https://github.com/pallets/quart/issues/288 is fixed +from quart.typing import ResponseTypes +from quart_trio import QuartTrio + +hypercorn_app = QuartTrio(__name__) + + +@hypercorn_app.route("/") +async def index() -> ResponseTypes: + return await make_response("Dummy server!") + + +@hypercorn_app.route("/echo", methods=["GET", "POST"]) +async def echo() -> ResponseTypes: + "Echo back the params" + if request.method == "GET": + return await make_response(request.query_string) + + return await make_response(await request.get_data()) + + +@hypercorn_app.route("/echo_json", methods=["POST"]) +async def echo_json() -> ResponseTypes: + "Echo back the JSON" + data = await request.get_data() + return await make_response(data, 200, request.headers) + + +@hypercorn_app.route("/echo_uri") +async def echo_uri() -> ResponseTypes: + "Echo back the requested URI" + assert request.full_path is not None + return await make_response(request.full_path) + + +@hypercorn_app.route("/headers", methods=["GET", "POST"]) +async def headers() -> ResponseTypes: + return await make_response(dict(request.headers.items())) + + +@hypercorn_app.route("/headers_and_params") +async def headers_and_params() -> ResponseTypes: + return await make_response( + { + "headers": dict(request.headers), + "params": request.args, + } + ) + + +@hypercorn_app.route("/multi_headers", methods=["GET", "POST"]) +async def multi_headers() -> ResponseTypes: + return await make_response({"headers": list(request.headers)}) + + +@hypercorn_app.route("/encodingrequest") +async def encodingrequest() -> ResponseTypes: + "Check for UA accepting gzip/deflate encoding" + data = b"hello, world!" + encoding = request.headers.get("Accept-Encoding", "") + headers = [] + if encoding == "gzip": + headers = [("Content-Encoding", "gzip")] + file_ = BytesIO() + with contextlib.closing(gzip.GzipFile("", mode="w", fileobj=file_)) as zipfile: + zipfile.write(data) + data = file_.getvalue() + elif encoding == "deflate": + headers = [("Content-Encoding", "deflate")] + data = zlib.compress(data) + elif encoding == "garbage-gzip": + headers = [("Content-Encoding", "gzip")] + data = b"garbage" + elif encoding == "garbage-deflate": + headers = [("Content-Encoding", "deflate")] + data = b"garbage" + return await make_response(data, 200, headers) + + +@hypercorn_app.route("/redirect", methods=["GET", "POST"]) +async def redirect() -> ResponseTypes: + "Perform a redirect to ``target``" + values = await request.values + target = values.get("target", "/") + status = values.get("status", "303 See Other") + status_code = status.split(" ")[0] + + headers = [("Location", target)] + return await make_response("", status_code, headers) + + +@hypercorn_app.route("/status") +async def status() -> ResponseTypes: + values = await request.values + status = values.get("status", "200 OK") + status_code = status.split(" ")[0] + return await make_response("", status_code) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 69e0ad8fce..86201a116f 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -13,7 +13,6 @@ from io import BytesIO from urllib.parse import urlsplit -from quart_trio import QuartTrio from tornado import httputil from tornado.web import RequestHandler @@ -363,11 +362,3 @@ def redirect_after(self, request: httputil.HTTPServerRequest) -> Response: def shutdown(self, request: httputil.HTTPServerRequest) -> typing.NoReturn: sys.exit() - - -hypercorn_app = QuartTrio(__name__) - - -@hypercorn_app.route("/") -async def hello() -> str: - return "Dummy Hypercorn server!" diff --git a/dummyserver/hypercornserver.py b/dummyserver/hypercornserver.py index 02afcd85b6..7c860ecb5d 100644 --- a/dummyserver/hypercornserver.py +++ b/dummyserver/hypercornserver.py @@ -69,7 +69,7 @@ def run_hypercorn_in_thread( def main() -> int: # For debugging dummyserver itself - PYTHONPATH=src python -m dummyserver.hypercornserver - from .handlers import hypercorn_app + from .app import hypercorn_app config = hypercorn.Config() config.bind = ["localhost:0"] diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index c3c8be7ebe..769b03f132 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -11,7 +11,8 @@ import pytest from tornado import httpserver, ioloop, web -from dummyserver.handlers import TestingApp, hypercorn_app +from dummyserver.app import hypercorn_app +from dummyserver.handlers import TestingApp from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.proxy import ProxyHandler from dummyserver.tornadoserver import ( @@ -297,8 +298,10 @@ class IPv6HTTPDummyProxyTestCase(HTTPDummyProxyTestCase): class HypercornDummyServerTestCase: host = "localhost" + host_alt = "127.0.0.1" port: typing.ClassVar[int] base_url: typing.ClassVar[str] + base_url_alt: typing.ClassVar[str] _stack: typing.ClassVar[contextlib.ExitStack] diff --git a/test/__init__.py b/test/__init__.py index 7eb50ef074..bf4bb67d3b 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -73,7 +73,7 @@ # 3. To test our timeout logic by using two different values, eg. by using different # values at the pool level and at the request level. SHORT_TIMEOUT = 0.001 -LONG_TIMEOUT = 0.01 +LONG_TIMEOUT = 0.1 if os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS") == "true": LONG_TIMEOUT = 0.5 diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index e272bdaea9..33b0b1ae64 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -8,7 +8,6 @@ import pytest from dummyserver.testcase import ( - HTTPDummyServerTestCase, HypercornDummyServerTestCase, IPv6HTTPDummyServerTestCase, ) @@ -20,20 +19,7 @@ from urllib3.util.retry import Retry -class TestHypercornPoolManager(HypercornDummyServerTestCase): - @classmethod - def setup_class(cls) -> None: - super().setup_class() - cls.base_url = f"http://{cls.host}:{cls.port}" - - def test_index(self) -> None: - with PoolManager() as http: - r = http.request("GET", self.base_url + "/") - assert r.status == 200 - assert r.data == b"Dummy Hypercorn server!" - - -class TestPoolManager(HTTPDummyServerTestCase): +class TestPoolManager(HypercornDummyServerTestCase): @classmethod def setup_class(cls) -> None: super().setup_class() @@ -466,7 +452,7 @@ def test_headers_http_multi_header_multipart(self) -> None: encode_multipart=True, ) returned_headers = r.json()["headers"] - assert returned_headers[4:] == [ + assert returned_headers[5:] == [ ["Multi", "1"], ["Multi", "2"], ["Content-Type", "multipart/form-data; boundary=b"], @@ -484,7 +470,7 @@ def test_headers_http_multi_header_multipart(self) -> None: encode_multipart=True, ) returned_headers = r.json()["headers"] - assert returned_headers[4:] == [ + assert returned_headers[5:] == [ ["Multi", "1"], ["Multi", "2"], # Uses the set value, not the one that would be generated. @@ -514,10 +500,12 @@ def test_http_with_ca_cert_dir(self) -> None: @pytest.mark.parametrize( ["target", "expected_target"], [ + # annoyingly quart.request.full_path adds a stray `?` + ("/echo_uri", b"/echo_uri?"), ("/echo_uri?q=1#fragment", b"/echo_uri?q=1"), ("/echo_uri?#", b"/echo_uri?"), - ("/echo_uri#?", b"/echo_uri"), - ("/echo_uri#?#", b"/echo_uri"), + ("/echo_uri#!", b"/echo_uri?"), + ("/echo_uri#!#", b"/echo_uri?"), ("/echo_uri??#", b"/echo_uri??"), ("/echo_uri?%3f#", b"/echo_uri?%3F"), ("/echo_uri?%3F#", b"/echo_uri?%3F"), From f1e86bb2ec518f16bed13a21f12af91d6fca3bc0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sat, 18 Nov 2023 07:50:43 +0400 Subject: [PATCH 025/131] Migrate run_server_in_thread to Hypercorn --- test/conftest.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index e4c6e8ee64..745c2e6267 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,11 +7,14 @@ import typing from pathlib import Path +import hypercorn import pytest import trustme from tornado import web +from dummyserver.app import hypercorn_app from dummyserver.handlers import TestingApp +from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.proxy import ProxyHandler from dummyserver.testcase import HTTPSDummyServerTestCase from dummyserver.tornadoserver import ( @@ -20,6 +23,7 @@ run_tornado_loop_in_thread, ) from urllib3.util import ssl_ +from urllib3.util.url import parse_url from .tz_stub import stub_timezone_ctx @@ -83,17 +87,13 @@ def run_server_in_thread( ca.cert_pem.write_to_path(ca_cert_path) server_certs = _write_cert_to_dir(server_cert, tmpdir) - with run_tornado_loop_in_thread() as io_loop: - - async def run_app() -> int: - app = web.Application([(r".*", TestingApp)]) - server, port = run_tornado_app(app, server_certs, scheme, host) - return port - - port = asyncio.run_coroutine_threadsafe( - run_app(), io_loop.asyncio_loop # type: ignore[attr-defined] - ).result() - yield ServerConfig("https", host, port, ca_cert_path) + config = hypercorn.Config() + config.certfile = server_certs["certfile"] + config.keyfile = server_certs["keyfile"] + config.bind = [f"{host}:0"] + with run_hypercorn_in_thread(config, hypercorn_app): + port = typing.cast(int, parse_url(config.bind[0]).port) + yield ServerConfig(scheme, host, port, ca_cert_path) @contextlib.contextmanager From d1caf89bfb5ebe75952acd42cc95bd79ee18004b Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 20 Nov 2023 17:23:25 -0600 Subject: [PATCH 026/131] Use curl to test our Hypercorn server can speak HTTP/2 Co-authored-by: Quentin Pradet --- test/with_dummyserver/test_http2.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/with_dummyserver/test_http2.py diff --git a/test/with_dummyserver/test_http2.py b/test/with_dummyserver/test_http2.py new file mode 100644 index 0000000000..fa05900c01 --- /dev/null +++ b/test/with_dummyserver/test_http2.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import subprocess +from test import notWindows + +from dummyserver.testcase import HypercornDummyServerTestCase + + +class TestHypercornDummyServerTestCase(HypercornDummyServerTestCase): + @classmethod + def setup_class(cls) -> None: + super().setup_class() + cls.base_url = f"http://{cls.host}:{cls.port}" + + @notWindows() # GitHub Actions Windows doesn't have HTTP/2 support. + def test_hypercorn_server_http2(self) -> None: + # This is a meta test to make sure our Hypercorn test server is actually using HTTP/2 + # before urllib3 is capable of speaking HTTP/2. Thanks, Daniel! <3 + output = subprocess.check_output( + ["curl", "-vvv", "--http2", self.base_url], stderr=subprocess.STDOUT + ) + + # curl does HTTP/1.1 and upgrades to HTTP/2 without TLS which is fine + # for us. Hypercorn supports this thankfully, but we should try with + # HTTPS as well once that's available. + assert b"< HTTP/2 200" in output + assert output.endswith(b"Dummy server!") From e77a7f9d1a3fffc2f22d92ddded11d4336fa09ac Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Tue, 21 Nov 2023 19:17:34 +0200 Subject: [PATCH 027/131] Fix tests by upgrading quart-trio and pinning Quart (#3204) --- dev-requirements.txt | 3 ++- mypy-requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 266c1be646..55d900e747 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -11,5 +11,6 @@ backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython" trio==0.23.1 -quart-trio==0.11.0 +Quart==0.19.4 +quart-trio==0.11.1 hypercorn==0.15.0 \ No newline at end of file diff --git a/mypy-requirements.txt b/mypy-requirements.txt index b8a3d34c34..461d5d24eb 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -5,7 +5,8 @@ tornado>=6.1 pytest>=6.2 trustme==0.9.0 trio==0.23.1 -quart-trio==0.11.0 +Quart==0.19.4 +quart-trio==0.11.1 hypercorn==0.15.0 types-backports types-requests From e29f5049a6061025086b2043c71b1ed6d658d017 Mon Sep 17 00:00:00 2001 From: Sean Gilligan <5122866+sg3-141-592@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:41:37 +0000 Subject: [PATCH 028/131] Fix hostname matching of FQDN with proxies (#3183) --- changelog/2244.bugfix.rst | 2 ++ dummyserver/testcase.py | 1 + src/urllib3/connection.py | 5 ++++- test/with_dummyserver/test_proxy_poolmanager.py | 9 ++++++++- 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 changelog/2244.bugfix.rst diff --git a/changelog/2244.bugfix.rst b/changelog/2244.bugfix.rst new file mode 100644 index 0000000000..f26f356a30 --- /dev/null +++ b/changelog/2244.bugfix.rst @@ -0,0 +1,2 @@ +Fixed issue where requests against urls with trailing dots were failing due to SSL errors +when using proxy. \ No newline at end of file diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 769b03f132..a6c870415d 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -225,6 +225,7 @@ class HTTPDummyProxyTestCase: https_port: typing.ClassVar[int] https_url: typing.ClassVar[str] https_url_alt: typing.ClassVar[str] + https_url_fqdn: typing.ClassVar[str] proxy_host: typing.ClassVar[str] = "localhost" proxy_host_alt: typing.ClassVar[str] = "127.0.0.1" diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 38a2fd6dfa..708743d1c7 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -639,6 +639,9 @@ def connect(self) -> None: SystemTimeWarning, ) + # Remove trailing '.' from fqdn hostnames to allow certificate validation + server_hostname_rm_dot = server_hostname.rstrip(".") + sock_and_verified = _ssl_wrap_socket_and_match_hostname( sock=sock, cert_reqs=self.cert_reqs, @@ -651,7 +654,7 @@ def connect(self) -> None: cert_file=self.cert_file, key_file=self.key_file, key_password=self.key_password, - server_hostname=server_hostname, + server_hostname=server_hostname_rm_dot, ssl_context=self.ssl_context, tls_in_tls=tls_in_tls, assert_hostname=self.assert_hostname, diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 5c306cdccf..917a8c484b 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -9,7 +9,7 @@ import socket import ssl import tempfile -from test import LONG_TIMEOUT, SHORT_TIMEOUT, withPyOpenSSL +from test import LONG_TIMEOUT, SHORT_TIMEOUT, resolvesLocalhostFQDN, withPyOpenSSL from test.conftest import ServerConfig import pytest @@ -47,6 +47,7 @@ def setup_class(cls) -> None: cls.http_url_alt = f"http://{cls.http_host_alt}:{int(cls.http_port)}" cls.https_url = f"https://{cls.https_host}:{int(cls.https_port)}" cls.https_url_alt = f"https://{cls.https_host_alt}:{int(cls.https_port)}" + cls.https_url_fqdn = f"https://{cls.https_host}.:{int(cls.https_port)}" cls.proxy_url = f"http://{cls.proxy_host}:{int(cls.proxy_port)}" cls.https_proxy_url = f"https://{cls.proxy_host}:{int(cls.https_proxy_port)}" @@ -173,6 +174,12 @@ def test_oldapi(self) -> None: r = http.request("GET", f"{self.https_url}/") assert r.status == 200 + @resolvesLocalhostFQDN() + def test_proxy_https_fqdn(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.https_url_fqdn}/") + assert r.status == 200 + def test_proxy_verified(self) -> None: with proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=self.bad_ca_path From 8ad1ab29ce7dd7ef3043f1f85d6a2c07b9401b98 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 21 Nov 2023 22:19:05 +0400 Subject: [PATCH 029/131] Move test_skip_header to test/test_connection.py It relies on sending requests that violate the HTTP spec, and that are refused by h11 as used by Hypercorn, our new test server. Thankfully, we don't need the return of the server, looking at what urllib3 would have sent is enough. --- test/test_connection.py | 58 ++++++++++++++++++++ test/with_dummyserver/test_connectionpool.py | 48 ---------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/test/test_connection.py b/test/test_connection.py index f49497bafe..a4bd8731fc 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -19,6 +19,7 @@ ) from urllib3.exceptions import HTTPError, ProxyError, SSLError from urllib3.util import ssl_ +from urllib3.util.request import SKIP_HEADER from urllib3.util.ssl_match_hostname import ( CertificateError as ImplementationCertificateError, ) @@ -265,3 +266,60 @@ def test_assert_hostname_closes_socket(self) -> None: conn.connect() context.wrap_socket.return_value.close.assert_called_once_with() + + @pytest.mark.parametrize( + "accept_encoding", + [ + "Accept-Encoding", + "accept-encoding", + b"Accept-Encoding", + b"accept-encoding", + None, + ], + ) + @pytest.mark.parametrize("host", ["Host", "host", b"Host", b"host", None]) + @pytest.mark.parametrize( + "user_agent", ["User-Agent", "user-agent", b"User-Agent", b"user-agent", None] + ) + @pytest.mark.parametrize("chunked", [True, False]) + def test_skip_header( + self, + accept_encoding: str | None, + host: str | None, + user_agent: str | None, + chunked: bool, + ) -> None: + headers = {} + if accept_encoding is not None: + headers[accept_encoding] = SKIP_HEADER + if host is not None: + headers[host] = SKIP_HEADER + if user_agent is not None: + headers[user_agent] = SKIP_HEADER + + # When dropping support for Python 3.9, this can be rewritten to parenthesized + # context managers + with mock.patch("urllib3.util.connection.create_connection"): + with mock.patch( + "urllib3.connection._HTTPConnection.putheader" + ) as http_client_putheader: + conn = HTTPConnection("") + conn.request("GET", "/headers", headers=headers, chunked=chunked) + + request_headers = {} + for call in http_client_putheader.call_args_list: + header, value = call.args + request_headers[header] = value + + if accept_encoding is None: + assert "Accept-Encoding" in request_headers + else: + assert accept_encoding not in request_headers + if host is None: + assert "Host" in request_headers + else: + assert host not in request_headers + if user_agent is None: + assert "User-Agent" in request_headers + else: + assert user_agent not in request_headers diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 8b1eb171c0..0fd1988db8 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -974,54 +974,6 @@ def test_no_user_agent_header(self) -> None: assert no_ua_headers["User-Agent"] == SKIP_HEADER assert pool_headers.get("User-Agent") == custom_ua - @pytest.mark.parametrize( - "accept_encoding", - [ - "Accept-Encoding", - "accept-encoding", - b"Accept-Encoding", - b"accept-encoding", - None, - ], - ) - @pytest.mark.parametrize("host", ["Host", "host", b"Host", b"host", None]) - @pytest.mark.parametrize( - "user_agent", ["User-Agent", "user-agent", b"User-Agent", b"user-agent", None] - ) - @pytest.mark.parametrize("chunked", [True, False]) - def test_skip_header( - self, - accept_encoding: str | None, - host: str | None, - user_agent: str | None, - chunked: bool, - ) -> None: - headers = {} - - if accept_encoding is not None: - headers[accept_encoding] = SKIP_HEADER - if host is not None: - headers[host] = SKIP_HEADER - if user_agent is not None: - headers[user_agent] = SKIP_HEADER - - with HTTPConnectionPool(self.host, self.port) as pool: - r = pool.request("GET", "/headers", headers=headers, chunked=chunked) - request_headers = r.json() - - if accept_encoding is None: - assert "Accept-Encoding" in request_headers - else: - assert accept_encoding not in request_headers - if host is None: - assert "Host" in request_headers - else: - assert host not in request_headers - if user_agent is None: - assert "User-Agent" in request_headers - else: - assert user_agent not in request_headers - @pytest.mark.parametrize("header", ["Content-Length", "content-length"]) @pytest.mark.parametrize("chunked", [True, False]) def test_skip_header_non_supported(self, header: str, chunked: bool) -> None: From 8afb98b3cede2ac14f7d20b3d9ac290cf57fa72d Mon Sep 17 00:00:00 2001 From: David Hotham Date: Tue, 21 Nov 2023 19:19:58 +0000 Subject: [PATCH 030/131] Do not modify passed HTTP-header dicts in-place (#3203) Co-authored-by: Illia Volochii --- changelog/3203.bugfix.rst | 1 + src/urllib3/_request_methods.py | 6 ++++-- test/with_dummyserver/test_poolmanager.py | 14 ++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 changelog/3203.bugfix.rst diff --git a/changelog/3203.bugfix.rst b/changelog/3203.bugfix.rst new file mode 100644 index 0000000000..9aa313b052 --- /dev/null +++ b/changelog/3203.bugfix.rst @@ -0,0 +1 @@ +Stopped updating passed HTTP-header dicts during requests with JSON. diff --git a/src/urllib3/_request_methods.py b/src/urllib3/_request_methods.py index 3ce7603bda..632042f037 100644 --- a/src/urllib3/_request_methods.py +++ b/src/urllib3/_request_methods.py @@ -119,9 +119,11 @@ def request( if json is not None: if headers is None: - headers = self.headers.copy() # type: ignore + headers = self.headers + if not ("content-type" in map(str.lower, headers.keys())): - headers["Content-Type"] = "application/json" # type: ignore + headers = HTTPHeaderDict(headers) + headers["Content-Type"] = "application/json" body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode( "utf-8" diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 33b0b1ae64..cf7d8c7eb8 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -619,18 +619,20 @@ def test_top_level_request_with_timeout(self) -> None: ], ) def test_request_with_json(self, headers: HTTPHeaderDict) -> None: + old_headers = None if headers is None else headers.copy() body = {"attribute": "value"} r = request( method="POST", url=f"{self.base_url}/echo_json", headers=headers, json=body ) assert r.status == 200 assert r.json() == body - if headers is not None and "application/json" not in headers.values(): - assert "text/plain" in r.headers["Content-Type"].replace(" ", "").split(",") - else: - assert "application/json" in r.headers["Content-Type"].replace( - " ", "" - ).split(",") + content_type = HTTPHeaderDict(old_headers).get( + "Content-Type", "application/json" + ) + assert content_type in r.headers["Content-Type"].replace(" ", "").split(",") + + # Ensure the header argument itself is not modified in-place. + assert headers == old_headers def test_top_level_request_with_json_with_httpheaderdict(self) -> None: body = {"attribute": "value"} From 8b4e56854344b43f035a13ae5397e595f163d55f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 23 Nov 2023 19:18:28 +0400 Subject: [PATCH 031/131] Migrate TestConnectionPool to Hypercorn --- dummyserver/app.py | 94 +++++++++++++++++++- test/with_dummyserver/test_connectionpool.py | 21 +++-- 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index f1dc58c517..965e832899 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -4,6 +4,7 @@ import gzip import zlib from io import BytesIO +from typing import Iterator from quart import make_response, request @@ -19,6 +20,80 @@ async def index() -> ResponseTypes: return await make_response("Dummy server!") +@hypercorn_app.route("/specific_method", methods=["GET", "POST", "PUT"]) +async def specific_method() -> ResponseTypes: + "Confirm that the request matches the desired method type" + method_param = (await request.values).get("method") + + if request.method != method_param: + return await make_response( + f"Wrong method: {method_param} != {request.method}", 400 + ) + return await make_response() + + +@hypercorn_app.route("/upload", methods=["POST"]) +async def upload() -> ResponseTypes: + "Confirm that the uploaded file conforms to specification" + params = await request.form + param = params.get("upload_param") + filename_param = params.get("upload_filename") + size = int(params.get("upload_size", "0")) + files_ = (await request.files).getlist(param) + assert files_ is not None + + if len(files_) != 1: + return await make_response( + f"Expected 1 file for '{param}', not {len(files_)}", 400 + ) + + file_ = files_[0] + # data is short enough to read synchronously without blocking the event loop + with contextlib.closing(file_.stream) as stream: + data = stream.read() + + if int(size) != len(data): + return await make_response(f"Wrong size: {int(size)} != {len(data)}", 400) + + if filename_param != file_.filename: + return await make_response( + f"Wrong filename: {filename_param} != {file_.filename}", 400 + ) + + return await make_response() + + +@hypercorn_app.route("/chunked") +async def chunked() -> ResponseTypes: + def generate() -> Iterator[str]: + for _ in range(4): + yield "123" + + return await make_response(generate()) + + +@hypercorn_app.route("/chunked_gzip") +async def chunked_gzip() -> ResponseTypes: + def generate() -> Iterator[bytes]: + compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + + for uncompressed in [b"123"] * 4: + yield compressor.compress(uncompressed) + yield compressor.flush() + + return await make_response(generate(), 200, [("Content-Encoding", "gzip")]) + + +@hypercorn_app.route("/keepalive") +async def keepalive() -> ResponseTypes: + if request.args.get("close", b"0") == b"1": + headers = [("Connection", "close")] + return await make_response("Closing", 200, headers) + + headers = [("Connection", "keep-alive")] + return await make_response("Keeping alive", 200, headers) + + @hypercorn_app.route("/echo", methods=["GET", "POST"]) async def echo() -> ResponseTypes: "Echo back the params" @@ -35,13 +110,22 @@ async def echo_json() -> ResponseTypes: return await make_response(data, 200, request.headers) -@hypercorn_app.route("/echo_uri") -async def echo_uri() -> ResponseTypes: +@hypercorn_app.route("/echo_uri/") +@hypercorn_app.route("/echo_uri", defaults={"rest": ""}) +async def echo_uri(rest: str) -> ResponseTypes: "Echo back the requested URI" assert request.full_path is not None return await make_response(request.full_path) +@hypercorn_app.route("/echo_params") +async def echo_params() -> ResponseTypes: + "Echo back the query parameters" + await request.get_data() + echod = sorted((k, v) for k, v in request.args.items()) + return await make_response(repr(echod)) + + @hypercorn_app.route("/headers", methods=["GET", "POST"]) async def headers() -> ResponseTypes: return await make_response(dict(request.headers.items())) @@ -104,3 +188,9 @@ async def status() -> ResponseTypes: status = values.get("status", "200 OK") status_code = status.split(" ")[0] return await make_response("", status_code) + + +@hypercorn_app.route("/source_address") +async def source_address() -> ResponseTypes: + """Return the requester's IP address.""" + return await make_response(request.remote_addr) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 0fd1988db8..486c8ada91 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -12,8 +12,12 @@ import pytest -from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase -from dummyserver.tornadoserver import HAS_IPV6_AND_DNS, NoIPv6Warning +from dummyserver.testcase import ( + HTTPDummyServerTestCase, + HypercornDummyServerTestCase, + SocketDummyServerTestCase, +) +from dummyserver.tornadoserver import NoIPv6Warning from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict from urllib3.connection import _get_default_user_agent @@ -198,7 +202,7 @@ def test_create_connection_timeout(self) -> None: conn.connect() -class TestConnectionPool(HTTPDummyServerTestCase): +class TestConnectionPool(HypercornDummyServerTestCase): def test_get(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/specific_method", fields={"method": "GET"}) @@ -780,7 +784,9 @@ def test_percent_encode_invalid_target_chars(self) -> None: def test_source_address(self) -> None: for addr, is_ipv6 in VALID_SOURCE_ADDRESSES: - if is_ipv6 and not HAS_IPV6_AND_DNS: + if is_ipv6: + # TODO enable if HAS_IPV6_AND_DNS when this is fixed: + # https://github.com/pgjones/hypercorn/issues/160 warnings.warn("No IPv6 support: skipping.", NoIPv6Warning) continue with HTTPConnectionPool( @@ -890,7 +896,7 @@ def test_preserves_path_dot_segments(self) -> None: """ConnectionPool preserves dot segments in the URI""" with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request("GET", "/echo_uri/seg0/../seg2") - assert response.data == b"/echo_uri/seg0/../seg2" + assert response.data == b"/echo_uri/seg0/../seg2?" def test_default_user_agent_header(self) -> None: """ConnectionPool has a default user agent""" @@ -1013,6 +1019,8 @@ def test_headers_not_modified_by_request( else: conn = pool._get_conn() conn.request("GET", "/headers", chunked=chunked) + conn.getresponse().close() + conn.close() assert pool.headers == {"key": "val"} assert type(pool.headers) is header_type @@ -1023,6 +1031,8 @@ def test_headers_not_modified_by_request( else: conn = pool._get_conn() conn.request("GET", "/headers", headers=headers, chunked=chunked) + conn.getresponse().close() + conn.close() assert headers == {"key": "val"} @@ -1042,6 +1052,7 @@ def test_request_chunked_is_deprecated( resp = conn.getresponse() assert resp.status == 200 assert resp.json()["Transfer-Encoding"] == "chunked" + conn.close() def test_bytes_header(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: From ac6ca9983b7d3c7e4e9f21554157e559b8f842a1 Mon Sep 17 00:00:00 2001 From: Sam Mason Date: Thu, 23 Nov 2023 21:24:35 +0000 Subject: [PATCH 032/131] Add support for read1 in HTTPResponse (#3186) --- changelog/3186.feature.rst | 1 + src/urllib3/response.py | 92 ++++++++++++++++++++++- test/test_response.py | 88 ++++++++++++++++++++++ test/with_dummyserver/test_socketlevel.py | 43 +++++++++-- 4 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 changelog/3186.feature.rst diff --git a/changelog/3186.feature.rst b/changelog/3186.feature.rst new file mode 100644 index 0000000000..3a6507b49b --- /dev/null +++ b/changelog/3186.feature.rst @@ -0,0 +1 @@ +Added support for HTTPResponse.read1() method. diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 37936f9397..4ed5a90b8d 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -280,6 +280,19 @@ def get(self, n: int) -> bytes: return ret.getvalue() + def get_all(self) -> bytes: + buffer = self.buffer + if not buffer: + assert self._size == 0 + return b"" + if len(buffer) == 1: + result = buffer.pop() + else: + result = b"".join(buffer) + buffer.clear() + self._size = 0 + return result + class BaseHTTPResponse(io.IOBase): CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] @@ -393,6 +406,13 @@ def read( ) -> bytes: raise NotImplementedError() + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + raise NotImplementedError() + def read_chunked( self, amt: int | None = None, @@ -752,7 +772,12 @@ def _error_catcher(self) -> typing.Generator[None, None, None]: if self._original_response and self._original_response.isclosed(): self.release_conn() - def _fp_read(self, amt: int | None = None) -> bytes: + def _fp_read( + self, + amt: int | None = None, + *, + read1: bool = False, + ) -> bytes: """ Read a response with the thought that reading the number of bytes larger than can fit in a 32-bit int at a time via SSL in some @@ -770,8 +795,14 @@ def _fp_read(self, amt: int | None = None) -> bytes: c_int_max = 2**31 - 1 if ( (amt and amt > c_int_max) - or (self.length_remaining and self.length_remaining > c_int_max) + or ( + amt is None + and self.length_remaining + and self.length_remaining > c_int_max + ) ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)): + if read1: + return self._fp.read1(c_int_max) buffer = io.BytesIO() # Besides `max_chunk_amt` being a maximum chunk size, it # affects memory overhead of reading a response by this @@ -792,6 +823,8 @@ def _fp_read(self, amt: int | None = None) -> bytes: buffer.write(data) del data # to reduce peak memory usage by `max_chunk_amt`. return buffer.getvalue() + elif read1: + return self._fp.read1(amt) if amt is not None else self._fp.read1() else: # StringIO doesn't like amt=None return self._fp.read(amt) if amt is not None else self._fp.read() @@ -799,6 +832,8 @@ def _fp_read(self, amt: int | None = None) -> bytes: def _raw_read( self, amt: int | None = None, + *, + read1: bool = False, ) -> bytes: """ Reads `amt` of bytes from the socket. @@ -809,7 +844,7 @@ def _raw_read( fp_closed = getattr(self._fp, "closed", False) with self._error_catcher(): - data = self._fp_read(amt) if not fp_closed else b"" + data = self._fp_read(amt, read1=read1) if not fp_closed else b"" if amt is not None and amt != 0 and not data: # Platform-specific: Buggy versions of Python. # Close the connection when no data is returned @@ -909,6 +944,57 @@ def read( return data + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + """ + Similar to ``http.client.HTTPResponse.read1`` and documented + in :meth:`io.BufferedReader.read1`, but with an additional parameter: + ``decode_content``. + + :param amt: + How much of the content to read. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + if decode_content is None: + decode_content = self.decode_content + # try and respond without going to the network + if self._has_decoded_content: + if not decode_content: + raise RuntimeError( + "Calling read1(decode_content=False) is not supported after " + "read1(decode_content=True) was called." + ) + if len(self._decoded_buffer) > 0: + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + if amt == 0: + return b"" + + # FIXME, this method's type doesn't say returning None is possible + data = self._raw_read(amt, read1=True) + if not decode_content or data is None: + return data + + self._init_decoder() + while True: + flush_decoder = not data + decoded_data = self._decode(data, decode_content, flush_decoder) + self._decoded_buffer.put(decoded_data) + if decoded_data or flush_decoder: + break + data = self._raw_read(8192, read1=True) + + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + def stream( self, amt: int | None = 2**16, decode_content: bool | None = None ) -> typing.Generator[bytes, None, None]: diff --git a/test/test_response.py b/test/test_response.py index 98ae544eb4..3bc6f53b6a 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -73,6 +73,23 @@ def test_multiple_chunks(self) -> None: assert buffer.get(4) == b"rbaz" assert len(buffer) == 0 + def test_get_all_empty(self) -> None: + q = BytesQueueBuffer() + assert q.get_all() == b"" + + def test_get_all_single(self) -> None: + q = BytesQueueBuffer() + q.put(b"a") + assert q.get_all() == b"a" + + def test_get_all_many(self) -> None: + q = BytesQueueBuffer() + q.put(b"a") + q.put(b"b") + q.put(b"c") + assert q.get_all() == b"abc" + assert len(q) == 0 + @pytest.mark.limit_memory("12.5 MB") # assert that we're not doubling memory usage def test_memory_usage(self) -> None: # Allocate 10 1MiB chunks @@ -185,6 +202,39 @@ def test_reference_read(self) -> None: assert r.read() == b"" assert r.read() == b"" + def test_reference_read1(self) -> None: + fp = BytesIO(b"foobar") + r = HTTPResponse(fp, preload_content=False) + + assert r.read1(0) == b"" + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + + def test_reference_read1_nodecode(self) -> None: + fp = BytesIO(b"foobar") + r = HTTPResponse(fp, preload_content=False, decode_content=False) + + assert r.read1(0) == b"" + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + + def test_decoding_read1(self) -> None: + data = zlib.compress(b"foobar") + + fp = BytesIO(data) + r = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + def test_decode_deflate(self) -> None: data = zlib.compress(b"foo") @@ -624,6 +674,35 @@ def test_read_with_illegal_mix_decode_toggle(self) -> None: ): resp.read(decode_content=False) + def test_read1_with_illegal_mix_decode_toggle(self) -> None: + data = zlib.compress(b"foo") + + fp = BytesIO(data) + + resp = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + + assert resp.read1(1) == b"f" + + with pytest.raises( + RuntimeError, + match=( + r"Calling read1\(decode_content=False\) is not supported after " + r"read1\(decode_content=True\) was called" + ), + ): + resp.read1(1, decode_content=False) + + with pytest.raises( + RuntimeError, + match=( + r"Calling read1\(decode_content=False\) is not supported after " + r"read1\(decode_content=True\) was called" + ), + ): + resp.read1(decode_content=False) + def test_read_with_mix_decode_toggle(self) -> None: data = zlib.compress(b"foo") @@ -726,6 +805,9 @@ def read(self, _: int) -> bytes: # type: ignore[override] return self.payloads.pop(0) return b"" + def read1(self, amt: int) -> bytes: # type: ignore[override] + return self.read(amt) + uncompressed_data = zlib.decompress(ZLIB_PAYLOAD) payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS @@ -922,6 +1004,9 @@ def read(self, amt: int) -> bytes: return data + def read1(self, amt: int) -> bytes: + return self.read(1) + def close(self) -> None: self.fp = None @@ -1294,6 +1379,9 @@ def readline(self) -> bytes: def read(self, amt: int = -1) -> bytes: return self.pop_current_chunk(amt) + def read1(self, amt: int = -1) -> bytes: + return self.pop_current_chunk(amt) + def flush(self) -> None: # Python 3 wants this method. pass diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 9422e5d885..711ce8f78b 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1583,12 +1583,47 @@ def socket_handler(listener: socket.socket) -> None: pool.request("GET", "/", retries=False, timeout=LONG_TIMEOUT) assert server_closed.wait(LONG_TIMEOUT), "The socket was not terminated" + def _run_preload(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/") + assert len(response.data) == content_length + + def _run_read_None(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + assert len(response.read(None)) == content_length + assert response.read(None) == b"" + + def _run_read_amt(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + assert len(response.read(content_length)) == content_length + assert response.read(5) == b"" + + def _run_read1_None(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + remaining = content_length + while True: + chunk = response.read1(None) + if not chunk: + break + remaining -= len(chunk) + assert remaining == 0 + + def _run_read1_amt(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + remaining = content_length + while True: + chunk = response.read1(content_length) + if not chunk: + break + remaining -= len(chunk) + assert remaining == 0 + @pytest.mark.integration @pytest.mark.parametrize( - "preload_content,read_amt", [(True, None), (False, None), (False, 2**31)] + "method", + [_run_preload, _run_read_None, _run_read_amt, _run_read1_None, _run_read1_amt], ) def test_requesting_large_resources_via_ssl( - self, preload_content: bool, read_amt: int | None + self, method: typing.Callable[[typing.Any, HTTPSConnectionPool, int], None] ) -> None: """ Ensure that it is possible to read 2 GiB or more via an SSL @@ -1630,9 +1665,7 @@ def socket_handler(listener: socket.socket) -> None: with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, retries=False ) as pool: - response = pool.request("GET", "/", preload_content=preload_content) - data = response.data if preload_content else response.read(read_amt) - assert len(data) == content_length + method(self, pool, content_length) class TestErrorWrapping(SocketDummyServerTestCase): From 4ece59b2d438f605dcb9a2448e1155075cd2fe07 Mon Sep 17 00:00:00 2001 From: David Hotham Date: Thu, 23 Nov 2023 21:33:28 +0000 Subject: [PATCH 033/131] Begin requiring error codes in all `# type: ignore` (#3134) --- pyproject.toml | 3 +++ src/urllib3/connection.py | 5 ----- src/urllib3/connectionpool.py | 4 ++-- test/with_dummyserver/test_connectionpool.py | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b850412a1..e2659207e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,3 +116,6 @@ warn_redundant_casts = true warn_return_any = true warn_unused_configs = true warn_unused_ignores = true +enable_error_code = [ + "ignore-without-code", +] diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 708743d1c7..2a988500d5 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -160,11 +160,6 @@ def __init__( self._tunnel_port: int | None = None self._tunnel_scheme: str | None = None - # https://github.com/python/mypy/issues/4125 - # Mypy treats this as LSP violation, which is considered a bug. - # If `host` is made a property it violates LSP, because a writeable attribute is overridden with a read-only one. - # However, there is also a `host` setter so LSP is not violated. - # Potentially, a `@host.deleter` might be needed depending on how this issue will be fixed. @property def host(self) -> str: """ diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 70048b7aed..abb4d4dcd4 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -748,8 +748,8 @@ def urlopen( # type: ignore[override] # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: - headers = headers.copy() # type: ignore[attr-defined] - headers.update(self.proxy_headers) # type: ignore[union-attr] + headers = HTTPHeaderDict(headers) + headers.update(self.proxy_headers) # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 486c8ada91..82b4a16c96 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -235,7 +235,7 @@ def test_upload(self) -> None: "upload_filename": "lolcat.txt", "filefield": ("lolcat.txt", data), } - fields["upload_size"] = len(data) # type: ignore + fields["upload_size"] = len(data) # type: ignore[assignment] with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("POST", "/upload", fields=fields) @@ -274,7 +274,7 @@ def test_unicode_upload(self) -> None: "upload_filename": filename, fieldname: (filename, data), } - fields["upload_size"] = size # type: ignore + fields["upload_size"] = size # type: ignore[assignment] with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("POST", "/upload", fields=fields) assert r.status == 200, r.data From a19ee0a796920d1310e70849a6f4e6004013a2a2 Mon Sep 17 00:00:00 2001 From: Dimitris Zlatanidis <77074526+dslackw@users.noreply.github.com> Date: Thu, 23 Nov 2023 23:55:50 +0200 Subject: [PATCH 034/131] Change dependency warning link for PySocks (#2964) --- src/urllib3/contrib/socks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py index 6c3bb764b2..8c3b57b645 100644 --- a/src/urllib3/contrib/socks.py +++ b/src/urllib3/contrib/socks.py @@ -51,7 +51,7 @@ ( "SOCKS support in urllib3 requires the installation of optional " "dependencies: specifically, PySocks. For more information, see " - "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies" + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies" ), DependencyWarning, ) From d32e1fcee8d93612985af5094c8b8615522a80ba Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 24 Nov 2023 06:27:45 +0400 Subject: [PATCH 035/131] Fix flaky time-based socketlevel tests (#3206) Instead, rely on the expected bytes to know when to stop calling recv(). --- test/with_dummyserver/test_socketlevel.py | 83 +++++++++++------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 711ce8f78b..795ba1f20d 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -12,7 +12,6 @@ import socket import ssl import tempfile -import time import typing import zlib from collections import OrderedDict @@ -1840,11 +1839,14 @@ def test_headers_sent_with_add( body: None | bytes | io.BytesIO if body_type is None: body = None + expected = b"\r\n\r\n" elif body_type == "bytes": body = b"my-body" + expected = b"\r\n\r\nmy-body" elif body_type == "bytes-io": body = io.BytesIO(b"bytes-io-body") body.seek(0, 0) + expected = b"bytes-io-body\r\n0\r\n\r\n" else: raise ValueError("Unknonw body type") @@ -1855,12 +1857,9 @@ def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock.settimeout(0) - start = time.time() - while time.time() - start < (LONG_TIMEOUT / 2): - try: + while expected not in buffer: + with contextlib.suppress(BlockingIOError): buffer += sock.recv(65536) - except OSError: - continue sock.sendall( b"HTTP/1.1 200 OK\r\n" @@ -2273,18 +2272,16 @@ def test_chunked_specified( self, method: str, chunked: bool, body_type: str ) -> None: buffer = bytearray() + expected_bytes = b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" def socket_handler(listener: socket.socket) -> None: nonlocal buffer sock = listener.accept()[0] sock.settimeout(0) - start = time.time() - while time.time() - start < (LONG_TIMEOUT / 2): - try: + while expected_bytes not in buffer: + with contextlib.suppress(BlockingIOError): buffer += sock.recv(65536) - except OSError: - continue sock.sendall( b"HTTP/1.1 200 OK\r\n" @@ -2323,7 +2320,7 @@ def body_generator() -> typing.Generator[bytes, None, None]: assert b"Transfer-Encoding: chunked\r\n" in sent_bytes assert b"User-Agent: python-urllib3/" in sent_bytes assert b"content-length" not in sent_bytes.lower() - assert b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" in sent_bytes + assert expected_bytes in sent_bytes @pytest.mark.parametrize("method", ["POST", "PUT", "PATCH"]) @pytest.mark.parametrize( @@ -2331,29 +2328,9 @@ def body_generator() -> typing.Generator[bytes, None, None]: ) def test_chunked_not_specified(self, method: str, body_type: str) -> None: buffer = bytearray() - - def socket_handler(listener: socket.socket) -> None: - nonlocal buffer - sock = listener.accept()[0] - sock.settimeout(0) - - start = time.time() - while time.time() - start < (LONG_TIMEOUT / 2): - try: - buffer += sock.recv(65536) - except OSError: - continue - - sock.sendall( - b"HTTP/1.1 200 OK\r\n" - b"Server: example.com\r\n" - b"Content-Length: 0\r\n\r\n" - ) - sock.close() - - self._start_server(socket_handler) - + expected_bytes: bytes body: typing.Any + if body_type == "generator": def body_generator() -> typing.Generator[bytes, None, None]: @@ -2361,25 +2338,44 @@ def body_generator() -> typing.Generator[bytes, None, None]: body = body_generator() should_be_chunked = True - elif body_type == "file": body = io.BytesIO(b"x" * 10) body.seek(0, 0) should_be_chunked = True - elif body_type == "file_text": body = io.StringIO("x" * 10) body.seek(0, 0) should_be_chunked = True - elif body_type == "bytearray": body = bytearray(b"x" * 10) should_be_chunked = False - else: body = b"x" * 10 should_be_chunked = False + if should_be_chunked: + expected_bytes = b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" + else: + expected_bytes = b"\r\n\r\nxxxxxxxxxx" + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + sock = listener.accept()[0] + sock.settimeout(0) + + while expected_bytes not in buffer: + with contextlib.suppress(BlockingIOError): + buffer += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler) + with HTTPConnectionPool( self.host, self.port, timeout=LONG_TIMEOUT, retries=False ) as pool: @@ -2395,12 +2391,12 @@ def body_generator() -> typing.Generator[bytes, None, None]: if should_be_chunked: assert b"content-length" not in sent_bytes.lower() assert b"Transfer-Encoding: chunked\r\n" in sent_bytes - assert b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" in sent_bytes + assert expected_bytes in sent_bytes else: assert b"Content-Length: 10\r\n" in sent_bytes assert b"transfer-encoding" not in sent_bytes.lower() - assert sent_bytes.endswith(b"\r\n\r\nxxxxxxxxxx") + assert sent_bytes.endswith(expected_bytes) @pytest.mark.parametrize( "header_transform", @@ -2431,12 +2427,9 @@ def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock.settimeout(0) - start = time.time() - while time.time() - start < (LONG_TIMEOUT / 2): - try: + while expected not in buffer: + with contextlib.suppress(BlockingIOError): buffer += sock.recv(65536) - except OSError: - continue sock.sendall( b"HTTP/1.1 200 OK\r\n" From 776049e8cfad989e9b5cbaf9de55b358adae2587 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sat, 25 Nov 2023 20:27:06 +0400 Subject: [PATCH 036/131] Migrate TestRetry to Hypercorn --- dummyserver/app.py | 36 ++++++++++++++++++++ test/with_dummyserver/test_connectionpool.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index 965e832899..453961303a 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -1,5 +1,6 @@ from __future__ import annotations +import collections import contextlib import gzip import zlib @@ -14,6 +15,8 @@ hypercorn_app = QuartTrio(__name__) +RETRY_TEST_NAMES: collections.Counter[str] = collections.Counter() + @hypercorn_app.route("/") async def index() -> ResponseTypes: @@ -146,6 +149,21 @@ async def multi_headers() -> ResponseTypes: return await make_response({"headers": list(request.headers)}) +@hypercorn_app.route("/multi_redirect") +async def multi_redirect() -> ResponseTypes: + "Performs a redirect chain based on ``redirect_codes``" + params = request.args + codes = params.get("redirect_codes", "200") + head, tail = codes.split(",", 1) if "," in codes else (codes, None) + assert head is not None + status = head + if not tail: + return await make_response("Done redirecting", status) + + headers = [("Location", f"/multi_redirect?redirect_codes={tail}")] + return await make_response("", status, headers) + + @hypercorn_app.route("/encodingrequest") async def encodingrequest() -> ResponseTypes: "Check for UA accepting gzip/deflate encoding" @@ -194,3 +212,21 @@ async def status() -> ResponseTypes: async def source_address() -> ResponseTypes: """Return the requester's IP address.""" return await make_response(request.remote_addr) + + +@hypercorn_app.route("/successful_retry") +async def successful_retry() -> ResponseTypes: + """First return an error and then success + + It's not currently very flexible as the number of retries is hard-coded. + """ + test_name = request.headers.get("test-name", None) + if not test_name: + return await make_response("test-name header not set", 400) + + RETRY_TEST_NAMES[test_name] += 1 + + if RETRY_TEST_NAMES[test_name] >= 2: + return await make_response("Retry successful!", 200) + else: + return await make_response("need to keep retrying!", 418) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 82b4a16c96..827af716e0 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -1077,7 +1077,7 @@ def test_user_agent_non_ascii_user_agent(self, user_agent: str) -> None: assert request_headers["User-Agent"] == "Schönefeld/1.18.0" -class TestRetry(HTTPDummyServerTestCase): +class TestRetry(HypercornDummyServerTestCase): def test_max_retry(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: with pytest.raises(MaxRetryError): From 76b0fd5b52c5529ef3ab5f5654c32cf8b2e2b990 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sat, 25 Nov 2023 20:28:32 +0400 Subject: [PATCH 037/131] Migrate TestHTTPWithoutSSL to Hypercorn --- test/with_dummyserver/test_no_ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py index b89f703fac..14ac80fec3 100644 --- a/test/with_dummyserver/test_no_ssl.py +++ b/test/with_dummyserver/test_no_ssl.py @@ -8,13 +8,13 @@ import pytest import urllib3 -from dummyserver.testcase import HTTPDummyServerTestCase, HTTPSDummyServerTestCase +from dummyserver.testcase import HTTPSDummyServerTestCase, HypercornDummyServerTestCase from urllib3.exceptions import InsecureRequestWarning from ..test_no_ssl import TestWithoutSSL -class TestHTTPWithoutSSL(HTTPDummyServerTestCase, TestWithoutSSL): +class TestHTTPWithoutSSL(HypercornDummyServerTestCase, TestWithoutSSL): def test_simple(self) -> None: with urllib3.HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/") From 1331b6483d18319c9311ee74f0e7b7863ac4b57c Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 26 Nov 2023 00:45:32 +0400 Subject: [PATCH 038/131] Migrate TestHTTPSWithoutSSL to Hypercorn --- dummyserver/testcase.py | 15 +++++++++++++++ test/with_dummyserver/test_no_ssl.py | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index a6c870415d..27fd155d78 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -303,6 +303,7 @@ class HypercornDummyServerTestCase: port: typing.ClassVar[int] base_url: typing.ClassVar[str] base_url_alt: typing.ClassVar[str] + certs: typing.ClassVar[dict[str, typing.Any]] = {} _stack: typing.ClassVar[contextlib.ExitStack] @@ -310,6 +311,12 @@ class HypercornDummyServerTestCase: def setup_class(cls) -> None: with contextlib.ExitStack() as stack: config = hypercorn.Config() + if cls.certs: + config.certfile = cls.certs["certfile"] + config.keyfile = cls.certs["keyfile"] + config.verify_mode = cls.certs["cert_reqs"] + config.ca_certs = cls.certs["ca_certs"] + config.alpn_protocols = cls.certs["alpn_protocols"] config.bind = [f"{cls.host}:0"] stack.enter_context(run_hypercorn_in_thread(config, hypercorn_app)) cls._stack = stack.pop_all() @@ -320,6 +327,14 @@ def teardown_class(cls) -> None: cls._stack.close() +class HTTPSHypercornDummyServerTestCase(HypercornDummyServerTestCase): + scheme = "https" + host = "localhost" + certs = DEFAULT_CERTS + certs_dir = "" + bad_ca_path = "" + + class ConnectionMarker: """ Marks an HTTP(S)Connection's socket after a request was made. diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py index 14ac80fec3..9a28119abf 100644 --- a/test/with_dummyserver/test_no_ssl.py +++ b/test/with_dummyserver/test_no_ssl.py @@ -8,7 +8,10 @@ import pytest import urllib3 -from dummyserver.testcase import HTTPSDummyServerTestCase, HypercornDummyServerTestCase +from dummyserver.testcase import ( + HTTPSHypercornDummyServerTestCase, + HypercornDummyServerTestCase, +) from urllib3.exceptions import InsecureRequestWarning from ..test_no_ssl import TestWithoutSSL @@ -21,7 +24,7 @@ def test_simple(self) -> None: assert r.status == 200, r.data -class TestHTTPSWithoutSSL(HTTPSDummyServerTestCase, TestWithoutSSL): +class TestHTTPSWithoutSSL(HTTPSHypercornDummyServerTestCase, TestWithoutSSL): def test_simple(self) -> None: with urllib3.HTTPSConnectionPool( self.host, self.port, cert_reqs="NONE" From 17bcdada7cddfa912e0b7e9093d9d29657cecde1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 26 Nov 2023 00:47:13 +0400 Subject: [PATCH 039/131] Migrate RetryAfter to Hypercorn --- dummyserver/app.py | 34 ++++++++++++++++++++ test/with_dummyserver/test_connectionpool.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index 453961303a..452627ef3e 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -2,6 +2,8 @@ import collections import contextlib +import datetime +import email.utils import gzip import zlib from io import BytesIO @@ -15,7 +17,9 @@ hypercorn_app = QuartTrio(__name__) +# Globals are not safe in Flask/Quart but work for our Hypercorn use case RETRY_TEST_NAMES: collections.Counter[str] = collections.Counter() +LAST_RETRY_AFTER_REQ: datetime.datetime = datetime.datetime.min @hypercorn_app.route("/") @@ -200,6 +204,36 @@ async def redirect() -> ResponseTypes: return await make_response("", status_code, headers) +@hypercorn_app.route("/redirect_after") +async def redirect_after() -> ResponseTypes: + "Perform a redirect to ``target``" + params = request.args + date = params.get("date") + if date: + dt = datetime.datetime.fromtimestamp(float(date), tz=datetime.timezone.utc) + http_dt = email.utils.format_datetime(dt, usegmt=True) + retry_after = str(http_dt) + else: + retry_after = "1" + target = params.get("target", "/") + headers = [("Location", target), ("Retry-After", retry_after)] + return await make_response("", 303, headers) + + +@hypercorn_app.route("/retry_after") +async def retry_after() -> ResponseTypes: + global LAST_RETRY_AFTER_REQ + params = request.args + if datetime.datetime.now() - LAST_RETRY_AFTER_REQ < datetime.timedelta(seconds=1): + status = params.get("status", "429 Too Many Requests") + status_code = status.split(" ")[0] + + return await make_response("", status_code, [("Retry-After", "1")]) + + LAST_RETRY_AFTER_REQ = datetime.datetime.now() + return await make_response("", 200) + + @hypercorn_app.route("/status") async def status() -> ResponseTypes: values = await request.values diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 827af716e0..e7a14557a8 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -1241,7 +1241,7 @@ def test_multi_redirect_history(self) -> None: assert actual == expected -class TestRetryAfter(HTTPDummyServerTestCase): +class TestRetryAfter(HypercornDummyServerTestCase): def test_retry_after(self) -> None: # Request twice in a second to get a 429 response. with HTTPConnectionPool(self.host, self.port) as pool: From 1812eac7115b3a4e9a5feece5fae0c9cffe8c585 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 25 Nov 2023 22:43:08 +0000 Subject: [PATCH 040/131] Add native Emscripten support --- .eslintrc.yml | 7 + .github/workflows/ci.yml | 10 + .pre-commit-config.yaml | 11 + changelog/2951.feature.rst | 1 + dummyserver/handlers.py | 1 + emscripten-requirements.txt | 3 + noxfile.py | 106 ++ src/urllib3/__init__.py | 7 + src/urllib3/connectionpool.py | 4 +- src/urllib3/contrib/emscripten/__init__.py | 16 + src/urllib3/contrib/emscripten/connection.py | 249 +++++ .../emscripten/emscripten_fetch_worker.js | 110 ++ src/urllib3/contrib/emscripten/fetch.py | 413 ++++++++ src/urllib3/contrib/emscripten/request.py | 22 + src/urllib3/contrib/emscripten/response.py | 276 +++++ src/urllib3/response.py | 5 +- test/contrib/emscripten/__init__.py | 0 test/contrib/emscripten/conftest.py | 269 +++++ test/contrib/emscripten/test_emscripten.py | 948 ++++++++++++++++++ 19 files changed, 2456 insertions(+), 2 deletions(-) create mode 100644 .eslintrc.yml create mode 100644 changelog/2951.feature.rst create mode 100644 emscripten-requirements.txt create mode 100644 src/urllib3/contrib/emscripten/__init__.py create mode 100644 src/urllib3/contrib/emscripten/connection.py create mode 100644 src/urllib3/contrib/emscripten/emscripten_fetch_worker.js create mode 100644 src/urllib3/contrib/emscripten/fetch.py create mode 100644 src/urllib3/contrib/emscripten/request.py create mode 100644 src/urllib3/contrib/emscripten/response.py create mode 100644 test/contrib/emscripten/__init__.py create mode 100644 test/contrib/emscripten/conftest.py create mode 100644 test/contrib/emscripten/test_emscripten.py diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000000..b1be2badbd --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,7 @@ +env: + es2020 : true + worker: true +rules: {} +extends: +- eslint:recommended + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a58a92c1e..9e244f5632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,10 @@ jobs: os: ubuntu-20.04 # CPython 3.9.2 is not available for ubuntu-22.04. experimental: false nox-session: test-3.9 + - python-version: "3.11" + os: ubuntu-latest + nox-session: emscripten + experimental: true exclude: # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it # https://github.com/python/cpython/issues/83001 @@ -104,6 +108,12 @@ jobs: - name: "Install dependencies" run: python -m pip install --upgrade pip setuptools nox + - name: "Install Chrome" + uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 # v1.3.0 + if: ${{ matrix.nox-session == 'emscripten' }} + - name: "Install Firefox" + uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0 + if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04e65f8cd4..7cb61dedcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,14 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-2020] + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.1.0" + hooks: + - id: prettier + types_or: [javascript] + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.53.0 + hooks: + - id: eslint + args: ["--fix"] \ No newline at end of file diff --git a/changelog/2951.feature.rst b/changelog/2951.feature.rst new file mode 100644 index 0000000000..43bba90dee --- /dev/null +++ b/changelog/2951.feature.rst @@ -0,0 +1 @@ +Added support for Emscripten, including streaming support in cross-origin isolated browser environments where threading is enabled. \ No newline at end of file diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 86201a116f..84493ab82d 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -247,6 +247,7 @@ def echo(self, request: httputil.HTTPServerRequest) -> Response: def echo_json(self, request: httputil.HTTPServerRequest) -> Response: "Echo back the JSON" + print("ECHO JSON:", request.body) return Response(json=request.body, headers=list(request.headers.items())) def echo_uri(self, request: httputil.HTTPServerRequest) -> Response: diff --git a/emscripten-requirements.txt b/emscripten-requirements.txt new file mode 100644 index 0000000000..7ed8739675 --- /dev/null +++ b/emscripten-requirements.txt @@ -0,0 +1,3 @@ +pytest-pyodide==0.54.0 +pyodide-build==0.24.1 +webdriver-manager==4.0.1 \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index c63c3de92f..d153d08e3e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,6 +3,8 @@ import os import shutil import sys +import typing +from pathlib import Path import nox @@ -14,6 +16,7 @@ def tests_impl( # https://github.com/python-hyper/h2/issues/1236 byte_string_comparisons: bool = False, integration: bool = False, + pytest_extra_args: list[str] = [], ) -> None: # Install deps and the package itself. session.install("-r", "dev-requirements.txt") @@ -53,6 +56,7 @@ def tests_impl( "--durations=10", "--strict-config", "--strict-markers", + *pytest_extra_args, *(session.posargs or ("test/",)), env={"PYTHONWARNINGS": "always::DeprecationWarning"}, ) @@ -152,6 +156,108 @@ def lint(session: nox.Session) -> None: mypy(session) +# TODO: node support is not tested yet - it should work if you require('xmlhttprequest') before +# loading pyodide, but there is currently no nice way to do this with pytest-pyodide +# because you can't override the test runner properties easily - see +# https://github.com/pyodide/pytest-pyodide/issues/118 for more +@nox.session(python="3.11") +@nox.parametrize("runner", ["firefox", "chrome"]) +def emscripten(session: nox.Session, runner: str) -> None: + """Test on Emscripten with Pyodide & Chrome / Firefox""" + session.install("-r", "emscripten-requirements.txt") + # build wheel into dist folder + session.run("python", "-m", "build") + # make sure we have a dist dir for pyodide + dist_dir = None + if "PYODIDE_ROOT" in os.environ: + # we have a pyodide build tree checked out + # use the dist directory from that + dist_dir = Path(os.environ["PYODIDE_ROOT"]) / "dist" + else: + # we don't have a build tree, get one + # that matches the version of pyodide build + pyodide_version = typing.cast( + str, + session.run( + "python", + "-c", + "import pyodide_build;print(pyodide_build.__version__)", + silent=True, + ), + ).strip() + + pyodide_artifacts_path = Path(session.cache_dir) / f"pyodide-{pyodide_version}" + if not pyodide_artifacts_path.exists(): + print("Fetching pyodide build artifacts") + session.run( + "wget", + f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2", + "-O", + f"{pyodide_artifacts_path}.tar.bz2", + ) + pyodide_artifacts_path.mkdir(parents=True) + session.run( + "tar", + "-xjf", + f"{pyodide_artifacts_path}.tar.bz2", + "-C", + str(pyodide_artifacts_path), + "--strip-components", + "1", + ) + + dist_dir = pyodide_artifacts_path + assert dist_dir is not None + assert dist_dir.exists() + if runner == "chrome": + # install chrome webdriver and add it to path + driver = typing.cast( + str, + session.run( + "python", + "-c", + "from webdriver_manager.chrome import ChromeDriverManager;print(ChromeDriverManager().install())", + silent=True, + ), + ).strip() + session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}" + + tests_impl( + session, + pytest_extra_args=[ + "--rt", + "chrome-no-host", + "--dist-dir", + str(dist_dir), + "test", + ], + ) + elif runner == "firefox": + driver = typing.cast( + str, + session.run( + "python", + "-c", + "from webdriver_manager.firefox import GeckoDriverManager;print(GeckoDriverManager().install())", + silent=True, + ), + ).strip() + session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}" + + tests_impl( + session, + pytest_extra_args=[ + "--rt", + "firefox-no-host", + "--dist-dir", + str(dist_dir), + "test", + ], + ) + else: + raise ValueError(f"Unknown runnner: {runner}") + + @nox.session(python="3.12") def mypy(session: nox.Session) -> None: """Run mypy.""" diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 1bd3010b3f..1e0bf37b33 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -6,6 +6,7 @@ # Set default logging handler to avoid "No handler found" warnings. import logging +import sys import typing import warnings from logging import NullHandler @@ -202,3 +203,9 @@ def request( timeout=timeout, json=json, ) + + +if sys.platform == "emscripten": + from .contrib.emscripten import inject_into_urllib3 # noqa: 401 + + inject_into_urllib3() diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index abb4d4dcd4..f7b0824cde 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -543,6 +543,8 @@ def _make_request( response._connection = response_conn # type: ignore[attr-defined] response._pool = self # type: ignore[attr-defined] + # emscripten connection doesn't have _http_vsn_str + http_version = getattr(conn, "_http_vsn_str", "HTTP/?") log.debug( '%s://%s:%s "%s %s %s" %s %s', self.scheme, @@ -551,7 +553,7 @@ def _make_request( method, url, # HTTP version - conn._http_vsn_str, # type: ignore[attr-defined] + http_version, response.status, response.length_remaining, # type: ignore[attr-defined] ) diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py new file mode 100644 index 0000000000..8a3c5bebdc --- /dev/null +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import urllib3.connection + +from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection + + +def inject_into_urllib3() -> None: + # override connection classes to use emscripten specific classes + # n.b. mypy complains about the overriding of classes below + # if it isn't ignored + HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection + HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection + urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment] + urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment] diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py new file mode 100644 index 0000000000..25d7baa42f --- /dev/null +++ b/src/urllib3/contrib/emscripten/connection.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import os +import typing + +# use http.client.HTTPException for consistency with non-emscripten +from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import ResponseNotReady + +from ..._base_connection import _TYPE_BODY +from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...exceptions import TimeoutError +from ...response import BaseHTTPResponse +from ...util.connection import _TYPE_SOCKET_OPTIONS +from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from ...util.url import Url +from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request +from .request import EmscriptenRequest +from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + + +class EmscriptenHTTPConnection: + default_port: typing.ClassVar[int] = port_by_scheme["http"] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + timeout: None | (float) + + host: str + port: int + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool = False + proxy_is_verified: bool | None = None + + _response: EmscriptenResponse | None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = None, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + self.host = host + self.port = port + self.timeout = timeout if isinstance(timeout, float) else 0.0 + self.scheme = "http" + self._closed = True + self._response = None + # ignore these things because we don't + # have control over that stuff + self.proxy = None + self.proxy_config = None + self.blocksize = blocksize + self.source_address = None + self.socket_options = None + + def set_tunnel( + self, + host: str, + port: int | None = 0, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + pass + + def connect(self) -> None: + pass + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: + self._closed = False + if url.startswith("/"): + # no scheme / host / port included, make a full url + url = f"{self.scheme}://{self.host}:{self.port}" + url + request = EmscriptenRequest( + url=url, + method=method, + timeout=self.timeout if self.timeout else 0, + decode_content=decode_content, + ) + request.set_body(body) + if headers: + for k, v in headers.items(): + request.set_header(k, v) + self._response = None + try: + if not preload_content: + self._response = send_streaming_request(request) + if self._response is None: + self._response = send_request(request) + except _TimeoutError as e: + raise TimeoutError(e.message) + except _RequestError as e: + raise HTTPException(e.message) + + def getresponse(self) -> BaseHTTPResponse: + if self._response is not None: + return EmscriptenHttpResponseWrapper( + internal_response=self._response, + url=self._response.request.url, + connection=self, + ) + else: + raise ResponseNotReady() + + def close(self) -> None: + self._closed = True + self._response = None + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + return self._closed + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + return True + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + return False + + +class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): + default_port = port_by_scheme["https"] + # all this is basically ignored, as browser handles https + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + cert_file: str | None + key_file: str | None + key_password: str | None + ssl_context: typing.Any | None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None = None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None + | _TYPE_SOCKET_OPTIONS = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: typing.Any | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) + self.scheme = "https" + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data + + self.cert_reqs = None + + def set_cert( + self, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: + pass + + +# verify that this class implements BaseHTTP(s) connection correctly +if typing.TYPE_CHECKING: + _supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0) + _supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0) diff --git a/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js new file mode 100644 index 0000000000..243b86222f --- /dev/null +++ b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js @@ -0,0 +1,110 @@ +let Status = { + SUCCESS_HEADER: -1, + SUCCESS_EOF: -2, + ERROR_TIMEOUT: -3, + ERROR_EXCEPTION: -4, +}; + +let connections = {}; +let nextConnectionID = 1; +const encoder = new TextEncoder(); + +self.addEventListener("message", async function (event) { + if (event.data.close) { + let connectionID = event.data.close; + delete connections[connectionID]; + return; + } else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader, intBuffer, byteBuffer } = + connections[connectionID]; + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try { + let readResponse = await reader.read(); + + if (readResponse.done) { + // read everything - clear connection and return + delete connections[connectionID]; + Atomics.store(intBuffer, 0, Status.SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; + } + curOffset = 0; + connections[connectionID].value = readResponse.value; + value = readResponse.value; + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } + + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0); + + Atomics.store(intBuffer, 0, curLen); // store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset += curLen; + connections[connectionID].curOffset = curOffset; + + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8); + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + let headerObj = { + headers: headers, + status: response.status, + connectionID, + }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections[connectionID] = { + reader: response.body.getReader(), + intBuffer: intBuffer, + byteBuffer: byteBuffer, + value: undefined, + curOffset: 0, + }; + // set header ready + Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } +}); +self.postMessage({ inited: true }); diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py new file mode 100644 index 0000000000..ecf845d931 --- /dev/null +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -0,0 +1,413 @@ +""" +Support for streaming http requests in emscripten. + +A few caveats - + +Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. +Streaming only works if you're running pyodide in a web worker. + +Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch +operation, so it requires that you have crossOriginIsolation enabled, by serving over https +(or from localhost) with the two headers below set: + + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp + +You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in +javascript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole +request into a buffer and then returning it. it shows a warning in the javascript console in this case. + +Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once +control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. + +NB: in this code, there are a lot of javascript objects. They are named js_* +to make it clear what type of object they are. +""" +from __future__ import annotations + +import io +import json +from email.parser import Parser +from importlib.resources import files +from typing import TYPE_CHECKING, Any + +import js # type: ignore[import] +from pyodide.ffi import JsArray, JsException, JsProxy, to_js # type: ignore[import] + +if TYPE_CHECKING: + from typing_extensions import Buffer + +from .request import EmscriptenRequest +from .response import EmscriptenResponse + +""" +There are some headers that trigger unintended CORS preflight requests. +See also https://github.com/koenvo/pyodide-http/issues/22 +""" +HEADERS_TO_IGNORE = ("user-agent",) + +SUCCESS_HEADER = -1 +SUCCESS_EOF = -2 +ERROR_TIMEOUT = -3 +ERROR_EXCEPTION = -4 + +_STREAMING_WORKER_CODE = ( + files(__package__) + .joinpath("emscripten_fetch_worker.js") + .read_text(encoding="utf-8") +) + + +class _RequestError(Exception): + def __init__( + self, + message: str | None = None, + *, + request: EmscriptenRequest | None = None, + response: EmscriptenResponse | None = None, + ): + self.request = request + self.response = response + self.message = message + super().__init__(self.message) + + +class _StreamingError(_RequestError): + pass + + +class _TimeoutError(_RequestError): + pass + + +def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy: + return to_js(dict_val, dict_converter=js.Object.fromEntries) + + +class _ReadStream(io.RawIOBase): + def __init__( + self, + int_buffer: JsArray, + byte_buffer: JsArray, + timeout: float, + worker: JsProxy, + connection_id: int, + request: EmscriptenRequest, + ): + self.int_buffer = int_buffer + self.byte_buffer = byte_buffer + self.read_pos = 0 + self.read_len = 0 + self.connection_id = connection_id + self.worker = worker + self.timeout = int(1000 * timeout) if timeout > 0 else None + self.is_live = True + self._is_closed = False + self.request: EmscriptenRequest | None = request + + def __del__(self) -> None: + self.close() + + # this is compatible with _base_connection + def is_closed(self) -> bool: + return self._is_closed + + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + + def close(self) -> None: + if not self.is_closed(): + self.read_len = 0 + self.read_pos = 0 + self.int_buffer = None + self.byte_buffer = None + self._is_closed = True + self.request = None + if self.is_live: + self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) + self.is_live = False + super().close() + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def readinto(self, byte_obj: Buffer) -> int: + if not self.int_buffer: + raise _StreamingError( + "No buffer for stream in _ReadStream.readinto", + request=self.request, + response=None, + ) + if self.read_len == 0: + # wait for the worker to send something + js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT) + self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) + if ( + js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout) + == "timed-out" + ): + raise _TimeoutError + data_len = self.int_buffer[0] + if data_len > 0: + self.read_len = data_len + self.read_pos = 0 + elif data_len == ERROR_EXCEPTION: + string_len = self.int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", + request=self.request, + response=None, + ) + else: + # EOF, free the buffers and return zero + # and free the request + self.is_live = False + self.close() + return 0 + # copy from int32array to python bytes + ret_length = min(self.read_len, len(memoryview(byte_obj))) + subarray = self.byte_buffer.subarray( + self.read_pos, self.read_pos + ret_length + ).to_py() + memoryview(byte_obj)[0:ret_length] = subarray + self.read_len -= ret_length + self.read_pos += ret_length + return ret_length + + +class _StreamingFetcher: + def __init__(self) -> None: + # make web-worker and data buffer on startup + self.streaming_ready = False + + js_data_blob = js.Blob.new( + [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) + ) + + def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None: + def onMsg(e: JsProxy) -> None: + self.streaming_ready = True + js_resolve_fn(e) + + def onErr(e: JsProxy) -> None: + js_reject_fn(e) # Defensive: never happens in ci + + self.js_worker.onmessage = onMsg + self.js_worker.onerror = onErr + + js_data_url = js.URL.createObjectURL(js_data_blob) + self.js_worker = js.globalThis.Worker.new(js_data_url) + self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver) + + def send(self, request: EmscriptenRequest) -> EmscriptenResponse: + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + + body = request.body + fetch_data = {"headers": headers, "body": to_js(body), "method": request.method} + # start the request off in the worker + timeout = int(1000 * request.timeout) if request.timeout > 0 else None + js_shared_buffer = js.SharedArrayBuffer.new(1048576) + js_int_buffer = js.Int32Array.new(js_shared_buffer) + js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8) + + js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT) + js.Atomics.notify(js_int_buffer, 0) + js_absolute_url = js.URL.new(request.url, js.location).href + self.js_worker.postMessage( + _obj_from_dict( + { + "buffer": js_shared_buffer, + "url": js_absolute_url, + "fetchParams": fetch_data, + } + ) + ) + # wait for the worker to send something + js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout) + if js_int_buffer[0] == ERROR_TIMEOUT: + raise _TimeoutError( + "Timeout connecting to streaming request", + request=request, + response=None, + ) + elif js_int_buffer[0] == SUCCESS_HEADER: + # got response + # header length is in second int of intBuffer + string_len = js_int_buffer[1] + # decode the rest to a JSON string + js_decoder = js.TextDecoder.new() + # this does a copy (the slice) because decode can't work on shared array + # for some silly reason + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + # get it as an object + response_obj = json.loads(json_str) + return EmscriptenResponse( + request=request, + status_code=response_obj["status"], + headers=response_obj["headers"], + body=_ReadStream( + js_int_buffer, + js_byte_buffer, + request.timeout, + self.js_worker, + response_obj["connectionID"], + request, + ), + ) + elif js_int_buffer[0] == ERROR_EXCEPTION: + string_len = js_int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", request=request, response=None + ) + else: + raise _StreamingError( + f"Unknown status from worker in fetch: {js_int_buffer[0]}", + request=request, + response=None, + ) + + +# check if we are in a worker or not +def is_in_browser_main_thread() -> bool: + return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + + +def is_cross_origin_isolated() -> bool: + return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated + + +def is_in_node() -> bool: + return ( + hasattr(js, "process") + and hasattr(js.process, "release") + and hasattr(js.process.release, "name") + and js.process.release.name == "node" + ) + + +def is_worker_available() -> bool: + return hasattr(js, "Worker") and hasattr(js, "Blob") + + +_fetcher: _StreamingFetcher | None = None + +if is_worker_available() and ( + (is_cross_origin_isolated() and not is_in_browser_main_thread()) + and (not is_in_node()) +): + _fetcher = _StreamingFetcher() +else: + _fetcher = None + + +def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None: + if _fetcher and streaming_ready(): + return _fetcher.send(request) + else: + _show_streaming_warning() + return None + + +_SHOWN_TIMEOUT_WARNING = False + + +def _show_timeout_warning() -> None: + global _SHOWN_TIMEOUT_WARNING + if not _SHOWN_TIMEOUT_WARNING: + _SHOWN_TIMEOUT_WARNING = True + message = "Warning: Timeout is not available on main browser thread" + js.console.warn(message) + + +_SHOWN_STREAMING_WARNING = False + + +def _show_streaming_warning() -> None: + global _SHOWN_STREAMING_WARNING + if not _SHOWN_STREAMING_WARNING: + _SHOWN_STREAMING_WARNING = True + message = "Can't stream HTTP requests because: \n" + if not is_cross_origin_isolated(): + message += " Page is not cross-origin isolated\n" + if is_in_browser_main_thread(): + message += " Python is running in main browser thread\n" + if not is_worker_available(): + message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in + if streaming_ready() is False: + message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch +is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" + from js import console + + console.warn(message) + + +def send_request(request: EmscriptenRequest) -> EmscriptenResponse: + try: + js_xhr = js.XMLHttpRequest.new() + + if not is_in_browser_main_thread(): + js_xhr.responseType = "arraybuffer" + if request.timeout: + js_xhr.timeout = int(request.timeout * 1000) + else: + js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + if request.timeout: + # timeout isn't available on the main thread - show a warning in console + # if it is set + _show_timeout_warning() + + js_xhr.open(request.method, request.url, False) + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + js_xhr.setRequestHeader(name, value) + + js_xhr.send(to_js(request.body)) + + headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders())) + + if not is_in_browser_main_thread(): + body = js_xhr.response.to_py().tobytes() + else: + body = js_xhr.response.encode("ISO-8859-15") + return EmscriptenResponse( + status_code=js_xhr.status, headers=headers, body=body, request=request + ) + except JsException as err: + if err.name == "TimeoutError": + raise _TimeoutError(err.message, request=request) + elif err.name == "NetworkError": + raise _RequestError(err.message, request=request) + else: + # general http error + raise _RequestError(err.message, request=request) + + +def streaming_ready() -> bool | None: + if _fetcher: + return _fetcher.streaming_ready + else: + return None # no fetcher, return None to signify that + + +async def wait_for_streaming_ready() -> bool: + if _fetcher: + await _fetcher.js_worker_ready_promise + return True + else: + return False diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py new file mode 100644 index 0000000000..e692e692bd --- /dev/null +++ b/src/urllib3/contrib/emscripten/request.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..._base_connection import _TYPE_BODY + + +@dataclass +class EmscriptenRequest: + method: str + url: str + params: dict[str, str] | None = None + body: _TYPE_BODY | None = None + headers: dict[str, str] = field(default_factory=dict) + timeout: float = 0 + decode_content: bool = True + + def set_header(self, name: str, value: str) -> None: + self.headers[name.capitalize()] = value + + def set_body(self, body: _TYPE_BODY | None) -> None: + self.body = body diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py new file mode 100644 index 0000000000..303b4ee011 --- /dev/null +++ b/src/urllib3/contrib/emscripten/response.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import json as _json +import logging +import typing +from contextlib import contextmanager +from dataclasses import dataclass +from http.client import HTTPException as HTTPException +from io import BytesIO, IOBase + +from ...exceptions import InvalidHeader, TimeoutError +from ...response import BaseHTTPResponse +from ...util.retry import Retry +from .request import EmscriptenRequest + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + +log = logging.getLogger(__name__) + + +@dataclass +class EmscriptenResponse: + status_code: int + headers: dict[str, str] + body: IOBase | bytes + request: EmscriptenRequest + + +class EmscriptenHttpResponseWrapper(BaseHTTPResponse): + def __init__( + self, + internal_response: EmscriptenResponse, + url: str | None = None, + connection: BaseHTTPConnection | BaseHTTPSConnection | None = None, + ): + self._pool = None # set by pool class + self._body = None + self._response = internal_response + self._url = url + self._connection = connection + self._closed = False + super().__init__( + headers=internal_response.headers, + status=internal_response.status_code, + request_url=url, + version=0, + reason="", + decode_content=True, + ) + self.length_remaining = self._init_length(self._response.request.method) + self.length_is_certain = False + + @property + def url(self) -> str | None: + return self._url + + @url.setter + def url(self, url: str | None) -> None: + self._url = url + + @property + def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None: + return self._connection + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + self._retries = retries + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes, None, None]: + """ + A generator wrapper for the read() method. A call will block until + ``amt`` bytes have been read from the connection or until the + connection is closed. + + :param amt: + How much of the content to read. The generator will return up to + much data per iteration, but may return less. This is particularly + likely when using compressed data. However, the empty string will + never be returned. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + while True: + data = self.read(amt=amt, decode_content=decode_content) + + if data: + yield data + else: + break + + def _init_length(self, request_method: str | None) -> int | None: + length: int | None + content_length: str | None = self.headers.get("content-length") + + if content_length is not None: + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = {int(val) for val in content_length.split(",")} + if len(lengths) > 1: + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % content_length + ) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + else: # if content_length is None + length = None + + # Check for responses that shouldn't include a body + if ( + self.status in (204, 304) + or 100 <= self.status < 200 + or request_method == "HEAD" + ): + length = 0 + + return length + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, # ignored because browser decodes always + cache_content: bool = False, + ) -> bytes: + if ( + self._closed + or self._response is None + or (isinstance(self._response.body, IOBase) and self._response.body.closed) + ): + return b"" + + with self._error_catcher(): + # body has been preloaded as a string by XmlHttpRequest + if not isinstance(self._response.body, IOBase): + self.length_remaining = len(self._response.body) + self.length_is_certain = True + # wrap body in IOStream + self._response.body = BytesIO(self._response.body) + if amt is not None: + # don't cache partial content + cache_content = False + data = self._response.body.read(amt) + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if (self.length_is_certain and self.length_remaining == 0) or len( + data + ) < amt: + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + else: # read all we can (and cache it) + data = self._response.body.read() + if cache_content: + self._body = data + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if len(data) == 0 or ( + self.length_is_certain and self.length_remaining == 0 + ): + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Generator[bytes, None, None]: + # chunked is handled by browser + while True: + bytes = self.read(amt, decode_content) + if not bytes: + break + yield bytes + + def release_conn(self) -> None: + if not self._pool or not self._connection: + return None + + self._pool._put_conn(self._connection) + self._connection = None + + def drain_conn(self) -> None: + self.close() + + @property + def data(self) -> bytes: + if self._body: + return self._body + else: + return self.read(cache_content=True) + + def json(self) -> typing.Any: + """ + Parses the body of the HTTP response as JSON. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder. + + This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`. + + Read more :ref:`here `. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + + def close(self) -> None: + if not self._closed: + if isinstance(self._response.body, IOBase): + self._response.body.close() + if self._connection: + self._connection.close() + self._connection = None + self._closed = True + + @contextmanager + def _error_catcher(self) -> typing.Generator[None, None, None]: + """ + Catch Emscripten specific exceptions thrown by fetch.py, + instead re-raising urllib3 variants, so that low-level exceptions + are not leaked in the high-level api. + + On exit, release the connection back to the pool. + """ + from .fetch import _RequestError, _TimeoutError # avoid circular import + + clean_exit = False + + try: + yield + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + except _TimeoutError as e: + raise TimeoutError(str(e)) + except _RequestError as e: + raise HTTPException(str(e)) + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now + if ( + isinstance(self._response.body, IOBase) + and not self._response.body.closed + ): + self._response.body.close() + # release the connection back to the pool + self.release_conn() + else: + # If we have read everything from the response stream, + # return the connection back to the pool. + if ( + isinstance(self._response.body, IOBase) + and self._response.body.closed + ): + self.release_conn() diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 4ed5a90b8d..01220c359e 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -14,6 +14,9 @@ from http.client import HTTPResponse as _HttplibHTTPResponse from socket import timeout as SocketTimeout +if typing.TYPE_CHECKING: + from ._base_connection import BaseHTTPConnection + try: try: import brotlicffi as brotli # type: ignore[import] @@ -379,7 +382,7 @@ def url(self, url: str | None) -> None: raise NotImplementedError() @property - def connection(self) -> HTTPConnection | None: + def connection(self) -> BaseHTTPConnection | None: raise NotImplementedError() @property diff --git a/test/contrib/emscripten/__init__.py b/test/contrib/emscripten/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py new file mode 100644 index 0000000000..abaa3bee6e --- /dev/null +++ b/test/contrib/emscripten/conftest.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import asyncio +import contextlib +import mimetypes +import os +import random +import textwrap +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Generator +from urllib.parse import urlsplit + +import pytest +from tornado import web +from tornado.httputil import HTTPServerRequest + +from dummyserver.handlers import Response, TestingApp +from dummyserver.testcase import HTTPDummyProxyTestCase +from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread + +_coverage_count = 0 + + +def _get_coverage_filename(prefix: str) -> str: + global _coverage_count + _coverage_count += 1 + rand_part = "".join([random.choice("1234567890") for x in range(20)]) + return prefix + rand_part + f".{_coverage_count}" + + +@pytest.fixture(scope="module") +def testserver_http( + request: pytest.FixtureRequest, +) -> Generator[PyodideServerInfo, None, None]: + dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) + server = PyodideDummyServerTestCase + server.setup_class(str(dist_dir)) + print( + f"Server:{server.http_host}:{server.http_port},https({server.https_port}) [{dist_dir}]" + ) + yield PyodideServerInfo( + http_host=server.http_host, + http_port=server.http_port, + https_port=server.https_port, + ) + print("Server teardown") + server.teardown_class() + + +@pytest.fixture() +def selenium_coverage(selenium: Any) -> Generator[Any, None, None]: + def _install_coverage(self: Any) -> None: + self.run_js( + """ + await pyodide.loadPackage("coverage") + await pyodide.runPythonAsync(`import coverage +_coverage= coverage.Coverage(source_pkgs=['urllib3']) +_coverage.start() + ` + )""" + ) + + setattr( + selenium, + "_install_coverage", + _install_coverage.__get__(selenium, selenium.__class__), + ) + selenium._install_coverage() + yield selenium + # on teardown, save _coverage output + coverage_out_binary = bytes( + selenium.run_js( + """ +return await pyodide.runPythonAsync(` +_coverage.stop() +_coverage.save() +_coverage_datafile = open(".coverage","rb") +_coverage_outdata = _coverage_datafile.read() +# avoid polluting main namespace too much +import js as _coverage_js +# convert to js Array (as default conversion is TypedArray which does +# bad things in firefox) +_coverage_js.Array.from_(_coverage_outdata) +`) + """ + ) + ) + with open(f"{_get_coverage_filename('.coverage.emscripten.')}", "wb") as outfile: + outfile.write(coverage_out_binary) + + +class ServerRunnerInfo: + def __init__(self, host: str, port: int, selenium: Any) -> None: + self.host = host + self.port = port + self.selenium = selenium + + def run_webworker(self, code: str) -> Any: + if isinstance(code, str) and code.startswith("\n"): + # we have a multiline string, fix indentation + code = textwrap.dedent(code) + # add coverage collection to this code + code = ( + textwrap.dedent( + """ + import coverage + _coverage= coverage.Coverage(source_pkgs=['urllib3']) + _coverage.start() + """ + ) + + code + ) + code += textwrap.dedent( + """ + _coverage.stop() + _coverage.save() + _coverage_datafile = open(".coverage","rb") + _coverage_outdata = _coverage_datafile.read() + # avoid polluting main namespace too much + import js as _coverage_js + # convert to js Array (as default conversion is TypedArray which does + # bad things in firefox) + _coverage_js.Array.from_(_coverage_outdata) + """ + ) + + coverage_out_binary = bytes( + self.selenium.run_js( + f""" + let worker = new Worker('https://{self.host}:{self.port}/pyodide/webworker_dev.js'); + let p = new Promise((res, rej) => {{ + worker.onmessageerror = e => rej(e); + worker.onerror = e => rej(e); + worker.onmessage = e => {{ + if (e.data.results) {{ + res(e.data.results); + }} else {{ + rej(e.data.error); + }} + }}; + worker.postMessage({{ python: {repr(code)} }}); + }}); + return await p; + """, + pyodide_checks=False, + ) + ) + with open( + f"{_get_coverage_filename('.coverage.emscripten.worker.')}", "wb" + ) as outfile: + outfile.write(coverage_out_binary) + + +# run pyodide on our test server instead of on the default +# pytest-pyodide one - this makes it so that +# we are at the same origin as web requests to server_host +@pytest.fixture() +def run_from_server( + selenium_coverage: Any, testserver_http: PyodideServerInfo +) -> Generator[ServerRunnerInfo, None, None]: + addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" + selenium_coverage.goto(addr) + selenium_coverage.javascript_setup() + selenium_coverage.load_pyodide() + selenium_coverage.initialize_pyodide() + selenium_coverage.save_state() + selenium_coverage.restore_state() + # install the wheel, which is served at /wheel/* + selenium_coverage.run_js( + """ +await pyodide.loadPackage('/wheel/dist.whl') +""" + ) + selenium_coverage._install_coverage() + yield ServerRunnerInfo( + testserver_http.http_host, testserver_http.https_port, selenium_coverage + ) + + +class PyodideTestingApp(TestingApp): + pyodide_dist_dir: str = "" + + def set_default_headers(self) -> None: + """Allow cross-origin requests for emscripten""" + self.set_header("Access-Control-Allow-Origin", "*") + self.set_header("Cross-Origin-Opener-Policy", "same-origin") + self.set_header("Cross-Origin-Embedder-Policy", "require-corp") + self.add_header("Feature-Policy", "sync-xhr *;") + self.add_header("Access-Control-Allow-Headers", "*") + + def slow(self, _req: HTTPServerRequest) -> Response: + import time + + time.sleep(10) + return Response("TEN SECONDS LATER") + + def bigfile(self, req: HTTPServerRequest) -> Response: + # great big text file, should force streaming + # if supported + bigdata = 1048576 * b"WOOO YAY BOOYAKAH" + return Response(bigdata) + + def mediumfile(self, req: HTTPServerRequest) -> Response: + # quite big file + bigdata = 1024 * b"WOOO YAY BOOYAKAH" + return Response(bigdata) + + def pyodide(self, req: HTTPServerRequest) -> Response: + path = req.path[:] + if not path.startswith("/"): + path = urlsplit(path).path + path_split = path.split("/") + file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path_split[2:]) + if file_path.exists(): + mime_type, encoding = mimetypes.guess_type(file_path) + if not mime_type: + mime_type = "text/plain" + self.set_header("Content-Type", mime_type) + return Response( + body=file_path.read_bytes(), + headers=[("Access-Control-Allow-Origin", "*")], + ) + else: + return Response(status="404 NOT FOUND") + + def wheel(self, _req: HTTPServerRequest) -> Response: + # serve our wheel + wheel_folder = Path(__file__).parent.parent.parent.parent / "dist" + wheels = list(wheel_folder.glob("*.whl")) + if len(wheels) > 0: + resp = Response( + body=wheels[0].read_bytes(), + headers=[ + ("Content-Disposition", f"inline; filename='{wheels[0].name}'") + ], + ) + return resp + else: + return Response(status="404 NOT FOUND") + + +class PyodideDummyServerTestCase(HTTPDummyProxyTestCase): + @classmethod + def setup_class(cls, pyodide_dist_dir: str) -> None: # type:ignore[override] + PyodideTestingApp.pyodide_dist_dir = pyodide_dist_dir + with contextlib.ExitStack() as stack: + io_loop = stack.enter_context(run_tornado_loop_in_thread()) + + async def run_app() -> None: + app = web.Application([(r".*", PyodideTestingApp)]) + cls.http_server, cls.http_port = run_tornado_app( + app, None, "http", cls.http_host + ) + + app = web.Application([(r".*", PyodideTestingApp)]) + cls.https_server, cls.https_port = run_tornado_app( + app, cls.https_certs, "https", cls.http_host + ) + + asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] + cls._stack = stack.pop_all() + + +@dataclass +class PyodideServerInfo: + http_port: int + https_port: int + http_host: str diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py new file mode 100644 index 0000000000..29b8cf08a2 --- /dev/null +++ b/test/contrib/emscripten/test_emscripten.py @@ -0,0 +1,948 @@ +from __future__ import annotations + +import sys +import typing + +import pytest + +from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE + +from ...port_helpers import find_unused_port + +if sys.version_info < (3, 11): + # pyodide only works on 3.11+ + pytest.skip(allow_module_level=True) + +# only run these tests if pytest_pyodide is installed +# so we don't break non-emscripten pytest running +pytest_pyodide = pytest.importorskip("pytest_pyodide") + +from pytest_pyodide import run_in_pyodide # type: ignore[import] # noqa: E402 +from pytest_pyodide.decorator import ( # type: ignore[import] # noqa: E402 + copy_files_to_pyodide, +) + +from .conftest import PyodideServerInfo, ServerRunnerInfo # noqa: E402 + +# make our ssl certificates work in chrome +pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") + + +# copy our wheel file to pyodide and install it +def install_urllib3_wheel() -> ( + typing.Callable[ + [typing.Callable[..., typing.Any]], typing.Callable[..., typing.Any] + ] +): + return copy_files_to_pyodide( # type: ignore[no-any-return] + file_list=[("dist/*.whl", "/tmp")], install_wheels=True + ) + + +@install_urllib3_wheel() +def test_index( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + url = f"http://{host}:{port}/" + conn.request("GET", url) + response = conn.getresponse() + # check methods of response + assert isinstance(response, BaseHTTPResponse) + assert response.url == url + response.url = "http://woo" + assert response.url == "http://woo" + assert response.connection == conn + assert response.retries is None + data1 = response.data + decoded1 = data1.decode("utf-8") + data2 = response.data # check that getting data twice works + decoded2 = data2.decode("utf-8") + assert decoded1 == decoded2 == "Dummy server!" + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_pool_requests( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, https_port: int) -> None: # type: ignore[no-untyped-def] + # first with PoolManager + import urllib3 + + http = urllib3.PoolManager() + resp = http.request("GET", f"http://{host}:{port}/") + assert resp.data.decode("utf-8") == "Dummy server!" + + resp2 = http.request("GET", f"http://{host}:{port}/index") + assert resp2.data.decode("utf-8") == "Dummy server!" + + # should all have come from one pool + assert len(http.pools) == 1 + + resp3 = http.request("GET", f"https://{host}:{https_port}/") + assert resp2.data.decode("utf-8") == "Dummy server!" + + # one http pool + one https pool + assert len(http.pools) == 2 + + # now with ConnectionPool + # because block == True, this will fail if the connection isn't + # returned to the pool correctly after the first request + pool = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + resp3 = pool.urlopen("GET", "/index") + assert resp3.data.decode("utf-8") == "Dummy server!" + + resp4 = pool.urlopen("GET", "/") + assert resp4.data.decode("utf-8") == "Dummy server!" + + # now with manual release of connection + # first - connection should be released once all + # data is read + pool2 = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + + resp5 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool is not None + # at this point, the connection should not be in the pool + assert pool2.pool.qsize() == 0 + assert resp5.data.decode("utf-8") == "Dummy server!" + # now we've read all the data, connection should be back to the pool + assert pool2.pool.qsize() == 1 + resp6 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool.qsize() == 0 + # force it back to the pool + resp6.release_conn() + assert pool2.pool.qsize() == 1 + read_str = resp6.read() + # for consistency with urllib3, this still returns the correct data even though + # we are in theory not using the connection any more + assert read_str.decode("utf-8") == "Dummy server!" + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + ) + + +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException +@install_urllib3_wheel() +def test_wrong_protocol( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("GET", f"http://{host}:{port}/") + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException +@install_urllib3_wheel() +def test_bad_method( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("TRACE", f"http://{host}:{port}/") + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +# no connection - should raise +@install_urllib3_wheel() +def test_no_response( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("GET", f"http://{host}:{port}/") + _ = conn.getresponse() + + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) + + +@install_urllib3_wheel() +def test_404(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/status?status=404 NOT FOUND") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + assert response.status == 404 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +# setting timeout should show a warning to js console +# if we're on the ui thread, because XMLHttpRequest doesn't +# support timeout in async mode if globalThis == Window +@install_urllib3_wheel() +def test_timeout_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide() # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js # type: ignore[import] + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + + old_log = js.console.warn + log_msgs = [] + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + + conn = HTTPConnection(host, port, timeout=1.0) + conn.request("GET", f"http://{host}:{port}/") + conn.getresponse() + js.console.warn = old_log + # should have shown timeout warning exactly once by now + assert len([x for x in log_msgs if x.find("Warning: Timeout") != -1]) == 1 + assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_timeout_in_worker_non_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + result=-1 + try: + conn.request("GET","/slow") + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except BaseException as e: + result=-2 + assert result == 1 +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_timeout_in_worker_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + result=-1 + try: + conn.request("GET","/slow",preload_content=False) + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except BaseException as e: + result=-2 + assert result == 1 +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_index_https( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@install_urllib3_wheel() +def test_non_streaming_no_fallback_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/", preload_content=True) + response = conn.getresponse() + js.console.warn = old_log + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + # no console warnings because we didn't ask it to stream the response + # check no log messages + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 0 + ) + assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@install_urllib3_wheel() +def test_streaming_fallback_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + # monkeypatch is_cross_origin_isolated so that it warns about that + # even if we're serving it so it is fine + urllib3.contrib.emscripten.fetch.is_cross_origin_isolated = lambda: False + + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/", preload_content=False) + response = conn.getresponse() + js.console.warn = old_log + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + # check that it has warned about falling back to non-streaming fetch exactly once + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 1 + ) + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@install_urllib3_wheel() +def test_specific_method( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3 import HTTPSConnectionPool + + with HTTPSConnectionPool(host, port) as pool: + path = "/specific_method?method=POST" + response = pool.request("POST", path) + assert response.status == 200 + + response = pool.request("PUT", path) + assert response.status == 400 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@install_urllib3_wheel() +def test_streaming_download( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # test streaming download, which must be in a webworker + # as you can't do it on main thread + + # this should return the 17mb big file, and + # should not log any warning about falling back + bigfile_url = ( + f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + ) + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{bigfile_url}",preload_content=False) + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False + data=response.data.decode('utf-8') + assert len(data) == 17825792 +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_close( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # test streaming download, which must be in a webworker + # as you can't do it on main thread + + # this should return the 17mb big file, and + # should not log any warning about falling back + url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + from io import RawIOBase + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{url}",preload_content=False) + response = conn.getresponse() + # check body is a RawIOBase stream and isn't seekable, writeable + body_internal = response._response.body + assert(isinstance(body_internal,RawIOBase)) + assert(body_internal.writable() is False) + assert(body_internal.seekable() is False) + assert(body_internal.readable() is True) + response.drain_conn() + x=response.read() + assert(not x) + response.close() + conn.close() + # try and make destructor be covered + # by killing everything + del response + del body_internal + del conn +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_bad_url( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"hsffsdfttp://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import http.client + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(http.client.HTTPException): + conn.request("GET", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_bad_method( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import http.client + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(http.client.HTTPException): + # TRACE method should throw SecurityError in Javascript + conn.request("TRACE", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_notready_warning( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # test streaming download but don't wait for + # worker to be ready - should fallback to non-streaming + # and log a warning + file_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import js + import urllib3 + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + log_msgs=[] + old_log=js.console.warn + def capture_log(*args): + log_msgs.append(str(args)) + old_log(*args) + js.console.warn=capture_log + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{file_url}",preload_content=False) + js.console.warn=old_log + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data=response.data.decode('utf-8') + assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True + """ + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_post_receive_json( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import json + + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + json_data = { + "Bears": "like", + "to": {"eat": "buns", "with": ["marmalade", "and custard"]}, + } + conn = HTTPConnection(host, port) + conn.request( + "POST", + f"http://{host}:{port}/echo_json", + body=json.dumps(json_data).encode("utf-8"), + ) + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data = response.json() + assert data == json_data + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_upload( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3 import HTTPConnectionPool + + data = "I'm in ur multipart form-data, hazing a cheezburgr" + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { + "upload_param": "filefield", + "upload_filename": "lolcat.txt", + "filefield": ("lolcat.txt", data), + } + fields["upload_size"] = str(len(data)) + with HTTPConnectionPool(host, port) as pool: + r = pool.request("POST", "/upload", fields=fields) + assert r.status == 200 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_streaming_not_ready_in_browser( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + # streaming ready should always be false + # if we're in the main browser thread + selenium_coverage.run_async( + """ + import urllib3.contrib.emscripten.fetch + result=await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + assert(result is False) + assert(urllib3.contrib.emscripten.fetch.streaming_ready() is None ) + """ + ) + + +@install_urllib3_wheel() +def test_requests_with_micropip( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + # this can't be @run_in_pyodide because of the async code + selenium_coverage.run_async( + f""" + import micropip + await micropip.install("requests") + import requests + import json + r = requests.get("http://{testserver_http.http_host}:{testserver_http.http_port}/") + assert(r.status_code == 200) + assert(r.text == "Dummy server!") + json_data={{"woo":"yay"}} + # try posting some json with requests + r = requests.post("http://{testserver_http.http_host}:{testserver_http.http_port}/echo_json",json=json_data) + import js + assert(r.json() == json_data) + """ + ) + + +@install_urllib3_wheel() +def test_open_close( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from http.client import ResponseNotReady + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + # initially connection should be closed + assert conn.is_closed is True + # connection should have no response + with pytest.raises(ResponseNotReady): + response = conn.getresponse() + # now make the response + conn.request("GET", f"http://{host}:{port}/") + # we never connect to proxy (or if we do, browser handles it) + assert conn.has_connected_to_proxy is False + # now connection should be open + assert conn.is_closed is False + # and should have a response + response = conn.getresponse() + assert response is not None + conn.close() + # now it is closed + assert conn.is_closed is True + # closed connection shouldn't have any response + with pytest.raises(ResponseNotReady): + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +# check that various ways that the worker may be broken +# throw exceptions nicely, by deliberately breaking things +# this is for coverage +@install_urllib3_wheel() +def test_break_worker_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import pytest + import urllib3.contrib.emscripten.fetch + import js + import http.client + + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + # make the fetch worker return a bad response by: + # 1) Clearing the int buffer + # in the receive stream + with pytest.raises(http.client.HTTPException): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + body_internal = response._response.body + assert(body_internal.int_buffer!=None) + body_internal.int_buffer=None + data=response.read() + # 2) Monkeypatch postMessage so that it just sets an + # exception status + old_pm= body_internal.worker.postMessage + with pytest.raises(http.client.HTTPException): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages set an exception + body_internal = response._response.body + def set_exception(*args): + body_internal.worker.postMessage = old_pm + body_internal.int_buffer[1]=4 + body_internal.byte_buffer[0]=ord("W") + body_internal.byte_buffer[1]=ord("O") + body_internal.byte_buffer[2]=ord("O") + body_internal.byte_buffer[3]=ord("!") + body_internal.byte_buffer[4]=0 + js.Atomics.store(body_internal.int_buffer, 0, -4) + js.Atomics.notify(body_internal.int_buffer,0) + body_internal.worker.postMessage = set_exception + data=response.read() + # monkeypatch so it returns an unknown value for the magic number on initial fetch call + with pytest.raises(http.client.HTTPException): + # make posted messages set an exception + worker=urllib3.contrib.emscripten.fetch._fetcher.js_worker + def set_exception(self,*args): + array=js.Int32Array.new(args[0].buffer) + array[0]=-1234 + worker.postMessage=set_exception.__get__(worker,worker.__class__) + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + data=response.read() + urllib3.contrib.emscripten.fetch._fetcher.js_worker.postMessage=old_pm + # 3) Stopping the worker receiving any messages which should cause a timeout error + # in the receive stream + with pytest.raises(TimeoutError): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages not be send + body_internal = response._response.body + def ignore_message(*args): + pass + old_pm= body_internal.worker.postMessage + body_internal.worker.postMessage = ignore_message + data=response.read() + body_internal.worker.postMessage = old_pm + +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_response_init_length( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + # head shouldn't have length + length = response._init_length("HEAD") + assert length == 0 + # multiple inconsistent lengths - should raise invalid header + with pytest.raises(urllib3.exceptions.InvalidHeader): + response.headers["Content-Length"] = "4,5,6" + length = response._init_length("GET") + # non-numeric length - should return None + response.headers["Content-Length"] = "anna" + length = response._init_length("GET") + assert length is None + # numeric length - should return it + response.headers["Content-Length"] = "54" + length = response._init_length("GET") + assert length == 54 + # negative length - should return None + response.headers["Content-Length"] = "-12" + length = response._init_length("GET") + assert length is None + # none -> None + del response.headers["Content-Length"] + length = response._init_length("GET") + assert length is None + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_response_close_connection( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + response.close() + assert conn.is_closed + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_read_chunked( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/mediumfile", preload_content=False) + response = conn.getresponse() + count = 0 + for x in response.read_chunked(512): + count += 1 + if count < 10: + assert len(x) == 512 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_retries( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3 + + pool = urllib3.HTTPConnectionPool( + host, + port, + maxsize=1, + block=True, + retries=urllib3.util.Retry(connect=5, read=5, redirect=5), + ) + + # monkeypatch connection class to count calls + old_request = urllib3.connection.HTTPConnection.request + count = 0 + + def count_calls(self, *args, **argv): # type: ignore[no-untyped-def] + nonlocal count + count += 1 + return old_request(self, *args, **argv) + + urllib3.connection.HTTPConnection.request = count_calls # type: ignore[method-assign] + with pytest.raises(urllib3.exceptions.MaxRetryError): + pool.urlopen("GET", "/") + # this should fail, but should have tried 6 times total + assert count == 6 + + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) From 14015dfdded46ef65a7e0ba7e55c449508cb25f5 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 26 Nov 2023 07:43:20 +0400 Subject: [PATCH 041/131] Migrate test_connection.py to Hypercorn --- test/with_dummyserver/test_connection.py | 74 +++++++++--------------- 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/test/with_dummyserver/test_connection.py b/test/with_dummyserver/test_connection.py index 2442c8ad2d..0a9f738328 100644 --- a/test/with_dummyserver/test_connection.py +++ b/test/with_dummyserver/test_connection.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import sys import typing from http.client import ResponseNotReady @@ -7,7 +8,7 @@ import pytest -from dummyserver.testcase import HTTPDummyServerTestCase as server +from dummyserver.testcase import HypercornDummyServerTestCase as server from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse @@ -23,16 +24,10 @@ def pool() -> typing.Generator[HTTPConnectionPool, None, None]: def test_returns_urllib3_HTTPResponse(pool: HTTPConnectionPool) -> None: - conn = pool._get_conn() - - method = "GET" - path = "/" - - conn.request(method, path) - - response = conn.getresponse() - - assert isinstance(response, HTTPResponse) + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() + assert isinstance(response, HTTPResponse) @pytest.mark.skipif(not hasattr(sys, "audit"), reason="requires python 3.8+") @@ -49,52 +44,37 @@ def test_audit_event(audit_mock: mock.Mock, pool: HTTPConnectionPool) -> None: def test_does_not_release_conn(pool: HTTPConnectionPool) -> None: - conn = pool._get_conn() - - method = "GET" - path = "/" - - conn.request(method, path) + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() - response = conn.getresponse() - - response.release_conn() - assert pool.pool.qsize() == 0 # type: ignore[union-attr] + response.release_conn() + assert pool.pool.qsize() == 0 # type: ignore[union-attr] def test_releases_conn(pool: HTTPConnectionPool) -> None: - conn = pool._get_conn() - assert conn is not None - - method = "GET" - path = "/" - - conn.request(method, path) + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() - response = conn.getresponse() - # If these variables are set by the pool - # then the response can release the connection - # back into the pool. - response._pool = pool # type: ignore[attr-defined] - response._connection = conn # type: ignore[attr-defined] + # If these variables are set by the pool + # then the response can release the connection + # back into the pool. + response._pool = pool # type: ignore[attr-defined] + response._connection = conn # type: ignore[attr-defined] - response.release_conn() - assert pool.pool.qsize() == 1 # type: ignore[union-attr] + response.release_conn() + assert pool.pool.qsize() == 1 # type: ignore[union-attr] def test_double_getresponse(pool: HTTPConnectionPool) -> None: - conn = pool._get_conn() - - method = "GET" - path = "/" - - conn.request(method, path) - - _ = conn.getresponse() + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + _ = conn.getresponse() - # Calling getrepsonse() twice should cause an error - with pytest.raises(ResponseNotReady): - conn.getresponse() + # Calling getrepsonse() twice should cause an error + with pytest.raises(ResponseNotReady): + conn.getresponse() def test_connection_state_properties(pool: HTTPConnectionPool) -> None: From 2b8d9a819f2f100348ed8d06079f9988b3ddf954 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 27 Nov 2023 00:05:09 +0400 Subject: [PATCH 042/131] Migrate final connection pool tests to Hypercorn --- dummyserver/app.py | 6 +++--- test/with_dummyserver/test_connectionpool.py | 14 +++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index 452627ef3e..f3a10c18f4 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -101,7 +101,7 @@ async def keepalive() -> ResponseTypes: return await make_response("Keeping alive", 200, headers) -@hypercorn_app.route("/echo", methods=["GET", "POST"]) +@hypercorn_app.route("/echo", methods=["GET", "POST", "PUT"]) async def echo() -> ResponseTypes: "Echo back the params" if request.method == "GET": @@ -192,7 +192,7 @@ async def encodingrequest() -> ResponseTypes: return await make_response(data, 200, headers) -@hypercorn_app.route("/redirect", methods=["GET", "POST"]) +@hypercorn_app.route("/redirect", methods=["GET", "POST", "PUT"]) async def redirect() -> ResponseTypes: "Perform a redirect to ``target``" values = await request.values @@ -248,7 +248,7 @@ async def source_address() -> ResponseTypes: return await make_response(request.remote_addr) -@hypercorn_app.route("/successful_retry") +@hypercorn_app.route("/successful_retry", methods=["GET", "PUT"]) async def successful_retry() -> ResponseTypes: """First return an error and then success diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index e7a14557a8..64123c9349 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -12,11 +12,7 @@ import pytest -from dummyserver.testcase import ( - HTTPDummyServerTestCase, - HypercornDummyServerTestCase, - SocketDummyServerTestCase, -) +from dummyserver.testcase import HypercornDummyServerTestCase, SocketDummyServerTestCase from dummyserver.tornadoserver import NoIPv6Warning from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict @@ -1327,7 +1323,7 @@ def test_redirect_after(self) -> None: assert delta < 1 -class TestFileBodiesOnRetryOrRedirect(HTTPDummyServerTestCase): +class TestFileBodiesOnRetryOrRedirect(HypercornDummyServerTestCase): def test_retries_put_filehandle(self) -> None: """HTTP PUT retry with a file-like object should not timeout""" with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: @@ -1353,7 +1349,7 @@ def test_retries_put_filehandle(self) -> None: def test_redirect_put_file(self) -> None: """PUT with file object should work with a redirection response""" - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: retry = Retry(total=3, status_forcelist=[418]) # httplib reads in 8k chunks; use a larger content length content_length = 65535 @@ -1395,7 +1391,7 @@ def tell(self) -> typing.NoReturn: pool.urlopen("PUT", url, headers=headers, body=body) -class TestRetryPoolSize(HTTPDummyServerTestCase): +class TestRetryPoolSize(HypercornDummyServerTestCase): def test_pool_size_retry(self) -> None: retries = Retry(total=1, raise_on_status=False, status_forcelist=[404]) with HTTPConnectionPool( @@ -1405,7 +1401,7 @@ def test_pool_size_retry(self) -> None: assert pool.num_connections == 1 -class TestRedirectPoolSize(HTTPDummyServerTestCase): +class TestRedirectPoolSize(HypercornDummyServerTestCase): def test_pool_size_redirect(self) -> None: retries = Retry( total=1, raise_on_status=False, status_forcelist=[404], redirect=True From e3e02f8c68f953af9c58ad990c8cc3af9032140f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 27 Nov 2023 08:00:37 +0400 Subject: [PATCH 043/131] Migrate TestIPv6PoolManager to Hypercorn --- dummyserver/testcase.py | 5 +++++ test/with_dummyserver/test_poolmanager.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 27fd155d78..d4c2714e73 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -335,6 +335,11 @@ class HTTPSHypercornDummyServerTestCase(HypercornDummyServerTestCase): bad_ca_path = "" +@pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") +class IPv6HypercornDummyServerTestCase(HypercornDummyServerTestCase): + host = "::1" + + class ConnectionMarker: """ Marks an HTTP(S)Connection's socket after a request was made. diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index cf7d8c7eb8..f9c5e1f846 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -9,7 +9,7 @@ from dummyserver.testcase import ( HypercornDummyServerTestCase, - IPv6HTTPDummyServerTestCase, + IPv6HypercornDummyServerTestCase, ) from dummyserver.tornadoserver import HAS_IPV6 from urllib3 import HTTPHeaderDict, HTTPResponse, request @@ -669,7 +669,7 @@ def __repr__(self) -> str: @pytest.mark.skipif(not HAS_IPV6, reason="IPv6 is not supported on this system") -class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase): +class TestIPv6PoolManager(IPv6HypercornDummyServerTestCase): @classmethod def setup_class(cls) -> None: super().setup_class() From bfaaa53070d3814d1da997295a740fca2e2a5a02 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 27 Nov 2023 08:01:56 +0400 Subject: [PATCH 044/131] Migrate supported_tls_versions to Hypercorn --- test/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 745c2e6267..d5e7f61d0c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -16,7 +16,7 @@ from dummyserver.handlers import TestingApp from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.proxy import ProxyHandler -from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from dummyserver.tornadoserver import ( HAS_IPV6, run_tornado_app, @@ -318,8 +318,8 @@ def supported_tls_versions() -> typing.AbstractSet[str | None]: # disables TLSv1 and TLSv1.1. tls_versions = set() - _server = HTTPSDummyServerTestCase() - _server._start_server() + _server = HTTPSHypercornDummyServerTestCase + _server.setup_class() for _ssl_version_name, min_max_version in ( ("PROTOCOL_TLSv1", ssl.TLSVersion.TLSv1), ("PROTOCOL_TLSv1_1", ssl.TLSVersion.TLSv1_1), @@ -344,7 +344,7 @@ def supported_tls_versions() -> typing.AbstractSet[str | None]: else: tls_versions.add(_sock.version()) _sock.close() - _server._stop_server() + _server.teardown_class() return tls_versions From a8e77376943244ed8526a86317891499a60721c3 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 29 Nov 2023 02:53:48 +0400 Subject: [PATCH 045/131] Fix two more LONG_TIMEOUT tests --- test/with_dummyserver/test_connectionpool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 64123c9349..63b1f8a013 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -1326,7 +1326,7 @@ def test_redirect_after(self) -> None: class TestFileBodiesOnRetryOrRedirect(HypercornDummyServerTestCase): def test_retries_put_filehandle(self) -> None: """HTTP PUT retry with a file-like object should not timeout""" - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: retry = Retry(total=3, status_forcelist=[418]) # httplib reads in 8k chunks; use a larger content length content_length = 65535 @@ -1384,7 +1384,7 @@ def tell(self) -> typing.NoReturn: # httplib uses fileno if Content-Length isn't supplied, # which is unsupported by BytesIO. headers = {"Content-Length": "8"} - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: with pytest.raises( UnrewindableBodyError, match="Unable to record file position for" ): From 057ccaf5816221fd72d68f7bec828619746cd733 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 00:07:10 +0000 Subject: [PATCH 046/131] Bump cryptography from 41.0.4 to 41.0.6 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.4 to 41.0.6. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.4...41.0.6) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 55d900e747..60e1283cad 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ pytest-timeout==2.1.0 pyOpenSSL==23.2.0 idna==3.4 trustme==1.1.0 -cryptography==41.0.4 +cryptography==41.0.6 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython" From 16d8e90e9575d5dc48a2a7ad7a8d236ad8cf977b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 29 Nov 2023 21:33:39 +0400 Subject: [PATCH 047/131] Migrate TestHTTPS to Hypercorn --- dev-requirements.txt | 3 ++- dummyserver/app.py | 16 ++++++++++++++++ test/with_dummyserver/test_https.py | 13 ++++++++++--- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 60e1283cad..b668505ae6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -13,4 +13,5 @@ pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython" trio==0.23.1 Quart==0.19.4 quart-trio==0.11.1 -hypercorn==0.15.0 \ No newline at end of file +# https://github.com/pgjones/hypercorn/issues/62 +hypercorn @ git+https://github.com/urllib3/hypercorn@tls-extension diff --git a/dummyserver/app.py b/dummyserver/app.py index f3a10c18f4..201fef7f32 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -27,6 +27,22 @@ async def index() -> ResponseTypes: return await make_response("Dummy server!") +@hypercorn_app.route("/alpn_protocol") +async def alpn_protocol() -> ResponseTypes: + """Return the requester's certificate.""" + alpn_protocol = request.scope["extensions"]["tls"]["alpn_protocol"] + return await make_response(alpn_protocol) + + +@hypercorn_app.route("/certificate") +async def certificate() -> ResponseTypes: + """Return the requester's certificate.""" + print("scope", request.scope) + subject = request.scope["extensions"]["tls"]["client_cert_name"] + subject_as_dict = dict(part.split("=") for part in subject.split(", ")) + return await make_response(subject_as_dict) + + @hypercorn_app.route("/specific_method", methods=["GET", "POST", "PUT"]) async def specific_method() -> ResponseTypes: "Confirm that the request matches the desired method type" diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 45f0a3b974..81032bbd13 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -23,7 +23,7 @@ import urllib3.util as util import urllib3.util.ssl_ -from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from dummyserver.tornadoserver import ( DEFAULT_CA, DEFAULT_CA_KEY, @@ -65,7 +65,7 @@ CLIENT_CERT = CLIENT_INTERMEDIATE_PEM -class TestHTTPS(HTTPSDummyServerTestCase): +class TestHTTPS(HTTPSHypercornDummyServerTestCase): tls_protocol_name: str | None = None def tls_protocol_not_default(self) -> bool: @@ -447,6 +447,8 @@ def test_server_hostname(self) -> None: # the python ssl module). if hasattr(conn.sock, "server_hostname"): # type: ignore[attr-defined] assert conn.sock.server_hostname == "localhost" # type: ignore[attr-defined] + conn.getresponse().close() + conn.close() def test_assert_fingerprint_md5(self) -> None: with HTTPSConnectionPool( @@ -794,6 +796,7 @@ def test_tls_protocol_name_of_socket(self) -> None: self.port, ca_certs=DEFAULT_CA, ssl_minimum_version=self.tls_version(), + ssl_maximum_version=self.tls_version(), ) as https_pool: conn = https_pool._get_conn() try: @@ -898,7 +901,11 @@ def test_tls_version_maximum_and_minimum(self) -> None: conn = https_pool._get_conn() try: conn.connect() - assert conn.sock.version() == self.tls_protocol_name # type: ignore[attr-defined] + if maximum_version == TLSVersion.MAXIMUM_SUPPORTED: + # A higher protocol than tls_protocol_name could be negotiated + assert conn.sock.version() >= self.tls_protocol_name # type: ignore[attr-defined] + else: + assert conn.sock.version() == self.tls_protocol_name # type: ignore[attr-defined] finally: conn.close() From 661072c4f805fc448bfe52c04fc6e26ef454cdaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:28:19 +0000 Subject: [PATCH 048/131] Bump browser-actions/setup-chrome from 1.3.0 to 1.4.0 Bumps [browser-actions/setup-chrome](https://github.com/browser-actions/setup-chrome) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/browser-actions/setup-chrome/releases) - [Changelog](https://github.com/browser-actions/setup-chrome/blob/master/CHANGELOG.md) - [Commits](https://github.com/browser-actions/setup-chrome/compare/11cef13cde73820422f9263a707fb8029808e191...52f10de5479c69bcbbab2eab094c9d373148005e) --- updated-dependencies: - dependency-name: browser-actions/setup-chrome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e244f5632..2f44eeeec8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: run: python -m pip install --upgrade pip setuptools nox - name: "Install Chrome" - uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 # v1.3.0 + uses: browser-actions/setup-chrome@52f10de5479c69bcbbab2eab094c9d373148005e # v1.4.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0 From 5d7979d3f03451c04c80995b8f35cb45ba37c003 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 6 Dec 2023 17:57:51 +0400 Subject: [PATCH 049/131] Migrate run_server_and_proxy_in_thread to Hypercorn (#3222) --- dev-requirements.txt | 5 +- dummyserver/asgi_proxy.py | 108 +++++++++++++++++++++++++++++++++ dummyserver/hypercornserver.py | 3 +- mypy-requirements.txt | 3 +- test/conftest.py | 50 +++++++-------- 5 files changed, 137 insertions(+), 32 deletions(-) create mode 100755 dummyserver/asgi_proxy.py diff --git a/dev-requirements.txt b/dev-requirements.txt index b668505ae6..0438cc8e22 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,4 +14,7 @@ trio==0.23.1 Quart==0.19.4 quart-trio==0.11.1 # https://github.com/pgjones/hypercorn/issues/62 -hypercorn @ git+https://github.com/urllib3/hypercorn@tls-extension +# https://github.com/pgjones/hypercorn/issues/168 +# https://github.com/pgjones/hypercorn/issues/169 +hypercorn @ git+https://github.com/urllib3/hypercorn@urllib3-changes +httpx==0.25.2 diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py new file mode 100755 index 0000000000..c6bb5162e4 --- /dev/null +++ b/dummyserver/asgi_proxy.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import typing + +import httpx +import trio +from hypercorn.typing import ( + ASGIReceiveCallable, + ASGISendCallable, + HTTPResponseBodyEvent, + HTTPResponseStartEvent, + HTTPScope, + Scope, +) + + +async def _read_body(receive: ASGIReceiveCallable) -> bytes: + body = bytearray() + body_consumed = False + while not body_consumed: + event = await receive() + if event["type"] == "http.request": + body.extend(event["body"]) + body_consumed = not event["more_body"] + else: + raise ValueError(event["type"]) + return bytes(body) + + +async def absolute_uri( + scope: HTTPScope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, +) -> None: + async with httpx.AsyncClient() as client: + client_response = await client.request( + method=scope["method"], + url=scope["path"], + headers=list(scope["headers"]), + content=await _read_body(receive), + ) + + headers = [] + for header in ( + "Date", + "Cache-Control", + "Server", + "Content-Type", + "Location", + ): + v = client_response.headers.get(header) + if v: + headers.append((header.encode(), v.encode())) + headers.append((b"Content-Length", str(len(client_response.content)).encode())) + + await send( + HTTPResponseStartEvent( + type="http.response.start", + status=client_response.status_code, + headers=headers, + ) + ) + await send( + HTTPResponseBodyEvent( + type="http.response.body", + body=client_response.content, + more_body=False, + ) + ) + + +async def connect(scope: HTTPScope, send: ASGISendCallable) -> None: + async def start_forward( + reader: trio.SocketStream, writer: trio.SocketStream + ) -> None: + while True: + try: + data = await reader.receive_some(4096) + except trio.ClosedResourceError: + break + if not data: + break + await writer.send_all(data) + await writer.aclose() + + host, port = scope["path"].split(":") + upstream = await trio.open_tcp_stream(host, int(port)) + + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": True}) + + client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) + + async with trio.open_nursery(strict_exception_groups=True) as nursery: + nursery.start_soon(start_forward, client, upstream) + nursery.start_soon(start_forward, upstream, client) + + +async def proxy_app( + scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable +) -> None: + assert scope["type"] == "http" + if scope["method"] in ["GET", "POST"]: + await absolute_uri(scope, receive, send) + elif scope["method"] == "CONNECT": + await connect(scope, send) + else: + raise ValueError(scope["method"]) diff --git a/dummyserver/hypercornserver.py b/dummyserver/hypercornserver.py index 7c860ecb5d..aef75d151f 100644 --- a/dummyserver/hypercornserver.py +++ b/dummyserver/hypercornserver.py @@ -9,6 +9,7 @@ import hypercorn import hypercorn.trio +import hypercorn.typing import trio from quart_trio import QuartTrio @@ -41,7 +42,7 @@ async def _start_server( @contextlib.contextmanager def run_hypercorn_in_thread( - config: hypercorn.Config, app: QuartTrio + config: hypercorn.Config, app: hypercorn.typing.ASGIFramework ) -> Generator[None, None, None]: ready_event = threading.Event() shutdown_event = threading.Event() diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 461d5d24eb..0ee5d6f2fe 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -3,11 +3,12 @@ idna>=2.0.0 cryptography>=1.3.4 tornado>=6.1 pytest>=6.2 -trustme==0.9.0 +trustme==1.1.0 trio==0.23.1 Quart==0.19.4 quart-trio==0.11.1 hypercorn==0.15.0 +httpx==0.25.2 types-backports types-requests nox diff --git a/test/conftest.py b/test/conftest.py index d5e7f61d0c..9a8cb4b263 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import contextlib import socket import ssl @@ -10,18 +9,12 @@ import hypercorn import pytest import trustme -from tornado import web from dummyserver.app import hypercorn_app -from dummyserver.handlers import TestingApp +from dummyserver.asgi_proxy import proxy_app from dummyserver.hypercornserver import run_hypercorn_in_thread -from dummyserver.proxy import ProxyHandler from dummyserver.testcase import HTTPSHypercornDummyServerTestCase -from dummyserver.tornadoserver import ( - HAS_IPV6, - run_tornado_app, - run_tornado_loop_in_thread, -) +from dummyserver.tornadoserver import HAS_IPV6 from urllib3.util import ssl_ from urllib3.util.url import parse_url @@ -111,26 +104,25 @@ def run_server_and_proxy_in_thread( server_certs = _write_cert_to_dir(server_cert, tmpdir) proxy_certs = _write_cert_to_dir(proxy_cert, tmpdir, "proxy") - with run_tornado_loop_in_thread() as io_loop: - - async def run_app() -> tuple[ServerConfig, ServerConfig]: - app = web.Application([(r".*", TestingApp)]) - server_app, port = run_tornado_app(app, server_certs, "https", "localhost") - server_config = ServerConfig("https", "localhost", port, ca_cert_path) - - proxy = web.Application([(r".*", ProxyHandler)]) - proxy_app, proxy_port = run_tornado_app( - proxy, proxy_certs, proxy_scheme, proxy_host - ) - proxy_config = ServerConfig( - proxy_scheme, proxy_host, proxy_port, ca_cert_path - ) - return proxy_config, server_config - - proxy_config, server_config = asyncio.run_coroutine_threadsafe( - run_app(), io_loop.asyncio_loop # type: ignore[attr-defined] - ).result() - yield (proxy_config, server_config) + with contextlib.ExitStack() as stack: + server_config = hypercorn.Config() + server_config.certfile = server_certs["certfile"] + server_config.keyfile = server_certs["keyfile"] + server_config.bind = ["localhost:0"] + stack.enter_context(run_hypercorn_in_thread(server_config, hypercorn_app)) + port = typing.cast(int, parse_url(server_config.bind[0]).port) + + proxy_config = hypercorn.Config() + proxy_config.certfile = proxy_certs["certfile"] + proxy_config.keyfile = proxy_certs["keyfile"] + proxy_config.bind = [f"{proxy_host}:0"] + stack.enter_context(run_hypercorn_in_thread(proxy_config, proxy_app)) + proxy_port = typing.cast(int, parse_url(proxy_config.bind[0]).port) + + yield ( + ServerConfig(proxy_scheme, proxy_host, proxy_port, ca_cert_path), + ServerConfig("https", "localhost", port, ca_cert_path), + ) @pytest.fixture(params=["localhost", "127.0.0.1", "::1"]) From 37c9496befb433ffef41631ce02ea0de9b050dcb Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 7 Dec 2023 20:26:04 +0400 Subject: [PATCH 050/131] Turn proxy_app into a class to allow configuring it (#3225) --- dummyserver/asgi_proxy.py | 150 +++++++++++++++++++------------------- test/conftest.py | 4 +- 2 files changed, 77 insertions(+), 77 deletions(-) diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py index c6bb5162e4..fcd7e20a7b 100755 --- a/dummyserver/asgi_proxy.py +++ b/dummyserver/asgi_proxy.py @@ -27,82 +27,82 @@ async def _read_body(receive: ASGIReceiveCallable) -> bytes: return bytes(body) -async def absolute_uri( - scope: HTTPScope, - receive: ASGIReceiveCallable, - send: ASGISendCallable, -) -> None: - async with httpx.AsyncClient() as client: - client_response = await client.request( - method=scope["method"], - url=scope["path"], - headers=list(scope["headers"]), - content=await _read_body(receive), - ) - - headers = [] - for header in ( - "Date", - "Cache-Control", - "Server", - "Content-Type", - "Location", - ): - v = client_response.headers.get(header) - if v: - headers.append((header.encode(), v.encode())) - headers.append((b"Content-Length", str(len(client_response.content)).encode())) +class ProxyApp: + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + assert scope["type"] == "http" + if scope["method"] in ["GET", "POST"]: + await self.absolute_uri(scope, receive, send) + elif scope["method"] == "CONNECT": + await self.connect(scope, send) + else: + raise ValueError(scope["method"]) - await send( - HTTPResponseStartEvent( - type="http.response.start", - status=client_response.status_code, - headers=headers, + async def absolute_uri( + self, + scope: HTTPScope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + ) -> None: + async with httpx.AsyncClient() as client: + client_response = await client.request( + method=scope["method"], + url=scope["path"], + headers=list(scope["headers"]), + content=await _read_body(receive), + ) + + headers = [] + for header in ( + "Date", + "Cache-Control", + "Server", + "Content-Type", + "Location", + ): + v = client_response.headers.get(header) + if v: + headers.append((header.encode(), v.encode())) + headers.append((b"Content-Length", str(len(client_response.content)).encode())) + + await send( + HTTPResponseStartEvent( + type="http.response.start", + status=client_response.status_code, + headers=headers, + ) ) - ) - await send( - HTTPResponseBodyEvent( - type="http.response.body", - body=client_response.content, - more_body=False, + await send( + HTTPResponseBodyEvent( + type="http.response.body", + body=client_response.content, + more_body=False, + ) ) - ) - - -async def connect(scope: HTTPScope, send: ASGISendCallable) -> None: - async def start_forward( - reader: trio.SocketStream, writer: trio.SocketStream - ) -> None: - while True: - try: - data = await reader.receive_some(4096) - except trio.ClosedResourceError: - break - if not data: - break - await writer.send_all(data) - await writer.aclose() - - host, port = scope["path"].split(":") - upstream = await trio.open_tcp_stream(host, int(port)) - - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": True}) - - client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) - - async with trio.open_nursery(strict_exception_groups=True) as nursery: - nursery.start_soon(start_forward, client, upstream) - nursery.start_soon(start_forward, upstream, client) - -async def proxy_app( - scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable -) -> None: - assert scope["type"] == "http" - if scope["method"] in ["GET", "POST"]: - await absolute_uri(scope, receive, send) - elif scope["method"] == "CONNECT": - await connect(scope, send) - else: - raise ValueError(scope["method"]) + async def connect(self, scope: HTTPScope, send: ASGISendCallable) -> None: + async def start_forward( + reader: trio.SocketStream, writer: trio.SocketStream + ) -> None: + while True: + try: + data = await reader.receive_some(4096) + except trio.ClosedResourceError: + break + if not data: + break + await writer.send_all(data) + await writer.aclose() + + host, port = scope["path"].split(":") + upstream = await trio.open_tcp_stream(host, int(port)) + + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": True}) + + client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) + + async with trio.open_nursery(strict_exception_groups=True) as nursery: + nursery.start_soon(start_forward, client, upstream) + nursery.start_soon(start_forward, upstream, client) diff --git a/test/conftest.py b/test/conftest.py index 9a8cb4b263..39c9b5b0c3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -11,7 +11,7 @@ import trustme from dummyserver.app import hypercorn_app -from dummyserver.asgi_proxy import proxy_app +from dummyserver.asgi_proxy import ProxyApp from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from dummyserver.tornadoserver import HAS_IPV6 @@ -116,7 +116,7 @@ def run_server_and_proxy_in_thread( proxy_config.certfile = proxy_certs["certfile"] proxy_config.keyfile = proxy_certs["keyfile"] proxy_config.bind = [f"{proxy_host}:0"] - stack.enter_context(run_hypercorn_in_thread(proxy_config, proxy_app)) + stack.enter_context(run_hypercorn_in_thread(proxy_config, ProxyApp())) proxy_port = typing.cast(int, parse_url(proxy_config.bind[0]).port) yield ( From 73d62d71ffa6132afe23459ef87aa47f02410975 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sat, 9 Dec 2023 22:33:18 +0400 Subject: [PATCH 051/131] Migrate remaining proxy tests to Hypercorn --- dummyserver/asgi_proxy.py | 5 +- dummyserver/testcase.py | 93 +++++++++++++++++++ .../test_proxy_poolmanager.py | 13 ++- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py index fcd7e20a7b..7b16303945 100755 --- a/dummyserver/asgi_proxy.py +++ b/dummyserver/asgi_proxy.py @@ -28,6 +28,9 @@ async def _read_body(receive: ASGIReceiveCallable) -> bytes: class ProxyApp: + def __init__(self, upstream_ca_certs: str | None = None): + self.upstream_ca_certs = upstream_ca_certs + async def __call__( self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable ) -> None: @@ -45,7 +48,7 @@ async def absolute_uri( receive: ASGIReceiveCallable, send: ASGISendCallable, ) -> None: - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(verify=self.upstream_ca_certs or True) as client: client_response = await client.request( method=scope["method"], url=scope["path"], diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index d4c2714e73..0628f8963a 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -12,6 +12,7 @@ from tornado import httpserver, ioloop, web from dummyserver.app import hypercorn_app +from dummyserver.asgi_proxy import ProxyApp from dummyserver.handlers import TestingApp from dummyserver.hypercornserver import run_hypercorn_in_thread from dummyserver.proxy import ProxyHandler @@ -335,11 +336,103 @@ class HTTPSHypercornDummyServerTestCase(HypercornDummyServerTestCase): bad_ca_path = "" +class HypercornDummyProxyTestCase: + http_host: typing.ClassVar[str] = "localhost" + http_host_alt: typing.ClassVar[str] = "127.0.0.1" + http_port: typing.ClassVar[int] + http_url: typing.ClassVar[str] + http_url_alt: typing.ClassVar[str] + + https_host: typing.ClassVar[str] = "localhost" + https_host_alt: typing.ClassVar[str] = "127.0.0.1" + https_certs: typing.ClassVar[dict[str, typing.Any]] = DEFAULT_CERTS + https_port: typing.ClassVar[int] + https_url: typing.ClassVar[str] + https_url_alt: typing.ClassVar[str] + https_url_fqdn: typing.ClassVar[str] + + proxy_host: typing.ClassVar[str] = "localhost" + proxy_host_alt: typing.ClassVar[str] = "127.0.0.1" + proxy_port: typing.ClassVar[int] + proxy_url: typing.ClassVar[str] + https_proxy_port: typing.ClassVar[int] + https_proxy_url: typing.ClassVar[str] + + certs_dir: typing.ClassVar[str] = "" + bad_ca_path: typing.ClassVar[str] = "" + + server_thread: typing.ClassVar[threading.Thread] + _stack: typing.ClassVar[contextlib.ExitStack] + + @classmethod + def setup_class(cls) -> None: + with contextlib.ExitStack() as stack: + http_server_config = hypercorn.Config() + http_server_config.bind = [f"{cls.http_host}:0"] + stack.enter_context( + run_hypercorn_in_thread(http_server_config, hypercorn_app) + ) + cls.http_port = typing.cast(int, parse_url(http_server_config.bind[0]).port) + + https_server_config = hypercorn.Config() + https_server_config.certfile = cls.https_certs["certfile"] + https_server_config.keyfile = cls.https_certs["keyfile"] + https_server_config.verify_mode = cls.https_certs["cert_reqs"] + https_server_config.ca_certs = cls.https_certs["ca_certs"] + https_server_config.alpn_protocols = cls.https_certs["alpn_protocols"] + https_server_config.bind = [f"{cls.https_host}:0"] + stack.enter_context( + run_hypercorn_in_thread(https_server_config, hypercorn_app) + ) + cls.https_port = typing.cast( + int, parse_url(https_server_config.bind[0]).port + ) + + http_proxy_config = hypercorn.Config() + http_proxy_config.bind = [f"{cls.proxy_host}:0"] + stack.enter_context(run_hypercorn_in_thread(http_proxy_config, ProxyApp())) + cls.proxy_port = typing.cast(int, parse_url(http_proxy_config.bind[0]).port) + + https_proxy_config = hypercorn.Config() + https_proxy_config.certfile = cls.https_certs["certfile"] + https_proxy_config.keyfile = cls.https_certs["keyfile"] + https_proxy_config.verify_mode = cls.https_certs["cert_reqs"] + https_proxy_config.ca_certs = cls.https_certs["ca_certs"] + https_proxy_config.alpn_protocols = cls.https_certs["alpn_protocols"] + https_proxy_config.bind = [f"{cls.proxy_host}:0"] + upstream_ca_certs = cls.https_certs.get("ca_certs") + stack.enter_context( + run_hypercorn_in_thread(https_proxy_config, ProxyApp(upstream_ca_certs)) + ) + cls.https_proxy_port = typing.cast( + int, parse_url(https_proxy_config.bind[0]).port + ) + + cls._stack = stack.pop_all() + + @classmethod + def teardown_class(cls) -> None: + cls._stack.close() + + @pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") class IPv6HypercornDummyServerTestCase(HypercornDummyServerTestCase): host = "::1" +@pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") +class IPv6HypercornDummyProxyTestCase(HypercornDummyProxyTestCase): + http_host = "localhost" + http_host_alt = "127.0.0.1" + + https_host = "localhost" + https_host_alt = "127.0.0.1" + https_certs = DEFAULT_CERTS + + proxy_host = "::1" + proxy_host_alt = "127.0.0.1" + + class ConnectionMarker: """ Marks an HTTP(S)Connection's socket after a request was made. diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 917a8c484b..dae1013f19 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -16,7 +16,10 @@ import trustme import urllib3.exceptions -from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase +from dummyserver.testcase import ( + HypercornDummyProxyTestCase, + IPv6HypercornDummyProxyTestCase, +) from dummyserver.tornadoserver import DEFAULT_CA, HAS_IPV6, get_unreachable_address from urllib3 import HTTPResponse from urllib3._collections import HTTPHeaderDict @@ -39,7 +42,7 @@ from .. import TARPIT_HOST, requires_network -class TestHTTPProxyManager(HTTPDummyProxyTestCase): +class TestHTTPProxyManager(HypercornDummyProxyTestCase): @classmethod def setup_class(cls) -> None: super().setup_class() @@ -132,7 +135,7 @@ def test_nagle_proxy(self) -> None: hc2 = http.connection_from_host(self.http_host, self.http_port) conn = hc2._get_conn() try: - hc2._make_request(conn, "GET", "/") + hc2._make_request(conn, "GET", f"{self.http_url}/") tcp_nodelay_setting = conn.sock.getsockopt( # type: ignore[attr-defined] socket.IPPROTO_TCP, socket.TCP_NODELAY ) @@ -638,10 +641,10 @@ def test_invalid_schema(self, url: str, error_msg: str) -> None: @pytest.mark.skipif(not HAS_IPV6, reason="Only runs on IPv6 systems") -class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase): +class TestIPv6HTTPProxyManager(IPv6HypercornDummyProxyTestCase): @classmethod def setup_class(cls) -> None: - HTTPDummyProxyTestCase.setup_class() + super().setup_class() cls.http_url = f"http://{cls.http_host}:{int(cls.http_port)}" cls.http_url_alt = f"http://{cls.http_host_alt}:{int(cls.http_port)}" cls.https_url = f"https://{cls.https_host}:{int(cls.https_port)}" From 22b0623c3a3f2582065639cec4851884d0316e4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 20:51:31 +0000 Subject: [PATCH 052/131] Bump actions/setup-python from 4.7.0 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/61a6322f88396a6271a6ee3565807d608ecaddd1...0a5c61591373683505ea898e09a3ea4f39ef2b9c) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f44eeeec8..563c0c0e33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: "3.x" cache: "pip" @@ -100,7 +100,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python ${{ matrix.python-version }}" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -138,7 +138,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: "3.x" diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 3cd2b2a068..b0fba2da8a 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: "3.x" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8cf8c69207..f7375a4d38 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: "3.x" cache: pip diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6fb36c29dc..e79066829d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: "3.x" From 90c30f5fdca56a54248614dc86570bf2692a4caa Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 12 Dec 2023 06:04:38 +0100 Subject: [PATCH 053/131] Fix some typos --- noxfile.py | 2 +- test/test_collections.py | 2 +- test/test_response.py | 2 +- test/with_dummyserver/test_socketlevel.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index d153d08e3e..1a2514b9aa 100644 --- a/noxfile.py +++ b/noxfile.py @@ -255,7 +255,7 @@ def emscripten(session: nox.Session, runner: str) -> None: ], ) else: - raise ValueError(f"Unknown runnner: {runner}") + raise ValueError(f"Unknown runner: {runner}") @nox.session(python="3.12") diff --git a/test/test_collections.py b/test/test_collections.py index 8d0c1ce26f..5f3b96cd72 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -282,7 +282,7 @@ def test_header_repeat(self, d: HTTPHeaderDict) -> None: ] assert list(d.items()) == expected_results - # make sure the values persist over copys + # make sure the values persist over copies assert list(d.copy().items()) == expected_results other_dict = HTTPHeaderDict() diff --git a/test/test_response.py b/test/test_response.py index 3bc6f53b6a..abb6a73cf3 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -830,7 +830,7 @@ def read1(self, amt: int) -> bytes: # type: ignore[override] assert uncompressed_data == payload # Check that the positions in the stream are correct - # It is difficult to determine programatically what the positions + # It is difficult to determine programmatically what the positions # returned by `tell` will be because the `HTTPResponse.read` method may # call socket `read` a couple of times if it doesn't have enough data # in the buffer or not call socket `read` at all if it has enough. All diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 795ba1f20d..275b06f197 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1848,7 +1848,7 @@ def test_headers_sent_with_add( body.seek(0, 0) expected = b"bytes-io-body\r\n0\r\n\r\n" else: - raise ValueError("Unknonw body type") + raise ValueError("Unknown body type") buffer: bytes = b"" From 767ad2f7369c90fd30262ee084fed46181e20499 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Dec 2023 12:52:25 +0200 Subject: [PATCH 054/131] Only run cron jobs for upstream (#3237) --- .github/workflows/codeql.yml | 1 + .github/workflows/scorecards.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a25c01daeb..e434dff2f3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,6 +12,7 @@ permissions: "read-all" jobs: analyze: + if: github.repository_owner == "urllib3" name: "Analyze" runs-on: "ubuntu-latest" permissions: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 18891ce8c1..218df29672 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -10,6 +10,7 @@ permissions: read-all jobs: analysis: + if: github.repository_owner == "urllib3" name: "Scorecard" runs-on: "ubuntu-latest" permissions: From 910c74d7f97ade35fc048b080584ca115c3bac06 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Dec 2023 13:24:00 +0200 Subject: [PATCH 055/131] Use single quotes, double quotes are invalid (#3238) --- .github/workflows/codeql.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e434dff2f3..a0de4fcc59 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,7 +12,7 @@ permissions: "read-all" jobs: analyze: - if: github.repository_owner == "urllib3" + if: github.repository_owner == 'urllib3' name: "Analyze" runs-on: "ubuntu-latest" permissions: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 218df29672..356be2d94d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -10,7 +10,7 @@ permissions: read-all jobs: analysis: - if: github.repository_owner == "urllib3" + if: github.repository_owner == 'urllib3' name: "Scorecard" runs-on: "ubuntu-latest" permissions: From eef1c06da8ef0737ef2a5632fcacc572234889a9 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 Dec 2023 13:55:25 +0400 Subject: [PATCH 056/131] Fix coverage by installing the same version everywhere (#3244) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 563c0c0e33..a4ac32f309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: python-version: "3.x" - name: "Install coverage" - run: "python -m pip install --upgrade coverage" + run: "python -m pip install -r dev-requirements.txt" - name: "Download artifact" uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From b660f61968579daba1ffb91731b04450a5b46d3d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 Dec 2023 16:14:53 +0400 Subject: [PATCH 057/131] Migrate emscripten test server to Hypercorn (#3230) Co-authored-by: Joe Marshall --- dummyserver/app.py | 106 +++++++++++++- src/urllib3/contrib/emscripten/connection.py | 4 +- test/contrib/emscripten/conftest.py | 140 +++++-------------- test/contrib/emscripten/test_emscripten.py | 1 + 4 files changed, 141 insertions(+), 110 deletions(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index 201fef7f32..a2eff8764b 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -5,10 +5,13 @@ import datetime import email.utils import gzip +import mimetypes import zlib from io import BytesIO +from pathlib import Path from typing import Iterator +import trio from quart import make_response, request # TODO switch to Response if https://github.com/pallets/quart/issues/288 is fixed @@ -22,7 +25,20 @@ LAST_RETRY_AFTER_REQ: datetime.datetime = datetime.datetime.min +pyodide_testing_app = QuartTrio(__name__) +DEFAULT_HEADERS = [ + # Allow cross-origin requests for emscripten + ("Access-Control-Allow-Origin", "*"), + ("Cross-Origin-Opener-Policy", "same-origin"), + ("Cross-Origin-Embedder-Policy", "require-corp"), + ("Feature-Policy", "sync-xhr *;"), + ("Access-Control-Allow-Headers", "*"), +] + + @hypercorn_app.route("/") +@pyodide_testing_app.route("/") +@pyodide_testing_app.route("/index") async def index() -> ResponseTypes: return await make_response("Dummy server!") @@ -44,15 +60,17 @@ async def certificate() -> ResponseTypes: @hypercorn_app.route("/specific_method", methods=["GET", "POST", "PUT"]) +@pyodide_testing_app.route("/specific_method", methods=["GET", "POST", "PUT"]) async def specific_method() -> ResponseTypes: "Confirm that the request matches the desired method type" - method_param = (await request.values).get("method") + method_param = (await request.values).get("method", "") - if request.method != method_param: + if request.method.upper() == method_param.upper(): + return await make_response("", 200) + else: return await make_response( f"Wrong method: {method_param} != {request.method}", 400 ) - return await make_response() @hypercorn_app.route("/upload", methods=["POST"]) @@ -127,8 +145,11 @@ async def echo() -> ResponseTypes: @hypercorn_app.route("/echo_json", methods=["POST"]) +@pyodide_testing_app.route("/echo_json", methods=["POST", "OPTIONS"]) async def echo_json() -> ResponseTypes: "Echo back the JSON" + if request.method == "OPTIONS": + return await make_response("", 200) data = await request.get_data() return await make_response(data, 200, request.headers) @@ -251,6 +272,7 @@ async def retry_after() -> ResponseTypes: @hypercorn_app.route("/status") +@pyodide_testing_app.route("/status") async def status() -> ResponseTypes: values = await request.values status = values.get("status", "200 OK") @@ -280,3 +302,81 @@ async def successful_retry() -> ResponseTypes: return await make_response("Retry successful!", 200) else: return await make_response("need to keep retrying!", 418) + + +@pyodide_testing_app.after_request +def apply_caching(response: ResponseTypes) -> ResponseTypes: + for header, value in DEFAULT_HEADERS: + response.headers[header] = value + return response + + +@pyodide_testing_app.route("/slow") +async def slow() -> ResponseTypes: + await trio.sleep(10) + return await make_response("TEN SECONDS LATER", 200) + + +@pyodide_testing_app.route("/bigfile") +async def bigfile() -> ResponseTypes: + # great big text file, should force streaming + # if supported + bigdata = 1048576 * b"WOOO YAY BOOYAKAH" + return await make_response(bigdata, 200) + + +@pyodide_testing_app.route("/mediumfile") +async def mediumfile() -> ResponseTypes: + # quite big file + bigdata = 1024 * b"WOOO YAY BOOYAKAH" + return await make_response(bigdata, 200) + + +@pyodide_testing_app.route("/upload", methods=["POST", "OPTIONS"]) +async def pyodide_upload() -> ResponseTypes: + if request.method == "OPTIONS": + return await make_response("", 200) + spare_data = await request.get_data(parse_form_data=True) + if len(spare_data) != 0: + return await make_response("Bad upload data", 404) + files = await request.files + form = await request.form + if form["upload_param"] != "filefield" or form["upload_filename"] != "lolcat.txt": + return await make_response("Bad upload form values", 404) + if len(files) != 1 or files.get("filefield") is None: + return await make_response("Missing file in form", 404) + file = files["filefield"] + if file.filename != "lolcat.txt": + return await make_response(f"File name incorrect {file.name}", 404) + data = file.read().decode("utf-8") + if data != "I'm in ur multipart form-data, hazing a cheezburgr": + return await make_response(f"File data incorrect {data}", 200) + return await make_response("Uploaded file correct", 200) + + +@pyodide_testing_app.route("/pyodide/") +async def pyodide(py_file: str) -> ResponseTypes: + file_path = Path(pyodide_testing_app.config["pyodide_dist_dir"], py_file) + if file_path.exists(): + mime_type, encoding = mimetypes.guess_type(file_path) + if not mime_type: + mime_type = "text/plain" + return await make_response( + file_path.read_bytes(), 200, [("Content-Type", mime_type)] + ) + else: + return await make_response("", 404) + + +@pyodide_testing_app.route("/wheel/dist.whl") +async def wheel() -> ResponseTypes: + # serve our wheel + wheel_folder = Path(__file__).parent.parent / "dist" + wheels = list(wheel_folder.glob("*.whl")) + if len(wheels) > 0: + wheel = wheels[0] + headers = [("Content-Disposition", f"inline; filename='{wheel.name}'")] + resp = await make_response(wheel.read_bytes(), 200, headers) + return resp + else: + return await make_response(f"NO WHEEL IN {wheel_folder}", 404) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 25d7baa42f..9090e51d18 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -116,9 +116,9 @@ def request( if self._response is None: self._response = send_request(request) except _TimeoutError as e: - raise TimeoutError(e.message) + raise TimeoutError(e.message) from e except _RequestError as e: - raise HTTPException(e.message) + raise HTTPException(e.message) from e def getresponse(self) -> BaseHTTPResponse: if self._response is not None: diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index abaa3bee6e..7438402b19 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -1,23 +1,21 @@ from __future__ import annotations -import asyncio import contextlib -import mimetypes import os import random import textwrap +import typing from dataclasses import dataclass from pathlib import Path from typing import Any, Generator -from urllib.parse import urlsplit +import hypercorn import pytest -from tornado import web -from tornado.httputil import HTTPServerRequest -from dummyserver.handlers import Response, TestingApp -from dummyserver.testcase import HTTPDummyProxyTestCase -from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread +from dummyserver.app import pyodide_testing_app +from dummyserver.hypercornserver import run_hypercorn_in_thread +from dummyserver.tornadoserver import DEFAULT_CERTS +from urllib3.util.url import parse_url _coverage_count = 0 @@ -33,19 +31,35 @@ def _get_coverage_filename(prefix: str) -> str: def testserver_http( request: pytest.FixtureRequest, ) -> Generator[PyodideServerInfo, None, None]: - dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) - server = PyodideDummyServerTestCase - server.setup_class(str(dist_dir)) - print( - f"Server:{server.http_host}:{server.http_port},https({server.https_port}) [{dist_dir}]" - ) - yield PyodideServerInfo( - http_host=server.http_host, - http_port=server.http_port, - https_port=server.https_port, - ) - print("Server teardown") - server.teardown_class() + pyodide_dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) + pyodide_testing_app.config["pyodide_dist_dir"] = str(pyodide_dist_dir) + http_host = "localhost" + with contextlib.ExitStack() as stack: + http_server_config = hypercorn.Config() + http_server_config.bind = [f"{http_host}:0"] + stack.enter_context( + run_hypercorn_in_thread(http_server_config, pyodide_testing_app) + ) + http_port = typing.cast(int, parse_url(http_server_config.bind[0]).port) + + https_server_config = hypercorn.Config() + https_server_config.certfile = DEFAULT_CERTS["certfile"] + https_server_config.keyfile = DEFAULT_CERTS["keyfile"] + https_server_config.verify_mode = DEFAULT_CERTS["cert_reqs"] + https_server_config.ca_certs = DEFAULT_CERTS["ca_certs"] + https_server_config.alpn_protocols = DEFAULT_CERTS["alpn_protocols"] + https_server_config.bind = [f"{http_host}:0"] + stack.enter_context( + run_hypercorn_in_thread(https_server_config, pyodide_testing_app) + ) + https_port = typing.cast(int, parse_url(https_server_config.bind[0]).port) + + yield PyodideServerInfo( + http_host=http_host, + http_port=http_port, + https_port=https_port, + ) + print("Server teardown") @pytest.fixture() @@ -178,90 +192,6 @@ def run_from_server( ) -class PyodideTestingApp(TestingApp): - pyodide_dist_dir: str = "" - - def set_default_headers(self) -> None: - """Allow cross-origin requests for emscripten""" - self.set_header("Access-Control-Allow-Origin", "*") - self.set_header("Cross-Origin-Opener-Policy", "same-origin") - self.set_header("Cross-Origin-Embedder-Policy", "require-corp") - self.add_header("Feature-Policy", "sync-xhr *;") - self.add_header("Access-Control-Allow-Headers", "*") - - def slow(self, _req: HTTPServerRequest) -> Response: - import time - - time.sleep(10) - return Response("TEN SECONDS LATER") - - def bigfile(self, req: HTTPServerRequest) -> Response: - # great big text file, should force streaming - # if supported - bigdata = 1048576 * b"WOOO YAY BOOYAKAH" - return Response(bigdata) - - def mediumfile(self, req: HTTPServerRequest) -> Response: - # quite big file - bigdata = 1024 * b"WOOO YAY BOOYAKAH" - return Response(bigdata) - - def pyodide(self, req: HTTPServerRequest) -> Response: - path = req.path[:] - if not path.startswith("/"): - path = urlsplit(path).path - path_split = path.split("/") - file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path_split[2:]) - if file_path.exists(): - mime_type, encoding = mimetypes.guess_type(file_path) - if not mime_type: - mime_type = "text/plain" - self.set_header("Content-Type", mime_type) - return Response( - body=file_path.read_bytes(), - headers=[("Access-Control-Allow-Origin", "*")], - ) - else: - return Response(status="404 NOT FOUND") - - def wheel(self, _req: HTTPServerRequest) -> Response: - # serve our wheel - wheel_folder = Path(__file__).parent.parent.parent.parent / "dist" - wheels = list(wheel_folder.glob("*.whl")) - if len(wheels) > 0: - resp = Response( - body=wheels[0].read_bytes(), - headers=[ - ("Content-Disposition", f"inline; filename='{wheels[0].name}'") - ], - ) - return resp - else: - return Response(status="404 NOT FOUND") - - -class PyodideDummyServerTestCase(HTTPDummyProxyTestCase): - @classmethod - def setup_class(cls, pyodide_dist_dir: str) -> None: # type:ignore[override] - PyodideTestingApp.pyodide_dist_dir = pyodide_dist_dir - with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_tornado_loop_in_thread()) - - async def run_app() -> None: - app = web.Application([(r".*", PyodideTestingApp)]) - cls.http_server, cls.http_port = run_tornado_app( - app, None, "http", cls.http_host - ) - - app = web.Application([(r".*", PyodideTestingApp)]) - cls.https_server, cls.https_port = run_tornado_app( - app, cls.https_certs, "https", cls.http_host - ) - - asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] - cls._stack = stack.pop_all() - - @dataclass class PyodideServerInfo: http_port: int diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 29b8cf08a2..66f4caedfe 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -630,6 +630,7 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno "POST", f"http://{host}:{port}/echo_json", body=json.dumps(json_data).encode("utf-8"), + headers={"Content-type": "application/json"}, ) response = conn.getresponse() assert isinstance(response, BaseHTTPResponse) From 8359450f8c57678c327ed85b8ba0eee1308c4ea5 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Tue, 19 Dec 2023 18:33:30 +0200 Subject: [PATCH 058/131] Make `BytesQueueBuffer.get_all` more memory efficient (#3236) --- src/urllib3/response.py | 5 +++-- test/test_response.py | 48 ++++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 01220c359e..6c73dcfa0c 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -291,8 +291,9 @@ def get_all(self) -> bytes: if len(buffer) == 1: result = buffer.pop() else: - result = b"".join(buffer) - buffer.clear() + ret = io.BytesIO() + ret.writelines(buffer.popleft() for _ in range(len(buffer))) + result = ret.getvalue() self._size = 0 return result diff --git a/test/test_response.py b/test/test_response.py index abb6a73cf3..b4939bb1a9 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -76,11 +76,13 @@ def test_multiple_chunks(self) -> None: def test_get_all_empty(self) -> None: q = BytesQueueBuffer() assert q.get_all() == b"" + assert len(q) == 0 def test_get_all_single(self) -> None: q = BytesQueueBuffer() q.put(b"a") assert q.get_all() == b"a" + assert len(q) == 0 def test_get_all_many(self) -> None: q = BytesQueueBuffer() @@ -90,15 +92,29 @@ def test_get_all_many(self) -> None: assert q.get_all() == b"abc" assert len(q) == 0 + @pytest.mark.parametrize( + "get_func", + (lambda b: b.get(len(b)), lambda b: b.get_all()), + ids=("get", "get_all"), + ) @pytest.mark.limit_memory("12.5 MB") # assert that we're not doubling memory usage - def test_memory_usage(self) -> None: + def test_memory_usage( + self, get_func: typing.Callable[[BytesQueueBuffer], str] + ) -> None: # Allocate 10 1MiB chunks buffer = BytesQueueBuffer() for i in range(10): # This allocates 2MiB, putting the max at around 12MiB. Not sure why. buffer.put(bytes(2**20)) - assert len(buffer.get(10 * 2**20)) == 10 * 2**20 + assert len(get_func(buffer)) == 10 * 2**20 + + @pytest.mark.limit_memory("10.01 MB") + def test_get_all_memory_usage_single_chunk(self) -> None: + buffer = BytesQueueBuffer() + chunk = bytes(10 * 2**20) # 10 MiB + buffer.put(chunk) + assert buffer.get_all() is chunk # A known random (i.e, not-too-compressible) payload generated with: @@ -891,12 +907,18 @@ def test_empty_stream(self) -> None: next(stream) @pytest.mark.parametrize( - "preload_content, amt", - [(True, None), (False, None), (False, 10 * 2**20)], + "preload_content, amt, read_meth", + [ + (True, None, "read"), + (False, None, "read"), + (False, 10 * 2**20, "read"), + (False, None, "read1"), + (False, 10 * 2**20, "read1"), + ], ) @pytest.mark.limit_memory("25 MB") def test_buffer_memory_usage_decode_one_chunk( - self, preload_content: bool, amt: int + self, preload_content: bool, amt: int, read_meth: str ) -> None: content_length = 10 * 2**20 # 10 MiB fp = BytesIO(zlib.compress(bytes(content_length))) @@ -905,21 +927,27 @@ def test_buffer_memory_usage_decode_one_chunk( preload_content=preload_content, headers={"content-encoding": "deflate"}, ) - data = resp.data if preload_content else resp.read(amt) + data = resp.data if preload_content else getattr(resp, read_meth)(amt) assert len(data) == content_length @pytest.mark.parametrize( - "preload_content, amt", - [(True, None), (False, None), (False, 10 * 2**20)], + "preload_content, amt, read_meth", + [ + (True, None, "read"), + (False, None, "read"), + (False, 10 * 2**20, "read"), + (False, None, "read1"), + (False, 10 * 2**20, "read1"), + ], ) @pytest.mark.limit_memory("10.5 MB") def test_buffer_memory_usage_no_decoding( - self, preload_content: bool, amt: int + self, preload_content: bool, amt: int, read_meth: str ) -> None: content_length = 10 * 2**20 # 10 MiB fp = BytesIO(bytes(content_length)) resp = HTTPResponse(fp, preload_content=preload_content, decode_content=False) - data = resp.data if preload_content else resp.read(amt) + data = resp.data if preload_content else getattr(resp, read_meth)(amt) assert len(data) == content_length def test_length_no_header(self) -> None: From c72ff2e988cc53b55c464e343125736f3de1346c Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 20 Dec 2023 22:30:10 +0000 Subject: [PATCH 059/131] Also shutdown Hypercorn server on setup failure --- dummyserver/hypercornserver.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dummyserver/hypercornserver.py b/dummyserver/hypercornserver.py index aef75d151f..c8b75ee8f0 100644 --- a/dummyserver/hypercornserver.py +++ b/dummyserver/hypercornserver.py @@ -62,10 +62,11 @@ def run_hypercorn_in_thread( if not ready_event.is_set(): raise Exception("most likely failed to start server") - yield - - shutdown_event.set() - future.result() + try: + yield + finally: + shutdown_event.set() + future.result() def main() -> int: From c7b9adcbbe911bec9249cb78111fa2561238dd70 Mon Sep 17 00:00:00 2001 From: zawan-ila <87228907+zawan-ila@users.noreply.github.com> Date: Thu, 28 Dec 2023 20:57:30 +0500 Subject: [PATCH 060/131] Fix TestBrokenPipe on macOS Co-authored-by: Seth Michael Larson --- src/urllib3/connectionpool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index f7b0824cde..549f0f431e 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -511,9 +511,10 @@ def _make_request( pass except OSError as e: # MacOS/Linux - # EPROTOTYPE is needed on macOS + # EPROTOTYPE and ECONNRESET are needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ - if e.errno != errno.EPROTOTYPE: + # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE. + if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET: raise # Reset the timeout for the recv() on the socket From f0eab79767ae7e8569a97ab7fdc17681f034a232 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 30 Dec 2023 21:19:18 +0000 Subject: [PATCH 061/131] Make sure test_ssl_failed_fingerprint_verification tests fingerprint verification Co-authored-by: Quentin Pradet Co-authored-by: Seth Michael Larson --- test/with_dummyserver/test_socketlevel.py | 42 ++++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 275b06f197..d89f311c33 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1353,19 +1353,21 @@ def socket_handler(listener: socket.socket) -> None: certfile=DEFAULT_CERTS["certfile"], ca_certs=DEFAULT_CA, ) - except (ssl.SSLError, ConnectionResetError): - if i == 1: - raise - return + except (ssl.SSLError, ConnectionResetError, ConnectionAbortedError): + pass - ssl_sock.send( - b"HTTP/1.1 200 OK\r\n" - b"Content-Type: text/plain\r\n" - b"Content-Length: 5\r\n\r\n" - b"Hello" - ) + else: + with ssl_sock: + try: + ssl_sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 5\r\n\r\n" + b"Hello" + ) + except (ssl.SSLEOFError, ConnectionResetError, BrokenPipeError): + pass - ssl_sock.close() sock.close() self._start_server(socket_handler) @@ -1374,7 +1376,10 @@ def socket_handler(listener: socket.socket) -> None: def request() -> None: pool = HTTPSConnectionPool( - self.host, self.port, assert_fingerprint=fingerprint + self.host, + self.port, + assert_fingerprint=fingerprint, + cert_reqs="CERT_NONE", ) try: timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT) @@ -1388,9 +1393,20 @@ def request() -> None: with pytest.raises(MaxRetryError) as cm: request() assert type(cm.value.reason) is SSLError + assert str(cm.value.reason) == ( + "Fingerprints did not match. Expected " + '"a0c4a74600eda72dc0becb9a8cb607ca58ee745e", got ' + '"728b554c9afc1e88a11cad1bb2e7cc3edbc8f98a"' + ) # Should not hang, see https://github.com/urllib3/urllib3/issues/529 - with pytest.raises(MaxRetryError): + with pytest.raises(MaxRetryError) as cm2: request() + assert type(cm2.value.reason) is SSLError + assert str(cm2.value.reason) == ( + "Fingerprints did not match. Expected " + '"a0c4a74600eda72dc0becb9a8cb607ca58ee745e", got ' + '"728b554c9afc1e88a11cad1bb2e7cc3edbc8f98a"' + ) def test_retry_ssl_error(self) -> None: def socket_handler(listener: socket.socket) -> None: From 56e606dfbf25f23f8bdb5f3d574d3278d3addab2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 31 Dec 2023 16:19:40 +0000 Subject: [PATCH 062/131] Raise ResourceWarnings as errors --- dummyserver/app.py | 3 +- dummyserver/asgi_proxy.py | 15 ++-- dummyserver/tornadoserver.py | 17 ++-- pyproject.toml | 3 +- test/test_ssltransport.py | 47 ++++++----- test/with_dummyserver/test_connection.py | 18 +++-- test/with_dummyserver/test_https.py | 56 ++++--------- .../test_proxy_poolmanager.py | 79 +++++++++++++++---- test/with_dummyserver/test_socketlevel.py | 64 +++++++++------ 9 files changed, 175 insertions(+), 127 deletions(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index a2eff8764b..692c31441b 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -348,7 +348,8 @@ async def pyodide_upload() -> ResponseTypes: file = files["filefield"] if file.filename != "lolcat.txt": return await make_response(f"File name incorrect {file.name}", 404) - data = file.read().decode("utf-8") + with contextlib.closing(file): + data = file.read().decode("utf-8") if data != "I'm in ur multipart form-data, hazing a cheezburgr": return await make_response(f"File data incorrect {data}", 200) return await make_response("Uploaded file correct", 200) diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py index 7b16303945..107c5e0af7 100755 --- a/dummyserver/asgi_proxy.py +++ b/dummyserver/asgi_proxy.py @@ -99,13 +99,12 @@ async def start_forward( await writer.aclose() host, port = scope["path"].split(":") - upstream = await trio.open_tcp_stream(host, int(port)) + async with await trio.open_tcp_stream(host, int(port)) as upstream: + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": True}) - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": True}) + client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) - client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) - - async with trio.open_nursery(strict_exception_groups=True) as nursery: - nursery.start_soon(start_forward, client, upstream) - nursery.start_soon(start_forward, upstream, client) + async with trio.open_nursery(strict_exception_groups=True) as nursery: + nursery.start_soon(start_forward, client, upstream) + nursery.start_soon(start_forward, upstream, client) diff --git a/dummyserver/tornadoserver.py b/dummyserver/tornadoserver.py index 64c31c7227..6fd874d282 100755 --- a/dummyserver/tornadoserver.py +++ b/dummyserver/tornadoserver.py @@ -134,17 +134,18 @@ def _start_server(self) -> None: sock = socket.socket(socket.AF_INET) if sys.platform != "win32": sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.host, 0)) - self.port = sock.getsockname()[1] - # Once listen() returns, the server socket is ready - sock.listen(1) + with sock: + sock.bind((self.host, 0)) + self.port = sock.getsockname()[1] - if self.ready_event: - self.ready_event.set() + # Once listen() returns, the server socket is ready + sock.listen(1) - self.socket_handler(sock) - sock.close() + if self.ready_event: + self.ready_event.set() + + self.socket_handler(sock) def run(self) -> None: self._start_server() diff --git a/pyproject.toml b/pyproject.toml index e2659207e9..962330940c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,8 +91,9 @@ filterwarnings = [ '''default:ssl\.TLSVersion\.TLSv1_1 is deprecated:DeprecationWarning''', '''default:ssl\.PROTOCOL_TLSv1_1 is deprecated:DeprecationWarning''', '''default:ssl\.PROTOCOL_TLSv1_2 is deprecated:DeprecationWarning''', - '''default:unclosed .*:ResourceWarning''', '''default:ssl NPN is deprecated, use ALPN instead:DeprecationWarning''', + '''default:Async generator 'quart\.wrappers\.response\.DataBody\.__aiter__\.\._aiter' was garbage collected.*:ResourceWarning''', # https://github.com/pallets/quart/issues/301 + '''default:unclosed file <_io\.BufferedWriter name='/dev/null'>:ResourceWarning''', # https://github.com/SeleniumHQ/selenium/issues/13328 ] [tool.isort] diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 4fa2d84cb6..49d45f6b02 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -184,33 +184,32 @@ def test_unwrap_existing_socket(self) -> None: """ def shutdown_handler(listener: socket.socket) -> None: - sock = listener.accept()[0] - ssl_sock = self.server_context.wrap_socket(sock, server_side=True) - - request = consume_socket(ssl_sock) - validate_request(request) - ssl_sock.sendall(sample_response()) + with listener.accept()[0] as sock, self.server_context.wrap_socket( + sock, server_side=True + ) as ssl_sock: + request = consume_socket(ssl_sock) + validate_request(request) + ssl_sock.sendall(sample_response()) + + with ssl_sock.unwrap() as unwrapped_sock: + request = consume_socket(unwrapped_sock) + validate_request(request) + unwrapped_sock.sendall(sample_response()) - unwrapped_sock = ssl_sock.unwrap() + self.start_dummy_server(shutdown_handler) + with socket.create_connection((self.host, self.port)) as sock: + ssock = SSLTransport(sock, self.client_context, server_hostname="localhost") - request = consume_socket(unwrapped_sock) - validate_request(request) - unwrapped_sock.sendall(sample_response()) + # request/response over TLS. + ssock.sendall(sample_request()) + response = consume_socket(ssock) + validate_response(response) - self.start_dummy_server(shutdown_handler) - sock = socket.create_connection((self.host, self.port)) - ssock = SSLTransport(sock, self.client_context, server_hostname="localhost") - - # request/response over TLS. - ssock.sendall(sample_request()) - response = consume_socket(ssock) - validate_response(response) - - # request/response over plaintext after unwrap. - ssock.unwrap() - sock.sendall(sample_request()) - response = consume_socket(sock) - validate_response(response) + # request/response over plaintext after unwrap. + ssock.unwrap() + sock.sendall(sample_request()) + response = consume_socket(sock) + validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) def test_ssl_object_attributes(self) -> None: diff --git a/test/with_dummyserver/test_connection.py b/test/with_dummyserver/test_connection.py index 0a9f738328..29f3786e27 100644 --- a/test/with_dummyserver/test_connection.py +++ b/test/with_dummyserver/test_connection.py @@ -33,14 +33,16 @@ def test_returns_urllib3_HTTPResponse(pool: HTTPConnectionPool) -> None: @pytest.mark.skipif(not hasattr(sys, "audit"), reason="requires python 3.8+") @mock.patch("urllib3.connection.sys.audit") def test_audit_event(audit_mock: mock.Mock, pool: HTTPConnectionPool) -> None: - conn = pool._get_conn() - conn.request("GET", "/") - audit_mock.assert_any_call("http.client.connect", conn, conn.host, conn.port) - # Ensure the event is raised only once. - connect_events = [ - call for call in audit_mock.mock_calls if call.args[0] == "http.client.connect" - ] - assert len(connect_events) == 1 + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + audit_mock.assert_any_call("http.client.connect", conn, conn.host, conn.port) + # Ensure the event is raised only once. + connect_events = [ + call + for call in audit_mock.mock_calls + if call.args[0] == "http.client.connect" + ] + assert len(connect_events) == 1 def test_does_not_release_conn(pool: HTTPConnectionPool) -> None: diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 81032bbd13..d99cdd486f 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -223,8 +223,8 @@ def test_verified(self) -> None: ca_certs=DEFAULT_CA, ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with warnings.catch_warnings(record=True) as w: r = https_pool.request("GET", "/") @@ -238,8 +238,8 @@ def test_verified_with_context(self) -> None: ) ctx.load_verify_locations(cafile=DEFAULT_CA) with HTTPSConnectionPool(self.host, self.port, ssl_context=ctx) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with mock.patch("warnings.warn") as warn: r = https_pool.request("GET", "/") @@ -253,8 +253,8 @@ def test_context_combines_with_ca_certs(self) -> None: with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, ssl_context=ctx ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with mock.patch("warnings.warn") as warn: r = https_pool.request("GET", "/") @@ -274,8 +274,8 @@ def test_ca_dir_verified(self, tmp_path: Path) -> None: ca_cert_dir=str(tmp_path), ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with warnings.catch_warnings(record=True) as w: r = https_pool.request("GET", "/") @@ -323,14 +323,11 @@ def test_wrap_socket_failure_resource_leak(self) -> None: ca_certs=self.bad_ca_path, ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with pytest.raises(ssl.SSLError): conn.connect() assert conn.sock is not None # type: ignore[attr-defined] - finally: - conn.close() def test_verified_without_ca_certs(self) -> None: # default is cert_reqs=None which is ssl.CERT_NONE @@ -614,8 +611,7 @@ def test_tunnel(self) -> None: cert_reqs="CERT_NONE", ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - try: + with contextlib.closing(https_pool._new_conn()) as conn: conn.set_tunnel(self.host, self.port) with mock.patch.object( conn, "_tunnel", create=True, return_value=None @@ -623,8 +619,6 @@ def test_tunnel(self) -> None: with pytest.warns(InsecureRequestWarning): https_pool._make_request(conn, "GET", "/") conn_tunnel.assert_called_once_with() - finally: - conn.close() @requires_network() def test_enhanced_timeout(self) -> None: @@ -635,14 +629,11 @@ def test_enhanced_timeout(self) -> None: retries=False, cert_reqs="CERT_REQUIRED", ) as https_pool: - conn = https_pool._new_conn() - try: + with contextlib.closing(https_pool._new_conn()) as conn: with pytest.raises(ConnectTimeoutError): https_pool.request("GET", "/") with pytest.raises(ConnectTimeoutError): https_pool._make_request(conn, "GET", "/") - finally: - conn.close() with HTTPSConnectionPool( TARPIT_HOST, @@ -661,14 +652,11 @@ def test_enhanced_timeout(self) -> None: retries=False, cert_reqs="CERT_REQUIRED", ) as https_pool: - conn = https_pool._new_conn() - try: + with contextlib.closing(https_pool._new_conn()) as conn: with pytest.raises(ConnectTimeoutError): https_pool.request( "GET", "/", timeout=Timeout(total=None, connect=SHORT_TIMEOUT) ) - finally: - conn.close() def test_enhanced_ssl_connection(self) -> None: fingerprint = "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" @@ -798,14 +786,11 @@ def test_tls_protocol_name_of_socket(self) -> None: ssl_minimum_version=self.tls_version(), ssl_maximum_version=self.tls_version(), ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: conn.connect() if not hasattr(conn.sock, "version"): # type: ignore[attr-defined] pytest.skip("SSLSocket.version() not available") assert conn.sock.version() == self.tls_protocol_name # type: ignore[attr-defined] - finally: - conn.close() def test_ssl_version_is_deprecated(self) -> None: if self.tls_protocol_name is None: @@ -814,12 +799,9 @@ def test_ssl_version_is_deprecated(self) -> None: with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, ssl_version=self.ssl_version() ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with pytest.warns(DeprecationWarning) as w: conn.connect() - finally: - conn.close() assert len(w) >= 1 assert any(x.category == DeprecationWarning for x in w) @@ -848,12 +830,9 @@ def test_ssl_version_with_protocol_tls_or_client_not_deprecated( with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, ssl_version=ssl_version ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with warnings.catch_warnings(record=True) as w: conn.connect() - finally: - conn.close() assert [str(wm) for wm in w if wm.category != ResourceWarning] == [] @@ -869,12 +848,9 @@ def test_no_tls_version_deprecation_with_ssl_context(self) -> None: ca_certs=DEFAULT_CA, ssl_context=ctx, ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with warnings.catch_warnings(record=True) as w: conn.connect() - finally: - conn.close() assert [str(wm) for wm in w if wm.category != ResourceWarning] == [] diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index dae1013f19..15164b59a9 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -1,6 +1,7 @@ from __future__ import annotations import binascii +import contextlib import hashlib import ipaddress import os.path @@ -187,9 +188,11 @@ def test_proxy_verified(self) -> None: with proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=self.bad_ca_path ) as http: - https_pool = http._new_pool("https", self.https_host, self.https_port) - with pytest.raises(MaxRetryError) as e: - https_pool.request("GET", "/", retries=0) + with http._new_pool( + "https", self.https_host, self.https_port + ) as https_pool: + with pytest.raises(MaxRetryError) as e: + https_pool.request("GET", "/", retries=0) assert isinstance(e.value.reason, SSLError) assert ( "certificate verify failed" in str(e.value.reason) @@ -200,22 +203,26 @@ def test_proxy_verified(self) -> None: http = proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=DEFAULT_CA ) - https_pool = http._new_pool("https", self.https_host, self.https_port) - - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection - https_pool.request("GET", "/") # Should succeed without exceptions. + with http._new_pool( + "https", self.https_host, self.https_port + ) as https_pool2: + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection + https_pool2.request( + "GET", "/" + ) # Should succeed without exceptions. http = proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=DEFAULT_CA ) - https_fail_pool = http._new_pool("https", "127.0.0.1", self.https_port) - - with pytest.raises( - MaxRetryError, match="doesn't match|IP address mismatch" - ) as e: - https_fail_pool.request("GET", "/", retries=0) - assert isinstance(e.value.reason, SSLError) + with http._new_pool( + "https", "127.0.0.1", self.https_port + ) as https_fail_pool: + with pytest.raises( + MaxRetryError, match="doesn't match|IP address mismatch" + ) as e: + https_fail_pool.request("GET", "/", retries=0) + assert isinstance(e.value.reason, SSLError) def test_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: @@ -488,6 +495,9 @@ def test_forwarding_proxy_request_timeout( # target so we put the blame on the target. assert isinstance(e.value.reason, ReadTimeoutError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")] @@ -508,6 +518,9 @@ def test_tunneling_proxy_request_timeout( assert isinstance(e.value.reason, ReadTimeoutError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme", "use_forwarding_for_https"], @@ -536,6 +549,9 @@ def test_forwarding_proxy_connect_timeout( assert isinstance(e.value.reason, ProxyError) assert isinstance(e.value.reason.original_error, ConnectTimeoutError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")] @@ -555,6 +571,9 @@ def test_tunneling_proxy_connect_timeout( assert isinstance(e.value.reason, ProxyError) assert isinstance(e.value.reason.original_error, ConnectTimeoutError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") @requires_network() @pytest.mark.parametrize( ["target_scheme", "use_forwarding_for_https"], @@ -579,6 +598,9 @@ def test_https_proxy_tls_error( assert isinstance(e.value.reason, ProxyError) assert isinstance(e.value.reason.original_error, SSLError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "use_forwarding_for_https"], @@ -609,6 +631,9 @@ def test_proxy_https_target_tls_error( proxy.request("GET", self.https_url) assert isinstance(e.value.reason, SSLError) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_scheme_host_case_insensitive(self) -> None: """Assert that upper-case schemes and hosts are normalized.""" with proxy_from_url(self.proxy_url.upper(), ca_certs=DEFAULT_CA) as http: @@ -635,6 +660,9 @@ def test_scheme_host_case_insensitive(self) -> None: ), ], ) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_invalid_schema(self, url: str, error_msg: str) -> None: with pytest.raises(ProxySchemeUnknown, match=error_msg): proxy_from_url(url) @@ -651,6 +679,9 @@ def setup_class(cls) -> None: cls.https_url_alt = f"https://{cls.https_host_alt}:{int(cls.https_port)}" cls.proxy_url = f"http://[{cls.proxy_host}]:{int(cls.proxy_port)}" + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_basic_ipv6_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -682,6 +713,9 @@ def _get_certificate_formatted_proxy_host(host: str) -> str: # Transform ipv6 like '::1' to 0:0:0:0:0:0:0:1 via '0000:0000:0000:0000:0000:0000:0000:0001' return addr.exploded.replace("0000", "0").replace("000", "") + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_assert_fingerprint_md5( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -697,6 +731,9 @@ def test_https_proxy_assert_fingerprint_md5( ) as https: https.request("GET", destination_url) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_assert_fingerprint_md5_non_matching( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -718,6 +755,9 @@ def test_https_proxy_assert_fingerprint_md5_non_matching( assert "Fingerprints did not match" in str(e) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_assert_hostname( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -729,6 +769,9 @@ def test_https_proxy_assert_hostname( ) as https: https.request("GET", destination_url) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_assert_hostname_non_matching( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -748,6 +791,9 @@ def test_https_proxy_assert_hostname_non_matching( msg = f"hostname \\'{proxy_hostname}\\' doesn\\'t match \\'{proxy_host}\\'" assert msg in str(e) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_hostname_verification( self, no_localhost_san_server: ServerConfig ) -> None: @@ -777,6 +823,9 @@ def test_https_proxy_hostname_verification( ssl_error ) or "Hostname mismatch" in str(ssl_error) + # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning + # see https://github.com/python/cpython/issues/103472 + @pytest.mark.filterwarnings("default::ResourceWarning") def test_https_proxy_ipv4_san( self, ipv4_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index d89f311c33..3d07793840 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1267,30 +1267,28 @@ def http_socket_handler(listener: socket.socket) -> None: class TestSSL(SocketDummyServerTestCase): def test_ssl_failure_midway_through_conn(self) -> None: def socket_handler(listener: socket.socket) -> None: - sock = listener.accept()[0] - sock2 = sock.dup() - ssl_sock = original_ssl_wrap_socket( - sock, - server_side=True, - keyfile=DEFAULT_CERTS["keyfile"], - certfile=DEFAULT_CERTS["certfile"], - ca_certs=DEFAULT_CA, - ) + with listener.accept()[0] as sock, sock.dup() as sock2: + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) - buf = b"" - while not buf.endswith(b"\r\n\r\n"): - buf += ssl_sock.recv(65536) + buf = b"" + while not buf.endswith(b"\r\n\r\n"): + buf += ssl_sock.recv(65536) - # Deliberately send from the non-SSL socket. - sock2.send( - b"HTTP/1.1 200 OK\r\n" - b"Content-Type: text/plain\r\n" - b"Content-Length: 2\r\n" - b"\r\n" - b"Hi" - ) - sock2.close() - ssl_sock.close() + # Deliberately send from the non-SSL socket. + sock2.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 2\r\n" + b"\r\n" + b"Hi" + ) + ssl_sock.close() self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA) as pool: @@ -1491,6 +1489,17 @@ def socket_handler(listener: socket.socket) -> None: context.load_default_certs = mock.Mock() context.options = 0 + class MockSSLSocket: + def __init__( + self, sock: socket.socket, *args: object, **kwargs: object + ) -> None: + self._sock = sock + + def close(self) -> None: + self._sock.close() + + context.wrap_socket = MockSSLSocket + with mock.patch("urllib3.util.ssl_.SSLContext", lambda *_, **__: context): self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port) as pool: @@ -1533,6 +1542,17 @@ def socket_handler(listener: socket.socket) -> None: context.load_default_certs = mock.Mock() context.options = 0 + class MockSSLSocket: + def __init__( + self, sock: socket.socket, *args: object, **kwargs: object + ) -> None: + self._sock = sock + + def close(self) -> None: + self._sock.close() + + context.wrap_socket = MockSSLSocket + with mock.patch("urllib3.util.ssl_.SSLContext", lambda *_, **__: context): for kwargs in [ {"ca_certs": "/a"}, From 59c122730badac2426dfbdecfe6be8afe0cd21c2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 1 Jan 2024 23:22:22 +0400 Subject: [PATCH 063/131] Bump RECENT_DATE --- src/urllib3/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 2a988500d5..42b1bbc5e5 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -73,7 +73,7 @@ class BaseSSLError(BaseException): # type: ignore[no-redef] # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2022, 1, 1) +RECENT_DATE = datetime.date(2023, 6, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") From 11a92fa416556c4a4d40306ecc7b67f83b20bac0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 1 Jan 2024 23:46:24 +0400 Subject: [PATCH 064/131] Remove Tornado from test suite This also removes all direct imports of asyncio as we're currently using Trio for our test server. Co-authored-by: Seth Michael Larson --- dev-requirements.txt | 1 - dummyserver/handlers.py | 365 ------------------ dummyserver/https_proxy.py | 48 --- dummyserver/proxy.py | 133 ------- .../{tornadoserver.py => socketserver.py} | 126 ------ dummyserver/testcase.py | 158 +------- mypy-requirements.txt | 1 - test/conftest.py | 2 +- test/contrib/emscripten/conftest.py | 2 +- test/contrib/test_socks.py | 2 +- test/test_collections.py | 2 +- test/test_connectionpool.py | 2 +- test/test_ssltransport.py | 2 +- test/with_dummyserver/test_connectionpool.py | 2 +- test/with_dummyserver/test_https.py | 4 +- test/with_dummyserver/test_poolmanager.py | 2 +- .../test_proxy_poolmanager.py | 2 +- test/with_dummyserver/test_socketlevel.py | 4 +- 18 files changed, 14 insertions(+), 844 deletions(-) delete mode 100644 dummyserver/handlers.py delete mode 100755 dummyserver/https_proxy.py delete mode 100755 dummyserver/proxy.py rename dummyserver/{tornadoserver.py => socketserver.py} (56%) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0438cc8e22..c0636b0bb1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,4 @@ coverage==7.3.2 -tornado==6.3.3 PySocks==1.7.1 pytest==7.4.2 pytest-timeout==2.1.0 diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py deleted file mode 100644 index 84493ab82d..0000000000 --- a/dummyserver/handlers.py +++ /dev/null @@ -1,365 +0,0 @@ -from __future__ import annotations - -import collections -import contextlib -import gzip -import json -import logging -import sys -import typing -import zlib -from datetime import datetime, timedelta, timezone -from http.client import responses -from io import BytesIO -from urllib.parse import urlsplit - -from tornado import httputil -from tornado.web import RequestHandler - -from urllib3.util.util import to_str - -log = logging.getLogger(__name__) - - -class Response: - def __init__( - self, - body: str | bytes | typing.Sequence[str | bytes] = "", - status: str = "200 OK", - headers: typing.Sequence[tuple[str, str | bytes]] | None = None, - json: typing.Any | None = None, - ) -> None: - self.body = body - self.status = status - if json is not None: - self.headers = headers or [("Content-type", "application/json")] - self.body = json - else: - self.headers = headers or [("Content-type", "text/plain")] - - def __call__(self, request_handler: RequestHandler) -> None: - status, reason = self.status.split(" ", 1) - request_handler.set_status(int(status), reason) - for header, value in self.headers: - request_handler.add_header(header, value) - - if isinstance(self.body, str): - request_handler.write(self.body.encode()) - elif isinstance(self.body, bytes): - request_handler.write(self.body) - # chunked - else: - for item in self.body: - if not isinstance(item, bytes): - item = item.encode("utf8") - request_handler.write(item) - request_handler.flush() - - -RETRY_TEST_NAMES: dict[str, int] = collections.defaultdict(int) - - -def request_params(request: httputil.HTTPServerRequest) -> dict[str, bytes]: - params = {} - for k, v in request.arguments.items(): - params[k] = next(iter(v)) - return params - - -class TestingApp(RequestHandler): - """ - Simple app that performs various operations, useful for testing an HTTP - library. - - Given any path, it will attempt to load a corresponding local method if - it exists. Status code 200 indicates success, 400 indicates failure. Each - method has its own conditions for success/failure. - """ - - def get(self) -> None: - """Handle GET requests""" - self._call_method() - - def post(self) -> None: - """Handle POST requests""" - self._call_method() - - def put(self) -> None: - """Handle PUT requests""" - self._call_method() - - def options(self) -> None: - """Handle OPTIONS requests""" - self._call_method() - - def head(self) -> None: - """Handle HEAD requests""" - self._call_method() - - def _call_method(self) -> None: - """Call the correct method in this class based on the incoming URI""" - req = self.request - - path = req.path[:] - if not path.startswith("/"): - path = urlsplit(path).path - - target = path[1:].split("/", 1)[0] - method = getattr(self, target, self.index) - - resp = method(req) - - if dict(resp.headers).get("Connection") == "close": - # FIXME: Can we kill the connection somehow? - pass - - resp(self) - - def index(self, _request: httputil.HTTPServerRequest) -> Response: - "Render simple message" - return Response("Dummy server!") - - def certificate(self, request: httputil.HTTPServerRequest) -> Response: - """Return the requester's certificate.""" - cert = request.get_ssl_certificate() - assert isinstance(cert, dict) - subject = {} - if cert is not None: - subject = {k: v for (k, v) in [y for z in cert["subject"] for y in z]} - return Response(json.dumps(subject)) - - def alpn_protocol(self, request: httputil.HTTPServerRequest) -> Response: - """Return the selected ALPN protocol.""" - assert request.connection is not None - proto = request.connection.stream.socket.selected_alpn_protocol() # type: ignore[attr-defined] - return Response(proto.encode("utf8") if proto is not None else "") - - def source_address(self, request: httputil.HTTPServerRequest) -> Response: - """Return the requester's IP address.""" - return Response(request.remote_ip) # type: ignore[arg-type] - - def set_up(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - test_type = params.get("test_type") - test_id = params.get("test_id") - if test_id: - print(f"\nNew test {test_type!r}: {test_id!r}") - else: - print(f"\nNew test {test_type!r}") - return Response("Dummy server is ready!") - - def specific_method(self, request: httputil.HTTPServerRequest) -> Response: - "Confirm that the request matches the desired method type" - params = request_params(request) - method = params.get("method") - method_str = method.decode() if method else None - - if request.method != method_str: - return Response( - f"Wrong method: {method_str} != {request.method}", - status="400 Bad Request", - ) - return Response() - - def upload(self, request: httputil.HTTPServerRequest) -> Response: - "Confirm that the uploaded file conforms to specification" - params = request_params(request) - # FIXME: This is a huge broken mess - param = params.get("upload_param", b"myfile").decode("ascii") - filename = params.get("upload_filename", b"").decode("utf-8") - size = int(params.get("upload_size", "0")) - files_ = request.files.get(param) - assert files_ is not None - - if len(files_) != 1: - return Response( - f"Expected 1 file for '{param}', not {len(files_)}", - status="400 Bad Request", - ) - file_ = files_[0] - - data = file_["body"] - if int(size) != len(data): - return Response( - f"Wrong size: {int(size)} != {len(data)}", status="400 Bad Request" - ) - - got_filename = file_["filename"] - if isinstance(got_filename, bytes): - got_filename = got_filename.decode("utf-8") - - # Tornado can leave the trailing \n in place on the filename. - if filename != got_filename: - return Response( - f"Wrong filename: {filename} != {file_.filename}", - status="400 Bad Request", - ) - - return Response() - - def redirect(self, request: httputil.HTTPServerRequest) -> Response: # type: ignore[override] - "Perform a redirect to ``target``" - params = request_params(request) - target = params.get("target", "/") - status = params.get("status", b"303 See Other").decode("latin-1") - if len(status) == 3: - status = f"{status} Redirect" - - headers = [("Location", target)] - return Response(status=status, headers=headers) - - def not_found(self, request: httputil.HTTPServerRequest) -> Response: - return Response("Not found", status="404 Not Found") - - def multi_redirect(self, request: httputil.HTTPServerRequest) -> Response: - "Performs a redirect chain based on ``redirect_codes``" - params = request_params(request) - codes = params.get("redirect_codes", b"200").decode("utf-8") - head, tail = codes.split(",", 1) if "," in codes else (codes, None) - assert head is not None - status = f"{head} {responses[int(head)]}" - if not tail: - return Response("Done redirecting", status=status) - - headers = [("Location", f"/multi_redirect?redirect_codes={tail}")] - return Response(status=status, headers=headers) - - def keepalive(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - if params.get("close", b"0") == b"1": - headers = [("Connection", "close")] - return Response("Closing", headers=headers) - - headers = [("Connection", "keep-alive")] - return Response("Keeping alive", headers=headers) - - def echo_params(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - echod = sorted((to_str(k), to_str(v)) for k, v in params.items()) - return Response(repr(echod)) - - def echo(self, request: httputil.HTTPServerRequest) -> Response: - "Echo back the params" - if request.method == "GET": - return Response(request.query) - - return Response(request.body) - - def echo_json(self, request: httputil.HTTPServerRequest) -> Response: - "Echo back the JSON" - print("ECHO JSON:", request.body) - return Response(json=request.body, headers=list(request.headers.items())) - - def echo_uri(self, request: httputil.HTTPServerRequest) -> Response: - "Echo back the requested URI" - assert request.uri is not None - return Response(request.uri) - - def encodingrequest(self, request: httputil.HTTPServerRequest) -> Response: - "Check for UA accepting gzip/deflate encoding" - data = b"hello, world!" - encoding = request.headers.get("Accept-Encoding", "") - headers = None - if encoding == "gzip": - headers = [("Content-Encoding", "gzip")] - file_ = BytesIO() - with contextlib.closing( - gzip.GzipFile("", mode="w", fileobj=file_) - ) as zipfile: - zipfile.write(data) - data = file_.getvalue() - elif encoding == "deflate": - headers = [("Content-Encoding", "deflate")] - data = zlib.compress(data) - elif encoding == "garbage-gzip": - headers = [("Content-Encoding", "gzip")] - data = b"garbage" - elif encoding == "garbage-deflate": - headers = [("Content-Encoding", "deflate")] - data = b"garbage" - return Response(data, headers=headers) - - def headers(self, request: httputil.HTTPServerRequest) -> Response: - return Response(json.dumps(dict(request.headers))) - - def headers_and_params(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - return Response( - json.dumps({"headers": dict(request.headers), "params": params}) - ) - - def multi_headers(self, request: httputil.HTTPServerRequest) -> Response: - return Response(json.dumps({"headers": list(request.headers.get_all())})) - - def successful_retry(self, request: httputil.HTTPServerRequest) -> Response: - """Handler which will return an error and then success - - It's not currently very flexible as the number of retries is hard-coded. - """ - test_name = request.headers.get("test-name", None) - if not test_name: - return Response("test-name header not set", status="400 Bad Request") - - RETRY_TEST_NAMES[test_name] += 1 - - if RETRY_TEST_NAMES[test_name] >= 2: - return Response("Retry successful!") - else: - return Response("need to keep retrying!", status="418 I'm A Teapot") - - def chunked(self, request: httputil.HTTPServerRequest) -> Response: - return Response(["123"] * 4) - - def chunked_gzip(self, request: httputil.HTTPServerRequest) -> Response: - chunks = [] - compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) - - for uncompressed in [b"123"] * 4: - chunks.append(compressor.compress(uncompressed)) - - chunks.append(compressor.flush()) - - return Response(chunks, headers=[("Content-Encoding", "gzip")]) - - def nbytes(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - length = int(params["length"]) - data = b"1" * length - return Response(data, headers=[("Content-Type", "application/octet-stream")]) - - def status(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - status = params.get("status", b"200 OK").decode("latin-1") - - return Response(status=status) - - def retry_after(self, request: httputil.HTTPServerRequest) -> Response: - params = request_params(request) - if datetime.now() - self.application.last_req < timedelta(seconds=1): # type: ignore[attr-defined] - status = params.get("status", b"429 Too Many Requests") - return Response( - status=status.decode("utf-8"), headers=[("Retry-After", "1")] - ) - - self.application.last_req = datetime.now() # type: ignore[attr-defined] - - return Response(status="200 OK") - - def redirect_after(self, request: httputil.HTTPServerRequest) -> Response: - "Perform a redirect to ``target``" - params = request_params(request) - date = params.get("date") - if date: - retry_after = str( - httputil.format_timestamp( - datetime.fromtimestamp(float(date), tz=timezone.utc) - ) - ) - else: - retry_after = "1" - target = params.get("target", "/") - headers = [("Location", target), ("Retry-After", retry_after)] - return Response(status="303 See Other", headers=headers) - - def shutdown(self, request: httputil.HTTPServerRequest) -> typing.NoReturn: - sys.exit() diff --git a/dummyserver/https_proxy.py b/dummyserver/https_proxy.py deleted file mode 100755 index 87c6eceeab..0000000000 --- a/dummyserver/https_proxy.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -from __future__ import annotations - -import sys -import typing - -import tornado.httpserver -import tornado.ioloop -import tornado.web - -from dummyserver.proxy import ProxyHandler -from dummyserver.tornadoserver import DEFAULT_CERTS, ssl_options_to_context - - -def run_proxy(port: int, certs: dict[str, typing.Any] = DEFAULT_CERTS) -> None: - """ - Run proxy on the specified port using the provided certs. - - Example usage: - - python -m dummyserver.https_proxy - - You'll need to ensure you have access to certain packages such as trustme, - tornado, urllib3. - """ - upstream_ca_certs = certs.get("ca_certs") - app = tornado.web.Application( - [(r".*", ProxyHandler)], upstream_ca_certs=upstream_ca_certs - ) - ssl_opts = ssl_options_to_context(**certs) - http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts) - http_server.listen(port) - - ioloop = tornado.ioloop.IOLoop.instance() - try: - ioloop.start() - except KeyboardInterrupt: - ioloop.stop() - - -if __name__ == "__main__": - port = 8443 - if len(sys.argv) > 1: - port = int(sys.argv[1]) - - print(f"Starting HTTPS proxy on port {port}") - run_proxy(port) diff --git a/dummyserver/proxy.py b/dummyserver/proxy.py deleted file mode 100755 index 45fb44e402..0000000000 --- a/dummyserver/proxy.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -# Simple asynchronous HTTP proxy with tunnelling (CONNECT). -# -# GET/POST proxying based on -# http://groups.google.com/group/python-tornado/msg/7bea08e7a049cf26 -# -# Copyright (C) 2012 Senko Rasic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import annotations - -import socket -import ssl -import sys - -import tornado.gen -import tornado.httpclient -import tornado.httpserver -import tornado.ioloop -import tornado.iostream -import tornado.web - -__all__ = ["ProxyHandler", "run_proxy"] - - -class ProxyHandler(tornado.web.RequestHandler): - SUPPORTED_METHODS = ["GET", "POST", "CONNECT"] # type: ignore[assignment] - - async def get(self) -> None: - upstream_ca_certs = self.application.settings.get("upstream_ca_certs", None) - ssl_options = None - - if upstream_ca_certs: - ssl_options = ssl.create_default_context(cafile=upstream_ca_certs) - - assert self.request.uri is not None - assert self.request.method is not None - req = tornado.httpclient.HTTPRequest( - url=self.request.uri, - method=self.request.method, - body=self.request.body, - headers=self.request.headers, - follow_redirects=False, - allow_nonstandard_methods=True, - ssl_options=ssl_options, - ) - - client = tornado.httpclient.AsyncHTTPClient() - response = await client.fetch(req, raise_error=False) - self.set_status(response.code) - for header in ( - "Date", - "Cache-Control", - "Server", - "Content-Type", - "Location", - ): - v = response.headers.get(header) - if v: - self.set_header(header, v) - if response.body: - self.write(response.body) - await self.finish() - - async def post(self) -> None: - await self.get() - - async def connect(self) -> None: - assert self.request.uri is not None - host, port = self.request.uri.split(":") - assert self.request.connection is not None - client: tornado.iostream.IOStream = self.request.connection.stream # type: ignore[attr-defined] - - async def start_forward( - reader: tornado.iostream.IOStream, writer: tornado.iostream.IOStream - ) -> None: - while True: - try: - data = await reader.read_bytes(4096, partial=True) - except tornado.iostream.StreamClosedError: - break - if not data: - break - writer.write(data) - writer.close() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - upstream = tornado.iostream.IOStream(s) - await upstream.connect((host, int(port))) - - client.write(b"HTTP/1.0 200 Connection established\r\n\r\n") - fu1 = start_forward(client, upstream) - fu2 = start_forward(upstream, client) - await tornado.gen.multi([fu1, fu2]) - - -def run_proxy(port: int, start_ioloop: bool = True) -> None: - """ - Run proxy on the specified port. If start_ioloop is True (default), - the tornado IOLoop will be started immediately. - """ - app = tornado.web.Application([(r".*", ProxyHandler)]) - app.listen(port) - ioloop = tornado.ioloop.IOLoop.instance() - if start_ioloop: - ioloop.start() - - -if __name__ == "__main__": - port = 8888 - if len(sys.argv) > 1: - port = int(sys.argv[1]) - - print(f"Starting HTTP proxy on port {port}") - run_proxy(port) diff --git a/dummyserver/tornadoserver.py b/dummyserver/socketserver.py similarity index 56% rename from dummyserver/tornadoserver.py rename to dummyserver/socketserver.py index 6fd874d282..488fc729f8 100755 --- a/dummyserver/tornadoserver.py +++ b/dummyserver/socketserver.py @@ -6,10 +6,6 @@ from __future__ import annotations -import asyncio -import concurrent.futures -import contextlib -import errno import logging import os import socket @@ -18,13 +14,7 @@ import threading import typing import warnings -from collections.abc import Coroutine, Generator -from datetime import datetime -import tornado.httpserver -import tornado.ioloop -import tornado.netutil -import tornado.web import trustme from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -184,46 +174,6 @@ def ssl_options_to_context( # type: ignore[no-untyped-def] return ctx -def run_tornado_app( - app: tornado.web.Application, - certs: dict[str, typing.Any] | None, - scheme: str, - host: str, -) -> tuple[tornado.httpserver.HTTPServer, int]: - # We can't use fromtimestamp(0) because of CPython issue 29097, so we'll - # just construct the datetime object directly. - app.last_req = datetime(1970, 1, 1) # type: ignore[attr-defined] - - if scheme == "https": - assert certs is not None - ssl_opts = ssl_options_to_context(**certs) - http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts) - else: - http_server = tornado.httpserver.HTTPServer(app) - - # When we request a socket with host localhost and port zero (None in Python), then - # Tornado gets a free IPv4 port and requests that same port in IPv6. But that port - # could easily be taken with IPv6, especially in crowded CI environments. For this - # reason we put bind_sockets in a retry loop. Full details: - # * https://github.com/urllib3/urllib3/issues/2171 - # * https://github.com/tornadoweb/tornado/issues/1860 - for i in range(10): - try: - sockets = tornado.netutil.bind_sockets(None, address=host) # type: ignore[arg-type] - except OSError as e: - if e.errno == errno.EADDRINUSE: - # TODO this should be a warning if there's a way for pytest to print it - print( - f"Retrying bind_sockets({host}) after EADDRINUSE", file=sys.stderr - ) - continue - break - - port = sockets[0].getsockname()[1] - http_server.add_sockets(sockets) - return http_server, port - - def get_unreachable_address() -> tuple[str, int]: # reserved as per rfc2606 return ("something.invalid", 54321) @@ -239,79 +189,3 @@ def encrypt_key_pem(private_key_pem: trustme.Blob, password: bytes) -> trustme.B serialization.BestAvailableEncryption(password), ) return trustme.Blob(encrypted_key) - - -R = typing.TypeVar("R") - - -def _run_and_close_tornado( - async_fn: typing.Callable[P, Coroutine[typing.Any, typing.Any, R]], - *args: P.args, - **kwargs: P.kwargs, -) -> R: - tornado_loop = None - - async def inner_fn() -> R: - nonlocal tornado_loop - tornado_loop = tornado.ioloop.IOLoop.current() - return await async_fn(*args, **kwargs) - - try: - return asyncio.run(inner_fn()) - finally: - tornado_loop.close(all_fds=True) # type: ignore[union-attr] - - -@contextlib.contextmanager -def run_tornado_loop_in_thread() -> Generator[tornado.ioloop.IOLoop, None, None]: - loop_started: concurrent.futures.Future[ - tuple[tornado.ioloop.IOLoop, asyncio.Event] - ] = concurrent.futures.Future() - with concurrent.futures.ThreadPoolExecutor( - 1, thread_name_prefix="test IOLoop" - ) as tpe: - - async def run() -> None: - io_loop = tornado.ioloop.IOLoop.current() - stop_event = asyncio.Event() - loop_started.set_result((io_loop, stop_event)) - await stop_event.wait() - - # run asyncio.run in a thread and collect exceptions from *either* - # the loop failing to start, or failing to close - ran = tpe.submit(_run_and_close_tornado, run) # type: ignore[arg-type] - for f in concurrent.futures.as_completed((loop_started, ran)): # type: ignore[misc] - if f is loop_started: - io_loop, stop_event = loop_started.result() - try: - yield io_loop - finally: - io_loop.add_callback(stop_event.set) - - elif f is ran: - # if this is the first iteration the loop failed to start - # if it's the second iteration the loop has finished or - # the loop failed to close and we need to raise the exception - ran.result() - return - - -def main() -> int: - # For debugging dummyserver itself - PYTHONPATH=src python -m dummyserver.tornadoserver - from .handlers import TestingApp - - host = "127.0.0.1" - - async def amain() -> int: - app = tornado.web.Application([(r".*", TestingApp)]) - server, port = run_tornado_app(app, None, "http", host) - - print(f"Listening on http://{host}:{port}") - await asyncio.Event().wait() - return 0 - - return asyncio.run(amain()) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 0628f8963a..7eed47668b 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import contextlib import socket import ssl @@ -9,20 +8,11 @@ import hypercorn import pytest -from tornado import httpserver, ioloop, web from dummyserver.app import hypercorn_app from dummyserver.asgi_proxy import ProxyApp -from dummyserver.handlers import TestingApp from dummyserver.hypercornserver import run_hypercorn_in_thread -from dummyserver.proxy import ProxyHandler -from dummyserver.tornadoserver import ( - DEFAULT_CERTS, - HAS_IPV6, - SocketServerThread, - run_tornado_app, - run_tornado_loop_in_thread, -) +from dummyserver.socketserver import DEFAULT_CERTS, HAS_IPV6, SocketServerThread from urllib3.connection import HTTPConnection from urllib3.util.ssltransport import SSLTransport from urllib3.util.url import parse_url @@ -152,152 +142,6 @@ def _start_server( cls.port = cls.server_thread.port -class HTTPDummyServerTestCase: - """A simple HTTP server that runs when your test class runs - - Have your test class inherit from this one, and then a simple server - will start when your tests run, and automatically shut down when they - complete. For examples of what test requests you can send to the server, - see the TestingApp in dummyserver/handlers.py. - """ - - scheme = "http" - host = "localhost" - host_alt = "127.0.0.1" # Some tests need two hosts - certs = DEFAULT_CERTS - base_url: typing.ClassVar[str] - base_url_alt: typing.ClassVar[str] - - io_loop: typing.ClassVar[ioloop.IOLoop] - server: typing.ClassVar[httpserver.HTTPServer] - port: typing.ClassVar[int] - server_thread: typing.ClassVar[threading.Thread] - _stack: typing.ClassVar[contextlib.ExitStack] - - @classmethod - def _start_server(cls) -> None: - with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_tornado_loop_in_thread()) - - async def run_app() -> None: - app = web.Application([(r".*", TestingApp)]) - cls.server, cls.port = run_tornado_app( - app, cls.certs, cls.scheme, cls.host - ) - - asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] - cls._stack = stack.pop_all() - - @classmethod - def _stop_server(cls) -> None: - cls._stack.close() - - @classmethod - def setup_class(cls) -> None: - cls._start_server() - - @classmethod - def teardown_class(cls) -> None: - cls._stop_server() - - -class HTTPSDummyServerTestCase(HTTPDummyServerTestCase): - scheme = "https" - host = "localhost" - certs = DEFAULT_CERTS - certs_dir = "" - bad_ca_path = "" - - -class HTTPDummyProxyTestCase: - io_loop: typing.ClassVar[ioloop.IOLoop] - - http_host: typing.ClassVar[str] = "localhost" - http_host_alt: typing.ClassVar[str] = "127.0.0.1" - http_server: typing.ClassVar[httpserver.HTTPServer] - http_port: typing.ClassVar[int] - http_url: typing.ClassVar[str] - http_url_alt: typing.ClassVar[str] - - https_host: typing.ClassVar[str] = "localhost" - https_host_alt: typing.ClassVar[str] = "127.0.0.1" - https_certs: typing.ClassVar[dict[str, typing.Any]] = DEFAULT_CERTS - https_server: typing.ClassVar[httpserver.HTTPServer] - https_port: typing.ClassVar[int] - https_url: typing.ClassVar[str] - https_url_alt: typing.ClassVar[str] - https_url_fqdn: typing.ClassVar[str] - - proxy_host: typing.ClassVar[str] = "localhost" - proxy_host_alt: typing.ClassVar[str] = "127.0.0.1" - proxy_server: typing.ClassVar[httpserver.HTTPServer] - proxy_port: typing.ClassVar[int] - proxy_url: typing.ClassVar[str] - https_proxy_server: typing.ClassVar[httpserver.HTTPServer] - https_proxy_port: typing.ClassVar[int] - https_proxy_url: typing.ClassVar[str] - - certs_dir: typing.ClassVar[str] = "" - bad_ca_path: typing.ClassVar[str] = "" - - server_thread: typing.ClassVar[threading.Thread] - _stack: typing.ClassVar[contextlib.ExitStack] - - @classmethod - def setup_class(cls) -> None: - with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_tornado_loop_in_thread()) - - async def run_app() -> None: - app = web.Application([(r".*", TestingApp)]) - cls.http_server, cls.http_port = run_tornado_app( - app, None, "http", cls.http_host - ) - - app = web.Application([(r".*", TestingApp)]) - cls.https_server, cls.https_port = run_tornado_app( - app, cls.https_certs, "https", cls.http_host - ) - - app = web.Application([(r".*", ProxyHandler)]) - cls.proxy_server, cls.proxy_port = run_tornado_app( - app, None, "http", cls.proxy_host - ) - - upstream_ca_certs = cls.https_certs.get("ca_certs") - app = web.Application( - [(r".*", ProxyHandler)], upstream_ca_certs=upstream_ca_certs - ) - cls.https_proxy_server, cls.https_proxy_port = run_tornado_app( - app, cls.https_certs, "https", cls.proxy_host - ) - - asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] - cls._stack = stack.pop_all() - - @classmethod - def teardown_class(cls) -> None: - cls._stack.close() - - -@pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") -class IPv6HTTPDummyServerTestCase(HTTPDummyServerTestCase): - host = "::1" - - -@pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") -class IPv6HTTPDummyProxyTestCase(HTTPDummyProxyTestCase): - http_host = "localhost" - http_host_alt = "127.0.0.1" - - https_host = "localhost" - https_host_alt = "127.0.0.1" - https_certs = DEFAULT_CERTS - - proxy_host = "::1" - proxy_host_alt = "127.0.0.1" - - class HypercornDummyServerTestCase: host = "localhost" host_alt = "127.0.0.1" diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 0ee5d6f2fe..7861642e6b 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,7 +1,6 @@ mypy==1.5.1 idna>=2.0.0 cryptography>=1.3.4 -tornado>=6.1 pytest>=6.2 trustme==1.1.0 trio==0.23.1 diff --git a/test/conftest.py b/test/conftest.py index 39c9b5b0c3..810938e329 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -13,8 +13,8 @@ from dummyserver.app import hypercorn_app from dummyserver.asgi_proxy import ProxyApp from dummyserver.hypercornserver import run_hypercorn_in_thread +from dummyserver.socketserver import HAS_IPV6 from dummyserver.testcase import HTTPSHypercornDummyServerTestCase -from dummyserver.tornadoserver import HAS_IPV6 from urllib3.util import ssl_ from urllib3.util.url import parse_url diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 7438402b19..ebe909251f 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -14,7 +14,7 @@ from dummyserver.app import pyodide_testing_app from dummyserver.hypercornserver import run_hypercorn_in_thread -from dummyserver.tornadoserver import DEFAULT_CERTS +from dummyserver.socketserver import DEFAULT_CERTS from urllib3.util.url import parse_url _coverage_count = 0 diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py index 5510f36214..c23e14bcbd 100644 --- a/test/contrib/test_socks.py +++ b/test/contrib/test_socks.py @@ -11,8 +11,8 @@ import pytest import socks as py_socks # type: ignore[import] +from dummyserver.socketserver import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import IPV4SocketDummyServerTestCase -from dummyserver.tornadoserver import DEFAULT_CA, DEFAULT_CERTS from urllib3.contrib import socks from urllib3.exceptions import ConnectTimeoutError, NewConnectionError diff --git a/test/test_collections.py b/test/test_collections.py index 5f3b96cd72..ae896e2689 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -385,7 +385,7 @@ def test_dict_conversion(self, d: HTTPHeaderDict) -> None: hdict = { "Content-Length": "0", "Content-type": "text/plain", - "Server": "TornadoServer/1.2.3", + "Server": "Hypercorn/1.2.3", } h = dict(HTTPHeaderDict(hdict).items()) assert hdict == h diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index dbf2208581..176fed4ae4 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -12,7 +12,7 @@ import pytest -from dummyserver.tornadoserver import DEFAULT_CA +from dummyserver.socketserver import DEFAULT_CA from urllib3 import Retry from urllib3.connection import HTTPConnection from urllib3.connectionpool import ( diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 49d45f6b02..4cce4def02 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -9,8 +9,8 @@ import pytest +from dummyserver.socketserver import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import SocketDummyServerTestCase, consume_socket -from dummyserver.tornadoserver import DEFAULT_CA, DEFAULT_CERTS from urllib3.util import ssl_ from urllib3.util.ssltransport import SSLTransport diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 63b1f8a013..8be9d392f2 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -12,8 +12,8 @@ import pytest +from dummyserver.socketserver import NoIPv6Warning from dummyserver.testcase import HypercornDummyServerTestCase, SocketDummyServerTestCase -from dummyserver.tornadoserver import NoIPv6Warning from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict from urllib3.connection import _get_default_user_agent diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index d99cdd486f..cf95ec8b9d 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -23,13 +23,13 @@ import urllib3.util as util import urllib3.util.ssl_ -from dummyserver.testcase import HTTPSHypercornDummyServerTestCase -from dummyserver.tornadoserver import ( +from dummyserver.socketserver import ( DEFAULT_CA, DEFAULT_CA_KEY, DEFAULT_CERTS, encrypt_key_pem, ) +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from urllib3 import HTTPSConnectionPool from urllib3.connection import RECENT_DATE, HTTPSConnection, VerifiedHTTPSConnection from urllib3.exceptions import ( diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index f9c5e1f846..4fa9ec850a 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -7,11 +7,11 @@ import pytest +from dummyserver.socketserver import HAS_IPV6 from dummyserver.testcase import ( HypercornDummyServerTestCase, IPv6HypercornDummyServerTestCase, ) -from dummyserver.tornadoserver import HAS_IPV6 from urllib3 import HTTPHeaderDict, HTTPResponse, request from urllib3.connectionpool import port_by_scheme from urllib3.exceptions import MaxRetryError, URLSchemeUnknown diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 15164b59a9..db9229895c 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -17,11 +17,11 @@ import trustme import urllib3.exceptions +from dummyserver.socketserver import DEFAULT_CA, HAS_IPV6, get_unreachable_address from dummyserver.testcase import ( HypercornDummyProxyTestCase, IPv6HypercornDummyProxyTestCase, ) -from dummyserver.tornadoserver import DEFAULT_CA, HAS_IPV6, get_unreachable_address from urllib3 import HTTPResponse from urllib3._collections import HTTPHeaderDict from urllib3.connection import VerifiedHTTPSConnection diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 3d07793840..2760a764f7 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -23,13 +23,13 @@ import pytest import trustme -from dummyserver.testcase import SocketDummyServerTestCase, consume_socket -from dummyserver.tornadoserver import ( +from dummyserver.socketserver import ( DEFAULT_CA, DEFAULT_CERTS, encrypt_key_pem, get_unreachable_address, ) +from dummyserver.testcase import SocketDummyServerTestCase, consume_socket from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager, util from urllib3._collections import HTTPHeaderDict from urllib3.connection import HTTPConnection, _get_default_user_agent From 13521e4712480ec3357d4b67da0d425d70834ff9 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 1 Jan 2024 15:10:42 -0600 Subject: [PATCH 065/131] Use coverage sys.monitoring for a speed-boost --- dev-requirements.txt | 2 +- noxfile.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index c0636b0bb1..adefa58df0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -coverage==7.3.2 +coverage==7.4.0 PySocks==1.7.1 pytest==7.4.2 pytest-timeout==2.1.0 diff --git a/noxfile.py b/noxfile.py index 1a2514b9aa..b3ef89938f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -36,6 +36,20 @@ def tests_impl( elif sys.platform == "win32": memray_supported = False + # Environment variables being passed to the pytest run. + pytest_session_envvars = { + "PYTHONWARNINGS": "always::DeprecationWarning", + } + + # In coverage 7.4.0 we can only set the setting for Python 3.12+ + # Future versions of coverage will use sys.monitoring based on availability. + if ( + isinstance(session.python, str) + and "." in session.python + and int(session.python.split(".")[1]) >= 12 + ): + pytest_session_envvars["COVERAGE_CORE"] = "sysmon" + # Inspired from https://hynek.me/articles/ditch-codecov-python/ # We use parallel mode and then combine in a later CI step session.run( @@ -58,7 +72,7 @@ def tests_impl( "--strict-markers", *pytest_extra_args, *(session.posargs or ("test/",)), - env={"PYTHONWARNINGS": "always::DeprecationWarning"}, + env=pytest_session_envvars, ) From b156fa9d88a061e4098aefc1e522e60e127f09f2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 2 Jan 2024 17:39:34 +0400 Subject: [PATCH 066/131] Bump CI to macOS 12 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4ac32f309..dfd20faab4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: - - macos-11 + - macos-12 - windows-latest - ubuntu-20.04 # OpenSSL 1.1.1 - ubuntu-22.04 # OpenSSL 3.0 @@ -92,7 +92,7 @@ jobs: os: ubuntu-22.04 runs-on: ${{ matrix.os }} - name: ${{ fromJson('{"macos-11":"macOS","windows-latest":"Windows","ubuntu-latest":"Ubuntu","ubuntu-20.04":"Ubuntu 20.04 (OpenSSL 1.1.1)","ubuntu-22.04":"Ubuntu 22.04 (OpenSSL 3.0)"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session}} + name: ${{ fromJson('{"macos-12":"macOS","windows-latest":"Windows","ubuntu-latest":"Ubuntu","ubuntu-20.04":"Ubuntu 20.04 (OpenSSL 1.1.1)","ubuntu-22.04":"Ubuntu 22.04 (OpenSSL 3.0)"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session}} continue-on-error: ${{ matrix.experimental }} timeout-minutes: 30 steps: From 2ac4efacfef5182aaef3e26a4f1e066ab228dfdd Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 3 Jan 2024 08:44:43 -0600 Subject: [PATCH 067/131] Remove unnecessary OpenSSL 1.1.1 matrix, push branch jobs (#3256) Co-authored-by: Quentin Pradet --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfd20faab4..a18902557a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,7 @@ jobs: os: - macos-12 - windows-latest - - ubuntu-20.04 # OpenSSL 1.1.1 - - ubuntu-22.04 # OpenSSL 3.0 + - ubuntu-22.04 nox-session: [''] include: - experimental: false @@ -58,6 +57,11 @@ jobs: os: ubuntu-latest experimental: false nox-session: test_integration + # OpenSSL 1.1.1 + - python-version: "3.8" + os: ubuntu-20.04 + experimental: false + nox-session: test-3.8 # pypy - python-version: "pypy-3.8" os: ubuntu-latest From e470d3b72429e95077ecd1015f029e1eec6df819 Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Tue, 9 Jan 2024 04:42:59 +0100 Subject: [PATCH 068/131] Fix handling of OpenSSL 3.2.0 new error message "record layer failure" Co-authored-by: Ruben Laguna Co-authored-by: Seth Michael Larson --- changelog/3268.bugfix.rst | 1 + src/urllib3/connection.py | 1 + test/with_dummyserver/test_socketlevel.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog/3268.bugfix.rst diff --git a/changelog/3268.bugfix.rst b/changelog/3268.bugfix.rst new file mode 100644 index 0000000000..9d3568a7d2 --- /dev/null +++ b/changelog/3268.bugfix.rst @@ -0,0 +1 @@ +Fixed handling of OpenSSL 3.2.0 new error message for misconfiguring an HTTP proxy as HTTPS. diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 42b1bbc5e5..0a2a5e04e1 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -862,6 +862,7 @@ def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError: is_likely_http_proxy = ( "wrong version number" in error_normalized or "unknown protocol" in error_normalized + or "record layer failure" in error_normalized ) http_proxy_warning = ( ". Your proxy appears to only use HTTP and not HTTPS, " diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 2760a764f7..69d8070b8b 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1293,7 +1293,8 @@ def socket_handler(listener: socket.socket) -> None: self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA) as pool: with pytest.raises( - SSLError, match=r"(wrong version number|record overflow)" + SSLError, + match=r"(wrong version number|record overflow|record layer failure)", ): pool.request("GET", "/", retries=False) From 3ca46ea7155139aafd3e3e6f73567a55a3ea2b35 Mon Sep 17 00:00:00 2001 From: abebeos <149062843+abebeos@users.noreply.github.com> Date: Tue, 9 Jan 2024 21:55:49 +0200 Subject: [PATCH 069/131] Set proper `proxy_is_verified` value after connecting to proxy (#3266) Co-authored-by: Tushar <30565750+tushar5526@users.noreply.github.com> Co-authored-by: Illia Volochii --- changelog/3130.bugfix.rst | 3 ++ src/urllib3/connection.py | 16 +++++++++ .../test_proxy_poolmanager.py | 35 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 changelog/3130.bugfix.rst diff --git a/changelog/3130.bugfix.rst b/changelog/3130.bugfix.rst new file mode 100644 index 0000000000..cf301fb406 --- /dev/null +++ b/changelog/3130.bugfix.rst @@ -0,0 +1,3 @@ +Fixed ``HTTPConnection.proxy_is_verified`` and ``HTTPSConnection.proxy_is_verified`` +to be always set to a boolean after connecting to a proxy. It could be +``None`` in some cases previously. \ No newline at end of file diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 0a2a5e04e1..a44bd0278f 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -248,6 +248,9 @@ def connect(self) -> None: # not using tunnelling. self._has_connected_to_proxy = bool(self.proxy) + if self._has_connected_to_proxy: + self.proxy_is_verified = False + @property def is_closed(self) -> bool: return self.sock is None @@ -611,8 +614,11 @@ def connect(self) -> None: if self._tunnel_host is not None: # We're tunneling to an HTTPS origin so need to do TLS-in-TLS. if self._tunnel_scheme == "https": + # _connect_tls_proxy will verify and assign proxy_is_verified self.sock = sock = self._connect_tls_proxy(self.host, sock) tls_in_tls = True + elif self._tunnel_scheme == "http": + self.proxy_is_verified = False # If we're tunneling it means we're connected to our proxy. self._has_connected_to_proxy = True @@ -656,6 +662,11 @@ def connect(self) -> None: assert_fingerprint=self.assert_fingerprint, ) self.sock = sock_and_verified.socket + + # TODO: Set correct `self.is_verified` in case of HTTPS proxy + + # HTTP destination, see + # `test_is_verified_https_proxy_to_http_target` and + # https://github.com/urllib3/urllib3/issues/3267. self.is_verified = sock_and_verified.is_verified # If there's a proxy to be connected to we are fully connected. @@ -663,6 +674,11 @@ def connect(self) -> None: # not using tunnelling. self._has_connected_to_proxy = bool(self.proxy) + # Set `self.proxy_is_verified` unless it's already set while + # establishing a tunnel. + if self._has_connected_to_proxy and self.proxy_is_verified is None: + self.proxy_is_verified = sock_and_verified.is_verified + def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket: """ Establish a TLS connection to the proxy using the provided SSL context. diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index db9229895c..e520f30063 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -43,6 +43,16 @@ from .. import TARPIT_HOST, requires_network +def assert_is_verified(pm: ProxyManager, *, proxy: bool, target: bool) -> None: + pool = list(pm.pools._container.values())[-1] # retrieve last pool entry + connection = ( + pool.pool.queue[-1] if pool.pool is not None else None + ) # retrieve last connection entry + + assert connection.proxy_is_verified is proxy + assert connection.is_verified is target + + class TestHTTPProxyManager(HypercornDummyProxyTestCase): @classmethod def setup_class(cls) -> None: @@ -83,6 +93,31 @@ def test_https_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 + def test_is_verified_http_proxy_to_http_target(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.http_url}/") + assert r.status == 200 + assert_is_verified(http, proxy=False, target=False) + + def test_is_verified_http_proxy_to_https_target(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.https_url}/") + assert r.status == 200 + assert_is_verified(http, proxy=False, target=True) + + @pytest.mark.xfail(reason="see https://github.com/urllib3/urllib3/issues/3267") + def test_is_verified_https_proxy_to_http_target(self) -> None: + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", f"{self.http_url}/") + assert r.status == 200 + assert_is_verified(https, proxy=True, target=False) + + def test_is_verified_https_proxy_to_https_target(self) -> None: + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", f"{self.https_url}/") + assert r.status == 200 + assert_is_verified(https, proxy=True, target=True) + def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: with open(DEFAULT_CA) as pem_file: pem_file_data = pem_file.read() From 7531aa0abff4c3673967ac78579d7028cb82bb57 Mon Sep 17 00:00:00 2001 From: abebeos <149062843+abebeos@users.noreply.github.com> Date: Tue, 9 Jan 2024 22:28:12 +0200 Subject: [PATCH 070/131] Extend the Zstandard-related test suite (#3265) Co-authored-by: Illia Volochii --- test/test_response.py | 63 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/test/test_response.py b/test/test_response.py index b4939bb1a9..2fbf7d570d 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -431,8 +431,13 @@ def test_chunked_decoding_zstd(self) -> None: break assert ret == b"foobarbaz" + decode_param_set = [ + b"foo", + b"x" * 100, + ] + @onlyZstd() - @pytest.mark.parametrize("data", [b"foo", b"x" * 100]) + @pytest.mark.parametrize("data", decode_param_set) def test_decode_zstd_error(self, data: bytes) -> None: fp = BytesIO(data) @@ -440,14 +445,66 @@ def test_decode_zstd_error(self, data: bytes) -> None: HTTPResponse(fp, headers={"content-encoding": "zstd"}) @onlyZstd() - @pytest.mark.parametrize("data", [b"foo", b"x" * 100]) - def test_decode_zstd_incomplete(self, data: bytes) -> None: + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None: data = zstd.compress(data) fp = BytesIO(data[:-1]) with pytest.raises(DecodeError): HTTPResponse(fp, headers={"content-encoding": "zstd"}) + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_read(self, data: bytes) -> None: + data = zstd.compress(data) + fp = BytesIO(data[:-1]) # shorten the data to trigger DecodeError + + # create response object without(!) reading/decoding the content + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + # read/decode, expecting DecodeError + with pytest.raises(DecodeError): + r.read(decode_content=True) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_read1(self, data: bytes) -> None: + data = zstd.compress(data) + fp = BytesIO(data[:-1]) + + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + # read/decode via read1(!), expecting DecodeError + with pytest.raises(DecodeError): + amt_decoded = 0 + # loop, as read1() may return just partial data + while amt_decoded < len(data): + part = r.read1(decode_content=True) + amt_decoded += len(part) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_read1(self, data: bytes) -> None: + encoded_data = zstd.compress(data) + fp = BytesIO(encoded_data) + + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + amt_decoded = 0 + decoded_data = b"" + # loop, as read1() may return just partial data + while amt_decoded < len(data): + part = r.read1(decode_content=True) + amt_decoded += len(part) + decoded_data += part + assert decoded_data == data + def test_multi_decoding_deflate_deflate(self) -> None: data = zlib.compress(zlib.compress(b"foo")) From 79b638600def5d446cb129f393d9099bfe16e4da Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Wed, 10 Jan 2024 04:02:02 +0100 Subject: [PATCH 071/131] Use -p and -m options in mypy Co-authored-by: Ruben Laguna --- noxfile.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index b3ef89938f..9c97b92069 100644 --- a/noxfile.py +++ b/noxfile.py @@ -279,9 +279,13 @@ def mypy(session: nox.Session) -> None: session.run("mypy", "--version") session.run( "mypy", + "-p", "dummyserver", - "noxfile.py", - "src/urllib3", + "-m", + "noxfile", + "-p", + "urllib3", + "-p", "test", ) From 745b002e3476f6c4ab707a2b1c82875038249be0 Mon Sep 17 00:00:00 2001 From: abebeos <149062843+abebeos@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:38:40 +0200 Subject: [PATCH 072/131] Remove watcher thread suggestion from docs (#3264) --- src/urllib3/util/timeout.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/urllib3/util/timeout.py b/src/urllib3/util/timeout.py index f044625c35..4bb1be11d9 100644 --- a/src/urllib3/util/timeout.py +++ b/src/urllib3/util/timeout.py @@ -101,10 +101,6 @@ class Timeout: the case; if a server streams one byte every fifteen seconds, a timeout of 20 seconds will not trigger, even though the request will take several minutes to complete. - - If your goal is to cut off any request after a set amount of wall clock - time, consider having a second "watcher" thread to cut off a slow - request. """ #: A sentinel object representing the default timeout value From f862bfeb3b7a100f86f4d17eb54f3b82c6921d16 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 16 Jan 2024 11:18:35 -0600 Subject: [PATCH 073/131] Link to fundraiser in docs banner --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ba67335880..138f99abbe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -84,8 +84,8 @@ html_theme_options = { "announcement": """ - Support urllib3 on GitHub Sponsors + href=\"https://opencollective.com/urllib3/updates/urllib3-is-fundraising-for-http-2-support\"> + urllib3 is fundraising for HTTP/2 support! """, "sidebar_hide_name": True, From bbba48753448a9e9b642cf32efe041683f3324f8 Mon Sep 17 00:00:00 2001 From: abebeos <149062843+abebeos@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:26:04 +0200 Subject: [PATCH 074/131] Set HTTPSConnection.is_verified to False when using a forwarding proxy Co-authored-by: abebeos <129396476+abebeos@users.noreply.github.com> Co-authored-by: Seth Michael Larson --- changelog/3267.bugfix.rst | 2 ++ src/urllib3/connection.py | 20 ++++++++++++++----- src/urllib3/connectionpool.py | 3 ++- .../test_proxy_poolmanager.py | 1 - 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 changelog/3267.bugfix.rst diff --git a/changelog/3267.bugfix.rst b/changelog/3267.bugfix.rst new file mode 100644 index 0000000000..ac805a8b92 --- /dev/null +++ b/changelog/3267.bugfix.rst @@ -0,0 +1,2 @@ +Fixed ``HTTPSConnection.is_verified`` to be set to ``False`` when connecting +from a https proxy to a http target. It was set to ``True`` previously. diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index a44bd0278f..0318601bbd 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -265,6 +265,13 @@ def is_connected(self) -> bool: def has_connected_to_proxy(self) -> bool: return self._has_connected_to_proxy + @property + def proxy_is_forwarding(self) -> bool: + """ + Return True if a forwarding proxy is configured, else return False + """ + return bool(self.proxy) and self._tunnel_host is None + def close(self) -> None: try: super().close() @@ -663,11 +670,14 @@ def connect(self) -> None: ) self.sock = sock_and_verified.socket - # TODO: Set correct `self.is_verified` in case of HTTPS proxy + - # HTTP destination, see - # `test_is_verified_https_proxy_to_http_target` and - # https://github.com/urllib3/urllib3/issues/3267. - self.is_verified = sock_and_verified.is_verified + # Forwarding proxies can never have a verified target since + # the proxy is the one doing the verification. Should instead + # use a CONNECT tunnel in order to verify the target. + # See: https://github.com/urllib3/urllib3/issues/3267. + if self.proxy_is_forwarding: + self.is_verified = False + else: + self.is_verified = sock_and_verified.is_verified # If there's a proxy to be connected to we are fully connected. # This is set twice (once above and here) due to forwarding proxies diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 549f0f431e..52bb8657fc 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -1098,7 +1098,8 @@ def _validate_conn(self, conn: BaseHTTPConnection) -> None: if conn.is_closed: conn.connect() - if not conn.is_verified: + # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791 + if not conn.is_verified and not conn.proxy_is_verified: warnings.warn( ( f"Unverified HTTPS request is being made to host '{conn.host}'. " diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index e520f30063..397181a9e6 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -105,7 +105,6 @@ def test_is_verified_http_proxy_to_https_target(self) -> None: assert r.status == 200 assert_is_verified(http, proxy=False, target=True) - @pytest.mark.xfail(reason="see https://github.com/urllib3/urllib3/issues/3267") def test_is_verified_https_proxy_to_http_target(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.http_url}/") From 8beb3502cf6c945485174d96d90f2f5e5929bcbd Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Sat, 20 Jan 2024 01:04:55 +0100 Subject: [PATCH 075/131] Add tolerance for test value of TestConnectionPoolTimeouts.test_timeout --- test/with_dummyserver/test_connectionpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 8be9d392f2..4fbe6a4f74 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -104,7 +104,7 @@ def test_timeout(self) -> None: delta = time.time() - now message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT" - assert delta >= LONG_TIMEOUT, message + assert delta >= (LONG_TIMEOUT - 1e-5), message block_event.set() # Release request # Timeout passed directly to request should raise a request timeout From 4e7be193564ffb21078453adb4e34dda780aa8a1 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 22 Jan 2024 09:27:03 -0600 Subject: [PATCH 076/131] Add rudimentary HTTP/2 connection and response --- changelog/3284.feature.rst | 1 + noxfile.py | 2 +- pyproject.toml | 3 + src/urllib3/http2.py | 182 ++++++++++++++++++++++++++++ test/with_dummyserver/test_http2.py | 60 +++++++-- 5 files changed, 240 insertions(+), 8 deletions(-) create mode 100644 changelog/3284.feature.rst create mode 100644 src/urllib3/http2.py diff --git a/changelog/3284.feature.rst b/changelog/3284.feature.rst new file mode 100644 index 0000000000..3105c9a8e2 --- /dev/null +++ b/changelog/3284.feature.rst @@ -0,0 +1 @@ +Added rudimentary support for HTTP/2 via ``urllib3.contrib.h2`` and ``HTTP2Connection`` class. diff --git a/noxfile.py b/noxfile.py index 9c97b92069..3a29d7fd04 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,7 +11,7 @@ def tests_impl( session: nox.Session, - extras: str = "socks,brotli,zstd", + extras: str = "socks,brotli,zstd,h2", # hypercorn dependency h2 compares bytes and strings # https://github.com/python-hyper/h2/issues/1236 byte_string_comparisons: bool = False, diff --git a/pyproject.toml b/pyproject.toml index 962330940c..1fe82937ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ zstd = [ socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] +h2 = [ + "h2>=4,<5" +] [project.urls] "Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst" diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py new file mode 100644 index 0000000000..3364500c48 --- /dev/null +++ b/src/urllib3/http2.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import contextlib +import threading +import typing + +import h2.config # type: ignore[import] +import h2.connection # type: ignore[import] +import h2.events # type: ignore[import] + +import urllib3.connection +import urllib3.util.ssl_ + +from ._collections import HTTPHeaderDict +from .connection import HTTPSConnection +from .connectionpool import HTTPSConnectionPool + +orig_HTTPSConnection = HTTPSConnection + + +class HTTP2Connection(HTTPSConnection): + def __init__( + self, host: str, port: int | None = None, **kwargs: typing.Any + ) -> None: + self._h2_lock = threading.RLock() + self._h2_conn = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=True) + ) + self._h2_stream: int | None = None + self._h2_headers: list[tuple[bytes, bytes]] = [] + + if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive: + raise NotImplementedError("Proxies aren't supported with HTTP/2") + + super().__init__(host, port, **kwargs) + + @contextlib.contextmanager + def _lock_h2_conn(self) -> typing.Generator[h2.connection.H2Connection, None, None]: + with self._h2_lock: + yield self._h2_conn + + def connect(self) -> None: + super().connect() + + with self._lock_h2_conn() as h2_conn: + h2_conn.initiate_connection() + self.sock.sendall(h2_conn.data_to_send()) + + def putrequest( + self, + method: str, + url: str, + skip_host: bool = False, + skip_accept_encoding: bool = False, + ) -> None: + with self._lock_h2_conn() as h2_conn: + self._h2_stream = h2_conn.get_next_available_stream_id() + + if ":" in self.host: + authority = f"[{self.host}]:{self.port or 443}" + else: + authority = f"{self.host}:{self.port or 443}" + + self._h2_headers.extend( + ( + (b":scheme", b"https"), + (b":method", method.encode()), + (b":authority", authority.encode()), + (b":path", url.encode()), + ) + ) + + def putheader(self, header: str, *values: str) -> None: + for value in values: + self._h2_headers.append( + (header.encode("utf-8").lower(), value.encode("utf-8")) + ) + + def endheaders(self) -> None: # type: ignore[override] + with self._lock_h2_conn() as h2_conn: + h2_conn.send_headers( + stream_id=self._h2_stream, + headers=self._h2_headers, + end_stream=True, + ) + if data_to_send := h2_conn.data_to_send(): + self.sock.sendall(data_to_send) + + def send(self, data: bytes) -> None: # type: ignore[override] # Defensive: + if not data: + return + raise NotImplementedError("Sending data isn't supported yet") + + def getresponse( # type: ignore[override] + self, + ) -> HTTP2Response: + status = None + data = bytearray() + with self._lock_h2_conn() as h2_conn: + end_stream = False + while not end_stream: + # TODO: Arbitrary read value. + if received_data := self.sock.recv(65535): + events = h2_conn.receive_data(received_data) + for event in events: + if isinstance( + event, h2.events.InformationalResponseReceived + ): # Defensive: + continue # TODO: Does the stdlib do anything with these responses? + + elif isinstance(event, h2.events.ResponseReceived): + headers = HTTPHeaderDict() + for header, value in event.headers: + if header == b":status": + status = int(value.decode()) + else: + headers.add( + header.decode("ascii"), value.decode("ascii") + ) + + elif isinstance(event, h2.events.DataReceived): + data += event.data + h2_conn.acknowledge_received_data( + event.flow_controlled_length, event.stream_id + ) + + elif isinstance(event, h2.events.StreamEnded): + end_stream = True + + if data_to_send := h2_conn.data_to_send(): + self.sock.sendall(data_to_send) + + # We always close to not have to handle connection management. + self.close() + + assert status is not None + return HTTP2Response(status=status, headers=headers, data=bytes(data)) + + def close(self) -> None: + with self._lock_h2_conn() as h2_conn: + try: + self._h2_conn.close_connection() + if data := h2_conn.data_to_send(): + self.sock.sendall(data) + except Exception: + pass + + # Reset all our HTTP/2 connection state. + self._h2_conn = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=True) + ) + self._h2_stream = None + self._h2_headers = [] + + super().close() + + +class HTTP2Response: + # TODO: This is a woefully incomplete response object, but works for non-streaming. + def __init__(self, status: int, headers: HTTPHeaderDict, data: bytes) -> None: + self.status = status + self.headers = headers + self.data = data + self.length_remaining = 0 + + def get_redirect_location(self) -> None: + return None + + +def inject_into_urllib3() -> None: + HTTPSConnectionPool.ConnectionCls = HTTP2Connection # type: ignore[assignment] + urllib3.connection.HTTPSConnection = HTTP2Connection # type: ignore[misc] + + # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy. + urllib3.util.ssl_.ALPN_PROTOCOLS = ["h2"] + + +def extract_from_urllib3() -> None: + HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection + urllib3.connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc] + + urllib3.util.ssl_.ALPN_PROTOCOLS = ["http/1.1"] diff --git a/test/with_dummyserver/test_http2.py b/test/with_dummyserver/test_http2.py index fa05900c01..54f2582cb5 100644 --- a/test/with_dummyserver/test_http2.py +++ b/test/with_dummyserver/test_http2.py @@ -2,26 +2,72 @@ import subprocess from test import notWindows +from test.conftest import ServerConfig -from dummyserver.testcase import HypercornDummyServerTestCase +import pytest +import urllib3 +from dummyserver.socketserver import DEFAULT_CERTS +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase + +DEFAULT_CERTS_HTTP2 = DEFAULT_CERTS.copy() +DEFAULT_CERTS_HTTP2["alpn_protocols"] = ["h2"] + + +def setup_module() -> None: + try: + from urllib3.http2 import inject_into_urllib3 + + inject_into_urllib3() + except ImportError as e: + pytest.skip(f"Could not import h2: {e!r}") + + +def teardown_module() -> None: + try: + from urllib3.http2 import extract_from_urllib3 + + extract_from_urllib3() + except ImportError: + pass + + +class TestHypercornDummyServerTestCase(HTTPSHypercornDummyServerTestCase): + certs = DEFAULT_CERTS_HTTP2 -class TestHypercornDummyServerTestCase(HypercornDummyServerTestCase): @classmethod def setup_class(cls) -> None: super().setup_class() - cls.base_url = f"http://{cls.host}:{cls.port}" + cls.base_url = f"https://{cls.host}:{cls.port}" @notWindows() # GitHub Actions Windows doesn't have HTTP/2 support. def test_hypercorn_server_http2(self) -> None: # This is a meta test to make sure our Hypercorn test server is actually using HTTP/2 # before urllib3 is capable of speaking HTTP/2. Thanks, Daniel! <3 output = subprocess.check_output( - ["curl", "-vvv", "--http2", self.base_url], stderr=subprocess.STDOUT + [ + "curl", + "-vvv", + "--http2", + "--cacert", + self.certs["ca_certs"], + self.base_url, + ], + stderr=subprocess.STDOUT, ) - # curl does HTTP/1.1 and upgrades to HTTP/2 without TLS which is fine - # for us. Hypercorn supports this thankfully, but we should try with - # HTTPS as well once that's available. assert b"< HTTP/2 200" in output assert output.endswith(b"Dummy server!") + + +def test_simple_http2(san_server: ServerConfig) -> None: + with urllib3.PoolManager(ca_certs=san_server.ca_certs) as http: + resp = http.request("HEAD", san_server.base_url, retries=False) + + assert resp.status == 200 + resp.headers.pop("date") + assert resp.headers == { + "content-type": "text/html; charset=utf-8", + "content-length": "13", + "server": "hypercorn-h2", + } From 6b2b377d9acc996f3cd8296af3b5d5cf7a255a09 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Mon, 22 Jan 2024 20:45:27 +0200 Subject: [PATCH 077/131] Make `HTTPResponse.read1` close response when all data is read (#3235) --- src/urllib3/response.py | 8 ++++ test/test_response.py | 95 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 6c73dcfa0c..6a811ad513 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -870,6 +870,14 @@ def _raw_read( # raised during streaming, so all calls with incorrect # Content-Length are caught. raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + elif read1 and ( + (amt != 0 and not data) or self.length_remaining == len(data) + ): + # All data has been read, but `self._fp.read1` in + # CPython 3.12 and older doesn't always close + # `http.client.HTTPResponse`, so we close it here. + # See https://github.com/python/cpython/issues/113199 + self._fp.close() if data: self._fp_bytes_read += len(data) diff --git a/test/test_response.py b/test/test_response.py index 2fbf7d570d..199ca80179 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -613,7 +613,7 @@ def test_io(self, sock: socket.socket) -> None: with pytest.raises(IOError): resp3.fileno() - def test_io_closed_consistently(self, sock: socket.socket) -> None: + def test_io_closed_consistently_by_read(self, sock: socket.socket) -> None: try: hlr = httplib.HTTPResponse(sock) hlr.fp = BytesIO(b"foo") # type: ignore[assignment] @@ -633,6 +633,99 @@ def test_io_closed_consistently(self, sock: socket.socket) -> None: finally: hlr.close() + @pytest.mark.parametrize("read_amt", (None, 3)) + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_closed_consistently_by_read1( + self, sock: socket.socket, length_known: bool, read_amt: int | None + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + hlr.length = 3 if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = 3 + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + resp.read1(read_amt) + # If content length is unknown, IO is not closed until + # the next read returning zero bytes. + if not length_known: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + resp.read1(read_amt) + assert resp.closed + assert resp._fp.isclosed() + assert is_fp_closed(resp._fp) + assert resp.isclosed() + + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_not_closed_until_all_data_is_read( + self, sock: socket.socket, length_known: bool + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + length_remaining = 3 + hlr.length = length_remaining if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = length_remaining + while length_remaining: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(1) + assert len(data) == 1 + length_remaining -= 1 + # If content length is unknown, IO is not closed until + # the next read returning zero bytes. + if not length_known: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(1) + assert len(data) == 0 + assert resp.closed + assert resp._fp.isclosed() # type: ignore[union-attr] + assert is_fp_closed(resp._fp) + assert resp.isclosed() + + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_not_closed_after_requesting_0_bytes( + self, sock: socket.socket, length_known: bool + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + length_remaining = 3 + hlr.length = length_remaining if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = length_remaining + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(0) + assert data == b"" + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + def test_io_bufferedreader(self) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) From 03f7b65a47d87036674a7909ca6c57d4b5cf1a81 Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Tue, 23 Jan 2024 07:07:19 +0100 Subject: [PATCH 078/131] Skip memray on pypy (#3286) Co-authored-by: Quentin Pradet --- noxfile.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 3a29d7fd04..a48afff1a6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -29,9 +29,18 @@ def tests_impl( session.run("python", "-c", "import struct; print(struct.calcsize('P') * 8)") # Print OpenSSL information. session.run("python", "-m", "OpenSSL.debug") + # Retrieve sys info from the Python implementation under test + # to avoid enabling memray when nox runs under CPython but tests PyPy + session_python_info = session.run( + "python", + "-c", + "import sys; print(sys.implementation.name, sys.version_info.releaselevel)", + silent=True, + ).strip() # type: ignore[union-attr] # mypy doesn't know that silent=True will return a string + implementation_name, release_level = session_python_info.split(" ") memray_supported = True - if sys.implementation.name != "cpython" or sys.version_info.releaselevel != "final": + if implementation_name != "cpython" or release_level != "final": memray_supported = False # pytest-memray requires CPython 3.8+ elif sys.platform == "win32": memray_supported = False From fb6cf2dba92bba8bdd734deb4d6fde44c19849b9 Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Tue, 23 Jan 2024 15:16:05 +0100 Subject: [PATCH 079/131] Pin to pypy-3.9-v7.3.13 to not timeout CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a18902557a..39ab4e9b4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: os: ubuntu-latest experimental: false nox-session: test-pypy - - python-version: "pypy-3.9" + - python-version: "pypy-3.9-v7.3.13" os: ubuntu-latest experimental: false nox-session: test-pypy From 89ed0d6a65138f6d641e92ae4e0da0cdc7d66870 Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Wed, 24 Jan 2024 06:43:08 +0100 Subject: [PATCH 080/131] Add test-pypy 3.8 3.9 3.10 nox sessions (#3304) --- .github/workflows/ci.yml | 8 ++++---- docs/contributing.rst | 29 ++++++++++++++++------------- noxfile.py | 6 +++++- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39ab4e9b4f..42b045b791 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,15 +66,15 @@ jobs: - python-version: "pypy-3.8" os: ubuntu-latest experimental: false - nox-session: test-pypy + nox-session: test-pypy3.8 - python-version: "pypy-3.9-v7.3.13" os: ubuntu-latest experimental: false - nox-session: test-pypy + nox-session: test-pypy3.9 - python-version: "pypy-3.10" os: ubuntu-latest experimental: false - nox-session: test-pypy + nox-session: test-pypy3.10 - python-version: "3.x" # brotli os: ubuntu-latest @@ -120,7 +120,7 @@ jobs: if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version - run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters + run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} env: PYTHON_VERSION: ${{ matrix.python-version }} NOX_SESSION: ${{ matrix.nox-session }} diff --git a/docs/contributing.rst b/docs/contributing.rst index f5e81082eb..76a1168a90 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -13,8 +13,8 @@ If you wish to add a new feature or fix a bug: to start making your changes. #. Write a test which shows that the bug was fixed or that the feature works as expected. -#. Format your changes with black using command `$ nox -rs format` and lint your - changes using command `nox -rs lint`. +#. Format your changes with black using command ``nox -rs format`` and lint your + changes using command ``nox -rs lint``. #. Add a `changelog entry `__. #. Send a pull request and bug the maintainer until it gets merged and published. @@ -36,18 +36,21 @@ We use some external dependencies, multiple interpreters and code coverage analysis while running test suite. Our ``noxfile.py`` handles much of this for you:: - $ nox --reuse-existing-virtualenvs --sessions test-3.8 test-3.9 + $ nox --reuse-existing-virtualenvs --sessions test-3.12 test-pypy3.10 [ Nox will create virtualenv if needed, install the specified dependencies, and run the commands in order.] - nox > Running session test-3.8 - ....... - ....... - nox > Session test-3.8 was successful. - ....... - ....... - nox > Running session test-3.9 - ....... - ....... - nox > Session test-3.9 was successful. + + +Note that for nox to test different interpreters, the interpreters must be on the +``PATH`` first. Check with ``which`` to see if the interpreter is on the ``PATH`` +like so:: + + + $ which python3.12 + ~/.pyenv/versions/3.12.1/bin/python3.12 + + $ which pypy3.10 + ~/.pyenv/versions/pypy3.10-7.3.13/bin/pypy3.10 + There is also a nox command for running all of our tests and multiple python versions.:: diff --git a/noxfile.py b/noxfile.py index a48afff1a6..9cea46ddd7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,6 +8,8 @@ import nox +nox.options.error_on_missing_interpreters = True + def tests_impl( session: nox.Session, @@ -85,7 +87,9 @@ def tests_impl( ) -@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12", "pypy"]) +@nox.session( + python=["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.8", "pypy3.9", "pypy3.10"] +) def test(session: nox.Session) -> None: tests_impl(session) From 71e7c35662a4b2ba8543194cc132616065dfc56a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 24 Jan 2024 17:28:41 +0400 Subject: [PATCH 081/131] Allow testing HTTP/1.1 and HTTP/2 in the same test (#3310) --- dev-requirements.txt | 1 + dummyserver/socketserver.py | 4 +- test/conftest.py | 12 +++++ test/with_dummyserver/test_http2.py | 73 ----------------------------- test/with_dummyserver/test_https.py | 6 ++- 5 files changed, 19 insertions(+), 77 deletions(-) delete mode 100644 test/with_dummyserver/test_http2.py diff --git a/dev-requirements.txt b/dev-requirements.txt index adefa58df0..ef1cb59f92 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ +h2==4.1.0 coverage==7.4.0 PySocks==1.7.1 pytest==7.4.2 diff --git a/dummyserver/socketserver.py b/dummyserver/socketserver.py index 488fc729f8..202915ce88 100755 --- a/dummyserver/socketserver.py +++ b/dummyserver/socketserver.py @@ -20,7 +20,7 @@ from cryptography.hazmat.primitives import serialization from urllib3.exceptions import HTTPWarning -from urllib3.util import ALPN_PROTOCOLS, resolve_cert_reqs, resolve_ssl_version +from urllib3.util import resolve_cert_reqs, resolve_ssl_version if typing.TYPE_CHECKING: from typing_extensions import ParamSpec @@ -35,7 +35,7 @@ "keyfile": os.path.join(CERTS_PATH, "server.key"), "cert_reqs": ssl.CERT_OPTIONAL, "ca_certs": os.path.join(CERTS_PATH, "cacert.pem"), - "alpn_protocols": ALPN_PROTOCOLS, + "alpn_protocols": ["h2", "http/1.1"], } DEFAULT_CA = os.path.join(CERTS_PATH, "cacert.pem") DEFAULT_CA_KEY = os.path.join(CERTS_PATH, "cacert.key") diff --git a/test/conftest.py b/test/conftest.py index 810938e329..b71c7d95e2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,6 +10,7 @@ import pytest import trustme +import urllib3.http2 from dummyserver.app import hypercorn_app from dummyserver.asgi_proxy import ProxyApp from dummyserver.hypercornserver import run_hypercorn_in_thread @@ -369,3 +370,14 @@ def requires_tlsv1_3(supported_tls_versions: typing.AbstractSet[str]) -> None: or "TLSv1.3" not in supported_tls_versions ): pytest.skip("Test requires TLSv1.3") + + +@pytest.fixture(params=["h11", "h2"]) +def http_version(request: pytest.FixtureRequest) -> typing.Generator[str, None, None]: + if request.param == "h2": + urllib3.http2.inject_into_urllib3() + + yield request.param + + if request.param == "h2": + urllib3.http2.extract_from_urllib3() diff --git a/test/with_dummyserver/test_http2.py b/test/with_dummyserver/test_http2.py deleted file mode 100644 index 54f2582cb5..0000000000 --- a/test/with_dummyserver/test_http2.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import subprocess -from test import notWindows -from test.conftest import ServerConfig - -import pytest - -import urllib3 -from dummyserver.socketserver import DEFAULT_CERTS -from dummyserver.testcase import HTTPSHypercornDummyServerTestCase - -DEFAULT_CERTS_HTTP2 = DEFAULT_CERTS.copy() -DEFAULT_CERTS_HTTP2["alpn_protocols"] = ["h2"] - - -def setup_module() -> None: - try: - from urllib3.http2 import inject_into_urllib3 - - inject_into_urllib3() - except ImportError as e: - pytest.skip(f"Could not import h2: {e!r}") - - -def teardown_module() -> None: - try: - from urllib3.http2 import extract_from_urllib3 - - extract_from_urllib3() - except ImportError: - pass - - -class TestHypercornDummyServerTestCase(HTTPSHypercornDummyServerTestCase): - certs = DEFAULT_CERTS_HTTP2 - - @classmethod - def setup_class(cls) -> None: - super().setup_class() - cls.base_url = f"https://{cls.host}:{cls.port}" - - @notWindows() # GitHub Actions Windows doesn't have HTTP/2 support. - def test_hypercorn_server_http2(self) -> None: - # This is a meta test to make sure our Hypercorn test server is actually using HTTP/2 - # before urllib3 is capable of speaking HTTP/2. Thanks, Daniel! <3 - output = subprocess.check_output( - [ - "curl", - "-vvv", - "--http2", - "--cacert", - self.certs["ca_certs"], - self.base_url, - ], - stderr=subprocess.STDOUT, - ) - - assert b"< HTTP/2 200" in output - assert output.endswith(b"Dummy server!") - - -def test_simple_http2(san_server: ServerConfig) -> None: - with urllib3.PoolManager(ca_certs=san_server.ca_certs) as http: - resp = http.request("HEAD", san_server.base_url, retries=False) - - assert resp.status == 200 - resp.headers.pop("date") - assert resp.headers == { - "content-type": "text/html; charset=utf-8", - "content-length": "13", - "server": "hypercorn-h2", - } diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index cf95ec8b9d..3ec313699c 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -129,7 +129,7 @@ def teardown_class(cls) -> None: shutil.rmtree(cls.certs_dir) - def test_simple(self) -> None: + def test_simple(self, http_version: str) -> None: with HTTPSConnectionPool( self.host, self.port, @@ -138,6 +138,7 @@ def test_simple(self) -> None: ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200, r.data + assert r.headers["server"] == f"hypercorn-{http_version}" @resolvesLocalhostFQDN() def test_dotted_fqdn(self) -> None: @@ -1130,7 +1131,7 @@ def test_can_validate_ip_san(self, ipv4_san_server: ServerConfig) -> None: class TestHTTPS_IPV6SAN: @pytest.mark.parametrize("host", ["::1", "[::1]"]) def test_can_validate_ipv6_san( - self, ipv6_san_server: ServerConfig, host: str + self, ipv6_san_server: ServerConfig, host: str, http_version: str ) -> None: """Ensure that urllib3 can validate SANs with IPv6 addresses in them.""" with HTTPSConnectionPool( @@ -1141,3 +1142,4 @@ def test_can_validate_ipv6_san( ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200 + assert r.headers["server"] == f"hypercorn-{http_version}" From 26a07dbd3267b174439f1fca0ab473dc79f473dc Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jan 2024 00:24:45 +0400 Subject: [PATCH 082/131] Make BaseHTTPResponse a base class of HTTP2Response (#3311) --- src/urllib3/http2.py | 40 ++++++++++++++++++++++++----- test/with_dummyserver/test_https.py | 1 + 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py index 3364500c48..05de106f73 100644 --- a/src/urllib3/http2.py +++ b/src/urllib3/http2.py @@ -10,6 +10,7 @@ import urllib3.connection import urllib3.util.ssl_ +from urllib3.response import BaseHTTPResponse from ._collections import HTTPHeaderDict from .connection import HTTPSConnection @@ -54,6 +55,7 @@ def putrequest( skip_accept_encoding: bool = False, ) -> None: with self._lock_h2_conn() as h2_conn: + self._request_url = url self._h2_stream = h2_conn.get_next_available_stream_id() if ":" in self.host: @@ -134,7 +136,12 @@ def getresponse( # type: ignore[override] self.close() assert status is not None - return HTTP2Response(status=status, headers=headers, data=bytes(data)) + return HTTP2Response( + status=status, + headers=headers, + request_url=self._request_url, + data=bytes(data), + ) def close(self) -> None: with self._lock_h2_conn() as h2_conn: @@ -155,20 +162,39 @@ def close(self) -> None: super().close() -class HTTP2Response: +class HTTP2Response(BaseHTTPResponse): # TODO: This is a woefully incomplete response object, but works for non-streaming. - def __init__(self, status: int, headers: HTTPHeaderDict, data: bytes) -> None: - self.status = status - self.headers = headers - self.data = data + def __init__( + self, + status: int, + headers: HTTPHeaderDict, + request_url: str, + data: bytes, + decode_content: bool = False, # TODO: support decoding + ) -> None: + super().__init__( + status=status, + headers=headers, + # Following CPython, we map HTTP versions to major * 10 + minor integers + version=20, + # No reason phrase in HTTP/2 + reason=None, + decode_content=decode_content, + request_url=request_url, + ) + self._data = data self.length_remaining = 0 + @property + def data(self) -> bytes: + return self._data + def get_redirect_location(self) -> None: return None def inject_into_urllib3() -> None: - HTTPSConnectionPool.ConnectionCls = HTTP2Connection # type: ignore[assignment] + HTTPSConnectionPool.ConnectionCls = HTTP2Connection urllib3.connection.HTTPSConnection = HTTP2Connection # type: ignore[misc] # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy. diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 3ec313699c..aa22f11879 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -139,6 +139,7 @@ def test_simple(self, http_version: str) -> None: r = https_pool.request("GET", "/") assert r.status == 200, r.data assert r.headers["server"] == f"hypercorn-{http_version}" + assert r.data == b"Dummy server!" @resolvesLocalhostFQDN() def test_dotted_fqdn(self) -> None: From 8c8e26dab401121e9fccc8e8e8958609ccb045f5 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jan 2024 17:39:34 +0400 Subject: [PATCH 083/131] Hide H2Connection inside _LockedObject (#3318) --- src/urllib3/http2.py | 59 ++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py index 05de106f73..3f1eeefcbf 100644 --- a/src/urllib3/http2.py +++ b/src/urllib3/http2.py @@ -1,7 +1,7 @@ from __future__ import annotations -import contextlib import threading +import types import typing import h2.config # type: ignore[import] @@ -18,15 +18,41 @@ orig_HTTPSConnection = HTTPSConnection +T = typing.TypeVar("T") + + +class _LockedObject(typing.Generic[T]): + """ + A wrapper class that hides a specific object behind a lock. + + The goal here is to provide a simple way to protect access to an object + that cannot safely be simultaneously accessed from multiple threads. The + intended use of this class is simple: take hold of it with a context + manager, which returns the protected object. + """ + + def __init__(self, obj: T): + self.lock = threading.RLock() + self._obj = obj + + def __enter__(self) -> T: + self.lock.acquire() + return self._obj + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + self.lock.release() + class HTTP2Connection(HTTPSConnection): def __init__( self, host: str, port: int | None = None, **kwargs: typing.Any ) -> None: - self._h2_lock = threading.RLock() - self._h2_conn = h2.connection.H2Connection( - config=h2.config.H2Configuration(client_side=True) - ) + self._h2_conn = self._new_h2_conn() self._h2_stream: int | None = None self._h2_headers: list[tuple[bytes, bytes]] = [] @@ -35,15 +61,14 @@ def __init__( super().__init__(host, port, **kwargs) - @contextlib.contextmanager - def _lock_h2_conn(self) -> typing.Generator[h2.connection.H2Connection, None, None]: - with self._h2_lock: - yield self._h2_conn + def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]: + config = h2.config.H2Configuration(client_side=True) + return _LockedObject(h2.connection.H2Connection(config=config)) def connect(self) -> None: super().connect() - with self._lock_h2_conn() as h2_conn: + with self._h2_conn as h2_conn: h2_conn.initiate_connection() self.sock.sendall(h2_conn.data_to_send()) @@ -54,7 +79,7 @@ def putrequest( skip_host: bool = False, skip_accept_encoding: bool = False, ) -> None: - with self._lock_h2_conn() as h2_conn: + with self._h2_conn as h2_conn: self._request_url = url self._h2_stream = h2_conn.get_next_available_stream_id() @@ -79,7 +104,7 @@ def putheader(self, header: str, *values: str) -> None: ) def endheaders(self) -> None: # type: ignore[override] - with self._lock_h2_conn() as h2_conn: + with self._h2_conn as h2_conn: h2_conn.send_headers( stream_id=self._h2_stream, headers=self._h2_headers, @@ -98,7 +123,7 @@ def getresponse( # type: ignore[override] ) -> HTTP2Response: status = None data = bytearray() - with self._lock_h2_conn() as h2_conn: + with self._h2_conn as h2_conn: end_stream = False while not end_stream: # TODO: Arbitrary read value. @@ -144,18 +169,16 @@ def getresponse( # type: ignore[override] ) def close(self) -> None: - with self._lock_h2_conn() as h2_conn: + with self._h2_conn as h2_conn: try: - self._h2_conn.close_connection() + h2_conn.close_connection() if data := h2_conn.data_to_send(): self.sock.sendall(data) except Exception: pass # Reset all our HTTP/2 connection state. - self._h2_conn = h2.connection.H2Connection( - config=h2.config.H2Configuration(client_side=True) - ) + self._h2_conn = self._new_h2_conn() self._h2_stream = None self._h2_headers = [] From d7bb83b48c23d8dd429c20a43ac3da74ea6e9df0 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Mon, 29 Jan 2024 01:51:57 +0000 Subject: [PATCH 084/131] Fix TLS 1.3 post-handshake auth Post-hanshake auth was accidentally broken while removing support for Python 3.7 --- changelog/3325.bugfix.rst | 1 + src/urllib3/util/ssl_.py | 11 +++-------- test/test_ssl.py | 19 +++++++++++++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 changelog/3325.bugfix.rst diff --git a/changelog/3325.bugfix.rst b/changelog/3325.bugfix.rst new file mode 100644 index 0000000000..22fd97c7c3 --- /dev/null +++ b/changelog/3325.bugfix.rst @@ -0,0 +1 @@ +Fixed TLS 1.3 Post Handshake Auth when the server certificate validation was disabled. diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index e0a7c04a3c..b14cf27b61 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -319,14 +319,9 @@ def create_urllib3_context( # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is # necessary for conditional client cert authentication with TLS 1.3. - # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older - # versions of Python. We only enable if certificate verification is enabled to work - # around Python issue #37428 - # See: https://bugs.python.org/issue37428 - if ( - cert_reqs == ssl.CERT_REQUIRED - and getattr(context, "post_handshake_auth", None) is not None - ): + # The attribute is None for OpenSSL <= 1.1.0 or does not exist when using + # an SSLContext created by pyOpenSSL. + if getattr(context, "post_handshake_auth", None) is not None: context.post_handshake_auth = True # The order of the below lines setting verify_mode and check_hostname diff --git a/test/test_ssl.py b/test/test_ssl.py index c886d4e51c..43073cb263 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -108,13 +108,28 @@ def test_wrap_socket_no_ssltransport(self) -> None: ssl_.ssl_wrap_socket(sock, tls_in_tls=True) @pytest.mark.parametrize( - ["pha", "expected_pha"], [(None, None), (False, True), (True, True)] + ["pha", "expected_pha", "cert_reqs"], + [ + (None, None, None), + (None, None, ssl.CERT_NONE), + (None, None, ssl.CERT_OPTIONAL), + (None, None, ssl.CERT_REQUIRED), + (False, True, None), + (False, True, ssl.CERT_NONE), + (False, True, ssl.CERT_OPTIONAL), + (False, True, ssl.CERT_REQUIRED), + (True, True, None), + (True, True, ssl.CERT_NONE), + (True, True, ssl.CERT_OPTIONAL), + (True, True, ssl.CERT_REQUIRED), + ], ) def test_create_urllib3_context_pha( self, monkeypatch: pytest.MonkeyPatch, pha: bool | None, expected_pha: bool | None, + cert_reqs: int | None, ) -> None: context = mock.create_autospec(ssl_.SSLContext) context.set_ciphers = mock.Mock() @@ -122,7 +137,7 @@ def test_create_urllib3_context_pha( context.post_handshake_auth = pha monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) - assert ssl_.create_urllib3_context() is context + assert ssl_.create_urllib3_context(cert_reqs=cert_reqs) is context assert context.post_handshake_auth == expected_pha From 6d2f0f63b066ce6064b16ba97dc5cf3f70d5d44a Mon Sep 17 00:00:00 2001 From: KE programmer Date: Mon, 29 Jan 2024 15:48:10 +0300 Subject: [PATCH 085/131] Annotate response attribute `length_remaining` in BaseHTTPResponse (#3317) --- src/urllib3/connectionpool.py | 2 +- src/urllib3/response.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 52bb8657fc..c952dcbebb 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -556,7 +556,7 @@ def _make_request( # HTTP version http_version, response.status, - response.length_remaining, # type: ignore[attr-defined] + response.length_remaining, ) return response diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 6a811ad513..47e75fd171 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -344,6 +344,7 @@ def __init__( self.chunked = True self._decoder: ContentDecoder | None = None + self.length_remaining: int | None def get_redirect_location(self) -> str | None | Literal[False]: """ From 2aec09f6a1727dd54f65b9069ef8f168ef40a885 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 29 Jan 2024 11:50:07 -0600 Subject: [PATCH 086/131] Add documentation for Emscripten support Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- docs/reference/contrib/emscripten.rst | 87 +++++++++++++++++++++++++++ docs/reference/contrib/index.rst | 1 + 2 files changed, 88 insertions(+) create mode 100644 docs/reference/contrib/emscripten.rst diff --git a/docs/reference/contrib/emscripten.rst b/docs/reference/contrib/emscripten.rst new file mode 100644 index 0000000000..6d7c719a37 --- /dev/null +++ b/docs/reference/contrib/emscripten.rst @@ -0,0 +1,87 @@ +Pyodide, Emscripten, and PyScript +================================= + +From the Pyodide documentation, `Pyodide `_ is a Python distribution for the browser and Node.js based on WebAssembly and `Emscripten `_. +This technology also underpins the `PyScript framework `_ and `Jupyterlite `_, so should work in those environments too. + +Starting in version 2.2.0 urllib3 supports being used in a Pyodide runtime utilizing +the `JavaScript fetch API `_ +or falling back on `XMLHttpRequest `_ +if the fetch API isn't available (such as when cross-origin isolation +isn't active). This means you can use Python libraries to make HTTP requests from your browser! + +Because urllib3's Emscripten support is API-compatible, this means that +libraries that depend on urllib3 may now be usable from Emscripten and Pyodide environments, too. + + .. warning:: + + **Support for Emscripten and Pyodide is experimental**. Report all bugs to the `urllib3 issue tracker `_. + Currently only supports browsers, does not yet support running in Node.js. + +It's recommended to `run Pyodide in a Web Worker `_ +in order to take full advantage of features like the fetch API which enables streaming of HTTP response bodies. + +Getting started +--------------- + +Using urllib3 with Pyodide means you need to `get started with Pyodide first `_. +The Pyodide project provides a `useful online REPL `_ to try in your browser without +any setup or installation to test out the code examples below. + +urllib3's Emscripten support is automatically enabled if ``sys.platform`` is ``"emscripten"``, so no setup is required beyond installation and importing the module. + +You can install urllib3 in a Pyodide environment using micropip. +Try using the following code in a Pyodide console or `` + + + + + + + +
+ + + From 12f923325a1794bab26c82dbfef2c47d44f054f8 Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Thu, 8 Feb 2024 05:34:42 +0100 Subject: [PATCH 095/131] Bump cryptography to 42.0.2 and PyOpenSSL to 24.0.0 (#3340) --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7320d27590..7c3dab295e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,10 +3,10 @@ coverage==7.4.1 PySocks==1.7.1 pytest==7.4.4 pytest-timeout==2.1.0 -pyOpenSSL==23.2.0 +pyOpenSSL==24.0.0 idna==3.4 trustme==1.1.0 -cryptography==41.0.6 +cryptography==42.0.2 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;python_version<"3.13" and sys_platform!="win32" and implementation_name=="cpython" From 25155d7d3b7d91ef8400bc3cb7600b9253b765a3 Mon Sep 17 00:00:00 2001 From: Mike Fiedler Date: Thu, 8 Feb 2024 16:26:25 -0500 Subject: [PATCH 096/131] Ensure no remote connections during testing (#3328) As a way to prevent the test suite from making remote calls to anywhere unexpected, use the `pytest-socket` plugin to disable any calls to hosts other than localhost. Signed-off-by: Mike Fiedler --- dev-requirements.txt | 2 ++ noxfile.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7c3dab295e..9ac700bbe1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -21,7 +21,9 @@ quart-trio==0.11.1 # https://github.com/pgjones/hypercorn/issues/169 hypercorn @ git+https://github.com/urllib3/hypercorn@urllib3-changes httpx==0.25.2 +pytest-socket==0.7.0 # CFFI is not going to support CPython 3.13 in an actual release until # there is a release candidate for 3.13. # https://github.com/python-cffi/cffi/issues/23#issuecomment-1845861410 cffi @ git+https://github.com/python-cffi/cffi@14723b0bbd127790c450945099db31018d80fa83; python_version == "3.13" + diff --git a/noxfile.py b/noxfile.py index f1676450b2..3b25894a31 100644 --- a/noxfile.py +++ b/noxfile.py @@ -88,6 +88,9 @@ def tests_impl( "--durations=10", "--strict-config", "--strict-markers", + "--disable-socket", + "--allow-unix-socket", + "--allow-hosts=localhost,::1,127.0.0.0,240.0.0.0", # See `TARPIT_HOST` *pytest_extra_args, *(session.posargs or ("test/",)), env=pytest_session_envvars, From cfe52f96fb65fe2269981d6bba4f22c2bce00b2d Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Thu, 8 Feb 2024 21:28:54 -0600 Subject: [PATCH 097/131] Fix InsecureRequestWarning for HTTPS Emscripten requests (#3333) --- changelog/3331.bugfix.rst | 1 + src/urllib3/contrib/emscripten/connection.py | 5 ++++ test/contrib/emscripten/test_emscripten.py | 25 ++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 changelog/3331.bugfix.rst diff --git a/changelog/3331.bugfix.rst b/changelog/3331.bugfix.rst new file mode 100644 index 0000000000..d309ee7b48 --- /dev/null +++ b/changelog/3331.bugfix.rst @@ -0,0 +1 @@ +Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 9090e51d18..2ceb4579eb 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -67,6 +67,7 @@ def __init__( self.blocksize = blocksize self.source_address = None self.socket_options = None + self.is_verified = False def set_tunnel( self, @@ -228,6 +229,10 @@ def __init__( self.cert_reqs = None + # The browser will automatically verify all requests. + # We have no control over that setting. + self.is_verified = True + def set_cert( self, key_file: str | None = None, diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index b4f8e3341b..17264d8c50 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -947,3 +947,28 @@ def count_calls(self, *args, **argv): # type: ignore[no-untyped-def] assert count == 6 pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) + + +@install_urllib3_wheel() +def test_insecure_requests_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, https_port: int) -> None: # type: ignore[no-untyped-def] + import warnings + + import urllib3 + import urllib3.exceptions + + http = urllib3.PoolManager() + + with warnings.catch_warnings(record=True) as w: + http.request("GET", f"https://{host}:{https_port}") + assert len(w) == 0 + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + ) From fa541793ad42f2f49846de0a9808ee0a484c53cf Mon Sep 17 00:00:00 2001 From: Andreas Hasenkopf Date: Mon, 12 Feb 2024 15:59:48 +0100 Subject: [PATCH 098/131] Distinguish between truncated and excess content in response (#3273) `HTTPResponse._raw_read` raises `IncompleteRead` if the content length does not match the expected content length. For malformed responses (e.g. 204 response with content) the re-raised `ProtocolError` was a bit too unclear about the unexpected excess content being the reason for the exception. With this change, the exception points out, that the client is not dealing with a connection error but a protocol violation. --- changelog/3261.misc.rst | 1 + src/urllib3/exceptions.py | 7 +++++-- src/urllib3/response.py | 12 +++++++++++- test/test_response.py | 23 ++++++++++++++++++++++- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 changelog/3261.misc.rst diff --git a/changelog/3261.misc.rst b/changelog/3261.misc.rst new file mode 100644 index 0000000000..5e44888bc9 --- /dev/null +++ b/changelog/3261.misc.rst @@ -0,0 +1 @@ +Made raised ``urllib3.exceptions.ProtocolError`` more verbose when a response contains content unexpectedly. diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py index 5bb9236961..b0792f00fd 100644 --- a/src/urllib3/exceptions.py +++ b/src/urllib3/exceptions.py @@ -252,13 +252,16 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for ``partial`` to avoid creating large objects on streamed reads. """ + partial: int # type: ignore[assignment] + expected: int + def __init__(self, partial: int, expected: int) -> None: - self.partial = partial # type: ignore[assignment] + self.partial = partial self.expected = expected def __repr__(self) -> str: return "IncompleteRead(%i bytes read, %i more expected)" % ( - self.partial, # type: ignore[str-format] + self.partial, self.expected, ) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 41dc9575b8..d31fac9ba0 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -749,8 +749,18 @@ def _error_catcher(self) -> typing.Generator[None, None, None]: raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] + except IncompleteRead as e: + if ( + e.expected is not None + and e.partial is not None + and e.expected == -e.partial + ): + arg = "Response may not contain content." + else: + arg = f"Connection broken: {e!r}" + raise ProtocolError(arg, e) from e + except (HTTPException, OSError) as e: - # This includes IncompleteRead. raise ProtocolError(f"Connection broken: {e!r}", e) from e # If no exception is thrown, we should avoid cleaning up diff --git a/test/test_response.py b/test/test_response.py index 414cde0e81..6a3c50e147 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -1293,7 +1293,7 @@ def test_buggy_incomplete_read(self) -> None: orig_ex = ctx.value.args[1] assert isinstance(orig_ex, IncompleteRead) - assert orig_ex.partial == 0 # type: ignore[comparison-overlap] + assert orig_ex.partial == 0 assert orig_ex.expected == content_length def test_incomplete_chunk(self) -> None: @@ -1504,6 +1504,27 @@ def make_bad_mac_fp() -> typing.Generator[BytesIO, None, None]: resp.read() assert e.value.args[0] == mac_error + def test_unexpected_body(self) -> None: + with pytest.raises(ProtocolError) as excinfo: + fp = BytesIO(b"12345") + headers = {"content-length": "5"} + resp = HTTPResponse(fp, status=204, headers=headers) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + + with pytest.raises(ProtocolError): + fp = BytesIO(b"12345") + headers = {"content-length": "0"} + resp = HTTPResponse(fp, status=204, headers=headers) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + + with pytest.raises(ProtocolError): + fp = BytesIO(b"12345") + resp = HTTPResponse(fp, status=204) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + class MockChunkedEncodingResponse: def __init__(self, content: list[bytes]) -> None: From e22f651079ae65d06efbb28222c27000256ce7a5 Mon Sep 17 00:00:00 2001 From: Tom Sparrow <793763+sparrowt@users.noreply.github.com> Date: Tue, 13 Feb 2024 03:04:52 +0000 Subject: [PATCH 099/131] Fix docstring of retries parameter --- src/urllib3/__init__.py | 2 +- src/urllib3/connectionpool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 1e0bf37b33..3fe782c8a4 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -167,7 +167,7 @@ def request( Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. - Pass ``None`` to retry until you receive a response. Pass a + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index c952dcbebb..1036f0d718 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -649,7 +649,7 @@ def urlopen( # type: ignore[override] Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. - Pass ``None`` to retry until you receive a response. Pass a + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, From 49b2ddaf07ec9ef65ef12d0218117f20e739ee6e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 16 Feb 2024 11:35:30 +0400 Subject: [PATCH 100/131] Stop casting request headers to HTTPHeaderDict (#3344) While this was done to fix a mypy error, we did not notice the consequences: * This breaks boto3 that subclasses HTTPConnection because HTTPHeaderDict does not support bytes values yet. * When proxying, headers are still a dictionary by default. We can decide to reintroduce a forced conversion to HTTPHeaderDict in urllib3 3.0 if the above issues are fixed. --- changelog/3343.bugfix.rst | 1 + src/urllib3/connectionpool.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog/3343.bugfix.rst diff --git a/changelog/3343.bugfix.rst b/changelog/3343.bugfix.rst new file mode 100644 index 0000000000..4f2df9e7a4 --- /dev/null +++ b/changelog/3343.bugfix.rst @@ -0,0 +1 @@ +Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 1036f0d718..bd58ff14dd 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -751,8 +751,8 @@ def urlopen( # type: ignore[override] # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: - headers = HTTPHeaderDict(headers) - headers.update(self.proxy_headers) + headers = headers.copy() # type: ignore[attr-defined] + headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. From 54d6edf2a671510a5c029d3b76ffe71a5b07147a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 18 Feb 2024 07:44:08 +0400 Subject: [PATCH 101/131] Release 2.2.1 --- CHANGES.rst | 9 +++++++++ changelog/2860.bugfix.rst | 1 - changelog/3261.misc.rst | 1 - changelog/3331.bugfix.rst | 1 - changelog/3343.bugfix.rst | 1 - src/urllib3/_version.py | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 changelog/2860.bugfix.rst delete mode 100644 changelog/3261.misc.rst delete mode 100644 changelog/3331.bugfix.rst delete mode 100644 changelog/3343.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index 80f439f521..319accb223 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +2.2.1 (2024-02-16) +================== + +- Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. (`#3331 `__) +- Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. (`#3343 `__) +- Changed ``ProtocolError`` to ``InvalidChunkLength`` when response terminates before the chunk length is sent. (`#2860 `__) +- Changed ``ProtocolError`` to be more verbose on incomplete reads with excess content. (`#3261 `__) + + 2.2.0 (2024-01-30) ================== diff --git a/changelog/2860.bugfix.rst b/changelog/2860.bugfix.rst deleted file mode 100644 index 3fc6ef7069..0000000000 --- a/changelog/2860.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Use ProtocolError instead of InvalidChunkLength if response terminates before the chunk length is sent. diff --git a/changelog/3261.misc.rst b/changelog/3261.misc.rst deleted file mode 100644 index 5e44888bc9..0000000000 --- a/changelog/3261.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Made raised ``urllib3.exceptions.ProtocolError`` more verbose when a response contains content unexpectedly. diff --git a/changelog/3331.bugfix.rst b/changelog/3331.bugfix.rst deleted file mode 100644 index d309ee7b48..0000000000 --- a/changelog/3331.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. diff --git a/changelog/3343.bugfix.rst b/changelog/3343.bugfix.rst deleted file mode 100644 index 4f2df9e7a4..0000000000 --- a/changelog/3343.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index f697e3e7ea..095cf3c16b 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.2.0" +__version__ = "2.2.1" From db8691302b24d3e2ae5e279d38ba2fdb40b85987 Mon Sep 17 00:00:00 2001 From: Kevin Paulson Date: Sun, 18 Feb 2024 14:43:19 -0500 Subject: [PATCH 102/131] Update docs for `.json()` --- changelog/3342.doc.rst | 1 + docs/user-guide.rst | 2 ++ src/urllib3/contrib/emscripten/response.py | 16 ++++++++++++---- src/urllib3/response.py | 16 ++++++++++++---- 4 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 changelog/3342.doc.rst diff --git a/changelog/3342.doc.rst b/changelog/3342.doc.rst new file mode 100644 index 0000000000..924d32e0d7 --- /dev/null +++ b/changelog/3342.doc.rst @@ -0,0 +1 @@ +Updated docs for ``.json()`` diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 9416fe1263..5c78c8af1c 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -99,6 +99,8 @@ The :class:`~response.HTTPResponse` object provides print(resp.headers) # HTTPHeaderDict({"Content-Length": "32", ...}) +.. _json_content: + JSON Content ~~~~~~~~~~~~ JSON content can be loaded by :meth:`~response.HTTPResponse.json` diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 303b4ee011..05ccd4d94d 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -211,13 +211,21 @@ def data(self) -> bytes: def json(self) -> typing.Any: """ - Parses the body of the HTTP response as JSON. + Deserializes the body of the HTTP response as a Python object. - To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder. + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 `_. - This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`. + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. - Read more :ref:`here `. + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here `. + + :returns: The body of the HTTP response as a Python object. """ data = self.data.decode("utf-8") return _json.loads(data) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index d31fac9ba0..580234a6d3 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -364,13 +364,21 @@ def data(self) -> bytes: def json(self) -> typing.Any: """ - Parses the body of the HTTP response as JSON. + Deserializes the body of the HTTP response as a Python object. - To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder. + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 `_. - This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`. + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. - Read more :ref:`here `. + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here `. + + :returns: The body of the HTTP response as a Python object. """ data = self.data.decode("utf-8") return _json.loads(data) From 0b0bfb07c99e7575734310f24cf2c75843c1ac47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 16:29:40 -0600 Subject: [PATCH 103/131] Bump browser-actions/setup-chrome from 1.4.0 to 1.5.0 Bumps [browser-actions/setup-chrome](https://github.com/browser-actions/setup-chrome) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/browser-actions/setup-chrome/releases) - [Changelog](https://github.com/browser-actions/setup-chrome/blob/master/CHANGELOG.md) - [Commits](https://github.com/browser-actions/setup-chrome/compare/52f10de5479c69bcbbab2eab094c9d373148005e...97349de5c98094d4fc9412f31c524d7697115ad8) --- updated-dependencies: - dependency-name: browser-actions/setup-chrome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ced0d9fe2..573f1e642e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: run: python -m pip install --upgrade pip setuptools nox - name: "Install Chrome" - uses: browser-actions/setup-chrome@52f10de5479c69bcbbab2eab094c9d373148005e # v1.4.0 + uses: browser-actions/setup-chrome@97349de5c98094d4fc9412f31c524d7697115ad8 # v1.5.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0 From b69a08ac76a32f2cee0069200e569ad6c1561cf1 Mon Sep 17 00:00:00 2001 From: Tom Sparrow <793763+sparrowt@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:51:11 +0000 Subject: [PATCH 104/131] Fix changelog entry which was inverted (#3348) The original changelog entry was correct but https://github.com/urllib3/urllib3/commit/54d6edf2a671510a5c029d3b76ffe71a5b07147a inverted its meaning. This puts it back, see https://github.com/urllib3/urllib3/pull/2860#issuecomment-1954488635 --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 319accb223..14fbd4cc1b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ - Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. (`#3331 `__) - Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. (`#3343 `__) -- Changed ``ProtocolError`` to ``InvalidChunkLength`` when response terminates before the chunk length is sent. (`#2860 `__) +- Changed ``InvalidChunkLength`` to ``ProtocolError`` when response terminates before the chunk length is sent. (`#2860 `__) - Changed ``ProtocolError`` to be more verbose on incomplete reads with excess content. (`#3261 `__) From d4ffa29ee1862b3d1afe584efb57d489a7659dac Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 21 Feb 2024 08:13:49 -0800 Subject: [PATCH 105/131] Remove Flask pins from requests downstream tests --- noxfile.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 3b25894a31..92565613f1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -178,9 +178,6 @@ def downstream_requests(session: nox.Session) -> None: session.install(".[socks]", silent=False) session.install("-r", "requirements-dev.txt", silent=False) - # Workaround until https://github.com/psf/httpbin/pull/29 gets released - session.install("flask<3", "werkzeug<3", silent=False) - session.cd(root) session.install(".", silent=False) session.cd(f"{tmp_dir}/requests") From 0628b6b290e52ef9a8b01c49f49dd218d2ee4fe2 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Mon, 26 Feb 2024 04:03:24 +0300 Subject: [PATCH 106/131] Fix type checking when zstandard is installed --- mypy-requirements.txt | 1 + src/urllib3/response.py | 25 +++++++++++++------------ src/urllib3/util/request.py | 2 +- test/__init__.py | 10 ++++++---- test/test_response.py | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 50105b8f3a..d6dae313d3 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -11,3 +11,4 @@ httpx==0.25.2 types-backports types-requests nox +zstandard diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 580234a6d3..7c942f4a0f 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -26,20 +26,21 @@ brotli = None try: - import zstandard as zstd # type: ignore[import-not-found] - + import zstandard as zstd +except (AttributeError, ImportError, ValueError): # Defensive: + HAS_ZSTD = False +else: # The package 'zstandard' added the 'eof' property starting # in v0.18.0 which we require to ensure a complete and # valid zstd stream was fed into the ZstdDecoder. # See: https://github.com/urllib3/urllib3/pull/2624 - _zstd_version = _zstd_version = tuple( + _zstd_version = tuple( map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] ) if _zstd_version < (0, 18): # Defensive: - zstd = None - -except (AttributeError, ImportError, ValueError): # Defensive: - zstd = None + HAS_ZSTD = False + else: + HAS_ZSTD = True from . import util from ._base_connection import _TYPE_BODY @@ -163,7 +164,7 @@ def flush(self) -> bytes: return b"" -if zstd is not None: +if HAS_ZSTD: class ZstdDecoder(ContentDecoder): def __init__(self) -> None: @@ -183,7 +184,7 @@ def flush(self) -> bytes: ret = self._obj.flush() # note: this is a no-op if not self._obj.eof: raise DecodeError("Zstandard data is incomplete") - return ret # type: ignore[no-any-return] + return ret class MultiDecoder(ContentDecoder): @@ -219,7 +220,7 @@ def _get_decoder(mode: str) -> ContentDecoder: if brotli is not None and mode == "br": return BrotliDecoder() - if zstd is not None and mode == "zstd": + if HAS_ZSTD and mode == "zstd": return ZstdDecoder() return DeflateDecoder() @@ -302,7 +303,7 @@ class BaseHTTPResponse(io.IOBase): CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] if brotli is not None: CONTENT_DECODERS += ["br"] - if zstd is not None: + if HAS_ZSTD: CONTENT_DECODERS += ["zstd"] REDIRECT_STATUSES = [301, 302, 303, 307, 308] @@ -310,7 +311,7 @@ class BaseHTTPResponse(io.IOBase): if brotli is not None: DECODER_ERROR_CLASSES += (brotli.error,) - if zstd is not None: + if HAS_ZSTD: DECODER_ERROR_CLASSES += (zstd.ZstdError,) def __init__( diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index fe0e3485e8..859597e276 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -29,7 +29,7 @@ else: ACCEPT_ENCODING += ",br" try: - import zstandard as _unused_module_zstd # type: ignore[import-not-found] # noqa: F401 + import zstandard as _unused_module_zstd # noqa: F401 except ImportError: pass else: diff --git a/test/__init__.py b/test/__init__.py index 12c0055493..f83de804e6 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -26,9 +26,11 @@ brotli = None try: - import zstandard as zstd # type: ignore[import-not-found] + import zstandard as _unused_module_zstd # noqa: F401 except ImportError: - zstd = None + HAS_ZSTD = False +else: + HAS_ZSTD = True from urllib3 import util from urllib3.connectionpool import ConnectionPool @@ -144,13 +146,13 @@ def notBrotli() -> typing.Callable[[_TestFuncT], _TestFuncT]: def onlyZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]: return pytest.mark.skipif( - zstd is None, reason="only run if a python-zstandard library is installed" + not HAS_ZSTD, reason="only run if a python-zstandard library is installed" ) def notZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]: return pytest.mark.skipif( - zstd is not None, + HAS_ZSTD, reason="only run if a python-zstandard library is not installed", ) diff --git a/test/test_response.py b/test/test_response.py index 6a3c50e147..7d00632714 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -30,7 +30,6 @@ BytesQueueBuffer, HTTPResponse, brotli, - zstd, ) from urllib3.util.response import is_fp_closed from urllib3.util.retry import RequestHistory, Retry @@ -389,6 +388,8 @@ def test_decode_brotli_error(self) -> None: @onlyZstd() def test_decode_zstd(self) -> None: + import zstandard as zstd + data = zstd.compress(b"foo") fp = BytesIO(data) @@ -397,6 +398,8 @@ def test_decode_zstd(self) -> None: @onlyZstd() def test_decode_multiframe_zstd(self) -> None: + import zstandard as zstd + data = ( # Zstandard frame zstd.compress(b"foo") @@ -416,6 +419,8 @@ def test_decode_multiframe_zstd(self) -> None: @onlyZstd() def test_chunked_decoding_zstd(self) -> None: + import zstandard as zstd + data = zstd.compress(b"foobarbaz") fp = BytesIO(data) @@ -447,6 +452,8 @@ def test_decode_zstd_error(self, data: bytes) -> None: @onlyZstd() @pytest.mark.parametrize("data", decode_param_set) def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None: + import zstandard as zstd + data = zstd.compress(data) fp = BytesIO(data[:-1]) @@ -456,6 +463,8 @@ def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None: @onlyZstd() @pytest.mark.parametrize("data", decode_param_set) def test_decode_zstd_incomplete_read(self, data: bytes) -> None: + import zstandard as zstd + data = zstd.compress(data) fp = BytesIO(data[:-1]) # shorten the data to trigger DecodeError @@ -471,6 +480,8 @@ def test_decode_zstd_incomplete_read(self, data: bytes) -> None: @onlyZstd() @pytest.mark.parametrize("data", decode_param_set) def test_decode_zstd_incomplete_read1(self, data: bytes) -> None: + import zstandard as zstd + data = zstd.compress(data) fp = BytesIO(data[:-1]) @@ -489,6 +500,8 @@ def test_decode_zstd_incomplete_read1(self, data: bytes) -> None: @onlyZstd() @pytest.mark.parametrize("data", decode_param_set) def test_decode_zstd_read1(self, data: bytes) -> None: + import zstandard as zstd + encoded_data = zstd.compress(data) fp = BytesIO(encoded_data) From 1956e3526cd16f944f4f84a0723196df2fc9ecfc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:39:52 +0000 Subject: [PATCH 107/131] Bump cryptography from 42.0.2 to 42.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 9ac700bbe1..0533f66d8a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ pytest-timeout==2.1.0 pyOpenSSL==24.0.0 idna==3.4 trustme==1.1.0 -cryptography==42.0.2 +cryptography==42.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;python_version<"3.13" and sys_platform!="win32" and implementation_name=="cpython" From a7b81f554863f6dfafe2102e38d16688fdd9869a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:38:20 +0000 Subject: [PATCH 108/131] Bump actions/checkout Bumps [actions/checkout](https://github.com/actions/checkout) from 3df4ab11eba7bda6032a0b82a6bb43b11571feac to b32f140b0c872d58512e0a66172253c302617b90. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/3df4ab11eba7bda6032a0b82a6bb43b11571feac...b32f140b0c872d58512e0a66172253c302617b90) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index a47ab4ffc7..79f9474eb8 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,7 +13,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 with: # `towncrier check` runs `git diff --name-only origin/main...`, which # needs a non-shallow clone. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 573f1e642e..45421939ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 @@ -103,7 +103,7 @@ jobs: timeout-minutes: 30 steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python ${{ matrix.python-version }}" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 @@ -141,7 +141,7 @@ jobs: needs: test steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a0de4fcc59..93f75b8070 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,7 +21,7 @@ jobs: security-events: write steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Run CodeQL init" uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index b0fba2da8a..81b4df056d 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f7375a4d38..420f53d54f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e79066829d..c64a8eb49b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,7 +19,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 356be2d94d..4dd59be6fc 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -21,7 +21,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 with: persist-credentials: false From 8bbc3db69eb05ec6cce8b3599c2d81b50dbffebf Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Wed, 28 Feb 2024 17:55:59 +0200 Subject: [PATCH 109/131] Allow passing negative integers as `amt` to read methods (#3356) --- changelog/3122.bugfix.rst | 2 ++ src/urllib3/contrib/emscripten/response.py | 2 +- src/urllib3/response.py | 13 ++++++++++++- test/test_response.py | 21 +++++++++++++++++++-- 4 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 changelog/3122.bugfix.rst diff --git a/changelog/3122.bugfix.rst b/changelog/3122.bugfix.rst new file mode 100644 index 0000000000..12fbbbf131 --- /dev/null +++ b/changelog/3122.bugfix.rst @@ -0,0 +1,2 @@ +Allowed passing negative integers as ``amt`` to read methods of +:class:`http.client.HTTPResponse` as an alternative to ``None``. diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 05ccd4d94d..958aaeaf74 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -155,7 +155,7 @@ def read( self.length_is_certain = True # wrap body in IOStream self._response.body = BytesIO(self._response.body) - if amt is not None: + if amt is not None and amt >= 0: # don't cache partial content cache_content = False data = self._response.body.read(amt) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 7c942f4a0f..c18d84a50e 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -935,7 +935,10 @@ def read( if decode_content is None: decode_content = self.decode_content - if amt is not None: + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + elif amt is not None: cache_content = False if len(self._decoded_buffer) >= amt: @@ -995,6 +998,9 @@ def read1( """ if decode_content is None: decode_content = self.decode_content + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None # try and respond without going to the network if self._has_decoded_content: if not decode_content: @@ -1189,6 +1195,11 @@ def read_chunked( if self._fp.fp is None: # type: ignore[union-attr] return None + if amt and amt < 0: + # Negative numbers and `None` should be treated the same, + # but httplib handles only `None` correctly. + amt = None + while True: self._update_chunk_length() if self.chunk_left == 0: diff --git a/test/test_response.py b/test/test_response.py index 7d00632714..7fcd3fa6c7 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -217,6 +217,12 @@ def test_reference_read(self) -> None: assert r.read() == b"" assert r.read() == b"" + @pytest.mark.parametrize("read_args", ((), (None,), (-1,))) + def test_reference_read_until_eof(self, read_args: tuple[typing.Any, ...]) -> None: + fp = BytesIO(b"foo") + r = HTTPResponse(fp, preload_content=False) + assert r.read(*read_args) == b"foo" + def test_reference_read1(self) -> None: fp = BytesIO(b"foobar") r = HTTPResponse(fp, preload_content=False) @@ -227,6 +233,14 @@ def test_reference_read1(self) -> None: assert r.read1() == b"bar" assert r.read1() == b"" + @pytest.mark.parametrize("read1_args", ((), (None,), (-1,))) + def test_reference_read1_without_limit( + self, read1_args: tuple[typing.Any, ...] + ) -> None: + fp = BytesIO(b"foo") + r = HTTPResponse(fp, preload_content=False) + assert r.read1(*read1_args) == b"foo" + def test_reference_read1_nodecode(self) -> None: fp = BytesIO(b"foobar") r = HTTPResponse(fp, preload_content=False, decode_content=False) @@ -1262,7 +1276,10 @@ def test_mock_transfer_encoding_chunked_custom_read(self) -> None: response = list(resp.read_chunked(2)) assert expected_response == response - def test_mock_transfer_encoding_chunked_unlmtd_read(self) -> None: + @pytest.mark.parametrize("read_chunked_args", ((), (None,), (-1,))) + def test_mock_transfer_encoding_chunked_unlmtd_read( + self, read_chunked_args: tuple[typing.Any, ...] + ) -> None: stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedEncodingResponse(stream) r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] @@ -1272,7 +1289,7 @@ def test_mock_transfer_encoding_chunked_unlmtd_read(self) -> None: resp = HTTPResponse( r, preload_content=False, headers={"transfer-encoding": "chunked"} ) - assert stream == list(resp.read_chunked()) + assert stream == list(resp.read_chunked(*read_chunked_args)) def test_read_not_chunked_response_as_chunks(self) -> None: fp = BytesIO(b"foo") From 733f638a2faa02b4ff8a9f3b5668949d39396b8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:29:43 +0200 Subject: [PATCH 110/131] Bump browser-actions/setup-firefox from 1.4.0 to 1.5.0 (#3359) Bumps [browser-actions/setup-firefox](https://github.com/browser-actions/setup-firefox) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/browser-actions/setup-firefox/releases) - [Changelog](https://github.com/browser-actions/setup-firefox/blob/master/CHANGELOG.md) - [Commits](https://github.com/browser-actions/setup-firefox/compare/29a706787c6fb2196f091563261e1273bf379ead...233224b712fc07910ded8c15fb95a555c86da76f) --- updated-dependencies: - dependency-name: browser-actions/setup-firefox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45421939ba..4522a05a37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: uses: browser-actions/setup-chrome@97349de5c98094d4fc9412f31c524d7697115ad8 # v1.5.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" - uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0 + uses: browser-actions/setup-firefox@233224b712fc07910ded8c15fb95a555c86da76f # v1.5.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version From cc892b0b81fdb63dc2bf0e99264dfbc8fb21410c Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 17 Mar 2024 16:12:13 -0400 Subject: [PATCH 111/131] Change method return types to `Self` where suitable (#3364) --- changelog/3363.bugfix.rst | 1 + src/urllib3/_collections.py | 6 +++--- src/urllib3/connectionpool.py | 6 +++--- src/urllib3/poolmanager.py | 6 +++--- src/urllib3/util/retry.py | 6 ++++-- src/urllib3/util/ssltransport.py | 5 +++-- 6 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 changelog/3363.bugfix.rst diff --git a/changelog/3363.bugfix.rst b/changelog/3363.bugfix.rst new file mode 100644 index 0000000000..579423950a --- /dev/null +++ b/changelog/3363.bugfix.rst @@ -0,0 +1 @@ +Consistently used ``typing.Self`` for return types representing copying actions. diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index 55b0324797..8a4409a122 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -427,7 +427,7 @@ def _copy_from(self, other: HTTPHeaderDict) -> None: val = other.getlist(key) self._container[key.lower()] = [key, *val] - def copy(self) -> HTTPHeaderDict: + def copy(self) -> Self: clone = type(self)() clone._copy_from(self) return clone @@ -462,7 +462,7 @@ def __ior__(self, other: object) -> HTTPHeaderDict: self.extend(maybe_constructable) return self - def __or__(self, other: object) -> HTTPHeaderDict: + def __or__(self, other: object) -> Self: # Supports merging header dicts using operator | # combining items with add instead of __setitem__ maybe_constructable = ensure_can_construct_http_header_dict(other) @@ -472,7 +472,7 @@ def __or__(self, other: object) -> HTTPHeaderDict: result.extend(maybe_constructable) return result - def __ror__(self, other: object) -> HTTPHeaderDict: + def __ror__(self, other: object) -> Self: # Supports merging header dicts using operator | when other is on left side # combining items with add instead of __setitem__ maybe_constructable = ensure_can_construct_http_header_dict(other) diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index bd58ff14dd..fb85a97201 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -55,14 +55,14 @@ import ssl from typing import Literal + from typing_extensions import Self + from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection log = logging.getLogger(__name__) _TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None] -_SelfT = typing.TypeVar("_SelfT") - # Pool objects class ConnectionPool: @@ -95,7 +95,7 @@ def __init__(self, host: str, port: int | None = None) -> None: def __str__(self) -> str: return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})" - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__( diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index 32da0a00ab..a7072595ee 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -28,6 +28,8 @@ import ssl from typing import Literal + from typing_extensions import Self + __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] @@ -51,8 +53,6 @@ # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 _DEFAULT_BLOCKSIZE = 16384 -_SelfT = typing.TypeVar("_SelfT") - class PoolKey(typing.NamedTuple): """ @@ -214,7 +214,7 @@ def __init__( self.pool_classes_by_scheme = pool_classes_by_scheme self.key_fn_by_scheme = key_fn_by_scheme.copy() - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__( diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index 7572bfd26a..7a76a4a6ad 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -21,6 +21,8 @@ from .util import reraise if typing.TYPE_CHECKING: + from typing_extensions import Self + from ..connectionpool import ConnectionPool from ..response import BaseHTTPResponse @@ -240,7 +242,7 @@ def __init__( ) self.backoff_jitter = backoff_jitter - def new(self, **kw: typing.Any) -> Retry: + def new(self, **kw: typing.Any) -> Self: params = dict( total=self.total, connect=self.connect, @@ -429,7 +431,7 @@ def increment( error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, - ) -> Retry: + ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py index fa9f2b37c5..e748582d9e 100644 --- a/src/urllib3/util/ssltransport.py +++ b/src/urllib3/util/ssltransport.py @@ -10,10 +10,11 @@ if typing.TYPE_CHECKING: from typing import Literal + from typing_extensions import Self + from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT -_SelfT = typing.TypeVar("_SelfT", bound="SSLTransport") _WriteBuffer = typing.Union[bytearray, memoryview] _ReturnValue = typing.TypeVar("_ReturnValue") @@ -70,7 +71,7 @@ def __init__( # Perform initial handshake. self._ssl_io_loop(self.sslobj.do_handshake) - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__(self, *_: typing.Any) -> None: From da892d67c083d354cf5c96fb543dd920f7ec82bb Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 17 Mar 2024 16:31:22 -0400 Subject: [PATCH 112/131] Use `typing.Literal` available in Python 3.8+ (#3365) --- docs/conf.py | 1 - src/urllib3/_base_connection.py | 8 ++++---- src/urllib3/connection.py | 8 +++----- src/urllib3/connectionpool.py | 5 ++--- src/urllib3/contrib/socks.py | 4 +--- src/urllib3/poolmanager.py | 5 ++--- src/urllib3/response.py | 4 +--- src/urllib3/util/ssl_.py | 4 ++-- src/urllib3/util/ssltransport.py | 6 ++---- test/__init__.py | 3 +-- test/test_ssltransport.py | 13 +++++-------- test/test_util.py | 9 +++------ 12 files changed, 26 insertions(+), 44 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 138f99abbe..0ab1b3a245 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,7 +110,6 @@ ("py:class", "_HttplibHTTPResponse"), ("py:class", "_HttplibHTTPMessage"), ("py:class", "TracebackType"), - ("py:class", "Literal"), ("py:class", "email.errors.MessageDefect"), ("py:class", "MessageDefect"), ("py:class", "http.client.HTTPMessage"), diff --git a/src/urllib3/_base_connection.py b/src/urllib3/_base_connection.py index bb349c744b..29ca334879 100644 --- a/src/urllib3/_base_connection.py +++ b/src/urllib3/_base_connection.py @@ -12,7 +12,7 @@ class ProxyConfig(typing.NamedTuple): ssl_context: ssl.SSLContext | None use_forwarding_for_https: bool - assert_hostname: None | str | Literal[False] + assert_hostname: None | str | typing.Literal[False] assert_fingerprint: str | None @@ -28,7 +28,7 @@ class _ResponseOptions(typing.NamedTuple): if typing.TYPE_CHECKING: import ssl - from typing import Literal, Protocol + from typing import Protocol from .response import BaseHTTPResponse @@ -124,7 +124,7 @@ class BaseHTTPSConnection(BaseHTTPConnection, Protocol): # Certificate verification methods cert_reqs: int | str | None - assert_hostname: None | str | Literal[False] + assert_hostname: None | str | typing.Literal[False] assert_fingerprint: str | None ssl_context: ssl.SSLContext | None @@ -155,7 +155,7 @@ def __init__( proxy: Url | None = None, proxy_config: ProxyConfig | None = None, cert_reqs: int | str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, server_hostname: str | None = None, ssl_context: ssl.SSLContext | None = None, diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index aa5c547c66..96e24baa93 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -14,8 +14,6 @@ from socket import timeout as SocketTimeout if typing.TYPE_CHECKING: - from typing import Literal - from .response import HTTPResponse from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT from .util.ssltransport import SSLTransport @@ -523,7 +521,7 @@ def __init__( proxy: Url | None = None, proxy_config: ProxyConfig | None = None, cert_reqs: int | str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, server_hostname: str | None = None, ssl_context: ssl.SSLContext | None = None, @@ -577,7 +575,7 @@ def set_cert( cert_reqs: int | str | None = None, key_password: str | None = None, ca_certs: str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, ca_cert_dir: str | None = None, ca_cert_data: None | str | bytes = None, @@ -742,7 +740,7 @@ def _ssl_wrap_socket_and_match_hostname( ca_certs: str | None, ca_cert_dir: str | None, ca_cert_data: None | str | bytes, - assert_hostname: None | str | Literal[False], + assert_hostname: None | str | typing.Literal[False], assert_fingerprint: str | None, server_hostname: str | None, ssl_context: ssl.SSLContext | None, diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index fb85a97201..2d3b563c50 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -53,7 +53,6 @@ if typing.TYPE_CHECKING: import ssl - from typing import Literal from typing_extensions import Self @@ -103,7 +102,7 @@ def __exit__( exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, - ) -> Literal[False]: + ) -> typing.Literal[False]: self.close() # Return False to re-raise any potential exceptions return False @@ -1002,7 +1001,7 @@ def __init__( ssl_version: int | str | None = None, ssl_minimum_version: ssl.TLSVersion | None = None, ssl_maximum_version: ssl.TLSVersion | None = None, - assert_hostname: str | Literal[False] | None = None, + assert_hostname: str | typing.Literal[False] | None = None, assert_fingerprint: str | None = None, ca_cert_dir: str | None = None, **conn_kw: typing.Any, diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py index 5a803916b0..c62b5e0332 100644 --- a/src/urllib3/contrib/socks.py +++ b/src/urllib3/contrib/socks.py @@ -71,10 +71,8 @@ except ImportError: ssl = None # type: ignore[assignment] -from typing import TypedDict - -class _TYPE_SOCKS_OPTIONS(TypedDict): +class _TYPE_SOCKS_OPTIONS(typing.TypedDict): socks_version: int proxy_host: str | None proxy_port: str | None diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index a7072595ee..085d1dbafd 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -26,7 +26,6 @@ if typing.TYPE_CHECKING: import ssl - from typing import Literal from typing_extensions import Self @@ -222,7 +221,7 @@ def __exit__( exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, - ) -> Literal[False]: + ) -> typing.Literal[False]: self.clear() # Return False to re-raise any potential exceptions return False @@ -553,7 +552,7 @@ def __init__( proxy_headers: typing.Mapping[str, str] | None = None, proxy_ssl_context: ssl.SSLContext | None = None, use_forwarding_for_https: bool = False, - proxy_assert_hostname: None | str | Literal[False] = None, + proxy_assert_hostname: None | str | typing.Literal[False] = None, proxy_assert_fingerprint: str | None = None, **connection_pool_kw: typing.Any, ) -> None: diff --git a/src/urllib3/response.py b/src/urllib3/response.py index c18d84a50e..f00342eafa 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -62,8 +62,6 @@ from .util.retry import Retry if typing.TYPE_CHECKING: - from typing import Literal - from .connectionpool import HTTPConnectionPool log = logging.getLogger(__name__) @@ -347,7 +345,7 @@ def __init__( self._decoder: ContentDecoder | None = None self.length_remaining: int | None - def get_redirect_location(self) -> str | None | Literal[False]: + def get_redirect_location(self) -> str | None | typing.Literal[False]: """ Should we redirect and where to? diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index b14cf27b61..c46dd83e53 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -78,7 +78,7 @@ def _is_has_never_check_common_name_reliable( if typing.TYPE_CHECKING: from ssl import VerifyMode - from typing import Literal, TypedDict + from typing import TypedDict from .ssltransport import SSLTransport as SSLTransportType @@ -365,7 +365,7 @@ def ssl_wrap_socket( ca_cert_dir: str | None = ..., key_password: str | None = ..., ca_cert_data: None | str | bytes = ..., - tls_in_tls: Literal[False] = ..., + tls_in_tls: typing.Literal[False] = ..., ) -> ssl.SSLSocket: ... diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py index e748582d9e..b52c477c77 100644 --- a/src/urllib3/util/ssltransport.py +++ b/src/urllib3/util/ssltransport.py @@ -8,8 +8,6 @@ from ..exceptions import ProxySchemeUnsupported if typing.TYPE_CHECKING: - from typing import Literal - from typing_extensions import Self from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT @@ -175,12 +173,12 @@ def close(self) -> None: @typing.overload def getpeercert( - self, binary_form: Literal[False] = ... + self, binary_form: typing.Literal[False] = ... ) -> _TYPE_PEER_CERT_RET_DICT | None: ... @typing.overload - def getpeercert(self, binary_form: Literal[True]) -> bytes | None: + def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ... def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET: diff --git a/test/__init__.py b/test/__init__.py index f83de804e6..62c26330e6 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -44,7 +44,6 @@ if typing.TYPE_CHECKING: import ssl - from typing import Literal _RT = typing.TypeVar("_RT") # return type @@ -266,7 +265,7 @@ def __exit__( exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, - ) -> Literal[False]: + ) -> typing.Literal[False]: self.uninstall() return False diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 4cce4def02..616890d20f 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -14,9 +14,6 @@ from urllib3.util import ssl_ from urllib3.util.ssltransport import SSLTransport -if typing.TYPE_CHECKING: - from typing import Literal - # consume_socket can iterate forever, we add timeouts to prevent halting. PER_TEST_TIMEOUT = 60 @@ -34,12 +31,12 @@ def server_client_ssl_contexts() -> tuple[ssl.SSLContext, ssl.SSLContext]: @typing.overload -def sample_request(binary: Literal[True] = ...) -> bytes: +def sample_request(binary: typing.Literal[True] = ...) -> bytes: ... @typing.overload -def sample_request(binary: Literal[False]) -> str: +def sample_request(binary: typing.Literal[False]) -> str: ... @@ -54,7 +51,7 @@ def sample_request(binary: bool = True) -> bytes | str: def validate_request( - provided_request: bytearray, binary: Literal[False, True] = True + provided_request: bytearray, binary: typing.Literal[False, True] = True ) -> None: assert provided_request is not None expected_request = sample_request(binary) @@ -62,12 +59,12 @@ def validate_request( @typing.overload -def sample_response(binary: Literal[True] = ...) -> bytes: +def sample_response(binary: typing.Literal[True] = ...) -> bytes: ... @typing.overload -def sample_response(binary: Literal[False]) -> str: +def sample_response(binary: typing.Literal[False]) -> str: ... diff --git a/test/test_util.py b/test/test_util.py index 8ed92ee189..268f79f0dc 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -41,9 +41,6 @@ from . import clear_warnings -if typing.TYPE_CHECKING: - from typing import Literal - # This number represents a time in seconds, it doesn't mean anything in # isolation. Setting to a high-ish value to avoid conflicts with the smaller # numbers used for timeouts @@ -516,7 +513,7 @@ def test_netloc(self, url: str, expected_netloc: str | None) -> None: @pytest.mark.parametrize("url, expected_url", url_vulnerabilities) def test_url_vulnerabilities( - self, url: str, expected_url: Literal[False] | Url + self, url: str, expected_url: typing.Literal[False] | Url ) -> None: if expected_url is False: with pytest.raises(LocationParseError): @@ -748,7 +745,7 @@ def test_timeout_elapsed(self, time_monotonic: MagicMock) -> None: def test_is_fp_closed_object_supports_closed(self) -> None: class ClosedFile: @property - def closed(self) -> Literal[True]: + def closed(self) -> typing.Literal[True]: return True assert is_fp_closed(ClosedFile()) @@ -764,7 +761,7 @@ def fp(self) -> None: def test_is_fp_closed_object_has_fp(self) -> None: class FpFile: @property - def fp(self) -> Literal[True]: + def fp(self) -> typing.Literal[True]: return True assert not is_fp_closed(FpFile()) From ea443d208815e3136c9541e8c22dbd5a0f1f43b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 23:21:55 +0200 Subject: [PATCH 113/131] Bump slsa-framework/slsa-github-generator from 1.9.0 to 1.10.0 (#3369) Bumps [slsa-framework/slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) from 1.9.0 to 1.10.0. - [Release notes](https://github.com/slsa-framework/slsa-github-generator/releases) - [Changelog](https://github.com/slsa-framework/slsa-github-generator/blob/main/CHANGELOG.md) - [Commits](https://github.com/slsa-framework/slsa-github-generator/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: slsa-framework/slsa-github-generator dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c64a8eb49b..2a2b4eb64f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,7 +53,7 @@ jobs: actions: read contents: write id-token: write # Needed to access the workflow's OIDC identity. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.9.0 + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.10.0 with: base64-subjects: "${{ needs.build.outputs.hashes }}" upload-assets: true From 8c2088622059860e5411c8e37b26e402a5dda0bb Mon Sep 17 00:00:00 2001 From: Ruben Laguna Date: Tue, 26 Mar 2024 21:02:11 +0100 Subject: [PATCH 114/131] Fix tests that leak threads (pytest 8) (#3358) --- dev-requirements.txt | 2 +- dummyserver/socketserver.py | 2 + dummyserver/testcase.py | 76 +++++++++++++++++++---- test/test_ssltransport.py | 18 ++++-- test/with_dummyserver/test_socketlevel.py | 59 ++++++++++++++---- 5 files changed, 128 insertions(+), 29 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0533f66d8a..406b8d2f5b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ h2==4.1.0 coverage==7.4.1 PySocks==1.7.1 -pytest==7.4.4 +pytest==8.0.2 pytest-timeout==2.1.0 pyOpenSSL==24.0.0 idna==3.4 diff --git a/dummyserver/socketserver.py b/dummyserver/socketserver.py index 202915ce88..b8524b914d 100755 --- a/dummyserver/socketserver.py +++ b/dummyserver/socketserver.py @@ -108,6 +108,7 @@ def __init__( socket_handler: typing.Callable[[socket.socket], None], host: str = "localhost", ready_event: threading.Event | None = None, + quit_event: threading.Event | None = None, ) -> None: super().__init__() self.daemon = True @@ -115,6 +116,7 @@ def __init__( self.socket_handler = socket_handler self.host = host self.ready_event = ready_event + self.quit_event = quit_event def _start_server(self) -> None: if self.USE_IPV6: diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 7eed47668b..66a43606a5 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -5,6 +5,7 @@ import ssl import threading import typing +from test import LONG_TIMEOUT import hypercorn import pytest @@ -19,11 +20,19 @@ def consume_socket( - sock: SSLTransport | socket.socket, chunks: int = 65536 + sock: SSLTransport | socket.socket, + chunks: int = 65536, + quit_event: threading.Event | None = None, ) -> bytearray: consumed = bytearray() + sock.settimeout(LONG_TIMEOUT) while True: - b = sock.recv(chunks) + if quit_event and quit_event.is_set(): + break + try: + b = sock.recv(chunks) + except (TimeoutError, socket.timeout): + continue assert isinstance(b, bytes) consumed += b if b.endswith(b"\r\n\r\n"): @@ -57,11 +66,16 @@ class SocketDummyServerTestCase: @classmethod def _start_server( - cls, socket_handler: typing.Callable[[socket.socket], None] + cls, + socket_handler: typing.Callable[[socket.socket], None], + quit_event: threading.Event | None = None, ) -> None: ready_event = threading.Event() cls.server_thread = SocketServerThread( - socket_handler=socket_handler, ready_event=ready_event, host=cls.host + socket_handler=socket_handler, + ready_event=ready_event, + host=cls.host, + quit_event=quit_event, ) cls.server_thread.start() ready_event.wait(5) @@ -71,23 +85,41 @@ def _start_server( @classmethod def start_response_handler( - cls, response: bytes, num: int = 1, block_send: threading.Event | None = None + cls, + response: bytes, + num: int = 1, + block_send: threading.Event | None = None, ) -> threading.Event: ready_event = threading.Event() + quit_event = threading.Event() def socket_handler(listener: socket.socket) -> None: for _ in range(num): ready_event.set() - sock = listener.accept()[0] - consume_socket(sock) + listener.settimeout(LONG_TIMEOUT) + while True: + if quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + consume_socket(sock, quit_event=quit_event) + if quit_event.is_set(): + sock.close() + return if block_send: - block_send.wait() + while not block_send.wait(LONG_TIMEOUT): + if quit_event.is_set(): + sock.close() + return block_send.clear() sock.send(response) sock.close() - cls._start_server(socket_handler) + cls._start_server(socket_handler, quit_event=quit_event) return ready_event @classmethod @@ -100,10 +132,25 @@ def start_basic_handler( block_send, ) + @staticmethod + def quit_server_thread(server_thread: SocketServerThread) -> None: + if server_thread.quit_event: + server_thread.quit_event.set() + # in principle the maximum time that the thread can take to notice + # the quit_event is LONG_TIMEOUT and the thread should terminate + # shortly after that, we give 5 seconds leeway just in case + server_thread.join(LONG_TIMEOUT * 2 + 5.0) + if server_thread.is_alive(): + raise Exception("server_thread did not exit") + @classmethod def teardown_class(cls) -> None: if hasattr(cls, "server_thread"): - cls.server_thread.join(0.1) + cls.quit_server_thread(cls.server_thread) + + def teardown_method(self) -> None: + if hasattr(self, "server_thread"): + self.quit_server_thread(self.server_thread) def assert_header_received( self, @@ -128,11 +175,16 @@ def assert_header_received( class IPV4SocketDummyServerTestCase(SocketDummyServerTestCase): @classmethod def _start_server( - cls, socket_handler: typing.Callable[[socket.socket], None] + cls, + socket_handler: typing.Callable[[socket.socket], None], + quit_event: threading.Event | None = None, ) -> None: ready_event = threading.Event() cls.server_thread = SocketServerThread( - socket_handler=socket_handler, ready_event=ready_event, host=cls.host + socket_handler=socket_handler, + ready_event=ready_event, + host=cls.host, + quit_event=quit_event, ) cls.server_thread.USE_IPV6 = False cls.server_thread.start() diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index 616890d20f..b6d1f861eb 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -4,6 +4,7 @@ import select import socket import ssl +import threading import typing from unittest import mock @@ -108,20 +109,29 @@ def setup_class(cls) -> None: cls.server_context, cls.client_context = server_client_ssl_contexts() def start_dummy_server( - self, handler: typing.Callable[[socket.socket], None] | None = None + self, + handler: typing.Callable[[socket.socket], None] | None = None, + validate: bool = True, ) -> None: + quit_event = threading.Event() + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] try: with self.server_context.wrap_socket(sock, server_side=True) as ssock: - request = consume_socket(ssock) + request = consume_socket( + ssock, + quit_event=quit_event, + ) + if not validate: + return validate_request(request) ssock.send(sample_response()) except (ConnectionAbortedError, ConnectionResetError): return chosen_handler = handler if handler else socket_handler - self._start_server(chosen_handler) + self._start_server(chosen_handler, quit_event=quit_event) @pytest.mark.timeout(PER_TEST_TIMEOUT) def test_start_closed_socket(self) -> None: @@ -135,7 +145,7 @@ def test_start_closed_socket(self) -> None: @pytest.mark.timeout(PER_TEST_TIMEOUT) def test_close_after_handshake(self) -> None: """Socket errors should be bubbled up""" - self.start_dummy_server() + self.start_dummy_server(validate=False) sock = socket.create_connection((self.host, self.port)) with SSLTransport( diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 69d8070b8b..dceb5ee0ee 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -12,6 +12,7 @@ import socket import ssl import tempfile +import threading import typing import zlib from collections import OrderedDict @@ -955,7 +956,11 @@ def socket_handler(listener: socket.socket) -> None: assert response.connection is None def test_socket_close_socket_then_file(self) -> None: - def consume_ssl_socket(listener: socket.socket) -> None: + quit_event = threading.Event() + + def consume_ssl_socket( + listener: socket.socket, + ) -> None: try: with listener.accept()[0] as sock, original_ssl_wrap_socket( sock, @@ -964,11 +969,11 @@ def consume_ssl_socket(listener: socket.socket) -> None: certfile=DEFAULT_CERTS["certfile"], ca_certs=DEFAULT_CA, ) as ssl_sock: - consume_socket(ssl_sock) + consume_socket(ssl_sock, quit_event=quit_event) except (ConnectionResetError, ConnectionAbortedError, OSError): pass - self._start_server(consume_ssl_socket) + self._start_server(consume_ssl_socket, quit_event=quit_event) with socket.create_connection( (self.host, self.port) ) as sock, contextlib.closing( @@ -983,6 +988,8 @@ def consume_ssl_socket(listener: socket.socket) -> None: assert ssl_sock.fileno() == -1 def test_socket_close_stays_open_with_makefile_open(self) -> None: + quit_event = threading.Event() + def consume_ssl_socket(listener: socket.socket) -> None: try: with listener.accept()[0] as sock, original_ssl_wrap_socket( @@ -992,11 +999,11 @@ def consume_ssl_socket(listener: socket.socket) -> None: certfile=DEFAULT_CERTS["certfile"], ca_certs=DEFAULT_CA, ) as ssl_sock: - consume_socket(ssl_sock) + consume_socket(ssl_sock, quit_event=quit_event) except (ConnectionResetError, ConnectionAbortedError, OSError): pass - self._start_server(consume_ssl_socket) + self._start_server(consume_ssl_socket, quit_event=quit_event) with socket.create_connection( (self.host, self.port) ) as sock, contextlib.closing( @@ -2232,11 +2239,28 @@ def socket_handler(listener: socket.socket) -> None: class TestMultipartResponse(SocketDummyServerTestCase): def test_multipart_assert_header_parsing_no_defects(self) -> None: + quit_event = threading.Event() + def socket_handler(listener: socket.socket) -> None: for _ in range(2): - sock = listener.accept()[0] - while not sock.recv(65536).endswith(b"\r\n\r\n"): - pass + listener.settimeout(LONG_TIMEOUT) + + while True: + if quit_event and quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + + sock.settimeout(LONG_TIMEOUT) + while True: + if quit_event and quit_event.is_set(): + sock.close() + return + if sock.recv(65536).endswith(b"\r\n\r\n"): + break sock.sendall( b"HTTP/1.1 404 Not Found\r\n" @@ -2252,7 +2276,7 @@ def socket_handler(listener: socket.socket) -> None: ) sock.close() - self._start_server(socket_handler) + self._start_server(socket_handler, quit_event=quit_event) from urllib3.connectionpool import log with mock.patch.object(log, "warning") as log_warning: @@ -2308,15 +2332,26 @@ def socket_handler(listener: socket.socket) -> None: def test_chunked_specified( self, method: str, chunked: bool, body_type: str ) -> None: + quit_event = threading.Event() buffer = bytearray() expected_bytes = b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" def socket_handler(listener: socket.socket) -> None: nonlocal buffer - sock = listener.accept()[0] - sock.settimeout(0) + listener.settimeout(LONG_TIMEOUT) + while True: + if quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + sock.settimeout(LONG_TIMEOUT) while expected_bytes not in buffer: + if quit_event.is_set(): + return with contextlib.suppress(BlockingIOError): buffer += sock.recv(65536) @@ -2327,7 +2362,7 @@ def socket_handler(listener: socket.socket) -> None: ) sock.close() - self._start_server(socket_handler) + self._start_server(socket_handler, quit_event=quit_event) body: typing.Any if body_type == "generator": From 6434c9972ad3eb3d69f0a614d37b9c44972466f2 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Wed, 27 Mar 2024 23:25:24 +0200 Subject: [PATCH 115/131] Upgrade Trio to 0.25.0 to drop a workaround (#3367) --- dev-requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 406b8d2f5b..773fd9e53f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,10 +10,7 @@ cryptography==42.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 pytest-memray==1.5.0;python_version<"3.13" and sys_platform!="win32" and implementation_name=="cpython" -trio==0.23.1;python_version<"3.13" -# We need a release of Trio newer than 0.24.0 to support CPython 3.13. -# https://github.com/python-trio/trio/issues/2903 -trio @ git+https://github.com/python-trio/trio@e4c8eb2d7ef59eeea1441656e392fe1b0870a374; python_version == "3.13" +trio==0.25.0 Quart==0.19.4 quart-trio==0.11.1 # https://github.com/pgjones/hypercorn/issues/62 From 2df200368a55d3652d8bef9c54989d856055aee1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 21:22:51 +0000 Subject: [PATCH 116/131] Bump actions/checkout Bumps [actions/checkout](https://github.com/actions/checkout) from b32f140b0c872d58512e0a66172253c302617b90 to cd7d8d697e10461458bc61a30d094dc601a8b017. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/b32f140b0c872d58512e0a66172253c302617b90...cd7d8d697e10461458bc61a30d094dc601a8b017) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 79f9474eb8..f65b7c33d1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,7 +13,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 with: # `towncrier check` runs `git diff --name-only origin/main...`, which # needs a non-shallow clone. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4522a05a37..8b6b40550f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 @@ -103,7 +103,7 @@ jobs: timeout-minutes: 30 steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python ${{ matrix.python-version }}" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 @@ -141,7 +141,7 @@ jobs: needs: test steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 93f75b8070..753fcb22fb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,7 +21,7 @@ jobs: security-events: write steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Run CodeQL init" uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 81b4df056d..5b347113db 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 420f53d54f..1c5ab1d870 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2a2b4eb64f..4bad924ac3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,7 +19,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 4dd59be6fc..9b6daf91ef 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -21,7 +21,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0 + uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 with: persist-credentials: false From 84b0c473fe186b49fb629b68e9b0a7f68908442a Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Thu, 28 Mar 2024 23:59:06 +0200 Subject: [PATCH 117/131] Adjust tests to temporary failures with CPython 3.13.0a5 (#3371) --- dev-requirements.txt | 5 ++- .../test_proxy_poolmanager.py | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 773fd9e53f..3319a14ce5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,10 @@ pytest==8.0.2 pytest-timeout==2.1.0 pyOpenSSL==24.0.0 idna==3.4 -trustme==1.1.0 +# As of v1.1.0, child CA certificates generated by trustme fail +# verification by CPython 3.13. +# https://github.com/python-trio/trustme/pull/642 +trustme @ git+https://github.com/python-trio/trustme@b3a767f336e20600f30c9ff78385a58352ff6ee3 cryptography==42.0.4 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==23.6.0 diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 397181a9e6..f477d3382a 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -9,6 +9,7 @@ import shutil import socket import ssl +import sys import tempfile from test import LONG_TIMEOUT, SHORT_TIMEOUT, resolvesLocalhostFQDN, withPyOpenSSL from test.conftest import ServerConfig @@ -42,6 +43,11 @@ from .. import TARPIT_HOST, requires_network +_broken_on_python313a5 = pytest.mark.xfail( + sys.version_info == (3, 13, 0, "alpha", 5), + reason="https://github.com/python/cpython/issues/116764", +) + def assert_is_verified(pm: ProxyManager, *, proxy: bool, target: bool) -> None: pool = list(pm.pools._container.values())[-1] # retrieve last pool entry @@ -77,6 +83,7 @@ def teardown_class(cls) -> None: super().teardown_class() shutil.rmtree(cls.certs_dir) + @_broken_on_python313a5 def test_basic_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -85,6 +92,7 @@ def test_basic_proxy(self) -> None: r = http.request("GET", f"{self.https_url}/") assert r.status == 200 + @_broken_on_python313a5 def test_https_proxy(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.https_url}/") @@ -93,6 +101,7 @@ def test_https_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 + @_broken_on_python313a5 def test_is_verified_http_proxy_to_http_target(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -105,6 +114,7 @@ def test_is_verified_http_proxy_to_https_target(self) -> None: assert r.status == 200 assert_is_verified(http, proxy=False, target=True) + @_broken_on_python313a5 def test_is_verified_https_proxy_to_http_target(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.http_url}/") @@ -117,6 +127,7 @@ def test_is_verified_https_proxy_to_https_target(self) -> None: assert r.status == 200 assert_is_verified(https, proxy=True, target=True) + @_broken_on_python313a5 def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: with open(DEFAULT_CA) as pem_file: pem_file_data = pem_file.read() @@ -127,6 +138,7 @@ def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 + @_broken_on_python313a5 def test_https_proxy_with_proxy_ssl_context(self) -> None: proxy_ssl_context = create_urllib3_context() proxy_ssl_context.load_verify_locations(DEFAULT_CA) @@ -142,6 +154,7 @@ def test_https_proxy_with_proxy_ssl_context(self) -> None: assert r.status == 200 @withPyOpenSSL + @_broken_on_python313a5 def test_https_proxy_pyopenssl_not_supported(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.http_url}/") @@ -152,6 +165,7 @@ def test_https_proxy_pyopenssl_not_supported(self) -> None: ): https.request("GET", f"{self.https_url}/") + @_broken_on_python313a5 def test_https_proxy_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -164,6 +178,7 @@ def test_https_proxy_forwarding_for_https(self) -> None: r = https.request("GET", f"{self.https_url}/") assert r.status == 200 + @_broken_on_python313a5 def test_nagle_proxy(self) -> None: """Test that proxy connections do not have TCP_NODELAY turned on""" with ProxyManager(self.proxy_url) as http: @@ -202,6 +217,7 @@ def test_proxy_conn_fail_from_dns( e.value.reason.original_error, urllib3.exceptions.NameResolutionError ) + @_broken_on_python313a5 def test_oldapi(self) -> None: with ProxyManager( connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA # type: ignore[arg-type] @@ -258,6 +274,7 @@ def test_proxy_verified(self) -> None: https_fail_pool.request("GET", "/", retries=0) assert isinstance(e.value.reason, SSLError) + @_broken_on_python313a5 def test_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: r = http.request( @@ -278,6 +295,7 @@ def test_redirect(self) -> None: assert r.status == 200 assert r.data == b"Dummy server!" + @_broken_on_python313a5 def test_cross_host_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: cross_host_location = f"{self.http_url_alt}/echo?a=b" @@ -299,6 +317,7 @@ def test_cross_host_redirect(self) -> None: assert r._pool is not None assert r._pool.host != self.http_host_alt + @_broken_on_python313a5 def test_cross_protocol_redirect(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: cross_protocol_location = f"{self.https_url}/echo?a=b" @@ -320,6 +339,7 @@ def test_cross_protocol_redirect(self) -> None: assert r._pool is not None assert r._pool.host == self.https_host + @_broken_on_python313a5 def test_headers(self) -> None: with proxy_from_url( self.proxy_url, @@ -395,6 +415,7 @@ def test_headers(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) + @_broken_on_python313a5 def test_https_headers(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -427,6 +448,7 @@ def test_https_headers(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) + @_broken_on_python313a5 def test_https_headers_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -443,6 +465,7 @@ def test_https_headers_forwarding_for_https(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) + @_broken_on_python313a5 def test_headerdict(self) -> None: default_headers = HTTPHeaderDict(a="b") proxy_headers = HTTPHeaderDict() @@ -500,6 +523,10 @@ def test_proxy_pooling_ext(self) -> None: assert sc3 == sc4 @requires_network() + @pytest.mark.xfail( + sys.version_info == (3, 13, 0, "alpha", 5) and sys.platform == "win32", + reason="https://github.com/python/cpython/issues/116764", + ) @pytest.mark.parametrize( ["proxy_scheme", "target_scheme", "use_forwarding_for_https"], [ @@ -668,6 +695,7 @@ def test_proxy_https_target_tls_error( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") + @_broken_on_python313a5 def test_scheme_host_case_insensitive(self) -> None: """Assert that upper-case schemes and hosts are normalized.""" with proxy_from_url(self.proxy_url.upper(), ca_certs=DEFAULT_CA) as http: @@ -716,6 +744,7 @@ def setup_class(cls) -> None: # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") + @_broken_on_python313a5 def test_basic_ipv6_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -750,6 +779,7 @@ def _get_certificate_formatted_proxy_host(host: str) -> str: # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") + @_broken_on_python313a5 def test_https_proxy_assert_fingerprint_md5( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -792,6 +822,7 @@ def test_https_proxy_assert_fingerprint_md5_non_matching( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") + @_broken_on_python313a5 def test_https_proxy_assert_hostname( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -809,6 +840,11 @@ def test_https_proxy_assert_hostname( def test_https_proxy_assert_hostname_non_matching( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: + if ( + sys.version_info == (3, 13, 0, "alpha", 5) + and san_proxy_with_server[0].host == "::1" + ): + pytest.xfail(reason="https://github.com/python/cpython/issues/116764") proxy, server = san_proxy_with_server destination_url = f"https://{server.host}:{server.port}" @@ -860,6 +896,7 @@ def test_https_proxy_hostname_verification( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") + @_broken_on_python313a5 def test_https_proxy_ipv4_san( self, ipv4_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -870,6 +907,7 @@ def test_https_proxy_ipv4_san( r = https.request("GET", destination_url) assert r.status == 200 + @_broken_on_python313a5 def test_https_proxy_ipv6_san( self, ipv6_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -903,6 +941,7 @@ def test_https_proxy_no_san( ssl_error ) + @_broken_on_python313a5 def test_https_proxy_no_san_hostname_checks_common_name( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: From e6714d1b66fe88a49e63060c919d3c127a8498ac Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 4 Apr 2024 13:48:39 +0400 Subject: [PATCH 118/131] Switch from ResponseTypes to ResponseReturnValue (#3373) As suggested in https://github.com/pallets/quart/issues/288 --- dummyserver/app.py | 66 ++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/dummyserver/app.py b/dummyserver/app.py index 692c31441b..9fc9d1b7ff 100644 --- a/dummyserver/app.py +++ b/dummyserver/app.py @@ -12,10 +12,8 @@ from typing import Iterator import trio -from quart import make_response, request - -# TODO switch to Response if https://github.com/pallets/quart/issues/288 is fixed -from quart.typing import ResponseTypes +from quart import Response, make_response, request +from quart.typing import ResponseReturnValue from quart_trio import QuartTrio hypercorn_app = QuartTrio(__name__) @@ -39,19 +37,19 @@ @hypercorn_app.route("/") @pyodide_testing_app.route("/") @pyodide_testing_app.route("/index") -async def index() -> ResponseTypes: +async def index() -> ResponseReturnValue: return await make_response("Dummy server!") @hypercorn_app.route("/alpn_protocol") -async def alpn_protocol() -> ResponseTypes: +async def alpn_protocol() -> ResponseReturnValue: """Return the requester's certificate.""" alpn_protocol = request.scope["extensions"]["tls"]["alpn_protocol"] return await make_response(alpn_protocol) @hypercorn_app.route("/certificate") -async def certificate() -> ResponseTypes: +async def certificate() -> ResponseReturnValue: """Return the requester's certificate.""" print("scope", request.scope) subject = request.scope["extensions"]["tls"]["client_cert_name"] @@ -61,7 +59,7 @@ async def certificate() -> ResponseTypes: @hypercorn_app.route("/specific_method", methods=["GET", "POST", "PUT"]) @pyodide_testing_app.route("/specific_method", methods=["GET", "POST", "PUT"]) -async def specific_method() -> ResponseTypes: +async def specific_method() -> ResponseReturnValue: "Confirm that the request matches the desired method type" method_param = (await request.values).get("method", "") @@ -74,7 +72,7 @@ async def specific_method() -> ResponseTypes: @hypercorn_app.route("/upload", methods=["POST"]) -async def upload() -> ResponseTypes: +async def upload() -> ResponseReturnValue: "Confirm that the uploaded file conforms to specification" params = await request.form param = params.get("upload_param") @@ -105,7 +103,7 @@ async def upload() -> ResponseTypes: @hypercorn_app.route("/chunked") -async def chunked() -> ResponseTypes: +async def chunked() -> ResponseReturnValue: def generate() -> Iterator[str]: for _ in range(4): yield "123" @@ -114,7 +112,7 @@ def generate() -> Iterator[str]: @hypercorn_app.route("/chunked_gzip") -async def chunked_gzip() -> ResponseTypes: +async def chunked_gzip() -> ResponseReturnValue: def generate() -> Iterator[bytes]: compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) @@ -126,7 +124,7 @@ def generate() -> Iterator[bytes]: @hypercorn_app.route("/keepalive") -async def keepalive() -> ResponseTypes: +async def keepalive() -> ResponseReturnValue: if request.args.get("close", b"0") == b"1": headers = [("Connection", "close")] return await make_response("Closing", 200, headers) @@ -136,7 +134,7 @@ async def keepalive() -> ResponseTypes: @hypercorn_app.route("/echo", methods=["GET", "POST", "PUT"]) -async def echo() -> ResponseTypes: +async def echo() -> ResponseReturnValue: "Echo back the params" if request.method == "GET": return await make_response(request.query_string) @@ -146,7 +144,7 @@ async def echo() -> ResponseTypes: @hypercorn_app.route("/echo_json", methods=["POST"]) @pyodide_testing_app.route("/echo_json", methods=["POST", "OPTIONS"]) -async def echo_json() -> ResponseTypes: +async def echo_json() -> ResponseReturnValue: "Echo back the JSON" if request.method == "OPTIONS": return await make_response("", 200) @@ -156,14 +154,14 @@ async def echo_json() -> ResponseTypes: @hypercorn_app.route("/echo_uri/") @hypercorn_app.route("/echo_uri", defaults={"rest": ""}) -async def echo_uri(rest: str) -> ResponseTypes: +async def echo_uri(rest: str) -> ResponseReturnValue: "Echo back the requested URI" assert request.full_path is not None return await make_response(request.full_path) @hypercorn_app.route("/echo_params") -async def echo_params() -> ResponseTypes: +async def echo_params() -> ResponseReturnValue: "Echo back the query parameters" await request.get_data() echod = sorted((k, v) for k, v in request.args.items()) @@ -171,12 +169,12 @@ async def echo_params() -> ResponseTypes: @hypercorn_app.route("/headers", methods=["GET", "POST"]) -async def headers() -> ResponseTypes: +async def headers() -> ResponseReturnValue: return await make_response(dict(request.headers.items())) @hypercorn_app.route("/headers_and_params") -async def headers_and_params() -> ResponseTypes: +async def headers_and_params() -> ResponseReturnValue: return await make_response( { "headers": dict(request.headers), @@ -186,12 +184,12 @@ async def headers_and_params() -> ResponseTypes: @hypercorn_app.route("/multi_headers", methods=["GET", "POST"]) -async def multi_headers() -> ResponseTypes: +async def multi_headers() -> ResponseReturnValue: return await make_response({"headers": list(request.headers)}) @hypercorn_app.route("/multi_redirect") -async def multi_redirect() -> ResponseTypes: +async def multi_redirect() -> ResponseReturnValue: "Performs a redirect chain based on ``redirect_codes``" params = request.args codes = params.get("redirect_codes", "200") @@ -206,7 +204,7 @@ async def multi_redirect() -> ResponseTypes: @hypercorn_app.route("/encodingrequest") -async def encodingrequest() -> ResponseTypes: +async def encodingrequest() -> ResponseReturnValue: "Check for UA accepting gzip/deflate encoding" data = b"hello, world!" encoding = request.headers.get("Accept-Encoding", "") @@ -230,7 +228,7 @@ async def encodingrequest() -> ResponseTypes: @hypercorn_app.route("/redirect", methods=["GET", "POST", "PUT"]) -async def redirect() -> ResponseTypes: +async def redirect() -> ResponseReturnValue: "Perform a redirect to ``target``" values = await request.values target = values.get("target", "/") @@ -242,7 +240,7 @@ async def redirect() -> ResponseTypes: @hypercorn_app.route("/redirect_after") -async def redirect_after() -> ResponseTypes: +async def redirect_after() -> ResponseReturnValue: "Perform a redirect to ``target``" params = request.args date = params.get("date") @@ -258,7 +256,7 @@ async def redirect_after() -> ResponseTypes: @hypercorn_app.route("/retry_after") -async def retry_after() -> ResponseTypes: +async def retry_after() -> ResponseReturnValue: global LAST_RETRY_AFTER_REQ params = request.args if datetime.datetime.now() - LAST_RETRY_AFTER_REQ < datetime.timedelta(seconds=1): @@ -273,7 +271,7 @@ async def retry_after() -> ResponseTypes: @hypercorn_app.route("/status") @pyodide_testing_app.route("/status") -async def status() -> ResponseTypes: +async def status() -> ResponseReturnValue: values = await request.values status = values.get("status", "200 OK") status_code = status.split(" ")[0] @@ -281,13 +279,13 @@ async def status() -> ResponseTypes: @hypercorn_app.route("/source_address") -async def source_address() -> ResponseTypes: +async def source_address() -> ResponseReturnValue: """Return the requester's IP address.""" return await make_response(request.remote_addr) @hypercorn_app.route("/successful_retry", methods=["GET", "PUT"]) -async def successful_retry() -> ResponseTypes: +async def successful_retry() -> ResponseReturnValue: """First return an error and then success It's not currently very flexible as the number of retries is hard-coded. @@ -305,20 +303,20 @@ async def successful_retry() -> ResponseTypes: @pyodide_testing_app.after_request -def apply_caching(response: ResponseTypes) -> ResponseTypes: +def apply_caching(response: Response) -> ResponseReturnValue: for header, value in DEFAULT_HEADERS: response.headers[header] = value return response @pyodide_testing_app.route("/slow") -async def slow() -> ResponseTypes: +async def slow() -> ResponseReturnValue: await trio.sleep(10) return await make_response("TEN SECONDS LATER", 200) @pyodide_testing_app.route("/bigfile") -async def bigfile() -> ResponseTypes: +async def bigfile() -> ResponseReturnValue: # great big text file, should force streaming # if supported bigdata = 1048576 * b"WOOO YAY BOOYAKAH" @@ -326,14 +324,14 @@ async def bigfile() -> ResponseTypes: @pyodide_testing_app.route("/mediumfile") -async def mediumfile() -> ResponseTypes: +async def mediumfile() -> ResponseReturnValue: # quite big file bigdata = 1024 * b"WOOO YAY BOOYAKAH" return await make_response(bigdata, 200) @pyodide_testing_app.route("/upload", methods=["POST", "OPTIONS"]) -async def pyodide_upload() -> ResponseTypes: +async def pyodide_upload() -> ResponseReturnValue: if request.method == "OPTIONS": return await make_response("", 200) spare_data = await request.get_data(parse_form_data=True) @@ -356,7 +354,7 @@ async def pyodide_upload() -> ResponseTypes: @pyodide_testing_app.route("/pyodide/") -async def pyodide(py_file: str) -> ResponseTypes: +async def pyodide(py_file: str) -> ResponseReturnValue: file_path = Path(pyodide_testing_app.config["pyodide_dist_dir"], py_file) if file_path.exists(): mime_type, encoding = mimetypes.guess_type(file_path) @@ -370,7 +368,7 @@ async def pyodide(py_file: str) -> ResponseTypes: @pyodide_testing_app.route("/wheel/dist.whl") -async def wheel() -> ResponseTypes: +async def wheel() -> ResponseReturnValue: # serve our wheel wheel_folder = Path(__file__).parent.parent / "dist" wheels = list(wheel_folder.glob("*.whl")) From 5c5fcb0556ab2c121e2a3723cf9d78cb5a780854 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Apr 2024 12:09:47 +0300 Subject: [PATCH 119/131] Bump actions/setup-python from 5.0.0 to 5.1.0 (#3372) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/0a5c61591373683505ea898e09a3ea4f39ef2b9c...82c7e631bb3cdc910f68e0081d67478d79c6982d) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b6b40550f..0903cd121e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.x" cache: "pip" @@ -106,7 +106,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python ${{ matrix.python-version }}" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -144,7 +144,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.x" diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 5b347113db..1a53d3fe14 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.x" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1c5ab1d870..ff481fbd9f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.x" cache: pip diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4bad924ac3..bfba3c7c59 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 - name: "Setup Python" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.x" From c9f8d395cc1aee60b7c4488556bfa280de427795 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:57:51 +0300 Subject: [PATCH 120/131] Bump idna from 3.4 to 3.7 (#3378) Bumps [idna](https://github.com/kjd/idna) from 3.4 to 3.7. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.4...v3.7) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3319a14ce5..ebd377e4ca 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ PySocks==1.7.1 pytest==8.0.2 pytest-timeout==2.1.0 pyOpenSSL==24.0.0 -idna==3.4 +idna==3.7 # As of v1.1.0, child CA certificates generated by trustme fail # verification by CPython 3.13. # https://github.com/python-trio/trustme/pull/642 From b7c2910a9f49ba0d58c8d4381500c208f9a24765 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Fri, 12 Apr 2024 18:02:07 +0300 Subject: [PATCH 121/131] Drop xfails for CPython 3.13.0a5 --- .../test_proxy_poolmanager.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index f477d3382a..397181a9e6 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -9,7 +9,6 @@ import shutil import socket import ssl -import sys import tempfile from test import LONG_TIMEOUT, SHORT_TIMEOUT, resolvesLocalhostFQDN, withPyOpenSSL from test.conftest import ServerConfig @@ -43,11 +42,6 @@ from .. import TARPIT_HOST, requires_network -_broken_on_python313a5 = pytest.mark.xfail( - sys.version_info == (3, 13, 0, "alpha", 5), - reason="https://github.com/python/cpython/issues/116764", -) - def assert_is_verified(pm: ProxyManager, *, proxy: bool, target: bool) -> None: pool = list(pm.pools._container.values())[-1] # retrieve last pool entry @@ -83,7 +77,6 @@ def teardown_class(cls) -> None: super().teardown_class() shutil.rmtree(cls.certs_dir) - @_broken_on_python313a5 def test_basic_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -92,7 +85,6 @@ def test_basic_proxy(self) -> None: r = http.request("GET", f"{self.https_url}/") assert r.status == 200 - @_broken_on_python313a5 def test_https_proxy(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.https_url}/") @@ -101,7 +93,6 @@ def test_https_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - @_broken_on_python313a5 def test_is_verified_http_proxy_to_http_target(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -114,7 +105,6 @@ def test_is_verified_http_proxy_to_https_target(self) -> None: assert r.status == 200 assert_is_verified(http, proxy=False, target=True) - @_broken_on_python313a5 def test_is_verified_https_proxy_to_http_target(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.http_url}/") @@ -127,7 +117,6 @@ def test_is_verified_https_proxy_to_https_target(self) -> None: assert r.status == 200 assert_is_verified(https, proxy=True, target=True) - @_broken_on_python313a5 def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: with open(DEFAULT_CA) as pem_file: pem_file_data = pem_file.read() @@ -138,7 +127,6 @@ def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - @_broken_on_python313a5 def test_https_proxy_with_proxy_ssl_context(self) -> None: proxy_ssl_context = create_urllib3_context() proxy_ssl_context.load_verify_locations(DEFAULT_CA) @@ -154,7 +142,6 @@ def test_https_proxy_with_proxy_ssl_context(self) -> None: assert r.status == 200 @withPyOpenSSL - @_broken_on_python313a5 def test_https_proxy_pyopenssl_not_supported(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", f"{self.http_url}/") @@ -165,7 +152,6 @@ def test_https_proxy_pyopenssl_not_supported(self) -> None: ): https.request("GET", f"{self.https_url}/") - @_broken_on_python313a5 def test_https_proxy_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -178,7 +164,6 @@ def test_https_proxy_forwarding_for_https(self) -> None: r = https.request("GET", f"{self.https_url}/") assert r.status == 200 - @_broken_on_python313a5 def test_nagle_proxy(self) -> None: """Test that proxy connections do not have TCP_NODELAY turned on""" with ProxyManager(self.proxy_url) as http: @@ -217,7 +202,6 @@ def test_proxy_conn_fail_from_dns( e.value.reason.original_error, urllib3.exceptions.NameResolutionError ) - @_broken_on_python313a5 def test_oldapi(self) -> None: with ProxyManager( connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA # type: ignore[arg-type] @@ -274,7 +258,6 @@ def test_proxy_verified(self) -> None: https_fail_pool.request("GET", "/", retries=0) assert isinstance(e.value.reason, SSLError) - @_broken_on_python313a5 def test_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: r = http.request( @@ -295,7 +278,6 @@ def test_redirect(self) -> None: assert r.status == 200 assert r.data == b"Dummy server!" - @_broken_on_python313a5 def test_cross_host_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: cross_host_location = f"{self.http_url_alt}/echo?a=b" @@ -317,7 +299,6 @@ def test_cross_host_redirect(self) -> None: assert r._pool is not None assert r._pool.host != self.http_host_alt - @_broken_on_python313a5 def test_cross_protocol_redirect(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: cross_protocol_location = f"{self.https_url}/echo?a=b" @@ -339,7 +320,6 @@ def test_cross_protocol_redirect(self) -> None: assert r._pool is not None assert r._pool.host == self.https_host - @_broken_on_python313a5 def test_headers(self) -> None: with proxy_from_url( self.proxy_url, @@ -415,7 +395,6 @@ def test_headers(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - @_broken_on_python313a5 def test_https_headers(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -448,7 +427,6 @@ def test_https_headers(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - @_broken_on_python313a5 def test_https_headers_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, @@ -465,7 +443,6 @@ def test_https_headers_forwarding_for_https(self) -> None: returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - @_broken_on_python313a5 def test_headerdict(self) -> None: default_headers = HTTPHeaderDict(a="b") proxy_headers = HTTPHeaderDict() @@ -523,10 +500,6 @@ def test_proxy_pooling_ext(self) -> None: assert sc3 == sc4 @requires_network() - @pytest.mark.xfail( - sys.version_info == (3, 13, 0, "alpha", 5) and sys.platform == "win32", - reason="https://github.com/python/cpython/issues/116764", - ) @pytest.mark.parametrize( ["proxy_scheme", "target_scheme", "use_forwarding_for_https"], [ @@ -695,7 +668,6 @@ def test_proxy_https_target_tls_error( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") - @_broken_on_python313a5 def test_scheme_host_case_insensitive(self) -> None: """Assert that upper-case schemes and hosts are normalized.""" with proxy_from_url(self.proxy_url.upper(), ca_certs=DEFAULT_CA) as http: @@ -744,7 +716,6 @@ def setup_class(cls) -> None: # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") - @_broken_on_python313a5 def test_basic_ipv6_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") @@ -779,7 +750,6 @@ def _get_certificate_formatted_proxy_host(host: str) -> str: # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") - @_broken_on_python313a5 def test_https_proxy_assert_fingerprint_md5( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -822,7 +792,6 @@ def test_https_proxy_assert_fingerprint_md5_non_matching( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") - @_broken_on_python313a5 def test_https_proxy_assert_hostname( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -840,11 +809,6 @@ def test_https_proxy_assert_hostname( def test_https_proxy_assert_hostname_non_matching( self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: - if ( - sys.version_info == (3, 13, 0, "alpha", 5) - and san_proxy_with_server[0].host == "::1" - ): - pytest.xfail(reason="https://github.com/python/cpython/issues/116764") proxy, server = san_proxy_with_server destination_url = f"https://{server.host}:{server.port}" @@ -896,7 +860,6 @@ def test_https_proxy_hostname_verification( # stdlib http.client.HTTPConnection._tunnel() causes a ResourceWarning # see https://github.com/python/cpython/issues/103472 @pytest.mark.filterwarnings("default::ResourceWarning") - @_broken_on_python313a5 def test_https_proxy_ipv4_san( self, ipv4_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -907,7 +870,6 @@ def test_https_proxy_ipv4_san( r = https.request("GET", destination_url) assert r.status == 200 - @_broken_on_python313a5 def test_https_proxy_ipv6_san( self, ipv6_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: @@ -941,7 +903,6 @@ def test_https_proxy_no_san( ssl_error ) - @_broken_on_python313a5 def test_https_proxy_no_san_hostname_checks_common_name( self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] ) -> None: From 9961d14de7c920091d42d42ed76d5d479b80064d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:57:13 +0300 Subject: [PATCH 122/131] Bump browser-actions/setup-chrome from 1.5.0 to 1.6.0 (#3386) Bumps [browser-actions/setup-chrome](https://github.com/browser-actions/setup-chrome) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/browser-actions/setup-chrome/releases) - [Changelog](https://github.com/browser-actions/setup-chrome/blob/master/CHANGELOG.md) - [Commits](https://github.com/browser-actions/setup-chrome/compare/97349de5c98094d4fc9412f31c524d7697115ad8...82b9ce628cc5595478a9ebadc480958a36457dc2) --- updated-dependencies: - dependency-name: browser-actions/setup-chrome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0903cd121e..a5bc2cfc1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: run: python -m pip install --upgrade pip setuptools nox - name: "Install Chrome" - uses: browser-actions/setup-chrome@97349de5c98094d4fc9412f31c524d7697115ad8 # v1.5.0 + uses: browser-actions/setup-chrome@82b9ce628cc5595478a9ebadc480958a36457dc2 # v1.6.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" uses: browser-actions/setup-firefox@233224b712fc07910ded8c15fb95a555c86da76f # v1.5.0 From b34619f94ece0c40e691a5aaf1304953d88089de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:06:11 +0400 Subject: [PATCH 123/131] Bump actions/checkout to 4.1.4 (#3387) --- .github/workflows/changelog.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index f65b7c33d1..105837dbfa 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,7 +13,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: # `towncrier check` runs `git diff --name-only origin/main...`, which # needs a non-shallow clone. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5bc2cfc1c..727df544f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 @@ -103,7 +103,7 @@ jobs: timeout-minutes: 30 steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python ${{ matrix.python-version }}" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 @@ -141,7 +141,7 @@ jobs: needs: test steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 753fcb22fb..f00a09cca6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,7 +21,7 @@ jobs: security-events: write steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Run CodeQL init" uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 1a53d3fe14..0419e036ee 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ff481fbd9f..ce11a0b456 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bfba3c7c59..d020de0515 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,7 +19,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Setup Python" uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 9b6daf91ef..2714f1056d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -21,7 +21,7 @@ jobs: steps: - name: "Checkout repository" - uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 # v4.0.0 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: persist-credentials: false From 52392654b30183129cf3ec06010306f517d9c146 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 10 May 2024 22:37:56 +0400 Subject: [PATCH 124/131] Fix HTTP version in debug log (#3316) --- src/urllib3/connection.py | 1 + src/urllib3/connectionpool.py | 7 ++----- src/urllib3/contrib/emscripten/response.py | 1 + src/urllib3/http2.py | 1 + src/urllib3/response.py | 4 ++++ test/test_response.py | 1 + 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 96e24baa93..1b16279209 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -480,6 +480,7 @@ def getresponse( # type: ignore[override] headers=headers, status=httplib_response.status, version=httplib_response.version, + version_string=getattr(self, "_http_vsn_str", "HTTP/?"), reason=httplib_response.reason, preload_content=resp_options.preload_content, decode_content=resp_options.decode_content, diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 2d3b563c50..a2c3cf6098 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -543,17 +543,14 @@ def _make_request( response._connection = response_conn # type: ignore[attr-defined] response._pool = self # type: ignore[attr-defined] - # emscripten connection doesn't have _http_vsn_str - http_version = getattr(conn, "_http_vsn_str", "HTTP/?") log.debug( - '%s://%s:%s "%s %s %s" %s %s', + '%s://%s:%s "%s %s HTTP/%s" %s %s', self.scheme, self.host, self.port, method, url, - # HTTP version - http_version, + response.version, response.status, response.length_remaining, ) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 958aaeaf74..cd3d80e430 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -45,6 +45,7 @@ def __init__( status=internal_response.status_code, request_url=url, version=0, + version_string="HTTP/?", reason="", decode_content=True, ) diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py index 15fa9d9157..ceb40602da 100644 --- a/src/urllib3/http2.py +++ b/src/urllib3/http2.py @@ -195,6 +195,7 @@ def __init__( headers=headers, # Following CPython, we map HTTP versions to major * 10 + minor integers version=20, + version_string="HTTP/2", # No reason phrase in HTTP/2 reason=None, decode_content=decode_content, diff --git a/src/urllib3/response.py b/src/urllib3/response.py index f00342eafa..a0273d65b0 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -318,6 +318,7 @@ def __init__( headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, status: int, version: int, + version_string: str, reason: str | None, decode_content: bool, request_url: str | None, @@ -329,6 +330,7 @@ def __init__( self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type] self.status = status self.version = version + self.version_string = version_string self.reason = reason self.decode_content = decode_content self._has_decoded_content = False @@ -574,6 +576,7 @@ def __init__( headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, status: int = 0, version: int = 0, + version_string: str = "HTTP/?", reason: str | None = None, preload_content: bool = True, decode_content: bool = True, @@ -591,6 +594,7 @@ def __init__( headers=headers, status=status, version=version, + version_string=version_string, reason=reason, decode_content=decode_content, request_url=request_url, diff --git a/test/test_response.py b/test/test_response.py index 7fcd3fa6c7..c0062771ec 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -591,6 +591,7 @@ def test_base_io(self) -> None: resp = BaseHTTPResponse( status=200, version=11, + version_string="HTTP/1.1", reason=None, decode_content=False, request_url=None, From f3bdc5585111429e22c81b5fb26c3ec164d98b81 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 10 May 2024 23:04:57 +0400 Subject: [PATCH 125/131] Allow triggering CI manually (#3391) This is helpful to check if CI issues reproduce on the main branch. --- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 1 + .github/workflows/downstream.yml | 2 +- .github/workflows/lint.yml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 727df544f9..b5ac498ebe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: CI -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: "read-all" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f00a09cca6..b850f0d9d4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,6 +7,7 @@ on: branches: ["main"] schedule: - cron: "0 0 * * 5" + workflow_dispatch: permissions: "read-all" diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 0419e036ee..8f4206bb06 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -1,6 +1,6 @@ name: Downstream -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: "read-all" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ce11a0b456..d28ea81816 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,6 @@ name: lint -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: "read-all" From b8589ec9f8c4da91511e601b632ac06af7e7c10e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 13 May 2024 22:41:53 +0400 Subject: [PATCH 126/131] Measure coverage with v4 of artifact actions (#3394) --- .github/workflows/ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ac498ebe..ded5578351 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,10 +127,10 @@ jobs: PYTHON_VERSION: ${{ matrix.python-version }} NOX_SESSION: ${{ matrix.nox-session }} - - name: "Upload artifact" - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + - name: "Upload coverage data" + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: coverage-data + name: coverage-data-${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.experimental }}-${{ matrix.nox-session }} path: ".coverage.*" if-no-files-found: error @@ -151,10 +151,11 @@ jobs: - name: "Install coverage" run: "python -m pip install -r dev-requirements.txt" - - name: "Download artifact" - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + - name: "Download coverage data" + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - name: coverage-data + pattern: coverage-data-* + merge-multiple: true - name: "Combine & check coverage" run: | @@ -164,7 +165,7 @@ jobs: - if: ${{ failure() }} name: "Upload report if check failed" - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: coverage-report path: htmlcov From b07a669bd970d69847801148286b726f0570b625 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 16:58:56 +0300 Subject: [PATCH 127/131] Bump github/codeql-action from 2.13.4 to 3.25.6 (#3396) updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b850f0d9d4..ad45723b99 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -25,13 +25,13 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: "Run CodeQL init" - uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 + uses: github/codeql-action/init@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 with: config-file: "./.github/codeql.yml" languages: "python" - name: "Run CodeQL autobuild" - uses: github/codeql-action/autobuild@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 + uses: github/codeql-action/autobuild@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 - name: "Run CodeQL analyze" - uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 + uses: github/codeql-action/analyze@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 From da410581b6b3df73da976b5ce5eb20a4bd030437 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 14:12:02 +0300 Subject: [PATCH 128/131] Bump browser-actions/setup-chrome from 1.6.0 to 1.7.1 (#3399) Bumps [browser-actions/setup-chrome](https://github.com/browser-actions/setup-chrome) from 1.6.0 to 1.7.1. - [Release notes](https://github.com/browser-actions/setup-chrome/releases) - [Changelog](https://github.com/browser-actions/setup-chrome/blob/master/CHANGELOG.md) - [Commits](https://github.com/browser-actions/setup-chrome/compare/82b9ce628cc5595478a9ebadc480958a36457dc2...db1b524c26f20a8d1a10f7fc385c92387e2d0477) --- updated-dependencies: - dependency-name: browser-actions/setup-chrome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ded5578351..b1f3723b6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: run: python -m pip install --upgrade pip setuptools nox - name: "Install Chrome" - uses: browser-actions/setup-chrome@82b9ce628cc5595478a9ebadc480958a36457dc2 # v1.6.0 + uses: browser-actions/setup-chrome@db1b524c26f20a8d1a10f7fc385c92387e2d0477 # v1.7.1 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" uses: browser-actions/setup-firefox@233224b712fc07910ded8c15fb95a555c86da76f # v1.5.0 From 34be4a57e59eb7365bcc37d52e9f8271b5b8d0d3 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Mon, 17 Jun 2024 09:32:47 +0300 Subject: [PATCH 129/131] Pin CFFI to a new release candidate instead of a Git commit (#3398) --- dev-requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index ebd377e4ca..f712fe2a9e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -22,8 +22,5 @@ quart-trio==0.11.1 hypercorn @ git+https://github.com/urllib3/hypercorn@urllib3-changes httpx==0.25.2 pytest-socket==0.7.0 -# CFFI is not going to support CPython 3.13 in an actual release until -# there is a release candidate for 3.13. -# https://github.com/python-cffi/cffi/issues/23#issuecomment-1845861410 -cffi @ git+https://github.com/python-cffi/cffi@14723b0bbd127790c450945099db31018d80fa83; python_version == "3.13" +cffi==1.17.0rc1 From accff72ecc2f6cf5a76d9570198a93ac7c90270e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 17 Jun 2024 11:09:06 +0400 Subject: [PATCH 130/131] Merge pull request from GHSA-34jh-p97f-mpxf * Strip Proxy-Authorization header on redirects * Fix test_retry_default_remove_headers_on_redirect * Set release date --- CHANGES.rst | 5 +++++ src/urllib3/util/retry.py | 4 +++- test/test_retry.py | 6 ++++- test/with_dummyserver/test_poolmanager.py | 27 ++++++++++++++++++++--- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 14fbd4cc1b..b9770aa8b9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +2.2.2 (2024-06-17) +================== + +- Added the ``Proxy-Authorization`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. + 2.2.1 (2024-02-16) ================== diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index 7a76a4a6ad..0456cceba4 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -189,7 +189,9 @@ class Retry: RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) #: Default maximum backoff time. DEFAULT_BACKOFF_MAX = 120 diff --git a/test/test_retry.py b/test/test_retry.py index f71e7acc9e..ac3ce4ca73 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -334,7 +334,11 @@ def test_retry_method_not_allowed(self) -> None: def test_retry_default_remove_headers_on_redirect(self) -> None: retry = Retry() - assert retry.remove_headers_on_redirect == {"authorization", "cookie"} + assert retry.remove_headers_on_redirect == { + "authorization", + "proxy-authorization", + "cookie", + } def test_retry_set_remove_headers_on_redirect(self) -> None: retry = Retry(remove_headers_on_redirect=["X-API-Secret"]) diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 4fa9ec850a..af77241d6c 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -144,7 +144,11 @@ def test_redirect_cross_host_remove_headers(self) -> None: "GET", f"{self.base_url}/redirect", fields={"target": f"{self.base_url_alt}/headers"}, - headers={"Authorization": "foo", "Cookie": "foo=bar"}, + headers={ + "Authorization": "foo", + "Proxy-Authorization": "bar", + "Cookie": "foo=bar", + }, ) assert r.status == 200 @@ -152,13 +156,18 @@ def test_redirect_cross_host_remove_headers(self) -> None: data = r.json() assert "Authorization" not in data + assert "Proxy-Authorization" not in data assert "Cookie" not in data r = http.request( "GET", f"{self.base_url}/redirect", fields={"target": f"{self.base_url_alt}/headers"}, - headers={"authorization": "foo", "cookie": "foo=bar"}, + headers={ + "authorization": "foo", + "proxy-authorization": "baz", + "cookie": "foo=bar", + }, ) assert r.status == 200 @@ -167,6 +176,8 @@ def test_redirect_cross_host_remove_headers(self) -> None: assert "authorization" not in data assert "Authorization" not in data + assert "proxy-authorization" not in data + assert "Proxy-Authorization" not in data assert "cookie" not in data assert "Cookie" not in data @@ -176,7 +187,11 @@ def test_redirect_cross_host_no_remove_headers(self) -> None: "GET", f"{self.base_url}/redirect", fields={"target": f"{self.base_url_alt}/headers"}, - headers={"Authorization": "foo", "Cookie": "foo=bar"}, + headers={ + "Authorization": "foo", + "Proxy-Authorization": "bar", + "Cookie": "foo=bar", + }, retries=Retry(remove_headers_on_redirect=[]), ) @@ -185,6 +200,7 @@ def test_redirect_cross_host_no_remove_headers(self) -> None: data = r.json() assert data["Authorization"] == "foo" + assert data["Proxy-Authorization"] == "bar" assert data["Cookie"] == "foo=bar" def test_redirect_cross_host_set_removed_headers(self) -> None: @@ -196,6 +212,7 @@ def test_redirect_cross_host_set_removed_headers(self) -> None: headers={ "X-API-Secret": "foo", "Authorization": "bar", + "Proxy-Authorization": "baz", "Cookie": "foo=bar", }, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), @@ -207,11 +224,13 @@ def test_redirect_cross_host_set_removed_headers(self) -> None: assert "X-API-Secret" not in data assert data["Authorization"] == "bar" + assert data["Proxy-Authorization"] == "baz" assert data["Cookie"] == "foo=bar" headers = { "x-api-secret": "foo", "authorization": "bar", + "proxy-authorization": "baz", "cookie": "foo=bar", } r = http.request( @@ -229,12 +248,14 @@ def test_redirect_cross_host_set_removed_headers(self) -> None: assert "x-api-secret" not in data assert "X-API-Secret" not in data assert data["Authorization"] == "bar" + assert data["Proxy-Authorization"] == "baz" assert data["Cookie"] == "foo=bar" # Ensure the header argument itself is not modified in-place. assert headers == { "x-api-secret": "foo", "authorization": "bar", + "proxy-authorization": "baz", "cookie": "foo=bar", } From 27e2a5c5a7ab6a517252cc8dcef3ffa6ffb8f61a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 17 Jun 2024 12:00:39 +0400 Subject: [PATCH 131/131] Release 2.2.2 (#3406) --- CHANGES.rst | 2 ++ changelog/3122.bugfix.rst | 2 -- changelog/3342.doc.rst | 1 - changelog/3363.bugfix.rst | 1 - src/urllib3/_version.py | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 changelog/3122.bugfix.rst delete mode 100644 changelog/3342.doc.rst delete mode 100644 changelog/3363.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index b9770aa8b9..0f4e5cc581 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,8 @@ ================== - Added the ``Proxy-Authorization`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. +- Allowed passing negative integers as ``amt`` to read methods of ``http.client.HTTPResponse`` as an alternative to ``None``. (`#3122 `__) +- Fixed return types representing copying actions to use ``typing.Self``. (`#3363 `__) 2.2.1 (2024-02-16) ================== diff --git a/changelog/3122.bugfix.rst b/changelog/3122.bugfix.rst deleted file mode 100644 index 12fbbbf131..0000000000 --- a/changelog/3122.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allowed passing negative integers as ``amt`` to read methods of -:class:`http.client.HTTPResponse` as an alternative to ``None``. diff --git a/changelog/3342.doc.rst b/changelog/3342.doc.rst deleted file mode 100644 index 924d32e0d7..0000000000 --- a/changelog/3342.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Updated docs for ``.json()`` diff --git a/changelog/3363.bugfix.rst b/changelog/3363.bugfix.rst deleted file mode 100644 index 579423950a..0000000000 --- a/changelog/3363.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Consistently used ``typing.Self`` for return types representing copying actions. diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index 095cf3c16b..7442f2b842 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.2.1" +__version__ = "2.2.2"