Merge branch 'main' of github.com:psf/requests into 3.13

Author: graingert

.github/workflows/codeql-analysis.yml

@@ -45,7 +45,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0
+      uses: github/codeql-action/init@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
       with:
         languages: "python"
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0
+      uses: github/codeql-action/autobuild@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -70,4 +70,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0
+      uses: github/codeql-action/analyze@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0

.github/workflows/lint.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
     - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
     - name: Set up Python
-      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+      uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
       with:
         python-version: "3.x"
     - name: Run pre-commit

.github/workflows/run-tests.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
     - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+      uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
@@ -47,7 +47,7 @@ jobs:
     steps:
       - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
       - name: 'Set up Python 3.8'
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: '3.8'
       - name: Install dependencies
@@ -57,3 +57,23 @@ jobs:
       - name: Run tests
         run: |
           make ci
+
+  urllib3:
+    name: 'urllib3 1.x'
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: true
+
+    steps:
+      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
+      - name: 'Set up Python 3.8'
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          make
+          python  -m pip install "urllib3<2"
+      - name: Run tests
+        run: |
+          make ci

HISTORY.md

@@ -6,6 +6,15 @@ dev
 
 - \[Short description of non-trivial change.\]
 
+2.32.3 (2024-05-29)
+-------------------
+
+**Bugfixes**
+- Fixed bug breaking the ability to specify custom SSLContexts in sub-classes of
+  HTTPAdapter. (#6716)
+- Fixed issue where Requests started failing to run on Python versions compiled
+  without the `ssl` module. (#6724)
+
 2.32.2 (2024-05-21)
 -------------------
 

setup.py

@@ -4,7 +4,6 @@
 from codecs import open
 
 from setuptools import setup
-from setuptools.command.test import test as TestCommand
 
 CURRENT_PYTHON = sys.version_info[:2]
 REQUIRED_PYTHON = (3, 8)
@@ -28,30 +27,6 @@
     sys.exit(1)
 
 
-class PyTest(TestCommand):
-    user_options = [("pytest-args=", "a", "Arguments to pass into py.test")]
-
-    def initialize_options(self):
-        TestCommand.initialize_options(self)
-        try:
-            from multiprocessing import cpu_count
-
-            self.pytest_args = ["-n", str(cpu_count()), "--boxed"]
-        except (ImportError, NotImplementedError):
-            self.pytest_args = ["-n", "1", "--boxed"]
-
-    def finalize_options(self):
-        TestCommand.finalize_options(self)
-        self.test_args = []
-        self.test_suite = True
-
-    def run_tests(self):
-        import pytest
-
-        errno = pytest.main(self.pytest_args)
-        sys.exit(errno)
-
-
 # 'setup.py publish' shortcut.
 if sys.argv[-1] == "publish":
     os.system("python setup.py sdist bdist_wheel")
@@ -118,7 +93,6 @@ def run_tests(self):
         "Topic :: Internet :: WWW/HTTP",
         "Topic :: Software Development :: Libraries",
     ],
-    cmdclass={"test": PyTest},
     tests_require=test_requirements,
     extras_require={
         "security": [],

src/requests/__version__.py

@@ -5,8 +5,8 @@
 __title__ = "requests"
 __description__ = "Python HTTP for Humans."
 __url__ = "https://requests.readthedocs.io"
-__version__ = "2.32.2"
-__build__ = 0x023202
+__version__ = "2.32.3"
+__build__ = 0x023203
 __author__ = "Kenneth Reitz"
 __author_email__ = "[email protected]"
 __license__ = "Apache-2.0"

src/requests/adapters.py

@@ -73,26 +73,44 @@ def SOCKSProxyManager(*args, **kwargs):
 DEFAULT_RETRIES = 0
 DEFAULT_POOL_TIMEOUT = None
 
-_preloaded_ssl_context = create_urllib3_context()
-_preloaded_ssl_context.load_verify_locations(
-    extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
-)
+
+try:
+    import ssl  # noqa: F401
+
+    _preloaded_ssl_context = create_urllib3_context()
+    _preloaded_ssl_context.load_verify_locations(
+        extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
+    )
+except ImportError:
+    # Bypass default SSLContext creation when Python
+    # interpreter isn't built with the ssl module.
+    _preloaded_ssl_context = None
 
 
 def _urllib3_request_context(
     request: "PreparedRequest",
     verify: "bool | str | None",
     client_cert: "typing.Tuple[str, str] | str | None",
+    poolmanager: "PoolManager",
 ) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])":
     host_params = {}
     pool_kwargs = {}
     parsed_request_url = urlparse(request.url)
     scheme = parsed_request_url.scheme.lower()
     port = parsed_request_url.port
+
+    # Determine if we have and should use our default SSLContext
+    # to optimize performance on standard requests.
+    poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {})
+    has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context")
+    should_use_default_ssl_context = (
+        _preloaded_ssl_context is not None and not has_poolmanager_ssl_context
+    )
+
     cert_reqs = "CERT_REQUIRED"
     if verify is False:
         cert_reqs = "CERT_NONE"
-    elif verify is True:
+    elif verify is True and should_use_default_ssl_context:
         pool_kwargs["ssl_context"] = _preloaded_ssl_context
     elif isinstance(verify, str):
         if not os.path.isdir(verify):
@@ -375,23 +393,83 @@ def build_response(self, req, resp):
 
         return response
 
+    def build_connection_pool_key_attributes(self, request, verify, cert=None):
+        """Build the PoolKey attributes used by urllib3 to return a connection.
+
+        This looks at the PreparedRequest, the user-specified verify value,
+        and the value of the cert parameter to determine what PoolKey values
+        to use to select a connection from a given urllib3 Connection Pool.
+
+        The SSL related pool key arguments are not consistently set. As of
+        this writing, use the following to determine what keys may be in that
+        dictionary:
+
+        * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
+          default Requests SSL Context
+        * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
+          ``"cert_reqs"`` will be set
+        * If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
+          ``"ca_certs"`` will be set if the string is not a directory recognized
+          by :py:func:`os.path.isdir`, otherwise ``"ca_certs_dir"`` will be
+          set.
+        * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
+          ``"cert"`` is a tuple with a second item, ``"key_file"`` will also
+          be present
+
+        To override these settings, one may subclass this class, call this
+        method and use the above logic to change parameters as desired. For
+        example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
+        must both set ``"ssl_context"`` and based on what else they require,
+        alter the other keys to ensure the desired behaviour.
+
+        :param request:
+            The PreparedReqest being sent over the connection.
+        :type request:
+            :class:`~requests.models.PreparedRequest`
+        :param verify:
+            Either a boolean, in which case it controls whether
+            we verify the server's TLS certificate, or a string, in which case it
+            must be a path to a CA bundle to use.
+        :param cert:
+            (optional) Any user-provided SSL certificate for client
+            authentication (a.k.a., mTLS). This may be a string (i.e., just
+            the path to a file which holds both certificate and key) or a
+            tuple of length 2 with the certificate file path and key file
+            path.
+        :returns:
+            A tuple of two dictionaries. The first is the "host parameters"
+            portion of the Pool Key including scheme, hostname, and port. The
+            second is a dictionary of SSLContext related parameters.
+        """
+        return _urllib3_request_context(request, verify, cert, self.poolmanager)
+
     def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
         """Returns a urllib3 connection for the given request and TLS settings.
         This should not be called from user code, and is only exposed for use
         when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 
-        :param request: The :class:`PreparedRequest <PreparedRequest>` object
-            to be sent over the connection.
-        :param verify: Either a boolean, in which case it controls whether
-            we verify the server's TLS certificate, or a string, in which case it
-            must be a path to a CA bundle to use.
-        :param proxies: (optional) The proxies dictionary to apply to the request.
-        :param cert: (optional) Any user-provided SSL certificate to be trusted.
-        :rtype: urllib3.ConnectionPool
+        :param request:
+            The :class:`PreparedRequest <PreparedRequest>` object to be sent
+            over the connection.
+        :param verify:
+            Either a boolean, in which case it controls whether we verify the
+            server's TLS certificate, or a string, in which case it must be a
+            path to a CA bundle to use.
+        :param proxies:
+            (optional) The proxies dictionary to apply to the request.
+        :param cert:
+            (optional) Any user-provided SSL certificate to be used for client
+            authentication (a.k.a., mTLS).
+        :rtype:
+            urllib3.ConnectionPool
         """
         proxy = select_proxy(request.url, proxies)
         try:
-            host_params, pool_kwargs = _urllib3_request_context(request, verify, cert)
+            host_params, pool_kwargs = self.build_connection_pool_key_attributes(
+                request,
+                verify,
+                cert,
+            )
         except ValueError as e:
             raise InvalidURL(e, request=request)
         if proxy:

src/requests/compat.py

@@ -10,6 +10,18 @@
 import importlib
 import sys
 
+# -------
+# urllib3
+# -------
+from urllib3 import __version__ as urllib3_version
+
+# Detect which major version of urllib3 is being used.
+try:
+    is_urllib3_1 = int(urllib3_version.split(".")[0]) == 1
+except (TypeError, AttributeError):
+    # If we can't discern a version, prefer old functionality.
+    is_urllib3_1 = True
+
 # -------------------
 # Character Detection
 # -------------------

src/requests/utils.py

@@ -38,6 +38,7 @@
     getproxies,
     getproxies_environment,
     integer_types,
+    is_urllib3_1,
 )
 from .compat import parse_http_list as _parse_list_header
 from .compat import (
@@ -136,7 +137,9 @@ def super_len(o):
     total_length = None
     current_position = 0
 
-    if isinstance(o, str):
+    if not is_urllib3_1 and isinstance(o, str):
+        # urllib3 2.x+ treats all strings as utf-8 instead
+        # of latin-1 (iso-8859-1) like http.client.
         o = o.encode("utf-8")
 
     if hasattr(o, "__len__"):

tests/test_requests.py

@@ -25,6 +25,7 @@
     builtin_str,
     cookielib,
     getproxies,
+    is_urllib3_1,
     urlparse,
 )
 from requests.cookies import cookiejar_from_dict, morsel_to_cookie
@@ -1810,23 +1811,6 @@ def test_autoset_header_values_are_native(self, httpbin):
 
         assert p.headers["Content-Length"] == length
 
-    def test_content_length_for_bytes_data(self, httpbin):
-        data = "This is a string containing multi-byte UTF-8 ☃️"
-        encoded_data = data.encode("utf-8")
-        length = str(len(encoded_data))
-        req = requests.Request("POST", httpbin("post"), data=encoded_data)
-        p = req.prepare()
-
-        assert p.headers["Content-Length"] == length
-
-    def test_content_length_for_string_data_counts_bytes(self, httpbin):
-        data = "This is a string containing multi-byte UTF-8 ☃️"
-        length = str(len(data.encode("utf-8")))
-        req = requests.Request("POST", httpbin("post"), data=data)
-        p = req.prepare()
-
-        assert p.headers["Content-Length"] == length
-
     def test_nonhttp_schemes_dont_check_URLs(self):
         test_urls = (
             "data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==",
@@ -2966,6 +2950,29 @@ def response_handler(sock):
         assert client_cert is not None
 
 
+def test_content_length_for_bytes_data(httpbin):
+    data = "This is a string containing multi-byte UTF-8 ☃️"
+    encoded_data = data.encode("utf-8")
+    length = str(len(encoded_data))
+    req = requests.Request("POST", httpbin("post"), data=encoded_data)
+    p = req.prepare()
+
+    assert p.headers["Content-Length"] == length
+
+
+@pytest.mark.skipif(
+    is_urllib3_1,
+    reason="urllib3 2.x encodes all strings to utf-8, urllib3 1.x uses latin-1",
+)
+def test_content_length_for_string_data_counts_bytes(httpbin):
+    data = "This is a string containing multi-byte UTF-8 ☃️"
+    length = str(len(data.encode("utf-8")))
+    req = requests.Request("POST", httpbin("post"), data=data)
+    p = req.prepare()
+
+    assert p.headers["Content-Length"] == length
+
+
 def test_json_decode_errors_are_serializable_deserializable():
     json_decode_error = requests.exceptions.JSONDecodeError(
         "Extra data",