diff --git a/docs/installation.rst b/docs/installation.rst index 01f1c9a9..a6025e1a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -22,7 +22,6 @@ The following HTTP libraries are supported: - ``urllib2`` - ``urllib3`` - ``httpx`` -- ``httpcore`` Speed ----- diff --git a/pyproject.toml b/pyproject.toml index e5b10a1c..602c1e13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,11 @@ tests = [ "boto3", "cryptography", "httpbin", - "httpcore", "httplib2", "httpx", - "pycurl; platform_python_implementation !='PyPy'", + "httpx-curl-cffi", + "pycurl; platform_python_implementation != 'PyPy'", + "pyreqwest; python_version >= '3.11'", "pytest", "pytest-aiohttp", "pytest-asyncio", diff --git a/setup.cfg b/setup.cfg index 339714b2..b17d1cd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,10 +43,11 @@ tests = boto3 cryptography httpbin - httpcore httplib2 httpx - pycurl; platform_python_implementation !='PyPy' + httpx-curl-cffi + pycurl; platform_python_implementation != 'PyPy' + pyreqwest; python_version >= '3.11' pytest pytest-aiohttp pytest-asyncio diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 79c15f0f..19f24f2a 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -48,11 +48,11 @@ def client(self): def __call__(self, *args, **kwargs): if hasattr(self, "_client"): - return self.client.request(*args, timeout=60, **kwargs) + return self.client.request(*args, **kwargs) # Use one-time context and dispose of the client afterwards with self: - return self.client.request(*args, timeout=60, **kwargs) + return self.client.request(*args, **kwargs) def stream(self, *args, **kwargs): if hasattr(self, "_client"): @@ -125,6 +125,19 @@ def yml(tmpdir, request): return str(tmpdir.join(request.function.__name__ + ".yaml")) +def test_response_elapsed(tmpdir, httpbin, do_request): + url = httpbin.url + + with vcr.use_cassette(str(tmpdir.join("elapsed.yaml"))): + response = do_request()("GET", url) + + with vcr.use_cassette(str(tmpdir.join("elapsed.yaml"))): + cassette_response = do_request()("GET", url) + + assert response.elapsed + assert cassette_response.elapsed + + def test_status(tmpdir, httpbin, do_request): url = httpbin.url @@ -365,3 +378,72 @@ async def run(): assert cassette.play_count == 1 asyncio.run(run()) + + +def test_custom_transports(tmpdir, httpbin): + url = httpbin.url + transport = httpx.MockTransport(lambda _: httpx.Response(200, json={"text": "Hello, world!"})) + + with vcr.use_cassette(str(tmpdir.join("mock_transport.yaml"))): + response = DoSyncRequest(transport=transport)("GET", url) + + with vcr.use_cassette(str(tmpdir.join("mock_transport.yaml"))) as cassette: + cassette_response = DoSyncRequest(transport=transport)("GET", url) + + assert cassette_response.status_code == response.status_code + assert cassette.play_count == 1 + + try: + from httpx_curl_cffi import AsyncCurlTransport, CurlTransport + except ImportError: + pass + else: + with vcr.use_cassette(str(tmpdir.join("curl_transport.yaml"))): + response = DoSyncRequest(transport=CurlTransport())("GET", url) + + with vcr.use_cassette(str(tmpdir.join("curl_transport.yaml"))) as cassette: + cassette_response = DoSyncRequest(transport=CurlTransport())("GET", url) + + assert cassette_response.status_code == response.status_code + assert cassette.play_count == 1 + + with vcr.use_cassette(str(tmpdir.join("async_curl_transport.yaml"))): + response = DoAsyncRequest(transport=AsyncCurlTransport())("GET", url) + + with vcr.use_cassette(str(tmpdir.join("async_curl_transport.yaml"))) as cassette: + cassette_response = DoAsyncRequest(transport=AsyncCurlTransport())("GET", url) + + assert cassette_response.status_code == response.status_code + assert cassette.play_count == 1 + + try: + from pyreqwest.compatibility.httpx import HttpxTransport, SyncHttpxTransport + except ImportError: + pass + else: + with vcr.use_cassette(str(tmpdir.join("pyreqwest_transport.yaml"))): + response = DoSyncRequest(transport=SyncHttpxTransport())("GET", url) + + with vcr.use_cassette(str(tmpdir.join("pyreqwest_transport.yaml"))) as cassette: + cassette_response = DoSyncRequest(transport=SyncHttpxTransport())("GET", url) + + assert cassette_response.status_code == response.status_code + assert cassette.play_count == 1 + + with vcr.use_cassette(str(tmpdir.join("async_pyreqwest_transport.yaml"))): + response = DoAsyncRequest(transport=HttpxTransport())("GET", url) + + with vcr.use_cassette(str(tmpdir.join("async_pyreqwest_transport.yaml"))) as cassette: + cassette_response = DoAsyncRequest(transport=HttpxTransport())("GET", url) + + assert cassette_response.status_code == response.status_code + assert cassette.play_count == 1 + + +def test_connection_limits(tmpdir, httpbin, do_request): + url = httpbin.url + limits = httpx.Limits(max_connections=1) + + with vcr.use_cassette(str(tmpdir.join("connection_limits.yaml"))), do_request(limits=limits) as client: + for _ in range(2): + client("GET", url) diff --git a/vcr/patch.py b/vcr/patch.py index 965ef85b..0bf64ee8 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -92,12 +92,36 @@ try: - import httpcore + import httpx except ImportError: # pragma: no cover pass else: - _HttpcoreConnectionPool_handle_request = httpcore.ConnectionPool.handle_request - _HttpcoreAsyncConnectionPool_handle_async_request = httpcore.AsyncConnectionPool.handle_async_request + _HttpxHttpTransport_handle_request = httpx.HTTPTransport.handle_request + _HttpxAsyncHttpTransport_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request + _HttpxWsgiTransport_handle_request = httpx.WSGITransport.handle_request + _HttpxAsgiTransport_handle_async_request = httpx.ASGITransport.handle_async_request + _HttpxMockTransport_handle_request = httpx.MockTransport.handle_request + _HttpxMockTransport_handle_async_request = httpx.MockTransport.handle_async_request + +try: + import httpx_curl_cffi +except ImportError: # pragma: no cover + pass +else: + _HttpxCurlTransport_handle_request = httpx_curl_cffi.CurlTransport.handle_request + _HttpxAsyncCurlTransport_handle_async_request = httpx_curl_cffi.AsyncCurlTransport.handle_async_request + +try: + import pyreqwest.compatibility.httpx +except ImportError: # pragma: no cover + pass +else: + _HttpxSyncPyreqwestTransport_handle_request = ( + pyreqwest.compatibility.httpx.SyncHttpxTransport.handle_request + ) + _HttpxPyreqwestTransport_handle_async_request = ( + pyreqwest.compatibility.httpx.HttpxTransport.handle_async_request + ) class CassettePatcherBuilder: @@ -121,7 +145,7 @@ def build(self): self._httplib2(), self._tornado(), self._aiohttp(), - self._httpcore(), + self._httpx(), self._build_patchers_from_mock_triples(self._cassette.custom_patches), ) @@ -304,22 +328,79 @@ def _aiohttp(self): yield client.ClientSession, "_request", new_request @_build_patchers_from_mock_triples_decorator - def _httpcore(self): + def _httpx(self): + try: + import httpx + except ImportError: # pragma: no cover + pass + else: + from .stubs.httpx_stubs import vcr_handle_async_request, vcr_handle_request + + new_handle_request = vcr_handle_request(self._cassette, _HttpxHttpTransport_handle_request) + yield httpx.HTTPTransport, "handle_request", new_handle_request + + new_handle_async_request = vcr_handle_async_request( + self._cassette, + _HttpxAsyncHttpTransport_handle_async_request, + ) + yield httpx.AsyncHTTPTransport, "handle_async_request", new_handle_async_request + + new_handle_request = vcr_handle_request(self._cassette, _HttpxWsgiTransport_handle_request) + yield httpx.WSGITransport, "handle_request", new_handle_request + + new_handle_async_request = vcr_handle_async_request( + self._cassette, + _HttpxAsgiTransport_handle_async_request, + ) + yield httpx.ASGITransport, "handle_async_request", new_handle_async_request + + new_handle_request = vcr_handle_request(self._cassette, _HttpxMockTransport_handle_request) + yield httpx.MockTransport, "handle_request", new_handle_request + + new_handle_async_request = vcr_handle_async_request( + self._cassette, + _HttpxMockTransport_handle_async_request, + ) + yield httpx.MockTransport, "handle_async_request", new_handle_async_request + try: - import httpcore + import httpx_curl_cffi except ImportError: # pragma: no cover pass else: - from .stubs.httpcore_stubs import vcr_handle_async_request, vcr_handle_request + from .stubs.httpx_stubs import vcr_handle_async_request, vcr_handle_request - new_handle_request = vcr_handle_request(self._cassette, _HttpcoreConnectionPool_handle_request) - yield httpcore.ConnectionPool, "handle_request", new_handle_request + new_handle_request = vcr_handle_request(self._cassette, _HttpxCurlTransport_handle_request) + yield httpx_curl_cffi.CurlTransport, "handle_request", new_handle_request new_handle_async_request = vcr_handle_async_request( self._cassette, - _HttpcoreAsyncConnectionPool_handle_async_request, + _HttpxAsyncCurlTransport_handle_async_request, + ) + yield httpx_curl_cffi.AsyncCurlTransport, "handle_async_request", new_handle_async_request + + try: + import pyreqwest.compatibility.httpx + except ImportError: # pragma: no cover + pass + else: + from .stubs.httpx_stubs import vcr_handle_async_request, vcr_handle_request + + new_handle_request = vcr_handle_request( + self._cassette, + _HttpxSyncPyreqwestTransport_handle_request, + ) + yield pyreqwest.compatibility.httpx.SyncHttpxTransport, "handle_request", new_handle_request + + new_handle_async_request = vcr_handle_async_request( + self._cassette, + _HttpxPyreqwestTransport_handle_async_request, + ) + yield ( + pyreqwest.compatibility.httpx.HttpxTransport, + "handle_async_request", + new_handle_async_request, ) - yield httpcore.AsyncConnectionPool, "handle_async_request", new_handle_async_request def _urllib3_patchers(self, cpool, conn, stubs): http_connection_remover = ConnectionRemover( @@ -448,19 +529,55 @@ def reset_patchers(): yield mock.patch.object(curl.CurlAsyncHTTPClient, "fetch_impl", _CurlAsyncHTTPClient_fetch_impl) try: - import httpcore + import httpx + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object( + httpx.HTTPTransport, + "handle_request", + _HttpxHttpTransport_handle_request, + ) + yield mock.patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + _HttpxAsyncHttpTransport_handle_async_request, + ) + yield mock.patch.object( + httpx.WSGITransport, + "handle_request", + _HttpxWsgiTransport_handle_request, + ) + yield mock.patch.object( + httpx.ASGITransport, + "handle_async_request", + _HttpxAsgiTransport_handle_async_request, + ) + yield mock.patch.object( + httpx.MockTransport, + "handle_request", + _HttpxMockTransport_handle_request, + ) + yield mock.patch.object( + httpx.MockTransport, + "handle_async_request", + _HttpxMockTransport_handle_async_request, + ) + + try: + import httpx_curl_cffi except ImportError: # pragma: no cover pass else: yield mock.patch.object( - httpcore.ConnectionPool, + httpx_curl_cffi.CurlTransport, "handle_request", - _HttpcoreConnectionPool_handle_request, + _HttpxCurlTransport_handle_request, ) yield mock.patch.object( - httpcore.AsyncConnectionPool, + httpx_curl_cffi.AsyncCurlTransport, "handle_async_request", - _HttpcoreAsyncConnectionPool_handle_async_request, + _HttpxAsyncCurlTransport_handle_async_request, ) diff --git a/vcr/stubs/httpcore_stubs.py b/vcr/stubs/httpx_stubs.py similarity index 77% rename from vcr/stubs/httpcore_stubs.py rename to vcr/stubs/httpx_stubs.py index 5591296c..609214ac 100644 --- a/vcr/stubs/httpcore_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -2,8 +2,8 @@ import logging from collections import defaultdict -from httpcore import Response -from httpcore._models import ByteStream +from httpx import ByteStream, Response +from httpx._exceptions import request_context from vcr.errors import CannotOverwriteExistingCassetteException from vcr.filters import decode_response @@ -21,21 +21,15 @@ def _serialize_headers(real_response): headers = defaultdict(list) - for name, value in real_response.headers: - headers[name.decode("ascii")].append(value.decode("ascii")) + for name, value in real_response.headers.multi_items(): + headers[name].append(value) return dict(headers) def _serialize_response(real_response, real_response_content): - # The reason_phrase may not exist - try: - reason_phrase = real_response.extensions["reason_phrase"].decode("ascii") - except KeyError: - reason_phrase = None - return { - "status": {"code": real_response.status, "message": reason_phrase}, + "status": {"code": real_response.status_code, "message": real_response.reason_phrase}, "headers": _serialize_headers(real_response), "body": {"string": real_response_content}, } @@ -43,12 +37,10 @@ def _serialize_response(real_response, real_response_content): def _deserialize_headers(headers): """ - httpcore accepts headers as list of tuples of header key and value. + httpx accepts headers as list of tuples of header key and value. """ - return [ - (name.encode("ascii"), value.encode("ascii")) for name, values in headers.items() for value in values - ] + return [(name, value) for name, values in headers.items() for value in values] def _deserialize_response(vcr_response): @@ -67,32 +59,20 @@ def _deserialize_response(vcr_response): ) extensions = None else: - extensions = ( - {"reason_phrase": vcr_response["status"]["message"].encode("ascii")} - if vcr_response["status"]["message"] - else None - ) + extensions = {"reason_phrase": vcr_response["status"]["message"].encode("ascii")} return Response( vcr_response["status"]["code"], headers=_deserialize_headers(vcr_response["headers"]), - content=vcr_response["body"]["string"], + # Don't use content, because that closes the response, + # which we do not want as the real response is unclosed + stream=ByteStream(vcr_response["body"]["string"]), extensions=extensions, ) def _make_vcr_request(real_request, real_request_body): - uri = bytes(real_request.url).decode("ascii") - - # As per HTTPX: If there are multiple headers with the same key, then we concatenate them with commas - headers = defaultdict(list) - - for name, value in real_request.headers: - headers[name.decode("ascii")].append(value.decode("ascii")) - - headers = {name: ", ".join(values) for name, values in headers.items()} - - return VcrRequest(real_request.method.decode("ascii"), uri, real_request_body, headers) + return VcrRequest(real_request.method, str(real_request.url), real_request_body, real_request.headers) def _vcr_request(cassette, real_request, real_request_body): @@ -131,9 +111,13 @@ async def _vcr_handle_async_request(cassette, real_handle_async_request, self, r return vcr_response real_response = await real_handle_async_request(self, real_request) - - # Reading the response stream consumes the iterator, so we need to restore it afterwards real_response_content = b"".join([part async for part in real_response.stream]) + + # Close the original stream so that the connection is released back to the connection pool + with request_context(request=real_response._request): + await real_response.stream.aclose() + + # Reading the response stream consumes the iterator, so we need to restore it real_response.stream = ByteStream(real_response_content) _record_responses(cassette, vcr_request, real_response, real_response_content) @@ -160,9 +144,13 @@ def _vcr_handle_request(cassette, real_handle_request, self, real_request): return vcr_response real_response = real_handle_request(self, real_request) - - # Reading the response stream consumes the iterator, so we need to restore it afterwards real_response_content = b"".join(real_response.stream) + + # Close the original stream so that the connection is released back to the connection pool + with request_context(request=real_response._request): + real_response.stream.close() + + # Reading the response stream consumes the iterator, so we need to restore it real_response.stream = ByteStream(real_response_content) _record_responses(cassette, vcr_request, real_response, real_response_content)