dev(narugo): add support for headers and binary body in recorder

.gitignore

@@ -14,3 +14,7 @@ venv
 /.pytest_cache
 /.tox
 /.artifacts
+/test_*
+/*.yaml
+*.bin
+!/example_bins/*.bin

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+exclude: ^example_bins
 repos:
   - repo: https://github.com/psf/black
     rev: 23.3.0

example_bins/202.bin

@@ -0,0 +1 @@
+OK

example_bins/400.bin

@@ -0,0 +1 @@
+Invalid status code

example_bins/500.bin

@@ -0,0 +1 @@
+500 Internal Server Error

responses/__init__.py

@@ -1,6 +1,7 @@
 import inspect
 import json as json_module
 import logging
+import os
 from functools import partialmethod
 from functools import wraps
 from http import client
@@ -56,7 +57,6 @@
 
 if TYPE_CHECKING:  # pragma: no cover
     # import only for linter run
-    import os
     from typing import Protocol
     from unittest.mock import _patch as _mock_patcher
 
@@ -529,6 +529,7 @@ def calls(self) -> CallList:
 
 
 def _form_response(
+    method: Optional[str],
     body: Union[BufferedReader, BytesIO],
     headers: Optional[Mapping[str, str]],
     status: int,
@@ -566,6 +567,7 @@ def _form_response(
         headers=headers,
         original_response=orig_response,  # type: ignore[arg-type]  # See comment above
         preload_content=False,
+        request_method=method,
     )
 
 
@@ -632,7 +634,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse:
             content_length = len(body.getvalue())
             headers["Content-Length"] = str(content_length)
 
-        return _form_response(body, headers, status)
+        return _form_response(request.method, body, headers, status)
 
     def __repr__(self) -> str:
         return (
@@ -695,7 +697,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse:
         body = _handle_body(body)
         headers.extend(r_headers)
 
-        return _form_response(body, headers, status)
+        return _form_response(request.method, body, headers, status)
 
 
 class PassthroughResponse(BaseResponse):
@@ -842,14 +844,24 @@ def _parse_response_file(
 
     def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> None:
         data = self._parse_response_file(file_path)
+        parent_directory = os.path.dirname(os.path.abspath(file_path))
 
         for rsp in data["responses"]:
             rsp = rsp["response"]
+            headers = dict(rsp.get("headers") or {})
+            if "Content-Type" in headers:
+                headers.pop("Content-Type")
+            if "body_file" in rsp:
+                with open(os.path.join(parent_directory, rsp["body_file"]), "rb") as f:
+                    body = f.read()
+            else:
+                body = rsp["body"]
             self.add(
                 method=rsp["method"],
                 url=rsp["url"],
-                body=rsp["body"],
+                body=body,
                 status=rsp["status"],
+                headers=headers,
                 content_type=rsp["content_type"],
                 auto_calculate_content_length=rsp["auto_calculate_content_length"],
             )

responses/_recorder.py

@@ -1,27 +1,28 @@
+import copy
+import os
+import pathlib
+import uuid
 from functools import wraps
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:  # pragma: no cover
-    import os
-
     from typing import Any
-    from typing import BinaryIO
     from typing import Callable
     from typing import Dict
     from typing import List
     from typing import Type
     from typing import Union
+    from typing import IO
     from responses import FirstMatchRegistry
     from responses import HTTPAdapter
     from responses import PreparedRequest
     from responses import models
     from responses import _F
     from responses import BaseResponse
 
-    from io import TextIOWrapper
-
 import yaml
 
+from responses import _UNSET
 from responses import RequestsMock
 from responses import Response
 from responses import _real_send
@@ -38,19 +39,43 @@ def _remove_nones(d: "Any") -> "Any":
 
 def _dump(
     registered: "List[BaseResponse]",
-    destination: "Union[BinaryIO, TextIOWrapper]",
-    dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[BinaryIO, TextIOWrapper]], Any]",
+    config_file: "Union[str, os.PathLike[str]]",
+    dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[IO[Any]]], None]",
+    dumper_mode: "str" = "w",
 ) -> None:
     data: Dict[str, Any] = {"responses": []}
+
+    # e.g. config_file = 'my/dir/responses.yaml'
+    # parent_directory = 'my/dir'
+    # binary_directory = 'my/dir/responses'
+    config_file = pathlib.Path(config_file)
+    fname, fext = os.path.splitext(config_file.name)
+    parent_directory = config_file.absolute().parent
+    binary_directory = parent_directory / (fname if fext else f"{fname}_bodies")
+
     for rsp in registered:
         try:
             content_length = rsp.auto_calculate_content_length  # type: ignore[attr-defined]
+            body = rsp.body
+            if isinstance(body, bytes):
+                os.makedirs(binary_directory, exist_ok=True)
+                bin_file = os.path.join(binary_directory, f"{uuid.uuid4()}.bin")
+                with open(bin_file, "wb") as bf:
+                    bf.write(body)
+
+                # make sure the stored binary file path is relative to config file
+                # or the config file and binary directory will be hard to move
+                body_file = os.path.relpath(bin_file, parent_directory)
+                body = None
+            else:
+                body_file = None
             data["responses"].append(
                 {
                     "response": {
                         "method": rsp.method,
                         "url": rsp.url,
-                        "body": rsp.body,
+                        "body": body,
+                        "body_file": body_file,
                         "status": rsp.status,
                         "headers": rsp.headers,
                         "content_type": rsp.content_type,
@@ -63,7 +88,9 @@ def _dump(
                 "Cannot dump response object."
                 "Probably you use custom Response object that is missing required attributes"
             ) from exc
-    dumper(_remove_nones(data), destination)
+
+    with open(config_file, dumper_mode) as cfile:
+        dumper(_remove_nones(data), cfile)
 
 
 class Recorder(RequestsMock):
@@ -79,7 +106,7 @@ def reset(self) -> None:
         self._registry = OrderedRegistry()
 
     def record(
-        self, *, file_path: "Union[str, bytes, os.PathLike[Any]]" = "response.yaml"
+        self, *, file_path: "Union[str, os.PathLike[str]]" = "response.yaml"
     ) -> "Union[Callable[[_F], _F], _F]":
         def deco_record(function: "_F") -> "Callable[..., Any]":
             @wraps(function)
@@ -99,11 +126,10 @@ def wrapper(*args: "Any", **kwargs: "Any") -> "Any":  # type: ignore[misc]
     def dump_to_file(
         self,
         *,
-        file_path: "Union[str, bytes, os.PathLike[Any]]",
+        file_path: "Union[str, os.PathLike[str]]",
         registered: "List[BaseResponse]",
     ) -> None:
-        with open(file_path, "w") as file:
-            _dump(registered, file, yaml.dump)
+        _dump(registered, file_path, yaml.dump)
 
     def _on_request(
         self,
@@ -116,11 +142,33 @@ def _on_request(
         request.params = self._parse_request_params(request.path_url)  # type: ignore[attr-defined]
         request.req_kwargs = kwargs  # type: ignore[attr-defined]
         requests_response = _real_send(adapter, request, **kwargs)
+        # the object is a requests.structures.CaseInsensitiveDict object,
+        # if you re-construct the headers with a primitive dict object,
+        # some lower case headers like 'content-type' will not be able to be processed properly
+        # the deepcopy is for making sure the original headers object
+        # not changed by the following operations
+        requests_headers = copy.deepcopy(requests_response.headers)
+        if "Content-Type" in requests_headers:
+            requests_content_type = requests_headers.pop("Content-Type")
+        else:
+            requests_content_type = _UNSET  # type: ignore[assignment]
+        # Content-Encoding should be removed to
+        # avoid 'Content-Encoding: gzip' causing the error in requests
+        if "Content-Encoding" in requests_headers:
+            requests_headers.pop("Content-Encoding")
+
+            # When something like 'Content-Encoding: gzip' is used
+            # the 'Content-Length' may be the length of compressed data,
+            # so we need to replace it with decompressed length
+            if "Content-Length" in requests_headers:
+                requests_headers["Content-Length"] = str(len(requests_response.content))
         responses_response = Response(
             method=str(request.method),
             url=str(requests_response.request.url),
             status=requests_response.status_code,
-            body=requests_response.text,
+            headers=dict(requests_headers),
+            body=requests_response.content,
+            content_type=requests_content_type,
         )
         self._registry.add(responses_response)
         return requests_response