diff --git a/CHANGES b/CHANGES
index fa4aeb5..6085c46 100644
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,7 @@
 * Fixed issue when `passthru_prefixes` persisted across tests.
   Now `add_passthru` is valid only within a context manager or for a single function and
   cleared on exit
+* Added support for non-UTF-8 bytes in `responses.matchers.multipart_matcher`
 
 0.16.0
 ------
diff --git a/responses/matchers.py b/responses/matchers.py
index f90edef..1c4ef35 100644
--- a/responses/matchers.py
+++ b/responses/matchers.py
@@ -275,12 +275,12 @@ def multipart_matcher(files, data=None):
         )
 
         request_body = request.body
-        if isinstance(request_body, bytes):
-            request_body = request_body.decode("utf-8")
-
         prepared_body = prepared.body
+
         if isinstance(prepared_body, bytes):
-            prepared_body = prepared_body.decode("utf-8")
+            # since headers always come as str, need to convert to bytes
+            prepared_boundary = prepared_boundary.encode("utf-8")
+            request_boundary = request_boundary.encode("utf-8")
 
         prepared_body = prepared_body.replace(prepared_boundary, request_boundary)
 

diff --git a/responses/test_matchers.py b/responses/test_matchers.py
index ba6afb3..4587b28 100644
--- a/responses/test_matchers.py
+++ b/responses/test_matchers.py
@@ -283,17 +283,32 @@ def test_fail_matchers_error():
     assert_reset()
 
 
-def test_multipart_matcher():
+@pytest.mark.parametrize(
+    "req_file,match_file",
+    [
+        (b"Old World!", "Old World!"),
+        ("Old World!", b"Old World!"),
+        (b"Old World!", b"Old World!"),
+        ("Old World!", "Old World!"),
+        (b"\xacHello World!", b"\xacHello World!"),
+    ],
+)
+def test_multipart_matcher(req_file, match_file):
     @responses.activate
     def run():
         req_data = {"some": "other", "data": "fields"}
-        req_files = {"file_name": b"Old World!"}
         responses.add(
             responses.POST,
             url="http://httpbin.org/post",
-            match=[matchers.multipart_matcher(req_files, data=req_data)],
+            match=[
+                matchers.multipart_matcher(
+                    files={"file_name": match_file}, data=req_data
+                )
+            ],
+        )
+        resp = requests.post(
+            "http://httpbin.org/post", data=req_data, files={"file_name": req_file}
         )
-        resp = requests.post("http://httpbin.org/post", data=req_data, files=req_files)
         assert resp.status_code == 200
 
         with pytest.raises(TypeError):
@@ -334,14 +349,24 @@ def test_multipart_matcher_fail():
 
             msg = str(excinfo.value)
             assert "multipart/form-data doesn't match. Request body differs." in msg
-            assert (
-                '\r\nContent-Disposition: form-data; name="file_name"; '
-                'filename="file_name"\r\n\r\nOld World!\r\n'
-            ) in msg
-            assert (
-                '\r\nContent-Disposition: form-data; name="file_name"; '
-                'filename="file_name"\r\n\r\nNew World!\r\n'
-            ) in msg
+            if six.PY2:
+                assert (
+                    '\r\nContent-Disposition: form-data; name="file_name"; '
+                    'filename="file_name"\r\n\r\nOld World!\r\n'
+                ) in msg
+                assert (
+                    '\r\nContent-Disposition: form-data; name="file_name"; '
+                    'filename="file_name"\r\n\r\nNew World!\r\n'
+                ) in msg
+            else:
+                assert (
+                    r'\r\nContent-Disposition: form-data; name="file_name"; '
+                    r'filename="file_name"\r\n\r\nOld World!\r\n'
+                ) in msg
+                assert (
+                    r'\r\nContent-Disposition: form-data; name="file_name"; '
+                    r'filename="file_name"\r\n\r\nNew World!\r\n'
+                ) in msg
 
         # x-www-form-urlencoded request
         with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:

multipart_matcher doesn't work with non-UTF-8 bytes
### Environment

`responses==0.16.0`, `python==3.8.12`

### Steps to Reproduce

```python
import requests
import responses
from responses.matchers import multipart_matcher

non_utf_bytes = b'\xacHello World!'

def post_non_utf_bytes():
    requests.post(url="https://httpbin.org/post", files={"file": non_utf_bytes})

@responses.activate
def test_non_utf_8():
    responses.add(
        method=responses.POST,
        url="https://httpbin.org/post",
        match=[
            multipart_matcher(
                files={"file": non_utf_bytes},
            )
        ],
    )
    post_non_utf_bytes()

if __name__ == '__main__':
    post_non_utf_bytes()  # works, allowed
    test_non_utf_8()  # fails, tries to decode rather than doing byte-byte comparison
```

### Expected Result

As the `post` itself is valid I'd expect the body comparison to be done as bytestring-bytestring comparison rather than trying to decode it.

### Actual Result

```
responses/matchers.py", line 279, in match
    request_body = request_body.decode("utf-8")
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xac in position 100: invalid start byte
```

responses/test_matchers.py::test_multipart_matcher[\xacHello responses/test_matchers.py::test_multipart_matcher_fail
responses/test_matchers.py::test_query_string_matcher responses/test_matchers.py::test_request_matches_post_params responses/test_matchers.py::test_request_matches_empty_body responses/test_matchers.py::test_request_matches_params responses/test_matchers.py::test_fail_matchers_error responses/test_matchers.py::test_multipart_matcher[Old responses/test_matchers.py::test_query_string_matcher_raises responses/test_matchers.py::test_request_matches_headers responses/test_matchers.py::test_request_matches_headers_no_match responses/test_matchers.py::test_request_matches_headers_strict_match responses/test_matchers.py::test_fragment_identifier_matcher responses/test_matchers.py::test_fragment_identifier_matcher_error responses/test_matchers.py::test_fragment_identifier_matcher_and_match_querystring responses/test_matchers.py::test_matchers_create_key_val_str