diff --git a/CHANGELOG.md b/CHANGELOG.md
index 79fd0d6..71aa19a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog #
 
+## Development ##
+
+* Fix `to_dict` output, which should always be JSON encodeable. #139 (fixes #127 and #137)
+
 ## 3.1.0 -- 2019-12-10 ##
 
 This is a greatly overdue release.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..e69de29
diff --git a/jose/backends/cryptography_backend.py b/jose/backends/cryptography_backend.py
index fb6205f..4e8ff7d 100644
--- a/jose/backends/cryptography_backend.py
+++ b/jose/backends/cryptography_backend.py
@@ -183,15 +183,15 @@ class CryptographyECKey(Key):
             'alg': self._algorithm,
             'kty': 'EC',
             'crv': crv,
-            'x': long_to_base64(public_key.public_numbers().x, size=key_size),
-            'y': long_to_base64(public_key.public_numbers().y, size=key_size),
+            'x': long_to_base64(public_key.public_numbers().x, size=key_size).decode('ASCII'),
+            'y': long_to_base64(public_key.public_numbers().y, size=key_size).decode('ASCII'),
         }
 
         if not self.is_public():
             data['d'] = long_to_base64(
                 self.prepared_key.private_numbers().private_value,
                 size=key_size
-            )
+            ).decode('ASCII')
 
         return data
 
@@ -354,18 +354,18 @@ class CryptographyRSAKey(Key):
         data = {
             'alg': self._algorithm,
             'kty': 'RSA',
-            'n': long_to_base64(public_key.public_numbers().n),
-            'e': long_to_base64(public_key.public_numbers().e),
+            'n': long_to_base64(public_key.public_numbers().n).decode('ASCII'),
+            'e': long_to_base64(public_key.public_numbers().e).decode('ASCII'),
         }
 
         if not self.is_public():
             data.update({
-                'd': long_to_base64(self.prepared_key.private_numbers().d),
-                'p': long_to_base64(self.prepared_key.private_numbers().p),
-                'q': long_to_base64(self.prepared_key.private_numbers().q),
-                'dp': long_to_base64(self.prepared_key.private_numbers().dmp1),
-                'dq': long_to_base64(self.prepared_key.private_numbers().dmq1),
-                'qi': long_to_base64(self.prepared_key.private_numbers().iqmp),
+                'd': long_to_base64(self.prepared_key.private_numbers().d).decode('ASCII'),
+                'p': long_to_base64(self.prepared_key.private_numbers().p).decode('ASCII'),
+                'q': long_to_base64(self.prepared_key.private_numbers().q).decode('ASCII'),
+                'dp': long_to_base64(self.prepared_key.private_numbers().dmp1).decode('ASCII'),
+                'dq': long_to_base64(self.prepared_key.private_numbers().dmq1).decode('ASCII'),
+                'qi': long_to_base64(self.prepared_key.private_numbers().iqmp).decode('ASCII'),
             })
 
         return data
diff --git a/jose/backends/ecdsa_backend.py b/jose/backends/ecdsa_backend.py
index dc9ebe7..73b3eab 100644
--- a/jose/backends/ecdsa_backend.py
+++ b/jose/backends/ecdsa_backend.py
@@ -131,14 +131,14 @@ class ECDSAECKey(Key):
             'alg': self._algorithm,
             'kty': 'EC',
             'crv': crv,
-            'x': long_to_base64(public_key.pubkey.point.x(), size=key_size),
-            'y': long_to_base64(public_key.pubkey.point.y(), size=key_size),
+            'x': long_to_base64(public_key.pubkey.point.x(), size=key_size).decode('ASCII'),
+            'y': long_to_base64(public_key.pubkey.point.y(), size=key_size).decode('ASCII'),
         }
 
         if not self.is_public():
             data['d'] = long_to_base64(
                 self.prepared_key.privkey.secret_multiplier,
                 size=key_size
-            )
+            ).decode('ASCII')
 
         return data
diff --git a/jose/backends/pycrypto_backend.py b/jose/backends/pycrypto_backend.py
index c002102..cf270a8 100644
--- a/jose/backends/pycrypto_backend.py
+++ b/jose/backends/pycrypto_backend.py
@@ -185,8 +185,8 @@ class RSAKey(Key):
         data = {
             'alg': self._algorithm,
             'kty': 'RSA',
-            'n': long_to_base64(self.prepared_key.n),
-            'e': long_to_base64(self.prepared_key.e),
+            'n': long_to_base64(self.prepared_key.n).decode('ASCII'),
+            'e': long_to_base64(self.prepared_key.e).decode('ASCII'),
         }
 
         if not self.is_public():
@@ -201,12 +201,12 @@ class RSAKey(Key):
             dp = self.prepared_key.d % (self.prepared_key.p - 1)
             dq = self.prepared_key.d % (self.prepared_key.q - 1)
             data.update({
-                'd': long_to_base64(self.prepared_key.d),
-                'p': long_to_base64(self.prepared_key.q),
-                'q': long_to_base64(self.prepared_key.p),
-                'dp': long_to_base64(dq),
-                'dq': long_to_base64(dp),
-                'qi': long_to_base64(self.prepared_key.u),
+                'd': long_to_base64(self.prepared_key.d).decode('ASCII'),
+                'p': long_to_base64(self.prepared_key.q).decode('ASCII'),
+                'q': long_to_base64(self.prepared_key.p).decode('ASCII'),
+                'dp': long_to_base64(dq).decode('ASCII'),
+                'dq': long_to_base64(dp).decode('ASCII'),
+                'qi': long_to_base64(self.prepared_key.u).decode('ASCII'),
             })
 
         return data
diff --git a/jose/backends/rsa_backend.py b/jose/backends/rsa_backend.py
index ca9e956..38e42bb 100644
--- a/jose/backends/rsa_backend.py
+++ b/jose/backends/rsa_backend.py
@@ -246,18 +246,18 @@ class RSAKey(Key):
         data = {
             'alg': self._algorithm,
             'kty': 'RSA',
-            'n': long_to_base64(public_key.n),
-            'e': long_to_base64(public_key.e),
+            'n': long_to_base64(public_key.n).decode('ASCII'),
+            'e': long_to_base64(public_key.e).decode('ASCII'),
         }
 
         if not self.is_public():
             data.update({
-                'd': long_to_base64(self._prepared_key.d),
-                'p': long_to_base64(self._prepared_key.p),
-                'q': long_to_base64(self._prepared_key.q),
-                'dp': long_to_base64(self._prepared_key.exp1),
-                'dq': long_to_base64(self._prepared_key.exp2),
-                'qi': long_to_base64(self._prepared_key.coef),
+                'd': long_to_base64(self._prepared_key.d).decode('ASCII'),
+                'p': long_to_base64(self._prepared_key.p).decode('ASCII'),
+                'q': long_to_base64(self._prepared_key.q).decode('ASCII'),
+                'dp': long_to_base64(self._prepared_key.exp1).decode('ASCII'),
+                'dq': long_to_base64(self._prepared_key.exp2).decode('ASCII'),
+                'qi': long_to_base64(self._prepared_key.coef).decode('ASCII'),
             })
 
         return data
diff --git a/jose/jwk.py b/jose/jwk.py
index ad89e6d..b2c1113 100644
--- a/jose/jwk.py
+++ b/jose/jwk.py
@@ -137,5 +137,5 @@ class HMACKey(Key):
         return {
             'alg': self._algorithm,
             'kty': 'oct',
-            'k': base64url_encode(self.prepared_key),
+            'k': base64url_encode(self.prepared_key).decode('ASCII'),
         }

diff --git a/tests/algorithms/test_EC.py b/tests/algorithms/test_EC.py
index 7f012af..62b952c 100644
--- a/tests/algorithms/test_EC.py
+++ b/tests/algorithms/test_EC.py
@@ -1,3 +1,4 @@
+import json
 
 from jose.constants import ALGORITHMS
 from jose.exceptions import JOSEError, JWKError
@@ -194,6 +195,9 @@ class TestECAlgorithm:
             # Private parameters should be absent
             assert 'd' not in as_dict
 
+        # as_dict should be serializable to JSON
+        json.dumps(as_dict)
+
     def test_to_dict(self):
         key = ECKey(private_key, ALGORITHMS.ES256)
         self.assert_parameters(key.to_dict(), private=True)
diff --git a/tests/algorithms/test_HMAC.py b/tests/algorithms/test_HMAC.py
index e84c2c0..843d3a2 100644
--- a/tests/algorithms/test_HMAC.py
+++ b/tests/algorithms/test_HMAC.py
@@ -1,3 +1,4 @@
+import json
 
 from jose.constants import ALGORITHMS
 from jose.exceptions import JOSEError
@@ -31,7 +32,7 @@ class TestHMACAlgorithm:
 
     def test_to_dict(self):
         passphrase = 'The quick brown fox jumps over the lazy dog'
-        encoded = b'VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw'
+        encoded = 'VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw'
         key = HMACKey(passphrase, ALGORITHMS.HS256)
 
         as_dict = key.to_dict()
@@ -43,3 +44,6 @@ class TestHMACAlgorithm:
 
         assert 'k' in as_dict
         assert as_dict['k'] == encoded
+
+        # as_dict should be serializable to JSON
+        json.dumps(as_dict)
diff --git a/tests/algorithms/test_RSA.py b/tests/algorithms/test_RSA.py
index 97aeb20..cdcb1da 100644
--- a/tests/algorithms/test_RSA.py
+++ b/tests/algorithms/test_RSA.py
@@ -1,4 +1,5 @@
 import base64
+import json
 import sys
 
 try:
@@ -370,6 +371,9 @@ class TestRSAAlgorithm:
             assert 'dq' not in as_dict
             assert 'qi' not in as_dict
 
+        # as_dict should be serializable to JSON
+        json.dumps(as_dict)
+
     def assert_roundtrip(self, key):
         assert RSAKey(
             key.to_dict(),

JWK can not be dumped to JSON
JOSE has a functionality that could be helpful for `.well-known/jwks.json` generation — ability to convert PEMs to dictionary compatible with JWK specification. However, produced dictionary utilizes `bytes`, instead of `str` in Python 3, which crashes `json.dumps`.

For example:
```python3
import json
from jose import jwk


key_pem = """-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEU7X2p6GjDeHWgvDkWZHcjKLXdNwM
ua9kolwyy3nv7k3xiRgX/jfX1QbXaQuvqr40PhoWjtJR2C9uTvyP6AMHkw==
-----END PUBLIC KEY-----"""
key = jwk.construct(key_pem, algorithm="ES256").to_dict()
json.dumps({"keys": [key]})
```

This will crash with `TypeError: Object of type bytes is not JSON serializable` due to all base64 encoded values being stored as `bytes` not `str`.

The solution to circumvent the issue is:
```python3
import json
from jose import jwk


class ByteEncoder(json.JSONEncoder):
    #pylint: disable=method-hidden
    def default(self, x):
        if isinstance(x, bytes):
            return str(x, "UTF-8")
        else:
            super().default(x)


key_pem = """-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEU7X2p6GjDeHWgvDkWZHcjKLXdNwM
ua9kolwyy3nv7k3xiRgX/jfX1QbXaQuvqr40PhoWjtJR2C9uTvyP6AMHkw==
-----END PUBLIC KEY-----"""
key = jwk.construct(key_pem, algorithm="ES256").to_dict()
json.dumps({"keys": [key]}, cls=ByteEncoder)
```

However, is there a better way?
tests/algorithms/test_HMAC.py::TestHMACAlgorithm::test_to_dict tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_to_dict[RSA_2048_PKCS1] tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_to_dict[RSA_4096_PKCS1]
tests/algorithms/test_EC.py::test_key_from_ecdsa tests/algorithms/test_EC.py::TestECAlgorithm::test_key_from_pem tests/algorithms/test_EC.py::TestECAlgorithm::test_get_public_key tests/algorithms/test_EC.py::TestECAlgorithm::test_string_secret tests/algorithms/test_EC.py::TestECAlgorithm::test_object tests/algorithms/test_EC.py::TestECAlgorithm::test_invalid_algorithm tests/algorithms/test_EC.py::TestECAlgorithm::test_EC_jwk tests/algorithms/test_EC.py::TestECAlgorithm::test_verify tests/algorithms/test_HMAC.py::TestHMACAlgorithm::test_non_string_key tests/algorithms/test_HMAC.py::TestHMACAlgorithm::test_RSA_key tests/algorithms/test_RSA.py::TestPurePythonRsa::test_python_rsa_legacy_pem_read tests/algorithms/test_RSA.py::TestPurePythonRsa::test_python_rsa_legacy_pem_invalid tests/algorithms/test_RSA.py::TestPurePythonRsa::test_python_rsa_legacy_private_key_pkcs8_to_pkcs1 tests/algorithms/test_RSA.py::TestPurePythonRsa::test_python_rsa_legacy_private_key_pkcs8_to_pkcs1_invalid tests/algorithms/test_RSA.py::test_pycrypto_RSA_key_instance tests/algorithms/test_RSA.py::test_pycrypto_unencoded_cleartext[RSA_2048_PKCS1] tests/algorithms/test_RSA.py::test_pycrypto_unencoded_cleartext[RSA_4096_PKCS1] tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_RSA_key tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_string_secret tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_object tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_bad_cert tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_invalid_algorithm tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_RSA_jwk tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_get_public_key[RSA_2048_PKCS1] tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_get_public_key[RSA_4096_PKCS1] tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_to_pem[RSA_2048_PKCS1] tests/algorithms/test_RSA.py::TestRSAAlgorithm::test_to_pem[RSA_4096_PKCS1]