diff --git a/jose/jwt.py b/jose/jwt.py
index b5767fe1..e12d162e 100644
--- a/jose/jwt.py
+++ b/jose/jwt.py
@@ -10,6 +10,7 @@
 
 from jose import jws
 
+from .exceptions import JWSError
 from .exceptions import JWTClaimsError
 from .exceptions import JWTError
 from .exceptions import ExpiredSignatureError
@@ -112,12 +113,14 @@ def decode(token, key, algorithms=None, options=None, audience=None, issuer=None
         defaults.update(options)
 
     verify_signature = defaults.get('verify_signature', True)
-    payload = jws.verify(token, key, algorithms, verify=verify_signature)
+
+    try:
+        payload = jws.verify(token, key, algorithms, verify=verify_signature)
+    except JWSError as e:
+        raise JWTError(e)
 
     try:
         claims = json.loads(payload.decode('utf-8'))
-    except (TypeError, binascii.Error):
-        raise JWTError('Invalid payload padding')
     except ValueError as e:
         raise JWTError('Invalid payload string: %s' % e)