bpo-40645: use C implementation of HMAC (GH-24920)

- [x] fix tests
- [ ] add test scenarios for old/new code.

Signed-off-by: Christian Heimes <[email protected]>

Author: tiran

Lib/hashlib.py

@@ -173,6 +173,7 @@ def __hash_new(name, data=b'', **kwargs):
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
 except ImportError:
+    _hashlib = None
     new = __py_new
     __get_hash = __get_builtin_constructor
 

Lib/hmac.py

@@ -8,11 +8,12 @@
     import _hashlib as _hashopenssl
 except ImportError:
     _hashopenssl = None
-    _openssl_md_meths = None
+    _functype = None
     from _operator import _compare_digest as compare_digest
 else:
-    _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names)
     compare_digest = _hashopenssl.compare_digest
+    _functype = type(_hashopenssl.openssl_sha256)  # builtin type
+
 import hashlib as _hashlib
 
 trans_5C = bytes((x ^ 0x5C) for x in range(256))
@@ -23,7 +24,6 @@
 digest_size = None
 
 
-
 class HMAC:
     """RFC 2104 HMAC class.  Also complies with RFC 4231.
 
@@ -32,7 +32,7 @@ class HMAC:
     blocksize = 64  # 512-bit HMAC; can be changed in subclasses.
 
     __slots__ = (
-        "_digest_cons", "_inner", "_outer", "block_size", "digest_size"
+        "_hmac", "_inner", "_outer", "block_size", "digest_size"
     )
 
     def __init__(self, key, msg=None, digestmod=''):
@@ -55,15 +55,30 @@ def __init__(self, key, msg=None, digestmod=''):
         if not digestmod:
             raise TypeError("Missing required parameter 'digestmod'.")
 
+        if _hashopenssl and isinstance(digestmod, (str, _functype)):
+            try:
+                self._init_hmac(key, msg, digestmod)
+            except _hashopenssl.UnsupportedDigestmodError:
+                self._init_old(key, msg, digestmod)
+        else:
+            self._init_old(key, msg, digestmod)
+
+    def _init_hmac(self, key, msg, digestmod):
+        self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod)
+        self.digest_size = self._hmac.digest_size
+        self.block_size = self._hmac.block_size
+
+    def _init_old(self, key, msg, digestmod):
         if callable(digestmod):
-            self._digest_cons = digestmod
+            digest_cons = digestmod
         elif isinstance(digestmod, str):
-            self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
+            digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
         else:
-            self._digest_cons = lambda d=b'': digestmod.new(d)
+            digest_cons = lambda d=b'': digestmod.new(d)
 
-        self._outer = self._digest_cons()
-        self._inner = self._digest_cons()
+        self._hmac = None
+        self._outer = digest_cons()
+        self._inner = digest_cons()
         self.digest_size = self._inner.digest_size
 
         if hasattr(self._inner, 'block_size'):
@@ -79,13 +94,13 @@ def __init__(self, key, msg=None, digestmod=''):
                            RuntimeWarning, 2)
             blocksize = self.blocksize
 
+        if len(key) > blocksize:
+            key = digest_cons(key).digest()
+
         # self.blocksize is the default blocksize. self.block_size is
         # effective block size as well as the public API attribute.
         self.block_size = blocksize
 
-        if len(key) > blocksize:
-            key = self._digest_cons(key).digest()
-
         key = key.ljust(blocksize, b'\0')
         self._outer.update(key.translate(trans_5C))
         self._inner.update(key.translate(trans_36))
@@ -94,23 +109,15 @@ def __init__(self, key, msg=None, digestmod=''):
 
     @property
     def name(self):
-        return "hmac-" + self._inner.name
-
-    @property
-    def digest_cons(self):
-        return self._digest_cons
-
-    @property
-    def inner(self):
-        return self._inner
-
-    @property
-    def outer(self):
-        return self._outer
+        if self._hmac:
+            return self._hmac.name
+        else:
+            return f"hmac-{self._inner.name}"
 
     def update(self, msg):
         """Feed data from msg into this hashing object."""
-        self._inner.update(msg)
+        inst = self._hmac or self._inner
+        inst.update(msg)
 
     def copy(self):
         """Return a separate copy of this hashing object.
@@ -119,20 +126,27 @@ def copy(self):
         """
         # Call __new__ directly to avoid the expensive __init__.
         other = self.__class__.__new__(self.__class__)
-        other._digest_cons = self._digest_cons
         other.digest_size = self.digest_size
-        other._inner = self._inner.copy()
-        other._outer = self._outer.copy()
+        if self._hmac:
+            other._hmac = self._hmac.copy()
+            other._inner = other._outer = None
+        else:
+            other._hmac = None
+            other._inner = self._inner.copy()
+            other._outer = self._outer.copy()
         return other
 
     def _current(self):
         """Return a hash object for the current state.
 
         To be used only internally with digest() and hexdigest().
         """
-        h = self._outer.copy()
-        h.update(self._inner.digest())
-        return h
+        if self._hmac:
+            return self._hmac
+        else:
+            h = self._outer.copy()
+            h.update(self._inner.digest())
+            return h
 
     def digest(self):
         """Return the hash value of this hashing object.
@@ -179,9 +193,11 @@ def digest(key, msg, digest):
             A hashlib constructor returning a new hash object. *OR*
             A module supporting PEP 247.
     """
-    if (_hashopenssl is not None and
-            isinstance(digest, str) and digest in _openssl_md_meths):
-        return _hashopenssl.hmac_digest(key, msg, digest)
+    if _hashopenssl is not None and isinstance(digest, (str, _functype)):
+        try:
+            return _hashopenssl.hmac_digest(key, msg, digest)
+        except _hashopenssl.UnsupportedDigestmodError:
+            pass
 
     if callable(digest):
         digest_cons = digest

Lib/test/test_hmac.py

@@ -11,14 +11,21 @@
 from _operator import _compare_digest as operator_compare_digest
 
 try:
+    import _hashlib as _hashopenssl
     from _hashlib import HMAC as C_HMAC
     from _hashlib import hmac_new as c_hmac_new
     from _hashlib import compare_digest as openssl_compare_digest
 except ImportError:
+    _hashopenssl = None
     C_HMAC = None
     c_hmac_new = None
     openssl_compare_digest = None
 
+try:
+    import _sha256 as sha256_module
+except ImportError:
+    sha256_module = None
+
 
 def ignore_warning(func):
     @functools.wraps(func)
@@ -32,22 +39,27 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
-    def asssert_hmac(
-        self, key, data, digest, hashfunc, hashname, digest_size, block_size
+    def assert_hmac_internals(
+            self, h, digest, hashname, digest_size, block_size
     ):
-        h = hmac.HMAC(key, data, digestmod=hashfunc)
         self.assertEqual(h.hexdigest().upper(), digest.upper())
         self.assertEqual(h.digest(), binascii.unhexlify(digest))
         self.assertEqual(h.name, f"hmac-{hashname}")
         self.assertEqual(h.digest_size, digest_size)
         self.assertEqual(h.block_size, block_size)
 
+    def assert_hmac(
+        self, key, data, digest, hashfunc, hashname, digest_size, block_size
+    ):
+        h = hmac.HMAC(key, data, digestmod=hashfunc)
+        self.assert_hmac_internals(
+            h, digest, hashname, digest_size, block_size
+        )
+
         h = hmac.HMAC(key, data, digestmod=hashname)
-        self.assertEqual(h.hexdigest().upper(), digest.upper())
-        self.assertEqual(h.digest(), binascii.unhexlify(digest))
-        self.assertEqual(h.name, f"hmac-{hashname}")
-        self.assertEqual(h.digest_size, digest_size)
-        self.assertEqual(h.block_size, block_size)
+        self.assert_hmac_internals(
+            h, digest, hashname, digest_size, block_size
+        )
 
         h = hmac.HMAC(key, digestmod=hashname)
         h2 = h.copy()
@@ -56,11 +68,9 @@ def asssert_hmac(
         self.assertEqual(h.hexdigest().upper(), digest.upper())
 
         h = hmac.new(key, data, digestmod=hashname)
-        self.assertEqual(h.hexdigest().upper(), digest.upper())
-        self.assertEqual(h.digest(), binascii.unhexlify(digest))
-        self.assertEqual(h.name, f"hmac-{hashname}")
-        self.assertEqual(h.digest_size, digest_size)
-        self.assertEqual(h.block_size, block_size)
+        self.assert_hmac_internals(
+            h, digest, hashname, digest_size, block_size
+        )
 
         h = hmac.new(key, None, digestmod=hashname)
         h.update(data)
@@ -81,36 +91,43 @@ def asssert_hmac(
             hmac.digest(key, data, digest=hashfunc),
             binascii.unhexlify(digest)
         )
-        with unittest.mock.patch('hmac._openssl_md_meths', {}):
-            self.assertEqual(
-                hmac.digest(key, data, digest=hashname),
-                binascii.unhexlify(digest)
-            )
-            self.assertEqual(
-                hmac.digest(key, data, digest=hashfunc),
-                binascii.unhexlify(digest)
-            )
+
+        h = hmac.HMAC.__new__(hmac.HMAC)
+        h._init_old(key, data, digestmod=hashname)
+        self.assert_hmac_internals(
+            h, digest, hashname, digest_size, block_size
+        )
 
         if c_hmac_new is not None:
             h = c_hmac_new(key, data, digestmod=hashname)
-            self.assertEqual(h.hexdigest().upper(), digest.upper())
-            self.assertEqual(h.digest(), binascii.unhexlify(digest))
-            self.assertEqual(h.name, f"hmac-{hashname}")
-            self.assertEqual(h.digest_size, digest_size)
-            self.assertEqual(h.block_size, block_size)
+            self.assert_hmac_internals(
+                h, digest, hashname, digest_size, block_size
+            )
 
             h = c_hmac_new(key, digestmod=hashname)
             h2 = h.copy()
             h2.update(b"test update")
             h.update(data)
             self.assertEqual(h.hexdigest().upper(), digest.upper())
 
+            func = getattr(_hashopenssl, f"openssl_{hashname}")
+            h = c_hmac_new(key, data, digestmod=func)
+            self.assert_hmac_internals(
+                h, digest, hashname, digest_size, block_size
+            )
+
+            h = hmac.HMAC.__new__(hmac.HMAC)
+            h._init_hmac(key, data, digestmod=hashname)
+            self.assert_hmac_internals(
+                h, digest, hashname, digest_size, block_size
+            )
+
     @hashlib_helper.requires_hashdigest('md5', openssl=True)
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
         def md5test(key, data, digest):
-            self.asssert_hmac(
+            self.assert_hmac(
                 key, data, digest,
                 hashfunc=hashlib.md5,
                 hashname="md5",
@@ -150,7 +167,7 @@ def md5test(key, data, digest):
     @hashlib_helper.requires_hashdigest('sha1', openssl=True)
     def test_sha_vectors(self):
         def shatest(key, data, digest):
-            self.asssert_hmac(
+            self.assert_hmac(
                 key, data, digest,
                 hashfunc=hashlib.sha1,
                 hashname="sha1",
@@ -191,7 +208,7 @@ def _rfc4231_test_cases(self, hashfunc, hash_name, digest_size, block_size):
         def hmactest(key, data, hexdigests):
             digest = hexdigests[hashfunc]
 
-            self.asssert_hmac(
+            self.assert_hmac(
                 key, data, digest,
                 hashfunc=hashfunc,
                 hashname=hash_name,
@@ -427,6 +444,15 @@ def test_internal_types(self):
         ):
             C_HMAC()
 
+    @unittest.skipUnless(sha256_module is not None, 'need _sha256')
+    def test_with_sha256_module(self):
+        h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256)
+        self.assertEqual(h.hexdigest(), self.expected)
+        self.assertEqual(h.name, "hmac-sha256")
+
+        digest = hmac.digest(b"key", b"hash this!", sha256_module.sha256)
+        self.assertEqual(digest, binascii.unhexlify(self.expected))
+
 
 class SanityTestCase(unittest.TestCase):
 
@@ -447,39 +473,37 @@ def test_exercise_all_methods(self):
 class CopyTestCase(unittest.TestCase):
 
     @hashlib_helper.requires_hashdigest('sha256')
-    def test_attributes(self):
+    def test_attributes_old(self):
         # Testing if attributes are of same type.
-        h1 = hmac.HMAC(b"key", digestmod="sha256")
+        h1 = hmac.HMAC.__new__(hmac.HMAC)
+        h1._init_old(b"key", b"msg", digestmod="sha256")
         h2 = h1.copy()
-        self.assertTrue(h1._digest_cons == h2._digest_cons,
-            "digest constructors don't match.")
         self.assertEqual(type(h1._inner), type(h2._inner),
             "Types of inner don't match.")
         self.assertEqual(type(h1._outer), type(h2._outer),
             "Types of outer don't match.")
 
     @hashlib_helper.requires_hashdigest('sha256')
-    def test_realcopy(self):
+    def test_realcopy_old(self):
         # Testing if the copy method created a real copy.
-        h1 = hmac.HMAC(b"key", digestmod="sha256")
+        h1 = hmac.HMAC.__new__(hmac.HMAC)
+        h1._init_old(b"key", b"msg", digestmod="sha256")
         h2 = h1.copy()
         # Using id() in case somebody has overridden __eq__/__ne__.
         self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
         self.assertTrue(id(h1._inner) != id(h2._inner),
             "No real copy of the attribute 'inner'.")
         self.assertTrue(id(h1._outer) != id(h2._outer),
             "No real copy of the attribute 'outer'.")
-        self.assertEqual(h1._inner, h1.inner)
-        self.assertEqual(h1._outer, h1.outer)
-        self.assertEqual(h1._digest_cons, h1.digest_cons)
+        self.assertIs(h1._hmac, None)
 
+    @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl")
     @hashlib_helper.requires_hashdigest('sha256')
-    def test_properties(self):
-        # deprecated properties
-        h1 = hmac.HMAC(b"key", digestmod="sha256")
-        self.assertEqual(h1._inner, h1.inner)
-        self.assertEqual(h1._outer, h1.outer)
-        self.assertEqual(h1._digest_cons, h1.digest_cons)
+    def test_realcopy_hmac(self):
+        h1 = hmac.HMAC.__new__(hmac.HMAC)
+        h1._init_hmac(b"key", b"msg", digestmod="sha256")
+        h2 = h1.copy()
+        self.assertTrue(id(h1._hmac) != id(h2._hmac))
 
     @hashlib_helper.requires_hashdigest('sha256')
     def test_equality(self):

Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst

@@ -0,0 +1,2 @@
+The :mod:`hmac` module now uses OpenSSL's HMAC implementation when digestmod
+argument is a hash name or builtin hash function.