gh-144001: Support ignorechars in binascii.a2b_base64() and base64.b64decode() (GH-144024)

Author: serhiy-storchaka

Doc/library/base64.rst

@@ -73,6 +73,7 @@ POST request.
 
 
 .. function:: b64decode(s, altchars=None, validate=False)
+              b64decode(s, altchars=None, validate=True, *, ignorechars)
 
    Decode the Base64 encoded :term:`bytes-like object` or ASCII string
    *s* and return the decoded :class:`bytes`.
@@ -84,11 +85,17 @@ POST request.
    A :exc:`binascii.Error` exception is raised
    if *s* is incorrectly padded.
 
-   If *validate* is false (the default), characters that are neither
+   If *ignorechars* is specified, it should be a :term:`bytes-like object`
+   containing characters to ignore from the input when *validate* is true.
+   The default value of *validate* is ``True`` if *ignorechars* is specified,
+   ``False`` otherwise.
+
+   If *validate* is false, characters that are neither
    in the normal base-64 alphabet nor the alternative alphabet are
    discarded prior to the padding check, but the ``+`` and ``/`` characters
    keep their meaning if they are not in *altchars* (they will be discarded
    in future Python versions).
+
    If *validate* is true, these non-alphabet characters in the input
    result in a :exc:`binascii.Error`.
 
@@ -99,6 +106,10 @@ POST request.
       is now deprecated.
 
 
+   .. versionchanged:: next
+      Added the *ignorechars* parameter.
+
+
 .. function:: standard_b64encode(s)
 
    Encode :term:`bytes-like object` *s* using the standard Base64 alphabet

Doc/library/binascii.rst

@@ -49,10 +49,16 @@ The :mod:`binascii` module defines the following functions:
 
 
 .. function:: a2b_base64(string, /, *, strict_mode=False)
+              a2b_base64(string, /, *, strict_mode=True, ignorechars)
 
    Convert a block of base64 data back to binary and return the binary data. More
    than one line may be passed at a time.
 
+   If *ignorechars* is specified, it should be a :term:`bytes-like object`
+   containing characters to ignore from the input when *strict_mode* is true.
+   The default value of *strict_mode* is ``True`` if *ignorechars* is specified,
+   ``False`` otherwise.
+
    If *strict_mode* is true, only valid base64 data will be converted. Invalid base64
    data will raise :exc:`binascii.Error`.
 
@@ -66,6 +72,9 @@ The :mod:`binascii` module defines the following functions:
    .. versionchanged:: 3.11
       Added the *strict_mode* parameter.
 
+   .. versionchanged:: next
+      Added the *ignorechars* parameter.
+
 
 .. function:: b2a_base64(data, *, wrapcol=0, newline=True)
 

Doc/whatsnew/3.15.rst

@@ -444,13 +444,18 @@ base64
 * Added the *wrapcol* parameter in :func:`~base64.b64encode`.
   (Contributed by Serhiy Storchaka in :gh:`143214`.)
 
+* Added the *ignorechars* parameter in :func:`~base64.b64decode`.
+  (Contributed by Serhiy Storchaka in :gh:`144001`.)
 
 binascii
 --------
 
 * Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`.
   (Contributed by Serhiy Storchaka in :gh:`143214`.)
 
+* Added the *ignorechars* parameter in :func:`~binascii.a2b_base64`.
+  (Contributed by Serhiy Storchaka in :gh:`144001`.)
+
 
 calendar
 --------

Include/internal/pycore_global_objects_fini_generated.h

@@ -520,6 +520,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(ident)
         STRUCT_FOR_ID(identity_hint)
         STRUCT_FOR_ID(ignore)
+        STRUCT_FOR_ID(ignorechars)
         STRUCT_FOR_ID(imag)
         STRUCT_FOR_ID(implieslink)
         STRUCT_FOR_ID(importlib)

Include/internal/pycore_global_strings.h

@@ -26,6 +26,8 @@
     ]
 
 
+_NOT_SPECIFIED = ['NOT SPECIFIED']
+
 bytes_types = (bytes, bytearray)  # Types acceptable as binary data
 
 def _bytes_from_decode_data(s):
@@ -62,7 +64,7 @@ def b64encode(s, altchars=None, *, wrapcol=0):
     return encoded
 
 
-def b64decode(s, altchars=None, validate=False):
+def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, *, ignorechars=_NOT_SPECIFIED):
     """Decode the Base64 encoded bytes-like object or ASCII string s.
 
     Optional altchars must be a bytes-like object or ASCII string of length 2
@@ -72,38 +74,64 @@ def b64decode(s, altchars=None, validate=False):
     The result is returned as a bytes object.  A binascii.Error is raised if
     s is incorrectly padded.
 
-    If validate is false (the default), characters that are neither in the
-    normal base-64 alphabet nor the alternative alphabet are discarded prior
-    to the padding check.  If validate is true, these non-alphabet characters
-    in the input result in a binascii.Error.
+    If ignorechars is specified, it should be a byte string containing
+    characters to ignore from the input.  The default value of validate is
+    True if ignorechars is specified, False otherwise.
+
+    If validate is false, characters that are neither in the normal base-64
+    alphabet nor the alternative alphabet are discarded prior to the
+    padding check.  If validate is true, these non-alphabet characters in
+    the input result in a binascii.Error if they are not in ignorechars.
     For more information about the strict base64 check, see:
 
     https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
     """
     s = _bytes_from_decode_data(s)
+    if validate is _NOT_SPECIFIED:
+        validate = ignorechars is not _NOT_SPECIFIED
+    if ignorechars is _NOT_SPECIFIED:
+        ignorechars = b''
     badchar = None
+    badchar_strict = False
     if altchars is not None:
         altchars = _bytes_from_decode_data(altchars)
         if len(altchars) != 2:
             raise ValueError(f'invalid altchars: {altchars!r}')
         for b in b'+/':
             if b not in altchars and b in s:
-                badchar = b
-                break
+                if badchar is None:
+                    badchar = b
+                if not validate:
+                    break
+                if not isinstance(ignorechars, (bytes, bytearray)):
+                    ignorechars = memoryview(ignorechars).cast('B')
+                if b not in ignorechars:
+                    badchar_strict = True
+                    badchar = b
+                    break
         s = s.translate(bytes.maketrans(altchars, b'+/'))
-    result = binascii.a2b_base64(s, strict_mode=validate)
+    result = binascii.a2b_base64(s, strict_mode=validate,
+                                 ignorechars=ignorechars)
     if badchar is not None:
         import warnings
-        if validate:
+        if badchar_strict:
             warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data '
                           f'with altchars={altchars!r} and validate=True '
                           f'will be an error in future Python versions',
                           DeprecationWarning, stacklevel=2)
         else:
-            warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data '
-                          f'with altchars={altchars!r} and validate=False '
-                          f'will be discarded in future Python versions',
-                          FutureWarning, stacklevel=2)
+            ignorechars = bytes(ignorechars)
+            if ignorechars:
+                warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data '
+                              f'with altchars={altchars!r} '
+                              f'and ignorechars={ignorechars!r} '
+                              f'will be discarded in future Python versions',
+                              FutureWarning, stacklevel=2)
+            else:
+                warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data '
+                              f'with altchars={altchars!r} and validate=False '
+                              f'will be discarded in future Python versions',
+                              FutureWarning, stacklevel=2)
     return result
 
 

Include/internal/pycore_runtime_init_generated.h

@@ -303,22 +303,26 @@ def test_b64decode_padding_error(self):
 
     def test_b64decode_invalid_chars(self):
         # issue 1466065: Test some invalid characters.
-        tests = ((b'%3d==', b'\xdd'),
-                 (b'$3d==', b'\xdd'),
-                 (b'[==', b''),
-                 (b'YW]3=', b'am'),
-                 (b'3{d==', b'\xdd'),
-                 (b'3d}==', b'\xdd'),
-                 (b'@@', b''),
-                 (b'!', b''),
-                 (b"YWJj\n", b"abc"),
-                 (b'YWJj\nYWI=', b'abcab'))
+        tests = ((b'%3d==', b'\xdd', b'%$'),
+                 (b'$3d==', b'\xdd', b'%$'),
+                 (b'[==', b'', None),
+                 (b'YW]3=', b'am', b']'),
+                 (b'3{d==', b'\xdd', b'{}'),
+                 (b'3d}==', b'\xdd', b'{}'),
+                 (b'@@', b'', b'@!'),
+                 (b'!', b'', b'@!'),
+                 (b"YWJj\n", b"abc", b'\n'),
+                 (b'YWJj\nYWI=', b'abcab', b'\n'),
+                 (b'YW\nJj', b'abc', b'\n'),
+                 (b'YW\nJj', b'abc', bytearray(b'\n')),
+                 (b'YW\nJj', b'abc', memoryview(b'\n')),
+        )
         funcs = (
             base64.b64decode,
             base64.standard_b64decode,
             base64.urlsafe_b64decode,
         )
-        for bstr, res in tests:
+        for bstr, res, ignorechars in tests:
             for func in funcs:
                 with self.subTest(bstr=bstr, func=func):
                     self.assertEqual(func(bstr), res)
@@ -327,24 +331,76 @@ def test_b64decode_invalid_chars(self):
                 base64.b64decode(bstr, validate=True)
             with self.assertRaises(binascii.Error):
                 base64.b64decode(bstr.decode('ascii'), validate=True)
+            with self.assertRaises(binascii.Error):
+                # Even empty ignorechars enables the strict mode.
+                base64.b64decode(bstr, ignorechars=b'')
+            if ignorechars is not None:
+                r = base64.b64decode(bstr, ignorechars=ignorechars)
+                self.assertEqual(r, res)
+
+        with self.assertRaises(TypeError):
+            base64.b64decode(b'', ignorechars='')
+        with self.assertRaises(TypeError):
+            base64.b64decode(b'', ignorechars=[])
+        with self.assertRaises(TypeError):
+            base64.b64decode(b'', ignorechars=None)
 
         # Normal alphabet characters will be discarded when alternative given
-        with self.assertWarns(FutureWarning):
-            self.assertEqual(base64.b64decode(b'++++', altchars=b'-_'),
-                             b'\xfb\xef\xbe')
-        with self.assertWarns(FutureWarning):
-            self.assertEqual(base64.b64decode(b'////', altchars=b'-_'),
-                             b'\xff\xff\xff')
-        with self.assertWarns(DeprecationWarning):
-            self.assertEqual(base64.b64decode(b'++++', altchars=b'-_', validate=True),
-                             b'\xfb\xef\xbe')
-        with self.assertWarns(DeprecationWarning):
-            self.assertEqual(base64.b64decode(b'////', altchars=b'-_', validate=True),
-                             b'\xff\xff\xff')
-        with self.assertWarns(FutureWarning):
+        discarded = ("invalid character %a in Base64 data with %s "
+                     "will be discarded in future Python versions")
+        error = ("invalid character %a in Base64 data with %s "
+                 "will be an error in future Python versions")
+        with self.assertWarns(FutureWarning) as cm:
+            r = base64.b64decode(b'++++', altchars=b'-_')
+        self.assertEqual(r, b'\xfb\xef\xbe')
+        self.assertEqual(str(cm.warning),
+                         discarded % ('+', "altchars=b'-_' and validate=False"))
+        with self.assertWarns(FutureWarning) as cm:
+            r = base64.b64decode(b'////', altchars=b'-_')
+        self.assertEqual(r, b'\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         discarded % ('/', "altchars=b'-_' and validate=False"))
+        with self.assertWarns(DeprecationWarning) as cm:
+            r = base64.b64decode(b'++++', altchars=b'-_', validate=True)
+        self.assertEqual(r, b'\xfb\xef\xbe')
+        self.assertEqual(str(cm.warning),
+                         error % ('+', "altchars=b'-_' and validate=True"))
+        with self.assertWarns(DeprecationWarning) as cm:
+            r = base64.b64decode(b'////', altchars=b'-_', validate=True)
+        self.assertEqual(r, b'\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         error % ('/', "altchars=b'-_' and validate=True"))
+        with self.assertWarns(FutureWarning) as cm:
+            r = base64.b64decode(b'++++', altchars=b'-_', ignorechars=b'+')
+        self.assertEqual(r, b'\xfb\xef\xbe')
+        self.assertEqual(str(cm.warning),
+                         discarded % ('+', "altchars=b'-_' and ignorechars=b'+'"))
+        with self.assertWarns(FutureWarning) as cm:
+            r = base64.b64decode(b'////', altchars=b'-_', ignorechars=b'/')
+        self.assertEqual(r, b'\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         discarded % ('/', "altchars=b'-_' and ignorechars=b'/'"))
+        with self.assertWarns(DeprecationWarning) as cm:
+            r = base64.b64decode(b'++++////', altchars=b'-_', ignorechars=b'+')
+        self.assertEqual(r, b'\xfb\xef\xbe\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         error % ('/', "altchars=b'-_' and validate=True"))
+        with self.assertWarns(DeprecationWarning) as cm:
+            r = base64.b64decode(b'++++////', altchars=b'-_', ignorechars=b'/')
+        self.assertEqual(r, b'\xfb\xef\xbe\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         error % ('+', "altchars=b'-_' and validate=True"))
+
+        with self.assertWarns(FutureWarning) as cm:
             self.assertEqual(base64.urlsafe_b64decode(b'++++'), b'\xfb\xef\xbe')
-        with self.assertWarns(FutureWarning):
+        self.assertEqual(str(cm.warning),
+                         "invalid character '+' in URL-safe Base64 data "
+                         "will be discarded in future Python versions")
+        with self.assertWarns(FutureWarning) as cm:
             self.assertEqual(base64.urlsafe_b64decode(b'////'), b'\xff\xff\xff')
+        self.assertEqual(str(cm.warning),
+                         "invalid character '/' in URL-safe Base64 data "
+                         "will be discarded in future Python versions")
         with self.assertRaises(binascii.Error):
             base64.b64decode(b'+/!', altchars=b'-_')