Merge branch 'main' into sem-macosx-multiprocessing-module-C

Author: YvesDup

Doc/whatsnew/3.16.rst

@@ -106,10 +106,13 @@ module_name
 Removed
 =======
 
-module_name
------------
+sysconfig
+---------
+
+* The :func:`!sysconfig.expand_makefile_vars` function
+  which has been deprecated since Python 3.14.
+  Use the ``vars`` argument of :func:`sysconfig.get_paths` instead.
 
-* TODO
 .. Add removals above alphabetically, not here at the end.
 
 
@@ -156,4 +159,3 @@ Deprecated C APIs
 
 Removed C APIs
 --------------
-

Include/cpython/bytesobject.h

@@ -5,7 +5,7 @@
 typedef struct {
     PyObject_VAR_HEAD
     Py_DEPRECATED(3.11) Py_hash_t ob_shash;
-    char ob_sval[1];
+    unsigned char ob_sval[1];
 
     /* Invariants:
      *     ob_sval contains space for 'ob_size+1' elements.
@@ -20,7 +20,7 @@ PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t);
 #define _PyBytes_CAST(op) \
     (assert(PyBytes_Check(op)), _Py_CAST(PyBytesObject*, op))
 
-static inline char* PyBytes_AS_STRING(PyObject *op)
+static inline unsigned char* PyBytes_AS_STRING(PyObject *op)
 {
     return _PyBytes_CAST(op)->ob_sval;
 }

Include/internal/pycore_magic_number.h

@@ -297,9 +297,10 @@ Known values:
     Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions)
     Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations)
     Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations)
+    Python 3.16a0 3700 (Initial version)
 
 
-    Python 3.16 will start with 3700
+    Python 3.17 will start with 3750
 
     Please don't copy-paste the same pre-release tag for new entries above!!!
     You should always use the *upcoming* tag. For example, if 3.12a6 came out
@@ -310,7 +311,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3666
+#define PYC_MAGIC_NUMBER 3700
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \

Lib/sysconfig/__init__.py

@@ -753,41 +753,3 @@ def get_python_version():
 
 def _get_python_version_abi():
     return _PY_VERSION_SHORT + get_config_var("abi_thread")
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    import warnings
-    warnings.warn(
-        'sysconfig.expand_makefile_vars is deprecated and will be removed in '
-        'Python 3.16. Use sysconfig.get_paths(vars=...) instead.',
-        DeprecationWarning,
-        stacklevel=2,
-    )
-
-    import re
-
-    _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
-    _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while True:
-        m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s

Lib/test/test_robotparser.py

@@ -646,26 +646,23 @@ def test_group_without_user_agent(self):
 )
 class BaseLocalNetworkTestCase:
 
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         # clear _opener global variable
-        self.addCleanup(urllib.request.urlcleanup)
+        cls.addClassCleanup(urllib.request.urlcleanup)
 
-        self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler)
+        cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler)
+        cls.addClassCleanup(cls.server.server_close)
 
-        self.t = threading.Thread(
+        t = threading.Thread(
             name='HTTPServer serving',
-            target=self.server.serve_forever,
+            target=cls.server.serve_forever,
             # Short poll interval to make the test finish quickly.
             # Time between requests is short enough that we won't wake
             # up spuriously too many times.
             kwargs={'poll_interval':0.01})
-        self.t.daemon = True  # In case this function raises.
-        self.t.start()
-
-    def tearDown(self):
-        self.server.shutdown()
-        self.t.join()
-        self.server.server_close()
+        cls.enterClassContext(threading_helper.start_threads([t]))
+        cls.addClassCleanup(cls.server.shutdown)
 
 
 SAMPLE_ROBOTS_TXT = b'''\
@@ -687,7 +684,6 @@ def do_GET(self):
         def log_message(self, format, *args):
             pass
 
-    @threading_helper.reap_threads
     def testRead(self):
         # Test that reading a weird robots.txt doesn't fail.
         addr = self.server.server_address
@@ -709,24 +705,62 @@ def testRead(self):
         self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path'))
 
 
-class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
+class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
     class RobotHandler(BaseHTTPRequestHandler):
 
         def do_GET(self):
-            self.send_error(403, "Forbidden access")
+            self.send_error(self.server.return_code)
 
         def log_message(self, format, *args):
             pass
 
-    @threading_helper.reap_threads
+    def setUp(self):
+        # Make sure that a valid code is set in the test.
+        self.server.return_code = None
+
     def testPasswordProtectedSite(self):
+        self.server.return_code = 403
         addr = self.server.server_address
         url = 'http://' + socket_helper.HOST + ':' + str(addr[1])
         robots_url = url + "/robots.txt"
         parser = urllib.robotparser.RobotFileParser()
         parser.set_url(url)
         parser.read()
         self.assertFalse(parser.can_fetch("*", robots_url))
+        self.assertFalse(parser.can_fetch("*", url + '/some/file.html'))
+
+    def testNotFound(self):
+        self.server.return_code = 404
+        addr = self.server.server_address
+        url = f'http://{socket_helper.HOST}:{addr[1]}'
+        robots_url = url + "/robots.txt"
+        parser = urllib.robotparser.RobotFileParser()
+        parser.set_url(url)
+        parser.read()
+        self.assertTrue(parser.can_fetch("*", robots_url))
+        self.assertTrue(parser.can_fetch("*", url + '/path/file.html'))
+
+    def testTeapot(self):
+        self.server.return_code = 418
+        addr = self.server.server_address
+        url = f'http://{socket_helper.HOST}:{addr[1]}'
+        robots_url = url + "/robots.txt"
+        parser = urllib.robotparser.RobotFileParser()
+        parser.set_url(url)
+        parser.read()
+        self.assertTrue(parser.can_fetch("*", robots_url))
+        self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream'))
+
+    def testServiceUnavailable(self):
+        self.server.return_code = 503
+        addr = self.server.server_address
+        url = f'http://{socket_helper.HOST}:{addr[1]}'
+        robots_url = url + "/robots.txt"
+        parser = urllib.robotparser.RobotFileParser()
+        parser.set_url(url)
+        parser.read()
+        self.assertFalse(parser.can_fetch("*", robots_url))
+        self.assertFalse(parser.can_fetch("*", url + '/path/file.html'))
 
 
 @support.requires_working_socket()

Lib/urllib/robotparser.py

@@ -65,9 +65,17 @@ def read(self):
             f = urllib.request.urlopen(self.url)
         except urllib.error.HTTPError as err:
             if err.code in (401, 403):
+                # If access to robot.txt has the status Unauthorized/Forbidden,
+                # then most likely this applies to the entire site.
                 self.disallow_all = True
-            elif err.code >= 400 and err.code < 500:
+            elif 400 <= err.code < 500:
+                # RFC 9309, Section 2.3.1.3: the crawler MAY access any
+                # resources on the server.
                 self.allow_all = True
+            elif 500 <= err.code < 600:
+                # RFC 9309, Section 2.3.1.4: the crawler MUST assume
+                # complete disallow.
+                self.disallow_all = True
             err.close()
         else:
             raw = f.read()

Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst

@@ -0,0 +1,2 @@
+Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file
+is unreachable due to server or network errors.

Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst

@@ -0,0 +1,3 @@
+Removed the :func:`!sysconfig.expand_makefile_vars` function which has been
+deprecated since Python 3.14. Use the ``vars`` argument of
+:func:`sysconfig.get_paths` instead.

PC/launcher.c

@@ -1273,6 +1273,7 @@ static PYC_MAGIC magic_values[] = {
     { 3550, 3599, L"3.13" },
     { 3600, 3649, L"3.14" },
     { 3650, 3699, L"3.15" },
+    { 3700, 3749, L"3.16" },
     { 0 }
 };
 

PC/pyconfig.h

@@ -330,21 +330,21 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
                         the linking is explicitly handled */
 #                       if defined(Py_GIL_DISABLED)
 #                       if defined(Py_DEBUG)
-#                               pragma comment(lib,"python315t_d.lib")
+#                               pragma comment(lib,"python316t_d.lib")
 #                       elif defined(Py_LIMITED_API) || defined(Py_TARGET_ABI3T)
 #                               pragma comment(lib,"python3t.lib")
 #                       else
-#                               pragma comment(lib,"python315t.lib")
+#                               pragma comment(lib,"python316t.lib")
 #                       endif /* Py_DEBUG */
 #                       else /* Py_GIL_DISABLED */
 #                       if defined(Py_DEBUG)
-#                               pragma comment(lib,"python315_d.lib")
+#                               pragma comment(lib,"python316_d.lib")
 #                       elif defined(Py_TARGET_ABI3T)
 #                               pragma comment(lib,"python3t.lib")
 #                       elif defined(Py_LIMITED_API)
 #                               pragma comment(lib,"python3.lib")
 #                       else
-#                               pragma comment(lib,"python315.lib")
+#                               pragma comment(lib,"python316.lib")
 #                       endif /* Py_DEBUG */
 #                       endif /* Py_GIL_DISABLED */
 #               endif /* _MSC_VER && !Py_NO_LINK_LIB */