Merge branch 'main' into gh-143511-remote-exec-docs

Author: Yashp002

Android/testbed/app/build.gradle.kts

@@ -92,7 +92,12 @@ android {
             }
             throw GradleException("Failed to find API level in $androidEnvFile")
         }
-        targetSdk = 35
+
+        // This controls the API level of the maxVersion managed emulator, which is used
+        // by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has
+        // issues connecting to the internet (#142387), and 36 and later are not
+        // available as aosp_atd images yet.
+        targetSdk = 33
 
         versionCode = 1
         versionName = "1.0"

Doc/c-api/memory.rst

@@ -677,7 +677,11 @@ The pymalloc allocator
 Python has a *pymalloc* allocator optimized for small objects (smaller or equal
 to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
 with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
-platforms. It falls back to :c:func:`PyMem_RawMalloc` and
+platforms. When Python is configured with :option:`--with-pymalloc-hugepages`,
+the arena size on 64-bit platforms is increased to 2 MiB to match the huge page
+size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on
+Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages.
+It falls back to :c:func:`PyMem_RawMalloc` and
 :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
 
 *pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the

Doc/library/subprocess.rst

@@ -803,14 +803,29 @@ Instances of the :class:`Popen` class have the following methods:
 
    .. note::
 
-      When the ``timeout`` parameter is not ``None``, then (on POSIX) the
-      function is implemented using a busy loop (non-blocking call and short
-      sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see
+      When ``timeout`` is not ``None`` and the platform supports it, an
+      efficient event-driven mechanism is used to wait for process termination:
+
+      - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`
+      - macOS and other BSD variants use :func:`select.kqueue` +
+        ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``
+      - Windows uses ``WaitForSingleObject``
+
+      If none of these mechanisms are available, the function falls back to a
+      busy loop (non-blocking call and short sleeps).
+
+   .. note::
+
+      Use the :mod:`asyncio` module for an asynchronous wait: see
       :class:`asyncio.create_subprocess_exec`.
 
    .. versionchanged:: 3.3
       *timeout* was added.
 
+   .. versionchanged:: 3.15
+      if *timeout* is not ``None``, use efficient event-driven implementation
+      on Linux >= 5.3 and macOS / BSD.
+
 .. method:: Popen.communicate(input=None, timeout=None)
 
    Interact with process: Send data to stdin.  Read data from stdout and stderr,

Doc/using/configure.rst

@@ -783,6 +783,21 @@ also be used to improve performance.
 
    See also :envvar:`PYTHONMALLOC` environment variable.
 
+.. option:: --with-pymalloc-hugepages
+
+   Enable huge page support for :ref:`pymalloc <pymalloc>` arenas (disabled by
+   default). When enabled, the arena size on 64-bit platforms is increased to
+   2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or
+   ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+
+   The configure script checks that the platform supports ``MAP_HUGETLB``
+   and emits a warning if it is not available.
+
+   On Windows, use the ``--pymalloc-hugepages`` flag with ``build.bat`` or
+   set the ``UsePymallocHugepages`` MSBuild property.
+
+   .. versionadded:: 3.15
+
 .. option:: --without-doc-strings
 
    Disable static documentation strings to reduce the memory footprint (enabled

Doc/whatsnew/3.15.rst

@@ -743,6 +743,20 @@ ssl
 
    (Contributed by Ron Frederick in :gh:`138252`.)
 
+subprocess
+----------
+
+* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the
+  platform supports it, an efficient event-driven mechanism is used to wait for
+  process termination:
+
+  - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`.
+  - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``.
+  - Windows keeps using ``WaitForSingleObject`` (unchanged).
+
+  If none of these mechanisms are available, the function falls back to the
+  traditional busy loop (non-blocking call and short sleeps).
+  (Contributed by Giampaolo Rodola in :gh:`83069`).
 
 symtable
 --------
@@ -1463,6 +1477,12 @@ Build changes
   modules that are missing or packaged separately.
   (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.)
 
+* The new configure option :option:`--with-pymalloc-hugepages` enables huge
+  page support for :ref:`pymalloc <pymalloc>` arenas. When enabled, arena size
+  increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or
+  ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+  On Windows, use ``build.bat --pymalloc-hugepages``.
+
 * Annotating anonymous mmap usage is now supported if Linux kernel supports
   :manpage:`PR_SET_VMA_ANON_NAME <PR_SET_VMA(2const)>` (Linux 5.17 or newer).
   Annotations are visible in ``/proc/<pid>/maps`` if the kernel supports the feature

Include/internal/pycore_code.h

@@ -292,17 +292,7 @@ extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr);
 extern void _PyCode_Clear_Executors(PyCodeObject *code);
 
 
-#ifdef Py_GIL_DISABLED
-// gh-115999 tracks progress on addressing this.
-#define ENABLE_SPECIALIZATION 0
-// Use this to enable specialization families once they are thread-safe. All
-// uses will be replaced with ENABLE_SPECIALIZATION once all families are
-// thread-safe.
-#define ENABLE_SPECIALIZATION_FT 1
-#else
 #define ENABLE_SPECIALIZATION 1
-#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION
-#endif
 
 /* Specialization functions, these are exported only for other re-generated
  * interpreters to call */

Include/internal/pycore_jit.h

@@ -26,6 +26,7 @@ typedef _Py_CODEUNIT *(*jit_func)(
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);
 void _PyJIT_Fini(void);
+PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr);
 
 #endif  // _Py_JIT
 

Include/internal/pycore_obmalloc.h

@@ -208,7 +208,11 @@ typedef unsigned int pymem_uint;  /* assuming >= 16 bits */
  * mappings to reduce heap fragmentation.
  */
 #ifdef USE_LARGE_ARENAS
-#define ARENA_BITS              20                    /* 1 MiB */
+#  ifdef PYMALLOC_USE_HUGEPAGES
+#    define ARENA_BITS            21                    /* 2 MiB */
+#  else
+#    define ARENA_BITS            20                    /* 1 MiB */
+#  endif
 #else
 #define ARENA_BITS              18                    /* 256 KiB */
 #endif
@@ -469,7 +473,7 @@ nfp free pools in usable_arenas.
 */
 
 /* How many arena_objects do we initially allocate?
- * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
+ * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the
  * `arenas` vector.
  */
 #define INITIAL_ARENA_OBJECTS 16
@@ -512,14 +516,26 @@ struct _obmalloc_mgmt {
 
    memory address bit allocation for keys
 
-   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size:
+   ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or
+   21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled.  All bit widths
+   below are derived from ARENA_BITS automatically.
+
+   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default):
      15 -> MAP_TOP_BITS
      15 -> MAP_MID_BITS
      14 -> MAP_BOT_BITS
      20 -> ideal aligned arena
    ----
      64
 
+   64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages):
+     15 -> MAP_TOP_BITS
+     15 -> MAP_MID_BITS
+     13 -> MAP_BOT_BITS
+     21 -> ideal aligned arena
+   ----
+     64
+
    64-bit pointers, IGNORE_BITS=16, and 2^20 arena size:
      16 -> IGNORE_BITS
      10 -> MAP_TOP_BITS

Lib/profiling/sampling/sample.py

@@ -42,6 +42,7 @@ def _pause_threads(unwinder, blocking):
     LiveStatsCollector = None
 
 _FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None
+
 # Minimum number of samples required before showing the TUI
 # If fewer samples are collected, we skip the TUI and just print a message
 MIN_SAMPLES_FOR_TUI = 200
@@ -64,19 +65,23 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD
         self.realtime_stats = False
 
     def _new_unwinder(self, native, gc, opcodes, skip_non_matching_threads):
-        if _FREE_THREADED_BUILD:
-            unwinder = _remote_debugging.RemoteUnwinder(
-                self.pid, all_threads=self.all_threads, mode=self.mode, native=native, gc=gc,
-                opcodes=opcodes, skip_non_matching_threads=skip_non_matching_threads,
-                cache_frames=True, stats=self.collect_stats
-            )
+        kwargs = {}
+        if _FREE_THREADED_BUILD or self.all_threads:
+            kwargs['all_threads'] = self.all_threads
         else:
-            unwinder = _remote_debugging.RemoteUnwinder(
-                self.pid, only_active_thread=bool(self.all_threads), mode=self.mode, native=native, gc=gc,
-                opcodes=opcodes, skip_non_matching_threads=skip_non_matching_threads,
-                cache_frames=True, stats=self.collect_stats
-            )
-        return unwinder
+            kwargs['only_active_thread'] = bool(self.all_threads)
+
+        return _remote_debugging.RemoteUnwinder(
+            self.pid,
+            mode=self.mode,
+            native=native,
+            gc=gc,
+            opcodes=opcodes,
+            skip_non_matching_threads=skip_non_matching_threads,
+            cache_frames=True,
+            stats=self.collect_stats,
+            **kwargs
+        )
 
     def sample(self, collector, duration_sec=None, *, async_aware=False):
         sample_interval_sec = self.sample_interval_usec / 1_000_000