Add adaptive GC RSS brake

Author: pablogsal

Doc/library/gc.rst

@@ -115,6 +115,13 @@ The :mod:`gc` module provides the following functions:
    With the third generation, things are a bit more complicated,
    see `Collecting the oldest generation <https://devguide.python.org/garbage_collector/#collecting-the-oldest-generation>`_ for more information.
 
+   If :envvar:`PYTHON_GC_ADAPTIVE` is enabled, the interpreter may adjust
+   *threshold0* at runtime based on recent generation ``0`` collection cost
+   and collection yield.  On Linux, adaptive mode also samples current process
+   RSS and page-fault activity as a soft brake on threshold growth.  It does
+   not target RSS directly.  *threshold1* and *threshold2* are not adjusted by
+   adaptive mode.
+
 
 .. function:: get_count()
 

Doc/using/cmdline.rst

@@ -888,6 +888,25 @@ conflict.
    .. versionadded:: 3.4
 
 
+.. envvar:: PYTHON_GC_ADAPTIVE
+
+   If this environment variable is set to a positive integer, Python enables
+   adaptive cyclic garbage collector threshold tuning at startup.  The
+   adaptive mode changes only the youngest-generation allocation threshold
+   according to recent collection cost and collection yield.  On Linux, it
+   also samples current RSS and page-fault activity as a soft brake on
+   threshold growth.  The middle- and oldest-generation thresholds keep their
+   configured values.
+
+
+.. envvar:: PYTHON_GC_ADAPTIVE_MAX_THRESHOLD0
+
+   If :envvar:`PYTHON_GC_ADAPTIVE` is enabled, this environment variable sets
+   the maximum adaptive value for the youngest-generation allocation
+   threshold.  The value must be a positive integer.  The default is
+   ``5000000``.
+
+
 .. envvar:: PYTHONPROFILEIMPORTTIME
 
    If this environment variable is set to a non-empty string, Python will

Include/internal/pycore_gc.h

@@ -154,6 +154,20 @@ struct gc_generation_stats {
     Py_ssize_t uncollectable;
 };
 
+struct gc_threshold_controller {
+    int enabled;
+    int min_threshold0;
+    int max_threshold0;
+    int window_collections;
+    Py_ssize_t window_candidates;
+    Py_ssize_t window_collected;
+    _PyTime_t window_start;
+    _PyTime_t gc_time;
+    Py_ssize_t last_rss_pages;
+    long last_minor_faults;
+    long last_major_faults;
+};
+
 struct _gc_runtime_state {
     /* List of objects that still need to be cleaned up, singly linked
      * via their gc headers' gc_prev pointers.  */
@@ -176,6 +190,7 @@ struct _gc_runtime_state {
     PyObject *garbage;
     /* a list of callbacks to be invoked when collection is performed */
     PyObject *callbacks;
+    struct gc_threshold_controller threshold_controller;
     /* This is the number of objects that survived the last full
        collection. It approximates the number of long lived objects
        tracked by the GC.

Include/internal/pycore_time.h

@@ -18,6 +18,10 @@ struct _time_runtime_state {
 #endif
 };
 
+extern int _PyTime_GetProcessTimeWithInfo(
+    _PyTime_t *t,
+    _Py_clock_info_t *info);
+
 
 #ifdef __cplusplus
 }

Lib/test/test_gc.py

@@ -397,6 +397,49 @@ def test_collect_generations(self):
         self.assertEqual((e, f), (0, 1))
         self.assertEqual((h, i), (0, 0))
 
+    def test_adaptive_threshold_env_var(self):
+        code = textwrap.dedent("""
+            import gc
+
+            start = gc.get_threshold()
+            keep = []
+            for _ in range(8):
+                keep.extend([] for _ in range(1000))
+                gc.collect(0)
+            end = gc.get_threshold()
+
+            assert start == (700, 10, 10), start
+            assert end[0] > start[0], end
+            assert end[1:] == start[1:], end
+        """)
+        assert_python_ok("-c", code, PYTHON_GC_ADAPTIVE="1")
+
+    def test_adaptive_threshold_env_var_ignored_by_E(self):
+        code = textwrap.dedent("""
+            import gc
+
+            keep = []
+            for _ in range(8):
+                keep.extend([] for _ in range(1000))
+                gc.collect(0)
+            assert gc.get_threshold() == (700, 10, 10)
+        """)
+        assert_python_ok("-E", "-c", code, PYTHON_GC_ADAPTIVE="1")
+
+    def test_adaptive_max_threshold_env_var(self):
+        code = textwrap.dedent("""
+            import gc
+
+            keep = []
+            for _ in range(8):
+                keep.extend([] for _ in range(1000))
+                gc.collect(0)
+            assert gc.get_threshold() == (900, 10, 10), gc.get_threshold()
+        """)
+        assert_python_ok("-c", code,
+                         PYTHON_GC_ADAPTIVE="1",
+                         PYTHON_GC_ADAPTIVE_MAX_THRESHOLD0="900")
+
     def test_trashcan(self):
         class Ouch:
             n = 0