Adding time interval configurations used by demand-based hugepage rel…

…ease in HugeCache.

PiperOrigin-RevId: 677404092
Change-Id: Ia375eae8a1ca9a5960315e1a0611ca76eb25eef5

tcmalloc/global_stats.cc

@@ -551,6 +551,12 @@ void DumpStats(Printer* out, int level) {
     out->printf("PARAMETER tcmalloc_skip_subrelease_long_interval %s\n",
                 absl::FormatDuration(
                     Parameters::filler_skip_subrelease_long_interval()));
+    out->printf("PARAMETER tcmalloc_cache_demand_release_short_interval %s\n",
+                absl::FormatDuration(
+                    Parameters::cache_demand_release_short_interval()));
+    out->printf(
+        "PARAMETER tcmalloc_cache_demand_release_long_interval %s\n",
+        absl::FormatDuration(Parameters::cache_demand_release_long_interval()));
     out->printf("PARAMETER tcmalloc_release_partial_alloc_pages %d\n",
                 Parameters::release_partial_alloc_pages() ? 1 : 0);
     out->printf("PARAMETER tcmalloc_huge_cache_demand_based_release %d\n",
@@ -756,6 +762,12 @@ void DumpStatsInPbtxt(Printer* out, int level) {
   region.PrintI64("tcmalloc_skip_subrelease_long_interval_ns",
                   absl::ToInt64Nanoseconds(
                       Parameters::filler_skip_subrelease_long_interval()));
+  region.PrintI64("tcmalloc_cache_demand_release_short_interval_ns",
+                  absl::ToInt64Nanoseconds(
+                      Parameters::cache_demand_release_short_interval()));
+  region.PrintI64("tcmalloc_cache_demand_release_long_interval_ns",
+                  absl::ToInt64Nanoseconds(
+                      Parameters::cache_demand_release_long_interval()));
   region.PrintBool("tcmalloc_release_partial_alloc_pages",
                    Parameters::release_partial_alloc_pages());
   region.PrintBool("tcmalloc_huge_cache_demand_based_release",

tcmalloc/huge_page_aware_allocator.h

@@ -70,6 +70,12 @@ class StaticForwarder {
   static absl::Duration filler_skip_subrelease_long_interval() {
     return Parameters::filler_skip_subrelease_long_interval();
   }
+  static absl::Duration cache_demand_release_short_interval() {
+    return Parameters::cache_demand_release_short_interval();
+  }
+  static absl::Duration cache_demand_release_long_interval() {
+    return Parameters::cache_demand_release_long_interval();
+  }
 
   static bool release_partial_alloc_pages() {
     return Parameters::release_partial_alloc_pages();
@@ -852,21 +858,22 @@ inline Length HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPages(
   // We use demand-based release for the background release but not for the
   // other cases (e.g., limit hit). We achieve this by configuring the intervals
   // and hit_limit accordingly.
-  SkipSubreleaseIntervals intervals;
+  SkipSubreleaseIntervals cache_release_intervals;
   if (reason == PageReleaseReason::kProcessBackgroundActions) {
-    intervals.peak_interval = forwarder_.filler_skip_subrelease_interval();
-    intervals.short_interval =
-        forwarder_.filler_skip_subrelease_short_interval();
-    intervals.long_interval = forwarder_.filler_skip_subrelease_long_interval();
+    cache_release_intervals.short_interval =
+        forwarder_.cache_demand_release_short_interval();
+    cache_release_intervals.long_interval =
+        forwarder_.cache_demand_release_long_interval();
   }
   bool hit_limit = (reason == PageReleaseReason::kSoftLimitExceeded ||
                     reason == PageReleaseReason::kHardLimitExceeded);
   Length released;
   if (forwarder_.huge_cache_demand_based_release()) {
-    released += cache_
-                    .ReleaseCachedPagesByDemand(HLFromPages(num_pages),
-                                                intervals, hit_limit)
-                    .in_pages();
+    released +=
+        cache_
+            .ReleaseCachedPagesByDemand(HLFromPages(num_pages),
+                                        cache_release_intervals, hit_limit)
+            .in_pages();
   } else {
     released += cache_.ReleaseCachedPages(HLFromPages(num_pages)).in_pages();
   }

tcmalloc/huge_page_aware_allocator_fuzz.cc

@@ -429,9 +429,24 @@ void FuzzHPAA(const std::string& s) {
               i += size;
               break;
             }
-            case 8:
+            case 8: {
+              // Flips the settings used by demand-based release in HugeCache:
+              // actual_value[0] - release enabled
+              // actual_value[1:16] - interval_1
+              // actual_value[17:32] - interval_2
               forwarder.set_huge_cache_demand_based_release(actual_value & 0x1);
+              if (forwarder.huge_cache_demand_based_release()) {
+                const uint64_t interval_1 = (actual_value >> 1) & 0xffff;
+                const uint64_t interval_2 = (actual_value >> 17) & 0xffff;
+                forwarder.set_cache_demand_release_long_interval(
+                    interval_1 >= interval_2 ? absl::Nanoseconds(interval_1)
+                                             : absl::Nanoseconds(interval_2));
+                forwarder.set_cache_demand_release_short_interval(
+                    interval_1 >= interval_2 ? absl::Nanoseconds(interval_2)
+                                             : absl::Nanoseconds(interval_1));
+              }
               break;
+            }
           }
           break;
         }

tcmalloc/huge_page_aware_allocator_test.cc

@@ -384,6 +384,30 @@ TEST_P(HugePageAwareAllocatorTest, ReleasingLargeForBackgroundActions) {
   }
 }
 
+TEST_P(HugePageAwareAllocatorTest,
+       ReleasingLargeForBackgroundActionsWithZeroIntervals) {
+  // Tests that the configured intervals can be passed to HugeCache: release is
+  // not being impacted by demand-based release when the intervals are zero.
+  const bool old_enabled = Parameters::huge_cache_demand_based_release();
+  Parameters::set_huge_cache_demand_based_release(true);
+  const absl::Duration old_cache_short_interval =
+      Parameters::cache_demand_release_short_interval();
+  Parameters::set_cache_demand_release_short_interval(absl::ZeroDuration());
+  const absl::Duration old_cache_long_interval =
+      Parameters::cache_demand_release_long_interval();
+  Parameters::set_cache_demand_release_long_interval(absl::ZeroDuration());
+  const SpanAllocInfo kSpanInfo = {1, AccessDensityPrediction::kSparse};
+  Delete(New(kPagesPerHugePage, kSpanInfo), kSpanInfo.objects_per_span);
+  // There is no history to reference so release all.
+  EXPECT_EQ(
+      ReleasePages(kPagesPerHugePage,
+                   /*reason=*/PageReleaseReason::kProcessBackgroundActions),
+      kPagesPerHugePage);
+  Parameters::set_huge_cache_demand_based_release(old_enabled);
+  Parameters::set_cache_demand_release_short_interval(old_cache_short_interval);
+  Parameters::set_cache_demand_release_long_interval(old_cache_long_interval);
+}
+
 TEST_P(HugePageAwareAllocatorTest, SettingDemandBasedReleaseFlags) {
   // Checks if we can set the demand-based release flags.
   Parameters::set_huge_cache_demand_based_release(false);

tcmalloc/internal/parameter_accessors.h

@@ -36,6 +36,10 @@ TCMalloc_Internal_GetHugePageFillerSkipSubreleaseShortInterval(
 ABSL_ATTRIBUTE_WEAK void
 TCMalloc_Internal_GetHugePageFillerSkipSubreleaseLongInterval(
     absl::Duration* v);
+ABSL_ATTRIBUTE_WEAK void
+TCMalloc_Internal_GetHugeCacheDemandReleaseShortInterval(absl::Duration* v);
+ABSL_ATTRIBUTE_WEAK void
+TCMalloc_Internal_GetHugeCacheDemandReleaseLongInterval(absl::Duration* v);
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetReleasePartialAllocPagesEnabled();
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHugeCacheDemandBasedRelease();
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHugeRegionDemandBasedRelease();
@@ -90,6 +94,10 @@ TCMalloc_Internal_SetHugePageFillerSkipSubreleaseShortInterval(
     absl::Duration v);
 ABSL_ATTRIBUTE_WEAK void
 TCMalloc_Internal_SetHugePageFillerSkipSubreleaseLongInterval(absl::Duration v);
+ABSL_ATTRIBUTE_WEAK void
+TCMalloc_Internal_SetHugeCacheDemandReleaseShortInterval(absl::Duration v);
+ABSL_ATTRIBUTE_WEAK void
+TCMalloc_Internal_SetHugeCacheDemandReleaseLongInterval(absl::Duration v);
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseColdRegionsNoHugepage();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(
     bool v);

tcmalloc/internal_malloc_extension.h

@@ -93,6 +93,11 @@ ABSL_ATTRIBUTE_WEAK void
 MallocExtension_Internal_GetSkipSubreleaseShortInterval(absl::Duration* ret);
 ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_GetSkipSubreleaseLongInterval(
     absl::Duration* ret);
+ABSL_ATTRIBUTE_WEAK void
+MallocExtension_Internal_GetCacheDemandReleaseShortInterval(
+    absl::Duration* ret);
+ABSL_ATTRIBUTE_WEAK void
+MallocExtension_Internal_GetCacheDemandReleaseLongInterval(absl::Duration* ret);
 ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_GetProperties(
     std::map<std::string, tcmalloc::MallocExtension::Property>* ret);
 ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_GetExperiments(
@@ -111,6 +116,12 @@ ABSL_ATTRIBUTE_WEAK void
 MallocExtension_Internal_SetSkipSubreleaseShortInterval(absl::Duration value);
 ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetSkipSubreleaseLongInterval(
     absl::Duration value);
+ABSL_ATTRIBUTE_WEAK void
+MallocExtension_Internal_SetCacheDemandReleaseShortInterval(
+    absl::Duration value);
+ABSL_ATTRIBUTE_WEAK void
+MallocExtension_Internal_SetCacheDemandReleaseLongInterval(
+    absl::Duration value);
 ABSL_ATTRIBUTE_WEAK size_t MallocExtension_Internal_ReleaseCpuMemory(int cpu);
 ABSL_ATTRIBUTE_WEAK size_t
 MallocExtension_Internal_ReleaseMemoryToSystem(size_t bytes);

tcmalloc/malloc_extension.cc

@@ -580,6 +580,58 @@ void MallocExtension::SetSkipSubreleaseLongInterval(absl::Duration value) {
 #endif
 }
 
+absl::Duration MallocExtension::GetCacheDemandReleaseShortInterval() {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+  if (MallocExtension_Internal_GetCacheDemandReleaseShortInterval == nullptr) {
+    return absl::ZeroDuration();
+  }
+
+  absl::Duration value;
+  MallocExtension_Internal_GetCacheDemandReleaseShortInterval(&value);
+  return value;
+#else
+  return absl::ZeroDuration();
+#endif
+}
+
+void MallocExtension::SetCacheDemandReleaseShortInterval(absl::Duration value) {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+  if (MallocExtension_Internal_SetCacheDemandReleaseShortInterval == nullptr) {
+    return;
+  }
+
+  MallocExtension_Internal_SetCacheDemandReleaseShortInterval(value);
+#else
+  (void)value;
+#endif
+}
+
+absl::Duration MallocExtension::GetCacheDemandReleaseLongInterval() {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+  if (MallocExtension_Internal_GetCacheDemandReleaseLongInterval == nullptr) {
+    return absl::ZeroDuration();
+  }
+
+  absl::Duration value;
+  MallocExtension_Internal_GetCacheDemandReleaseLongInterval(&value);
+  return value;
+#else
+  return absl::ZeroDuration();
+#endif
+}
+
+void MallocExtension::SetCacheDemandReleaseLongInterval(absl::Duration value) {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+  if (MallocExtension_Internal_SetCacheDemandReleaseLongInterval == nullptr) {
+    return;
+  }
+
+  MallocExtension_Internal_SetCacheDemandReleaseLongInterval(value);
+#else
+  (void)value;
+#endif
+}
+
 std::optional<size_t> MallocExtension::GetNumericProperty(
     absl::string_view property) {
 #if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS

tcmalloc/malloc_extension.h

@@ -514,6 +514,15 @@ class MallocExtension final {
   static absl::Duration GetSkipSubreleaseLongInterval();
   static void SetSkipSubreleaseLongInterval(absl::Duration value);
 
+  // Gets and sets intervals used for finding the recent short-term demand
+  // fluctuation and long-term demand trend in HugeCache. Zero duration means
+  // not considering corresponding demand history for delayed (demand-based)
+  // hugepage release. The feature is disabled if both intervals are zero.
+  static absl::Duration GetCacheDemandReleaseShortInterval();
+  static void SetCacheDemandReleaseShortInterval(absl::Duration value);
+  static absl::Duration GetCacheDemandReleaseLongInterval();
+  static void SetCacheDemandReleaseLongInterval(absl::Duration value);
+
   // Returns the estimated number of bytes that will be allocated for a request
   // of "size" bytes.  This is an estimate: an allocation of "size" bytes may
   // reserve more bytes, but will never reserve fewer.

tcmalloc/mock_huge_page_static_forwarder.h

@@ -54,6 +54,12 @@ class FakeStaticForwarder {
   absl::Duration filler_skip_subrelease_long_interval() {
     return long_interval_;
   }
+  absl::Duration cache_demand_release_short_interval() {
+    return cache_demand_release_short_interval_;
+  }
+  absl::Duration cache_demand_release_long_interval() {
+    return cache_demand_release_long_interval_;
+  }
   bool release_partial_alloc_pages() { return release_partial_alloc_pages_; }
   bool hpaa_subrelease() const { return hpaa_subrelease_; }
 
@@ -66,6 +72,12 @@ class FakeStaticForwarder {
   void set_filler_skip_subrelease_long_interval(absl::Duration v) {
     long_interval_ = v;
   }
+  void set_cache_demand_release_short_interval(absl::Duration v) {
+    cache_demand_release_short_interval_ = v;
+  }
+  void set_cache_demand_release_long_interval(absl::Duration v) {
+    cache_demand_release_long_interval_ = v;
+  }
   void set_release_partial_alloc_pages(bool v) {
     release_partial_alloc_pages_ = v;
   }
@@ -166,6 +178,8 @@ class FakeStaticForwarder {
   }
 
   absl::Duration subrelease_interval_, short_interval_, long_interval_;
+  absl::Duration cache_demand_release_short_interval_;
+  absl::Duration cache_demand_release_long_interval_;
   bool release_partial_alloc_pages_ = false;
   bool hpaa_subrelease_ = true;
   bool release_succeeds_ = true;

tcmalloc/parameters.cc

@@ -187,6 +187,42 @@ static std::atomic<int64_t>& skip_subrelease_long_interval_ns() {
   return v;
 }
 
+static std::atomic<int64_t>& cache_demand_release_short_interval_ns() {
+  ABSL_CONST_INIT static absl::once_flag flag;
+  ABSL_CONST_INIT static std::atomic<int64_t> v{0};
+  absl::base_internal::LowLevelCallOnce(&flag, [&]() {
+    // clang-format off
+    v.store(absl::ToInt64Nanoseconds(
+#if defined(TCMALLOC_INTERNAL_SMALL_BUT_SLOW)
+                absl::ZeroDuration()
+#else
+                absl::Seconds(60)
+#endif
+                    ),
+            std::memory_order_relaxed);
+    // clang-format on
+  });
+  return v;
+}
+
+static std::atomic<int64_t>& cache_demand_release_long_interval_ns() {
+  ABSL_CONST_INIT static absl::once_flag flag;
+  ABSL_CONST_INIT static std::atomic<int64_t> v{0};
+  absl::base_internal::LowLevelCallOnce(&flag, [&]() {
+    // clang-format off
+    v.store(absl::ToInt64Nanoseconds(
+#if defined(TCMALLOC_INTERNAL_SMALL_BUT_SLOW)
+                absl::ZeroDuration()
+#else
+                absl::Seconds(300)
+#endif
+                    ),
+            std::memory_order_relaxed);
+    // clang-format on
+  });
+  return v;
+}
+
 uint64_t Parameters::heap_size_hard_limit() {
   return tc_globals.page_allocator().limit(PageAllocator::kHard);
 }
@@ -300,6 +336,16 @@ absl::Duration Parameters::filler_skip_subrelease_long_interval() {
       skip_subrelease_long_interval_ns().load(std::memory_order_relaxed));
 }
 
+absl::Duration Parameters::cache_demand_release_short_interval() {
+  return absl::Nanoseconds(
+      cache_demand_release_short_interval_ns().load(std::memory_order_relaxed));
+}
+
+absl::Duration Parameters::cache_demand_release_long_interval() {
+  return absl::Nanoseconds(
+      cache_demand_release_long_interval_ns().load(std::memory_order_relaxed));
+}
+
 bool Parameters::huge_region_demand_based_release() {
   return huge_region_demand_based_release_enabled().load(
       std::memory_order_relaxed);
@@ -419,6 +465,26 @@ void MallocExtension_Internal_SetSkipSubreleaseLongInterval(
   Parameters::set_filler_skip_subrelease_long_interval(value);
 }
 
+void MallocExtension_Internal_GetCacheDemandReleaseShortInterval(
+    absl::Duration* ret) {
+  *ret = Parameters::cache_demand_release_short_interval();
+}
+
+void MallocExtension_Internal_SetCacheDemandReleaseShortInterval(
+    absl::Duration value) {
+  Parameters::set_cache_demand_release_short_interval(value);
+}
+
+void MallocExtension_Internal_GetCacheDemandReleaseLongInterval(
+    absl::Duration* ret) {
+  *ret = Parameters::cache_demand_release_long_interval();
+}
+
+void MallocExtension_Internal_SetCacheDemandReleaseLongInterval(
+    absl::Duration value) {
+  Parameters::set_cache_demand_release_long_interval(value);
+}
+
 tcmalloc::MallocExtension::BytesPerSecond
 MallocExtension_Internal_GetBackgroundReleaseRate() {
   return Parameters::background_release_rate();
@@ -625,6 +691,27 @@ void TCMalloc_Internal_SetHugePageFillerSkipSubreleaseLongInterval(
       absl::ToInt64Nanoseconds(v), std::memory_order_relaxed);
 }
 
+void TCMalloc_Internal_GetHugeCacheDemandReleaseShortInterval(
+    absl::Duration* v) {
+  *v = Parameters::cache_demand_release_short_interval();
+}
+
+void TCMalloc_Internal_SetHugeCacheDemandReleaseShortInterval(
+    absl::Duration v) {
+  tcmalloc::tcmalloc_internal::cache_demand_release_short_interval_ns().store(
+      absl::ToInt64Nanoseconds(v), std::memory_order_relaxed);
+}
+
+void TCMalloc_Internal_GetHugeCacheDemandReleaseLongInterval(
+    absl::Duration* v) {
+  *v = Parameters::cache_demand_release_long_interval();
+}
+
+void TCMalloc_Internal_SetHugeCacheDemandReleaseLongInterval(absl::Duration v) {
+  tcmalloc::tcmalloc_internal::cache_demand_release_long_interval_ns().store(
+      absl::ToInt64Nanoseconds(v), std::memory_order_relaxed);
+}
+
 bool TCMalloc_Internal_GetPerCpuCachesDynamicSlabEnabled() {
   return Parameters::per_cpu_caches_dynamic_slab_enabled();
 }