WIP: Decrease memory usage in video playback

dom/media/AdaptiveBufferingPolicy.cpp

@@ -0,0 +1,153 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AdaptiveBufferingPolicy.h"
+#include "mozilla/Logging.h"
+
+namespace mozilla {
+
+extern LazyLogModule gMediaDecoderLog;
+
+#define LOG(arg, ...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Debug, ("AdaptiveBuffering::%s: " arg, __func__, ##__VA_ARGS__))
+
+/**
+ * Compute estimated memory usage for a single frame.
+ */
+static uint64_t EstimateFrameMemory(const gfx::IntSize& aSize)
+{
+  // YUV420 format: 1.5 bytes per pixel
+  uint64_t width = aSize.width;
+  uint64_t height = aSize.height;
+  return (width * height * 3) / 2;
+}
+
+/**
+ * Calculate resolution tier (affects quality and frame size).
+ */
+static uint32_t GetResolutionTier(const gfx::IntSize& aSize)
+{
+  uint32_t pixels = aSize.width * aSize.height;
+
+  // Classify resolution
+  if (pixels < 480 * 360) return 0;          // SD-like
+  if (pixels < 1280 * 720) return 1;         // HD-ready
+  if (pixels < 1920 * 1080) return 2;        // Full HD
+  if (pixels < 3840 * 2160) return 3;        // 4K
+  return 4;                                  // 8K+
+}
+
+AdaptiveBufferingPolicy::Config 
+AdaptiveBufferingPolicy::ComputeOptimalConfig(
+    const gfx::IntSize& aVideoSize,
+    uint32_t aFrameRate,
+    uint64_t aVideoBitrate,
+    bool aHardwareAccelerated,
+    uint64_t aAvailableMemory)
+{
+  Config config;
+  uint32_t resolutionTier = GetResolutionTier(aVideoSize);
+  uint64_t frameMemory = EstimateFrameMemory(aVideoSize);
+
+  // Adjust video queue size based on resolution and memory availability
+  uint32_t recommendedQueueSize = config.mDefaultVideoQueueFrames;
+
+  if (aHardwareAccelerated) {
+    recommendedQueueSize = config.mHWAccelVideoQueueFrames;
+  }
+
+  // For high-resolution content, reduce queue size to save memory
+  if (resolutionTier >= 3) {  // 4K or higher
+    recommendedQueueSize = std::max(config.mMinVideoQueueFrames, 
+                                    recommendedQueueSize / 2);
+    config.mMaxVideoBufferMemory = 50 * 1024 * 1024;  // 50MB for 4K
+  } else if (resolutionTier == 2) {  // Full HD
+    config.mMaxVideoBufferMemory = 75 * 1024 * 1024;  // 75MB
+  }
+
+  // Constrain by available memory
+  uint64_t maxFramesInMemory = config.mMaxVideoBufferMemory / 
+                               std::max(frameMemory, (uint64_t)1);
+  recommendedQueueSize = std::min((uint32_t)maxFramesInMemory, recommendedQueueSize);
+  recommendedQueueSize = std::max(config.mMinVideoQueueFrames, recommendedQueueSize);
+
+  config.mDefaultVideoQueueFrames = recommendedQueueSize;
+
+  // Adjust decode speed threshold based on bitrate
+  if (aVideoBitrate > 0) {
+    // High bitrate content may need faster decode speed
+    if (aVideoBitrate > 10000000) {  // >10 Mbps
+      config.mDecodeSpeedRatio = 2.0;
+    } else if (aVideoBitrate > 5000000) {  // >5 Mbps
+      config.mDecodeSpeedRatio = 1.75;
+    }
+  }
+
+  // For network streams, use more conservative buffering
+  if (aVideoBitrate > 0) {
+    config.mLowAudioThreshold = 500000;     // 500ms
+    config.mAmpleAudioThreshold = 3000000;  // 3s
+  }
+
+  LOG("Computed config: queueSize=%u, maxMemory=%llu, decodeRatio=%.2f (res=%u)",
+      recommendedQueueSize,
+      config.mMaxVideoBufferMemory,
+      config.mDecodeSpeedRatio,
+      resolutionTier);
+
+  return config;
+}
+
+uint32_t AdaptiveBufferingPolicy::GetOptimalVideoQueueSize(
+    const gfx::IntSize& aVideoSize,
+    bool aHardwareAccelerated,
+    uint64_t aAvailableMemory)
+{
+  // Start with base size
+  uint32_t queueSize = aHardwareAccelerated ? 5 : 10;
+
+  uint64_t frameMemory = EstimateFrameMemory(aVideoSize);
+  uint64_t maxFramesInMemory = (100 * 1024 * 1024) / frameMemory;
+
+  // Constrain by available memory
+  queueSize = std::min((uint32_t)maxFramesInMemory, queueSize);
+
+  return std::max(3u, queueSize);
+}
+
+uint64_t AdaptiveBufferingPolicy::GetOptimalAudioThreshold(bool aIsNetworkStream)
+{
+  if (aIsNetworkStream) {
+    return 500000;  // 500ms for network (more conservative)
+  }
+  return 300000;    // 300ms for local files
+}
+
+bool AdaptiveBufferingPolicy::ShouldSkipFrame(
+    int64_t aAudioTimeUs,
+    int64_t aFrameDisplayTimeUs,
+    int64_t aVideoDecodeTimeUs,
+    uint32_t aThresholdMs)
+{
+  // Skip if frame is already in the past relative to audio
+  int64_t thresholdUs = aThresholdMs * 1000;
+
+  // Calculate how late this frame is
+  int64_t timeLatenessUs = aAudioTimeUs - aFrameDisplayTimeUs;
+
+  // Skip if frame is significantly behind audio timeline
+  if (timeLatenessUs > thresholdUs) {
+    return true;
+  }
+
+  // Also skip if decoding this frame would take us further behind
+  int64_t projectedLatenessUs = timeLatenessUs + aVideoDecodeTimeUs;
+  if (projectedLatenessUs > (thresholdUs * 2)) {
+    return true;
+  }
+
+  return false;
+}
+
+} // namespace mozilla

dom/media/AdaptiveBufferingPolicy.h

@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_ADAPTIVEBUFFERINGPOLICY_H_
+#define MOZILLA_ADAPTIVEBUFFERINGPOLICY_H_
+
+#include "mozilla/Atomics.h"
+#include "TimeUnits.h"
+#include "gfxTypes.h"
+
+namespace mozilla {
+
+/**
+ * Adaptive buffering policy manager for video playback.
+ * 
+ * Dynamically adjusts buffering thresholds based on:
+ * - Available system memory
+ * - Video resolution and bitrate
+ * - Decode speed vs real-time ratio
+ * - Network conditions (if applicable)
+ * 
+ * This reduces memory usage while maintaining smooth playback.
+ */
+class AdaptiveBufferingPolicy {
+public:
+  struct Config {
+    // Audio buffering thresholds (microseconds)
+    uint64_t mLowAudioThreshold = 300000;     // 300ms - trigger faster decoding
+    uint64_t mAmpleAudioThreshold = 2000000;  // 2s - target comfortable level
+
+    // Video frame queue sizing
+    uint32_t mMinVideoQueueFrames = 3;        // Absolute minimum
+    uint32_t mDefaultVideoQueueFrames = 10;   // Default for standard content
+    uint32_t mMaxVideoQueueFrames = 20;       // Absolute maximum
+
+    // Hardware acceleration thresholds
+    uint32_t mHWAccelVideoQueueFrames = 5;    // Target for HW-accelerated playback
+
+    // Memory limits (bytes)
+    uint64_t mMaxVideoBufferMemory = 100 * 1024 * 1024;    // 100MB default
+    uint64_t mTargetVideoBufferMemory = 50 * 1024 * 1024;  // 50MB target
+
+    // Decode speed thresholds
+    double mDecodeSpeedRatio = 1.5;           // Must decode 1.5x realtime speed
+
+    // Frame skip configuration
+    bool mEnableAdaptiveFrameSkip = true;
+    bool mSkipNonKeyframes = true;
+    uint32_t mFrameSkipThresholdMs = 60;      // Skip if >60ms behind audio
+  };
+
+  /**
+   * Adjust buffering policy based on system state and video properties.
+   * Returns optimized configuration for the given video specifications.
+   */
+  static Config ComputeOptimalConfig(
+    const gfx::IntSize& aVideoSize,
+    uint32_t aFrameRate,
+    uint64_t aVideoBitrate,
+    bool aHardwareAccelerated,
+    uint64_t aAvailableMemory);
+
+  /**
+   * Get the recommended video queue size based on context.
+   */
+  static uint32_t GetOptimalVideoQueueSize(
+    const gfx::IntSize& aVideoSize,
+    bool aHardwareAccelerated,
+    uint64_t aAvailableMemory);
+
+  /**
+   * Get audio buffering threshold based on network conditions.
+   */
+  static uint64_t GetOptimalAudioThreshold(bool aIsNetworkStream);
+
+  /**
+   * For HD+ content with limited memory, suggest lower priority video frames
+   * that can be safely skipped without visible quality loss.
+   */
+  static bool ShouldSkipFrame(
+    int64_t aAudioTimeUs,
+    int64_t aFrameDisplayTimeUs,
+    int64_t aVideoDecodeTimeUs,
+    uint32_t aThresholdMs);
+};
+
+} // namespace mozilla
+
+#endif // MOZILLA_ADAPTIVEBUFFERINGPOLICY_H_

dom/media/FramePool.cpp

@@ -0,0 +1,95 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FramePool.h"
+#include "mozilla/Logging.h"
+#include "ImageContainer.h"
+
+namespace mozilla {
+
+extern LazyLogModule gMediaDecoderLog;
+
+#undef LOG
+#define LOG(arg, ...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Debug, ("FramePool::%s: " arg, __func__, ##__VA_ARGS__))
+#define LOGV(arg, ...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Verbose, ("FramePool::%s: " arg, __func__, ##__VA_ARGS__))
+
+FramePool::FramePool(size_t aMaxFrames)
+  : mMutex("FramePool::mMutex")
+  , mMaxPoolSize(aMaxFrames > 0 ? aMaxFrames : 16)
+{
+  LOG("FramePool created with max size: %zu", mMaxPoolSize);
+}
+
+FramePool::~FramePool()
+{
+  Clear();
+}
+
+RefPtr<layers::Image> FramePool::AcquireFrame(const gfx::IntSize& aSize)
+{
+  MutexAutoLock lock(mMutex);
+
+  // Try to reuse a frame from pool
+  if (!mFramePool.IsEmpty()) {
+    RefPtr<layers::Image> frame = mFramePool[mFramePool.Length() - 1];
+    mFramePool.RemoveElementAt(mFramePool.Length() - 1);
+    mFramesReused++;
+    LOGV("Reused frame from pool, pool size now: %zu", mFramePool.Length());
+    return frame;
+  }
+
+  // No pooled frame available; we cannot reliably construct a specific
+  // Image subclass here (requires platform-specific recycle bin). Return
+  // nullptr and let the caller allocate an appropriate Image if needed.
+  mTotalAllocated++;
+  LOG("No pooled frame available, returning nullptr (alloc attempts: %zu)", mTotalAllocated);
+  return nullptr;
+}
+
+void FramePool::ReleaseFrame(layers::Image* aFrame)
+{
+  if (!aFrame) {
+    return;
+  }
+
+  MutexAutoLock lock(mMutex);
+
+  // Only pool frame if we haven't reached max capacity
+  if (mFramePool.Length() < mMaxPoolSize) {
+    mFramePool.AppendElement(RefPtr<layers::Image>(aFrame));
+
+    // Track peak pool size
+    if (mFramePool.Length() > mPeakPoolSize) {
+      mPeakPoolSize = mFramePool.Length();
+    }
+
+    LOGV("Released frame to pool, pool size: %zu/%zu", 
+         mFramePool.Length(), mMaxPoolSize);
+  } else {
+    // Frame exceeds pool capacity, let it be released
+    LOGV("Frame pool at capacity, releasing frame");
+  }
+}
+
+void FramePool::Clear()
+{
+  MutexAutoLock lock(mMutex);
+  mFramePool.Clear();
+  LOG("Frame pool cleared");
+}
+
+size_t FramePool::PoolSize() const
+{
+  MutexAutoLock lock(mMutex);
+  return mFramePool.Length();
+}
+
+size_t FramePool::TotalFrames() const
+{
+  MutexAutoLock lock(mMutex);
+  return mTotalAllocated;
+}
+
+} // namespace mozilla