diff --git a/flutter_inappwebview_windows/windows/custom_platform_view/graphics_context.h b/flutter_inappwebview_windows/windows/custom_platform_view/graphics_context.h index 883bffe71..3cc0a7b67 100644 --- a/flutter_inappwebview_windows/windows/custom_platform_view/graphics_context.h +++ b/flutter_inappwebview_windows/windows/custom_platform_view/graphics_context.h @@ -5,6 +5,8 @@ #include #include +#include + #include "util/rohelper.h" namespace flutter_inappwebview_plugin @@ -24,6 +26,10 @@ namespace flutter_inappwebview_plugin { return device_context_.get(); } + // The immediate context is shared by every WebView of the plugin and is + // not thread-safe. Hold this while issuing a sequence of calls on it from + // a thread other than the one that created it (capture threads). + std::mutex& device_context_mutex() const { return device_context_mutex_; } winrt::com_ptr CreateCompositor(); @@ -50,5 +56,6 @@ namespace flutter_inappwebview_plugin device_winrt_; winrt::com_ptr device_{ nullptr }; winrt::com_ptr device_context_{ nullptr }; + mutable std::mutex device_context_mutex_; }; } \ No newline at end of file diff --git a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.cc b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.cc index 658df19fc..8ba2bda96 100644 --- a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.cc +++ b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.cc @@ -128,7 +128,7 @@ namespace flutter_inappwebview_plugin if (SUCCEEDED(frame->get_Surface(frame_surface.put()))) { last_frame_ = TryGetDXGIInterfaceFromObject(frame_surface); - has_frame = !ShouldDropFrame(); + has_frame = last_frame_ && !ShouldDropFrame() && AcceptFrame(last_frame_); } } @@ -148,6 +148,11 @@ namespace flutter_inappwebview_plugin } } + bool TextureBridge::AcceptFrame(const winrt::com_ptr&) + { + return true; + } + bool TextureBridge::ShouldDropFrame() { if (!frame_duration_.has_value()) { @@ -186,4 +191,4 @@ namespace flutter_inappwebview_plugin last_frame_timestamp_.reset(); } } -} \ No newline at end of file +} diff --git a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.h b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.h index 122c69388..126e20d79 100644 --- a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.h +++ b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge.h @@ -69,6 +69,13 @@ namespace flutter_inappwebview_plugin EventRegistrationToken on_frame_arrived_token_ = {}; virtual void StopInternal(); + // Called under |mutex_| for every captured frame that survived the fps + // limit. Returning false skips the frame: Flutter is not notified and the + // frame is not drawn. The base bridge accepts everything; the GPU bridge + // rejects frames whose pixels equal the frame Flutter already shows, since + // Windows.Graphics.Capture delivers a frame on every compositor tick even + // when the captured visual has not changed. + virtual bool AcceptFrame(const winrt::com_ptr& frame); void OnFrameArrived(); bool ShouldDropFrame(); diff --git a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.cc b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.cc index c3bdee266..8e4c96524 100644 --- a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.cc +++ b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.cc @@ -1,108 +1,398 @@ #include "texture_bridge_gpu.h" +#include + +#include +#include #include +#include +#include +#include #include "util/direct3d11.interop.h" namespace flutter_inappwebview_plugin { + namespace + { + // Writes 1 into a raw buffer if any pixel of |a| differs from |b|. Both + // textures are B8G8R8A8_UNORM of the same size; unorm loads of equal bytes + // yield equal floats, so the comparison is exact. + constexpr char kCompareShader[] = R"hlsl( +Texture2D a : register(t0); +Texture2D b : register(t1); +RWByteAddressBuffer result : register(u0); + +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) { + uint width, height; + a.GetDimensions(width, height); + if (id.x >= width || id.y >= height) { + return; + } + if (any(a.Load(int3(id.xy, 0)) != b.Load(int3(id.xy, 0)))) { + result.Store(0, 1u); + } +} +)hlsl"; + + typedef HRESULT(WINAPI* D3DCompileFn)(LPCVOID, SIZE_T, LPCSTR, + const D3D_SHADER_MACRO*, ID3DInclude*, LPCSTR, LPCSTR, UINT, UINT, + ID3DBlob**, ID3DBlob**); + + constexpr UINT kCompareResultBytes = 16; + constexpr size_t kSurfacePoolSize = 2; + } + + struct TextureBridgeGpu::SurfacePool { + struct Surface { + winrt::com_ptr texture; + HANDLE shared_handle = nullptr; + bool is_leased = false; + }; + + std::mutex mutex; + Size size = { 0, 0 }; + std::vector surfaces; + size_t published_surface = 0; + bool has_published_surface = false; + }; + + struct TextureBridgeGpu::FrameLease { + std::shared_ptr pool; + size_t surface_index = 0; + FlutterDesktopGpuSurfaceDescriptor descriptor = {}; + }; + TextureBridgeGpu::TextureBridgeGpu( GraphicsContext* graphics_context, ABI::Windows::UI::Composition::IVisual* visual) : TextureBridge(graphics_context, visual) { - surface_descriptor_.struct_size = sizeof(FlutterDesktopGpuSurfaceDescriptor); - surface_descriptor_.format = - kFlutterDesktopPixelFormatNone; // no format required for DXGI surfaces + if (!InitComparer()) { + std::cerr << "WebView frame compare unavailable; every captured frame " + "is forwarded to Flutter." << std::endl; + } } - void TextureBridgeGpu::ProcessFrame( - winrt::com_ptr src_texture) + bool TextureBridgeGpu::InitComparer() { - D3D11_TEXTURE2D_DESC desc; - src_texture->GetDesc(&desc); + auto device = graphics_context_->d3d_device(); + if (!device || device->GetFeatureLevel() < D3D_FEATURE_LEVEL_11_0) { + return false; + } + + // d3dcompiler_47.dll ships with Windows 8.1 and later. Loaded on demand so + // the plugin has no link-time dependency on it. + const HMODULE compiler = LoadLibraryW(L"d3dcompiler_47.dll"); + if (!compiler) { + return false; + } + const auto compile = reinterpret_cast( + GetProcAddress(compiler, "D3DCompile")); + winrt::com_ptr code; + winrt::com_ptr errors; + const HRESULT compiled = compile + ? compile(kCompareShader, sizeof(kCompareShader) - 1, "frame_compare", + nullptr, nullptr, "main", "cs_5_0", 0, 0, code.put(), errors.put()) + : E_FAIL; + FreeLibrary(compiler); + if (FAILED(compiled) || !code) { + if (errors) { + std::cerr << "frame compare shader: "; + std::cerr.write(static_cast(errors->GetBufferPointer()), + static_cast(errors->GetBufferSize())); + std::cerr << std::endl; + } + return false; + } + if (FAILED(device->CreateComputeShader(code->GetBufferPointer(), + code->GetBufferSize(), nullptr, compare_shader_.put()))) { + return false; + } + + D3D11_BUFFER_DESC result_desc = {}; + result_desc.ByteWidth = kCompareResultBytes; + result_desc.Usage = D3D11_USAGE_DEFAULT; + result_desc.BindFlags = D3D11_BIND_UNORDERED_ACCESS; + result_desc.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS; + if (FAILED(device->CreateBuffer(&result_desc, nullptr, + compare_result_.put()))) { + compare_shader_ = nullptr; + return false; + } + D3D11_UNORDERED_ACCESS_VIEW_DESC uav_desc = {}; + uav_desc.Format = DXGI_FORMAT_R32_TYPELESS; + uav_desc.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + uav_desc.Buffer.FirstElement = 0; + uav_desc.Buffer.NumElements = kCompareResultBytes / 4; + uav_desc.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + if (FAILED(device->CreateUnorderedAccessView(compare_result_.get(), + &uav_desc, compare_result_uav_.put()))) { + compare_shader_ = nullptr; + return false; + } + D3D11_BUFFER_DESC staging_desc = {}; + staging_desc.ByteWidth = kCompareResultBytes; + staging_desc.Usage = D3D11_USAGE_STAGING; + staging_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + if (FAILED(device->CreateBuffer(&staging_desc, nullptr, + compare_staging_.put()))) { + compare_shader_ = nullptr; + return false; + } + return true; + } - const auto width = desc.Width; - const auto height = desc.Height; + bool TextureBridgeGpu::FramesDiffer(ID3D11Texture2D* a, ID3D11Texture2D* b, + uint32_t width, uint32_t height) + { + if (!compare_shader_) { + return true; + } + auto device = graphics_context_->d3d_device(); + auto context = graphics_context_->d3d_device_context(); - EnsureSurface(width, height); + winrt::com_ptr view_a; + winrt::com_ptr view_b; + if (FAILED(device->CreateShaderResourceView(a, nullptr, view_a.put())) || + FAILED(device->CreateShaderResourceView(b, nullptr, view_b.put()))) { + return true; + } - auto device_context = graphics_context_->d3d_device_context(); + const UINT zeros[4] = { 0, 0, 0, 0 }; + context->ClearUnorderedAccessViewUint(compare_result_uav_.get(), zeros); + ID3D11ShaderResourceView* views[2] = { view_a.get(), view_b.get() }; + ID3D11UnorderedAccessView* uavs[1] = { compare_result_uav_.get() }; + context->CSSetShader(compare_shader_.get(), nullptr, 0); + context->CSSetShaderResources(0, 2, views); + context->CSSetUnorderedAccessViews(0, 1, uavs, nullptr); + context->Dispatch((width + 15) / 16, (height + 15) / 16, 1); + ID3D11ShaderResourceView* no_views[2] = { nullptr, nullptr }; + ID3D11UnorderedAccessView* no_uavs[1] = { nullptr }; + context->CSSetShaderResources(0, 2, no_views); + context->CSSetUnorderedAccessViews(0, 1, no_uavs, nullptr); + context->CSSetShader(nullptr, nullptr, 0); - device_context->CopyResource(surface_.get(), src_texture.get()); - device_context->Flush(); + // Never block the capture dispatcher waiting for the GPU. When the flag + // is not ready, fail open and forward the frame, which is the previous + // behaviour and preserves correctness under GPU pressure. + context->CopyResource(compare_staging_.get(), compare_result_.get()); + D3D11_MAPPED_SUBRESOURCE mapped; + if (FAILED(context->Map(compare_staging_.get(), 0, D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT, &mapped))) { + return true; + } + const bool differ = *static_cast(mapped.pData) != 0; + context->Unmap(compare_staging_.get(), 0); + return differ; } - void TextureBridgeGpu::EnsureSurface(uint32_t width, uint32_t height) + bool TextureBridgeGpu::AcceptFrame( + const winrt::com_ptr& frame) { - if (!surface_ || surface_size_.width != width || - surface_size_.height != height) { - D3D11_TEXTURE2D_DESC dstDesc = {}; - dstDesc.ArraySize = 1; - dstDesc.MipLevels = 1; - dstDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; - dstDesc.CPUAccessFlags = 0; - dstDesc.Format = static_cast(kPixelFormat); - dstDesc.Width = width; - dstDesc.Height = height; - dstDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; - dstDesc.SampleDesc.Count = 1; - dstDesc.SampleDesc.Quality = 0; - dstDesc.Usage = D3D11_USAGE_DEFAULT; - - surface_ = nullptr; - if (!SUCCEEDED(graphics_context_->d3d_device()->CreateTexture2D( - &dstDesc, nullptr, surface_.put()))) { - std::cerr << "Creating intermediate texture failed" << std::endl; - return; + D3D11_TEXTURE2D_DESC desc; + frame->GetDesc(&desc); + const bool reference_created = !reference_ || + reference_size_.width != desc.Width || + reference_size_.height != desc.Height; + if (!EnsureSurfacePool(desc.Width, desc.Height) || + (compare_shader_ && !EnsureReference(desc.Width, desc.Height))) { + return false; + } + // Several WebViews share one immediate context, and with the free-threaded + // frame pool their frames arrive on different threads. + const std::lock_guard context_lock( + graphics_context_->device_context_mutex()); + auto context = graphics_context_->d3d_device_context(); + + std::shared_ptr pool = surface_pool_; + size_t writable_surface; + { + const std::lock_guard pool_lock(pool->mutex); + writable_surface = pool->surfaces.size(); + for (size_t i = 0; i < pool->surfaces.size(); ++i) { + if (!pool->surfaces[i].is_leased) { + writable_surface = i; + break; + } } + } + // Flutter is still using every shared texture. Keep the reference frame + // unchanged so the next capture can publish the newest frame once a + // buffer is released. + if (writable_surface == pool->surfaces.size()) { + return false; + } - HANDLE shared_handle; - surface_.try_as(dxgi_surface_); - assert(dxgi_surface_); - dxgi_surface_->GetSharedHandle(&shared_handle); + bool changed = true; + if (compare_shader_ && EnsureIncoming(desc.Width, desc.Height)) { + context->CopyResource(incoming_.get(), frame.get()); + changed = !reference_created && FramesDiffer(incoming_.get(), + reference_.get(), desc.Width, desc.Height); + if (changed) { + context->CopyResource(reference_.get(), incoming_.get()); + context->CopyResource(pool->surfaces[writable_surface].texture.get(), + incoming_.get()); + } + } + else { + // First frame, or no compare available. + if (compare_shader_) { + context->CopyResource(reference_.get(), frame.get()); + } + context->CopyResource(pool->surfaces[writable_surface].texture.get(), + frame.get()); + } + if (changed) { + context->Flush(); + const std::lock_guard pool_lock(pool->mutex); + pool->published_surface = writable_surface; + pool->has_published_surface = true; + } + return changed; + } - surface_descriptor_.handle = shared_handle; - surface_descriptor_.width = surface_descriptor_.visible_width = width; - surface_descriptor_.height = surface_descriptor_.visible_height = height; - surface_descriptor_.release_context = surface_.get(); - surface_descriptor_.release_callback = [](void* release_context) - { - auto texture = reinterpret_cast(release_context); - texture->Release(); - }; + bool TextureBridgeGpu::EnsureSurfacePool(uint32_t width, uint32_t height) + { + if (surface_pool_ && surface_pool_->size.width == width && + surface_pool_->size.height == height) { + return true; + } + D3D11_TEXTURE2D_DESC dstDesc = {}; + dstDesc.ArraySize = 1; + dstDesc.MipLevels = 1; + dstDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + dstDesc.CPUAccessFlags = 0; + dstDesc.Format = static_cast(kPixelFormat); + dstDesc.Width = width; + dstDesc.Height = height; + dstDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + dstDesc.SampleDesc.Count = 1; + dstDesc.SampleDesc.Quality = 0; + dstDesc.Usage = D3D11_USAGE_DEFAULT; - surface_size_ = { width, height }; + auto pool = std::make_shared(); + pool->size = { width, height }; + pool->surfaces.resize(kSurfacePoolSize); + for (auto& surface : pool->surfaces) { + if (FAILED(graphics_context_->d3d_device()->CreateTexture2D( + &dstDesc, nullptr, surface.texture.put()))) { + std::cerr << "Creating intermediate texture failed" << std::endl; + return false; + } + winrt::com_ptr dxgi_surface; + surface.texture.try_as(dxgi_surface); + assert(dxgi_surface); + if (FAILED(dxgi_surface->GetSharedHandle(&surface.shared_handle)) || + !surface.shared_handle) { + std::cerr << "Creating shared texture handle failed" << std::endl; + return false; + } } + surface_pool_ = std::move(pool); + return true; + } + + bool TextureBridgeGpu::EnsureReference(uint32_t width, uint32_t height) + { + if (reference_ && reference_size_.width == width && + reference_size_.height == height) { + return true; + } + D3D11_TEXTURE2D_DESC desc = {}; + desc.ArraySize = 1; + desc.MipLevels = 1; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.Format = static_cast(kPixelFormat); + desc.Width = width; + desc.Height = height; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + reference_ = nullptr; + reference_size_ = { 0, 0 }; + if (FAILED(graphics_context_->d3d_device()->CreateTexture2D( + &desc, nullptr, reference_.put()))) { + std::cerr << "Creating frame reference texture failed" << std::endl; + return false; + } + reference_size_ = { width, height }; + return true; + } + + bool TextureBridgeGpu::EnsureIncoming(uint32_t width, uint32_t height) + { + if (incoming_ && incoming_size_.width == width && + incoming_size_.height == height) { + return true; + } + D3D11_TEXTURE2D_DESC desc = {}; + desc.ArraySize = 1; + desc.MipLevels = 1; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.Format = static_cast(kPixelFormat); + desc.Width = width; + desc.Height = height; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + incoming_ = nullptr; + incoming_size_ = { 0, 0 }; + if (FAILED(graphics_context_->d3d_device()->CreateTexture2D( + &desc, nullptr, incoming_.put()))) { + return false; + } + incoming_size_ = { width, height }; + return true; } const FlutterDesktopGpuSurfaceDescriptor* - TextureBridgeGpu::GetSurfaceDescriptor(size_t width, size_t height) + TextureBridgeGpu::GetSurfaceDescriptor(size_t, size_t) { const std::lock_guard lock(mutex_); - - if (!is_running_) { + if (!is_running_ || !surface_pool_) { return nullptr; } - - if (last_frame_) { - ProcessFrame(last_frame_); + auto pool = surface_pool_; + const std::lock_guard pool_lock(pool->mutex); + if (!pool->has_published_surface || + pool->surfaces[pool->published_surface].is_leased) { + return nullptr; } - if (surface_) { - // Gets released in the SurfaceDescriptor's release callback. - surface_->AddRef(); - } + auto* lease = new FrameLease(); + lease->pool = std::move(pool); + lease->surface_index = lease->pool->published_surface; + const auto& surface = lease->pool->surfaces[lease->surface_index]; + lease->descriptor.struct_size = sizeof(FlutterDesktopGpuSurfaceDescriptor); + lease->descriptor.handle = surface.shared_handle; + lease->descriptor.width = lease->descriptor.visible_width = + lease->pool->size.width; + lease->descriptor.height = lease->descriptor.visible_height = + lease->pool->size.height; + lease->descriptor.format = kFlutterDesktopPixelFormatNone; + lease->descriptor.release_context = lease; + lease->descriptor.release_callback = ReleaseSurface; + lease->pool->surfaces[lease->surface_index].is_leased = true; + return &lease->descriptor; + } - return &surface_descriptor_; + void TextureBridgeGpu::ReleaseSurface(void* release_context) + { + std::unique_ptr lease( + static_cast(release_context)); + const std::lock_guard pool_lock(lease->pool->mutex); + lease->pool->surfaces[lease->surface_index].is_leased = false; } void TextureBridgeGpu::StopInternal() { TextureBridge::StopInternal(); - - // For some reason, the destination surface needs to be recreated upon - // resuming. Force |EnsureSurface| to create a new one by resetting it here. - surface_ = nullptr; + // Outstanding Flutter leases retain the old pool until their release + // callbacks run; a resumed capture creates a new pool. + surface_pool_.reset(); + reference_ = nullptr; + reference_size_ = { 0, 0 }; + incoming_ = nullptr; + incoming_size_ = { 0, 0 }; } -} \ No newline at end of file +} diff --git a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.h b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.h index 8b29a3216..029c004a7 100644 --- a/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.h +++ b/flutter_inappwebview_windows/windows/custom_platform_view/texture_bridge_gpu.h @@ -1,29 +1,70 @@ #pragma once +#include #include +#include + #include "texture_bridge.h" namespace flutter_inappwebview_plugin { + // Feeds captured WebView frames to Flutter as a DXGI shared texture. + // + // Windows.Graphics.Capture produces a frame on every compositor tick for a + // captured visual, whether or not its content changed. Forwarding each one + // made Flutter re-rasterize at the display refresh rate for as long as a + // WebView was on screen. This bridge therefore compares every incoming frame + // with the frame Flutter already has (a small compute shader, no CPU + // readback of pixels) and only copies + notifies when at least one pixel + // differs. Flutter then renders only when the page actually repainted. class TextureBridgeGpu : public TextureBridge { public: TextureBridgeGpu(GraphicsContext* graphics_context, ABI::Windows::UI::Composition::IVisual* visual); + // Called by Flutter on its raster thread. The returned buffer is leased + // until Flutter invokes its release callback, so capture never writes a + // texture that Flutter may still be sampling. const FlutterDesktopGpuSurfaceDescriptor* GetSurfaceDescriptor(size_t width, size_t height); protected: void StopInternal() override; + bool AcceptFrame(const winrt::com_ptr& frame) override; private: - FlutterDesktopGpuSurfaceDescriptor surface_descriptor_ = {}; - Size surface_size_ = { 0, 0 }; - winrt::com_ptr surface_{ nullptr }; - winrt::com_ptr dxgi_surface_; + struct SurfacePool; + struct FrameLease; + + // Flutter samples a buffer from this pool. A buffer is selected only when + // it is not leased to Flutter; leases retain the pool through shutdown. + std::shared_ptr surface_pool_; + // Private copy of the last accepted frame. It is deliberately separate + // from |surface_pool_| so it remains safe to compare while Flutter owns + // the last published shared texture. + winrt::com_ptr reference_{ nullptr }; + Size reference_size_ = { 0, 0 }; + // Our own copy of the incoming frame: the capture pool's textures may not + // be bindable as shader resources, and copying releases the pool buffer + // early. + winrt::com_ptr incoming_{ nullptr }; + Size incoming_size_ = { 0, 0 }; + // Compare pass. Null when the compute shader could not be built; every + // frame is then treated as changed (the previous behaviour). + winrt::com_ptr compare_shader_; + winrt::com_ptr compare_result_; + winrt::com_ptr compare_result_uav_; + winrt::com_ptr compare_staging_; - void ProcessFrame(winrt::com_ptr src_texture); - void EnsureSurface(uint32_t width, uint32_t height); + bool EnsureSurfacePool(uint32_t width, uint32_t height); + bool EnsureReference(uint32_t width, uint32_t height); + bool EnsureIncoming(uint32_t width, uint32_t height); + bool InitComparer(); + static void ReleaseSurface(void* release_context); + // GPU compare of two same-sized B8G8R8A8 textures. Returns true when any + // pixel differs, and also when the compare itself could not run. + bool FramesDiffer(ID3D11Texture2D* a, ID3D11Texture2D* b, uint32_t width, + uint32_t height); }; }