VideoView: detect rendering / freeze (#90)

hiroshihorie · web-flow · commit 853c12f090e7 · 2022-06-14T04:54:31.000+09:00
* impl

* expose properties

* clean up

* revert to private

* update isRendering for SwiftUIVideoView

* update delegates

* adjust access

* mutate can be async for render

* swift ui fixes
diff --git a/Sources/LiveKit/Protocols/TrackDelegate.swift b/Sources/LiveKit/Protocols/TrackDelegate.swift
@@ -17,15 +17,9 @@
 import Foundation
 import WebRTC
 
-// TODO: Make this internal
-// Currently used for internal purposes
 public protocol TrackDelegate: AnyObject {
     /// Dimensions of the video track has updated
     func track(_ track: VideoTrack, didUpdate dimensions: Dimensions?)
-    /// Dimensions of the VideoView has updated
-    func track(_ track: VideoTrack, videoView: VideoView, didUpdate size: CGSize)
-    /// VideoView updated the render state
-    func track(_ track: VideoTrack, videoView: VideoView, didUpdate renderState: VideoView.RenderState)
     /// A ``VideoView`` was attached to the ``VideoTrack``
     func track(_ track: VideoTrack, didAttach videoView: VideoView)
     /// A ``VideoView`` was detached from the ``VideoTrack``
@@ -38,12 +32,10 @@ public protocol TrackDelegate: AnyObject {
 
 // MARK: - Optional
 
-extension TrackDelegate {
-    public func track(_ track: VideoTrack, didUpdate dimensions: Dimensions?) {}
-    public func track(_ track: VideoTrack, videoView: VideoView, didUpdate size: CGSize) {}
-    public func track(_ track: VideoTrack, videoView: VideoView, didUpdate renderState: VideoView.RenderState) {}
-    public func track(_ track: VideoTrack, didAttach videoView: VideoView) {}
-    public func track(_ track: VideoTrack, didDetach videoView: VideoView) {}
-    public func track(_ track: Track, didUpdate muted: Bool, shouldSendSignal: Bool) {}
-    public func track(_ track: Track, didUpdate stats: TrackStats) {}
+public extension TrackDelegate {
+    func track(_ track: VideoTrack, didUpdate dimensions: Dimensions?) {}
+    func track(_ track: VideoTrack, didAttach videoView: VideoView) {}
+    func track(_ track: VideoTrack, didDetach videoView: VideoView) {}
+    func track(_ track: Track, didUpdate muted: Bool, shouldSendSignal: Bool) {}
+    func track(_ track: Track, didUpdate stats: TrackStats) {}
 }
diff --git a/Sources/LiveKit/Protocols/VideoViewDelegate.swift b/Sources/LiveKit/Protocols/VideoViewDelegate.swift
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+import WebRTC
+
+public protocol VideoViewDelegate: AnyObject {
+    /// Dimensions of the VideoView itself has updated
+    func videoView(_ videoView: VideoView, didUpdate size: CGSize)
+    /// VideoView updated the isRendering property
+    func videoView(_ videoView: VideoView, didUpdate isRendering: Bool)
+}
+
+// MARK: - Optional
+
+public extension VideoViewDelegate {
+    func videoView(_ videoView: VideoView, didUpdate size: CGSize) {}
+    func videoView(_ videoView: VideoView, didUpdate isRendering: Bool) {}
+}
diff --git a/Sources/LiveKit/Publications/TrackPublication.swift b/Sources/LiveKit/Publications/TrackPublication.swift
@@ -133,18 +133,6 @@ public class TrackPublication: TrackDelegate, Loggable {
 
     // MARK: - TrackDelegate
 
-    public func track(_ track: VideoTrack, videoView: VideoView, didUpdate size: CGSize) {
-        //
-    }
-
-    public func track(_ track: VideoTrack, didAttach videoView: VideoView) {
-        //
-    }
-
-    public func track(_ track: VideoTrack, didDetach videoView: VideoView) {
-        //
-    }
-
     public func track(_ track: Track, didUpdate muted: Bool, shouldSendSignal: Bool) {
 
         log("muted: \(muted) shouldSendSignal: \(shouldSendSignal)")
diff --git a/Sources/LiveKit/Support/MulticastDelegate.swift b/Sources/LiveKit/Support/MulticastDelegate.swift
@@ -104,7 +104,7 @@ public protocol MulticastDelegateCapable {
     var delegates: MulticastDelegate<DelegateType> { get }
     func add(delegate: DelegateType)
     func remove(delegate: DelegateType)
-    func notify(_ fnc: @escaping (DelegateType) throws -> Void) rethrows
+    func notify(_ fnc: @escaping (DelegateType) -> Void)
 }
 
 extension MulticastDelegateCapable {
diff --git a/Sources/LiveKit/SwiftUI/SwiftUIVideoView.swift b/Sources/LiveKit/SwiftUI/SwiftUIVideoView.swift
@@ -16,20 +16,18 @@
 
 import SwiftUI
 
-/// This class receives delegate events since a struct can't be used for a delegate
-class SwiftUIVideoViewDelegateReceiver: TrackDelegate, Loggable {
+/// This class receives ``TrackDelegate`` events since a struct can't be used for a delegate
+internal class TrackDelegateReceiver: TrackDelegate, Loggable {
 
     @Binding var dimensions: Dimensions?
     @Binding var stats: TrackStats?
 
-    init(dimensions: Binding<Dimensions?> = .constant(nil),
-         stats: Binding<TrackStats?> = .constant(nil)) {
+    init(dimensions: Binding<Dimensions?>, stats: Binding<TrackStats?>) {
         self._dimensions = dimensions
         self._stats = stats
     }
 
-    func track(_ track: VideoTrack,
-               didUpdate dimensions: Dimensions?) {
+    func track(_ track: VideoTrack, didUpdate dimensions: Dimensions?) {
         DispatchQueue.main.async {
             self.dimensions = dimensions
         }
@@ -42,6 +40,22 @@ class SwiftUIVideoViewDelegateReceiver: TrackDelegate, Loggable {
     }
 }
 
+/// This class receives ``VideoViewDelegate`` events since a struct can't be used for a delegate
+internal class VideoViewDelegateReceiver: VideoViewDelegate, Loggable {
+
+    @Binding var isRendering: Bool
+
+    init(isRendering: Binding<Bool>) {
+        self._isRendering = isRendering
+    }
+
+    func videoView(_ videoView: VideoView, didUpdate isRendering: Bool) {
+        DispatchQueue.main.async {
+            self.isRendering = isRendering
+        }
+    }
+}
+
 /// A ``VideoView`` that can be used in SwiftUI.
 /// Supports both iOS and macOS.
 public struct SwiftUIVideoView: NativeViewRepresentable {
@@ -54,25 +68,32 @@ public struct SwiftUIVideoView: NativeViewRepresentable {
     let mirrorMode: VideoView.MirrorMode
     let debugMode: Bool
 
+    @Binding var isRendering: Bool
     @Binding var dimensions: Dimensions?
 
-    let delegateReceiver: SwiftUIVideoViewDelegateReceiver
+    let trackDelegateReceiver: TrackDelegateReceiver
+    let videoViewDelegateReceiver: VideoViewDelegateReceiver
 
     public init(_ track: VideoTrack,
                 layoutMode: VideoView.LayoutMode = .fill,
                 mirrorMode: VideoView.MirrorMode = .auto,
                 debugMode: Bool = false,
+                isRendering: Binding<Bool> = .constant(false),
                 dimensions: Binding<Dimensions?> = .constant(nil),
                 trackStats: Binding<TrackStats?> = .constant(nil)) {
 
         self.track = track
         self.layoutMode = layoutMode
         self.mirrorMode = mirrorMode
         self.debugMode = debugMode
+
+        self._isRendering = isRendering
         self._dimensions = dimensions
 
-        self.delegateReceiver = SwiftUIVideoViewDelegateReceiver(dimensions: dimensions,
-                                                                 stats: trackStats)
+        self.trackDelegateReceiver = TrackDelegateReceiver(dimensions: dimensions,
+                                                           stats: trackStats)
+
+        self.videoViewDelegateReceiver = VideoViewDelegateReceiver(isRendering: isRendering)
 
         // update binding value
         DispatchQueue.main.async {
@@ -81,11 +102,12 @@ public struct SwiftUIVideoView: NativeViewRepresentable {
         }
 
         // listen for TrackDelegate
-        track.add(delegate: delegateReceiver)
+        track.add(delegate: trackDelegateReceiver)
     }
 
     public func makeView(context: Context) -> VideoView {
         let view = VideoView()
+        view.add(delegate: videoViewDelegateReceiver)
         updateView(view, context: context)
         return view
     }
@@ -95,6 +117,11 @@ public struct SwiftUIVideoView: NativeViewRepresentable {
         videoView.layoutMode = layoutMode
         videoView.mirrorMode = mirrorMode
         videoView.debugMode = debugMode
+
+        // update
+        DispatchQueue.main.async {
+            self.isRendering = videoView.isRendering
+        }
     }
 
     public static func dismantleView(_ videoView: VideoView, coordinator: ()) {
diff --git a/Sources/LiveKit/Views/VideoView.swift b/Sources/LiveKit/Views/VideoView.swift
@@ -21,23 +21,14 @@ import MetalKit
 /// A ``NativeViewType`` that conforms to ``RTCVideoRenderer``.
 public typealias NativeRendererView = NativeViewType & RTCVideoRenderer
 
-public class VideoView: NativeView, Loggable {
+public class VideoView: NativeView, MulticastDelegateCapable, Loggable {
 
-    private static let mirrorTransform = CATransform3DMakeScale(-1.0, 1.0, 1.0)
+    public var delegates = MulticastDelegate<DelegateType>()
 
-    /// A set of bool values describing the state of rendering.
-    public struct RenderState: OptionSet {
-        public let rawValue: Int
-        public init(rawValue: Int) {
-            self.rawValue = rawValue
-        }
+    public typealias DelegateType = VideoViewDelegate
 
-        /// Received first frame and already rendered to the ``VideoView``.
-        /// This can be used to trigger smooth transition of the UI.
-        static let didRenderFirstFrame  = RenderState(rawValue: 1 << 0)
-        /// ``VideoView`` skipped rendering of a frame that could lead to crashes.
-        static let didSkipUnsafeFrame   = RenderState(rawValue: 1 << 1)
-    }
+    private static let mirrorTransform = CATransform3DMakeScale(-1.0, 1.0, 1.0)
+    private static let _freezeDetectThreshold = 2.0
 
     public enum LayoutMode: String, Codable, CaseIterable {
         case fit
@@ -69,7 +60,9 @@ public class VideoView: NativeView, Loggable {
             _state.mutate {
                 // reset states if track updated
                 if !Self.track($0.track, isEqualWith: newValue) {
-                    $0.renderState = []
+                    $0.renderDate = nil
+                    $0.didRenderFirstFrame = false
+                    $0.isRendering = false
                     $0.rendererSize = nil
                 }
                 $0.track = newValue
@@ -96,6 +89,9 @@ public class VideoView: NativeView, Loggable {
         set { _state.mutate { $0.debugMode = newValue } }
     }
 
+    public var isRendering: Bool { _state.isRendering }
+    public var didRenderFirstFrame: Bool { _state.didRenderFirstFrame }
+
     private var nativeRenderer: NativeRendererView?
 
     private var _debugTextView: TextView?
@@ -113,15 +109,21 @@ public class VideoView: NativeView, Loggable {
         var rendererSize: CGSize?
         var didLayout: Bool = false
         var layoutMode: LayoutMode = .fill
-
         var mirrorMode: MirrorMode = .auto
-        var renderState = RenderState()
 
         var debugMode: Bool = false
+
+        // render states
+        var renderDate: Date?
+        var didRenderFirstFrame: Bool = false
+        var isRendering: Bool = false
     }
 
     internal var _state: StateSync<State>
 
+    // used for stats timer
+    private lazy var _renderTimer = DispatchQueueTimer(timeInterval: 0.1)
+
     public override init(frame: CGRect = .zero) {
 
         // should always be on main thread
@@ -203,14 +205,23 @@ public class VideoView: NativeView, Loggable {
                 }
             }
 
-            // renderState updated
-            if state.renderState != oldState.renderState, let track = state.track {
-                track.notify { $0.track(track, videoView: self, didUpdate: state.renderState) }
+            // isRendering updated
+            if state.isRendering != oldState.isRendering {
+
+                self.log("isRendering \(oldState.isRendering) -> \(state.isRendering)")
+
+                if state.isRendering {
+                    self._renderTimer.restart()
+                } else {
+                    self._renderTimer.suspend()
+                }
+
+                self.notify { $0.videoView(self, didUpdate: state.isRendering) }
             }
 
             // viewSize updated
-            if state.viewSize != oldState.viewSize, let track = state.track {
-                track.notify { $0.track(track, videoView: self, didUpdate: state.viewSize) }
+            if state.viewSize != oldState.viewSize {
+                self.notify { $0.videoView(self, didUpdate: state.viewSize) }
             }
 
             // toggle MTKView's isPaused property
@@ -222,6 +233,7 @@ public class VideoView: NativeView, Loggable {
             if state.debugMode != oldState.debugMode ||
                 state.layoutMode != oldState.layoutMode ||
                 state.mirrorMode != oldState.mirrorMode ||
+                state.didRenderFirstFrame != oldState.didRenderFirstFrame ||
                 shouldRenderDidUpdate || trackDidUpdate {
 
                 // must be on main
@@ -230,6 +242,18 @@ public class VideoView: NativeView, Loggable {
                 }
             }
         }
+
+        _renderTimer.handler = { [weak self] in
+
+            guard let self = self else { return }
+
+            if self._state.isRendering, let renderDate = self._state.renderDate {
+                let diff = Date().timeIntervalSince(renderDate)
+                if diff >= Self._freezeDetectThreshold {
+                    self._state.mutate { $0.isRendering = false }
+                }
+            }
+        }
     }
 
     required init?(coder: NSCoder) {
@@ -261,11 +285,12 @@ public class VideoView: NativeView, Loggable {
         if _state.debugMode {
             let _trackSid = _state.track?.sid ?? "nil"
             let _dimensions = _state.track?.dimensions ?? .zero
-            let _didRenderFirstFrame = _state.renderState.contains(.didRenderFirstFrame) ? "true" : "false"
+            let _didRenderFirstFrame = _state.didRenderFirstFrame ? "true" : "false"
+            let _isRendering = _state.isRendering ? "true" : "false"
             let _viewCount = _state.track?.videoViews.count ?? 0
             let _didLayout = _state.didLayout
             let debugView = ensureDebugTextView()
-            debugView.text = "#\(hashValue)\n" + "\(_trackSid)\n" + "\(_dimensions.width)x\(_dimensions.height)\n" + "enabled: \(isEnabled)\n" + "firstFrame: \(_didRenderFirstFrame)\n" + "viewCount: \(_viewCount)\n" + "layout: \(_didLayout)"
+            debugView.text = "#\(hashValue)\n" + "\(_trackSid)\n" + "\(_dimensions.width)x\(_dimensions.height)\n" + "enabled: \(isEnabled)\n" + "firstFrame: \(_didRenderFirstFrame)\n" + "isRendering: \(_isRendering)\n" + "viewCount: \(_viewCount)\n" + "layout: \(_didLayout)"
             debugView.frame = bounds
             #if os(iOS)
             debugView.layer.borderColor = (_state.shouldRender ? UIColor.green : UIColor.red).withAlphaComponent(0.5).cgColor
@@ -456,10 +481,10 @@ extension VideoView: RTCVideoRenderer {
         // cache last rendered frame
         track?.set(videoFrame: frame)
 
-        if !_state.renderState.contains(.didRenderFirstFrame) {
-            _state.mutate { $0.renderState.insert(.didRenderFirstFrame) }
-            self.log("did render first frame, track: \(String(describing: track))")
-            _needsLayout = true
+        _state.mutateAsync {
+            $0.didRenderFirstFrame = true
+            $0.isRendering = true
+            $0.renderDate = Date()
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -104,7 +104,7 @@ public protocol MulticastDelegateCapable {`
`104`	`104`	`var delegates: MulticastDelegate<DelegateType> { get }`
`105`	`105`	`func add(delegate: DelegateType)`
`106`	`106`	`func remove(delegate: DelegateType)`
`107`		`- func notify(_ fnc: @escaping (DelegateType) throws -> Void) rethrows`
	`107`	`+ func notify(_ fnc: @escaping (DelegateType) -> Void)`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`extension MulticastDelegateCapable {`