diff --git a/dogfooding/assets/bg1.jpg b/dogfooding/assets/bg1.jpg new file mode 100644 index 00000000..99108528 Binary files /dev/null and b/dogfooding/assets/bg1.jpg differ diff --git a/dogfooding/assets/bg2.jpg b/dogfooding/assets/bg2.jpg new file mode 100644 index 00000000..6e497df6 Binary files /dev/null and b/dogfooding/assets/bg2.jpg differ diff --git a/dogfooding/assets/bg3.jpg b/dogfooding/assets/bg3.jpg new file mode 100644 index 00000000..fb11e9a8 Binary files /dev/null and b/dogfooding/assets/bg3.jpg differ diff --git a/dogfooding/lib/widgets/settings_menu.dart b/dogfooding/lib/widgets/settings_menu.dart index 0b366c10..7d3c46ce 100644 --- a/dogfooding/lib/widgets/settings_menu.dart +++ b/dogfooding/lib/widgets/settings_menu.dart @@ -53,6 +53,7 @@ class SettingsMenu extends StatefulWidget { class _SettingsMenuState extends State { final _deviceNotifier = RtcMediaDeviceNotifier.instance; StreamSubscription>? _deviceChangeSubscription; + late StreamVideoEffectsManager _videoEffectsManager; var _audioOutputs = []; var _audioInputs = []; @@ -60,12 +61,18 @@ class _SettingsMenuState extends State { bool showAudioOutputs = false; bool showAudioInputs = false; bool showIncomingQuality = false; + bool showBackgroundEffects = false; + bool get showMainSettings => - !showAudioOutputs && !showAudioInputs && !showIncomingQuality; + !showAudioOutputs && + !showAudioInputs && + !showIncomingQuality && + !showBackgroundEffects; @override void initState() { super.initState(); + _videoEffectsManager = StreamVideoEffectsManager(widget.call); _deviceChangeSubscription = _deviceNotifier.onDeviceChange.listen( (devices) { _audioOutputs = devices @@ -105,6 +112,7 @@ class _SettingsMenuState extends State { if (showAudioOutputs) ..._buildAudioOutputsMenu(), if (showAudioInputs) ..._buildAudioInputsMenu(), if (showIncomingQuality) ..._buildIncomingQualityMenu(), + if (showBackgroundEffects) ..._buildBackgroundFiltersMenu(), ]), ); } @@ -182,6 +190,24 @@ class _SettingsMenuState extends State { }, ), const SizedBox(height: 16), + StandardActionMenuItem( + icon: Icons.auto_awesome, + label: 'Set Background Effect', + trailing: Text( + _videoEffectsManager.currentEffect != null ? 'On' : 'Off', + style: TextStyle( + color: _videoEffectsManager.currentEffect != null + ? AppColorPalette.appGreen + : null, + ), + ), + onPressed: () { + setState(() { + showBackgroundEffects = true; + }); + }, + ), + const SizedBox(height: 16), StandardActionMenuItem( icon: Icons.high_quality_sharp, label: 'Incoming video quality', @@ -322,6 +348,144 @@ class _SettingsMenuState extends State { ]; } + List _buildBackgroundFiltersMenu() { + return [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + GestureDetector( + onTap: () { + setState(() { + showBackgroundEffects = false; + }); + }, + child: const Align( + alignment: Alignment.centerLeft, + child: Icon(Icons.arrow_back, size: 24), + ), + ), + TextButton( + child: const Text('Clear'), + onPressed: () { + _videoEffectsManager.disableAllFilters(); + }, + ) + ], + ), + const SizedBox(height: 16), + const Text('Background Blur', + style: TextStyle(fontWeight: FontWeight.bold)), + const SizedBox(height: 16), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + Column( + children: [ + SizedBox( + height: 60, + child: Center( + child: IconButton( + icon: const Icon( + Icons.blur_on, + size: 30, + ), + onPressed: () => _videoEffectsManager + .applyBackgroundBlurFilter(BlurIntensity.light), + ), + ), + ), + const Text('Light'), + ], + ), + Column( + children: [ + SizedBox( + height: 60, + child: Center( + child: IconButton( + icon: const Icon( + Icons.blur_on, + size: 40, + ), + onPressed: () => _videoEffectsManager + .applyBackgroundBlurFilter(BlurIntensity.medium), + ), + ), + ), + const Text('Medium'), + ], + ), + Column( + children: [ + SizedBox( + height: 60, + child: Center( + child: IconButton( + icon: const Icon( + Icons.blur_on, + size: 50, + ), + onPressed: () => _videoEffectsManager + .applyBackgroundBlurFilter(BlurIntensity.heavy), + ), + ), + ), + const Text('Heavy'), + ], + ) + ], + ), + const SizedBox(height: 16), + const Text('Image Background', + style: TextStyle(fontWeight: FontWeight.bold)), + const SizedBox(height: 16), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + InkWell( + onTap: () => _videoEffectsManager + .applyBackgroundImageFilter('assets/bg1.jpg'), + child: ClipRRect( + borderRadius: BorderRadius.circular(8.0), + child: Image.asset( + 'assets/bg1.jpg', + fit: BoxFit.cover, + width: 72, + height: 102, + ), + ), + ), + InkWell( + onTap: () => _videoEffectsManager + .applyBackgroundImageFilter('assets/bg2.jpg'), + child: ClipRRect( + borderRadius: BorderRadius.circular(8.0), + child: Image.asset( + 'assets/bg2.jpg', + fit: BoxFit.cover, + width: 72, + height: 102, + ), + ), + ), + InkWell( + onTap: () => _videoEffectsManager + .applyBackgroundImageFilter('assets/bg3.jpg'), + child: ClipRRect( + borderRadius: BorderRadius.circular(8.0), + child: Image.asset( + 'assets/bg3.jpg', + fit: BoxFit.cover, + width: 72, + height: 102, + ), + ), + ) + ], + ), + ]; + } + VideoResolution? getIncomingVideoResolution(IncomingVideoQuality quality) { switch (quality) { case IncomingVideoQuality.auto: diff --git a/packages/stream_video/lib/src/webrtc/peer_connection.dart b/packages/stream_video/lib/src/webrtc/peer_connection.dart index 344450aa..fd1ad253 100644 --- a/packages/stream_video/lib/src/webrtc/peer_connection.dart +++ b/packages/stream_video/lib/src/webrtc/peer_connection.dart @@ -348,6 +348,8 @@ class StreamPeerConnection extends Disposable { Duration(milliseconds: _reportingIntervalMs), (_) async { try { + if (_statsController.isClosed) return; + final stats = await pc.getStats(); final rtcPrintableStats = stats.toPrintableRtcStats(); final rawStats = stats.toRawStats(); @@ -386,6 +388,7 @@ class StreamPeerConnection extends Disposable { onIceCandidate = null; onTrack = null; _pendingCandidates.clear(); + await _statsController.close(); await pc.dispose(); return await super.dispose(); } diff --git a/packages/stream_video_flutter/android/build.gradle b/packages/stream_video_flutter/android/build.gradle index 8ba5582f..8033db7f 100644 --- a/packages/stream_video_flutter/android/build.gradle +++ b/packages/stream_video_flutter/android/build.gradle @@ -58,6 +58,12 @@ android { implementation 'androidx.media:media:1.1.0' implementation 'androidx.appcompat:appcompat:1.1.0' implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2' + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation "io.github.crow-misia.libyuv:libyuv-android:0.34.0" + implementation "androidx.annotation:annotation:1.8.0" + implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta5' + implementation "com.github.android:renderscript-intrinsics-replacement-toolkit:344be3f" + implementation 'io.github.webrtc-sdk:android:125.6422.03' } testOptions { diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/MethodCallHandlerImpl.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/MethodCallHandlerImpl.kt index f162ee70..b126fbd7 100644 --- a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/MethodCallHandlerImpl.kt +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/MethodCallHandlerImpl.kt @@ -25,6 +25,10 @@ import io.getstream.video.flutter.stream_video_flutter.service.StreamCallService import io.getstream.video.flutter.stream_video_flutter.service.StreamScreenShareService import io.getstream.video.flutter.stream_video_flutter.service.notification.NotificationPayload import io.getstream.video.flutter.stream_video_flutter.service.utils.putBoolean +import com.cloudwebrtc.webrtc.videoEffects.ProcessorProvider +import io.getstream.video.flutter.stream_video_flutter.videoFilters.factories.BackgroundBlurFactory +import io.getstream.video.flutter.stream_video_flutter.videoFilters.factories.BlurIntensity +import io.getstream.video.flutter.stream_video_flutter.videoFilters.factories.VirtualBackgroundFactory class MethodCallHandlerImpl( appContext: Context, @@ -34,6 +38,7 @@ class MethodCallHandlerImpl( private val logger by taggedLogger(tag = "StreamMethodHandler") private val serviceManager: ServiceManager = ServiceManagerImpl(appContext.applicationContext) + private val applicationContext = appContext.applicationContext private var permissionCallback: ((Result) -> Unit)? = null @@ -68,6 +73,38 @@ class MethodCallHandlerImpl( override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { logger.d { "[onMethodCall] method: ${call.method}" } when (call.method) { + "isBackgroundEffectSupported" -> { + result.success(true) + } + "registerBlurEffectProcessors" -> { + ProcessorProvider.addProcessor( + "BackgroundBlurLight", + BackgroundBlurFactory(BlurIntensity.LIGHT) + ) + + ProcessorProvider.addProcessor( + "BackgroundBlurMedium", + BackgroundBlurFactory(BlurIntensity.MEDIUM) + ) + + ProcessorProvider.addProcessor( + "BackgroundBlurHeavy", + BackgroundBlurFactory(BlurIntensity.HEAVY) + ) + + result.success(null) + } + "registerImageEffectProcessors" -> { + val backgroundImageUrl = call.argument("backgroundImageUrl") + backgroundImageUrl?.let { + ProcessorProvider.addProcessor( + "VirtualBackground-$backgroundImageUrl", + VirtualBackgroundFactory(applicationContext, backgroundImageUrl) + ) + } + + result.success(null) + } "enablePictureInPictureMode" -> { val activity = getActivity() putBoolean(activity, PictureInPictureHelper.PIP_ENABLED_PREF_KEY, true) diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/BitmapVideoFilter.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/BitmapVideoFilter.kt new file mode 100644 index 00000000..e81b8391 --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/BitmapVideoFilter.kt @@ -0,0 +1,11 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.common + +import android.graphics.Bitmap + +/** + * A filter that provides a Bitmap of each frame. It's less performant than using the + * RawVideoFilter because we do YUV<->ARGB conversions internally. + */ +abstract class BitmapVideoFilter { + abstract fun applyFilter(videoFrameBitmap: Bitmap) +} diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/FilterUtils.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/FilterUtils.kt new file mode 100644 index 00000000..4f074b2c --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/FilterUtils.kt @@ -0,0 +1,67 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.common + +import android.graphics.Bitmap +import android.graphics.Matrix +import com.google.mlkit.vision.segmentation.SegmentationMask + +internal fun copySegment( + segment: Segment, + source: Bitmap, + destination: Bitmap, + segmentationMask: SegmentationMask, + confidenceThreshold: Double, +) { + val scaleBetweenSourceAndMask = getScalingFactors( + widths = Pair(source.width, segmentationMask.width), + heights = Pair(source.height, segmentationMask.height), + ) + + segmentationMask.buffer.rewind() + + val sourcePixels = IntArray(source.width * source.height) + source.getPixels(sourcePixels, 0, source.width, 0, 0, source.width, source.height) + val destinationPixels = IntArray(destination.width * destination.height) + + for (y in 0 until segmentationMask.height) { + for (x in 0 until segmentationMask.width) { + val confidence = segmentationMask.buffer.float + + if (((segment == Segment.BACKGROUND) && confidence < confidenceThreshold) || + ((segment == Segment.FOREGROUND) && confidence >= confidenceThreshold) + ) { + val scaledX = (x * scaleBetweenSourceAndMask.first).toInt() + val scaledY = (y * scaleBetweenSourceAndMask.second).toInt() + destinationPixels[y * destination.width + x] = + sourcePixels[scaledY * source.width + scaledX] + } + } + } + + destination.setPixels( + destinationPixels, + 0, + destination.width, + 0, + 0, + destination.width, + destination.height, + ) +} + +internal enum class Segment { + FOREGROUND, BACKGROUND +} + +private fun getScalingFactors(widths: Pair, heights: Pair) = + Pair(widths.first.toFloat() / widths.second, heights.first.toFloat() / heights.second) + +internal fun newSegmentationMaskMatrix(bitmap: Bitmap, mask: SegmentationMask): Matrix { + val isRawSizeMaskEnabled = mask.width != bitmap.width || mask.height != bitmap.height + return if (!isRawSizeMaskEnabled) { + Matrix() + } else { + val scale = + getScalingFactors(Pair(bitmap.width, mask.width), Pair(bitmap.height, mask.height)) + Matrix().apply { preScale(scale.first, scale.second) } + } +} diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/VideoFrameWithBitmapFilter.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/VideoFrameWithBitmapFilter.kt new file mode 100644 index 00000000..f8042b5a --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/VideoFrameWithBitmapFilter.kt @@ -0,0 +1,98 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.common + +import android.graphics.Bitmap +import android.graphics.Matrix +import android.opengl.GLES20 +import android.opengl.GLUtils +import android.util.Log +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessor +import org.webrtc.SurfaceTextureHelper +import org.webrtc.TextureBufferImpl +import org.webrtc.VideoFrame +import org.webrtc.YuvConverter + +// Original Sources +// https://github.com/GetStream/stream-video-android/blob/9a3b8e92b74bc4408781b5274fc602034d616983/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/video/FilterVideoProcessor.kt +// https://github.com/SHIVAJIKUMAR007/real-time-VideoProcessing/blob/5bb96a8b0c3c602a458ece1774f68ea913336f9f/android/app/src/main/java/com/vchat/backgroundEffect/BackgroundBlurFactory.java + +class VideoFrameProcessorWithBitmapFilter(bitmapVideoFilterFunc: () -> BitmapVideoFilter) : + VideoFrameProcessor { + private val yuvConverter = YuvConverter() + private var inputWidth = 0 + private var inputHeight = 0 + private var inputBuffer: VideoFrame.TextureBuffer? = null + private var yuvBuffer: VideoFrame.I420Buffer? = null + private val textures = IntArray(1) + private var inputFrameBitmap: Bitmap? = null + + private val bitmapVideoFilter by lazy { + bitmapVideoFilterFunc.invoke() + } + + init { + GLES20.glGenTextures(1, textures, 0) + } + + override fun process(frame: VideoFrame, surfaceTextureHelper: SurfaceTextureHelper): VideoFrame { + // Step 1: Video Frame to Bitmap + val inputFrameBitmap = YuvFrame.bitmapFromVideoFrame(frame) ?: return frame + + // Prepare helpers (runs only once or if the dimensions change) + initialize( + inputFrameBitmap.width, + inputFrameBitmap.height, + surfaceTextureHelper, + ) + + // Step 2: Apply filter + bitmapVideoFilter.applyFilter(inputFrameBitmap) + + // Step 3: Bitmap to Video Frame + // feed back the modified bitmap + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, + GLES20.GL_TEXTURE_MIN_FILTER, + GLES20.GL_NEAREST, + ) + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, + GLES20.GL_TEXTURE_MAG_FILTER, + GLES20.GL_NEAREST, + ) + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, inputFrameBitmap, 0) + // Convert the buffer back to YUV (VideoFrame needs YUV) + yuvBuffer = yuvConverter.convert(inputBuffer) + return VideoFrame(yuvBuffer, 0, frame.timestampNs) + } + + private fun initialize(width: Int, height: Int, textureHelper: SurfaceTextureHelper) { + // TODO: temporarily disabled due to crash: java.lang.IllegalStateException: release() called on an object with refcount < 1 +// yuvBuffer?.release() + + if (this.inputWidth != width || this.inputHeight != height) { + Log.d(TAG, "initialize - width: $width height: $height") + this.inputWidth = width + this.inputHeight = height + inputFrameBitmap?.recycle() + inputBuffer?.release() + + val type = VideoFrame.TextureBuffer.Type.RGB + + val matrix = Matrix() + // This is vertical flip - we need to investigate why the image is flipped vertically and + // why we need to correct it here. + matrix.preScale(1.0f, -1.0f) + val surfaceTextureHelper: SurfaceTextureHelper = textureHelper + this.inputBuffer = TextureBufferImpl( + inputWidth, inputHeight, type, textures[0], matrix, surfaceTextureHelper.handler, + yuvConverter, null as Runnable?, + ) + this.inputFrameBitmap = + Bitmap.createBitmap(this.inputWidth, this.inputHeight, Bitmap.Config.ARGB_8888) + } + } + + companion object { + private const val TAG = "VideoFrameProcessorWithBitmapFilter" + } +} diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/YuvFrame.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/YuvFrame.kt new file mode 100644 index 00000000..d2f0f345 --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/common/YuvFrame.kt @@ -0,0 +1,97 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.common + +import android.graphics.Bitmap +import android.util.Log +import io.github.crow_misia.libyuv.AbgrBuffer +import io.github.crow_misia.libyuv.I420Buffer +import io.github.crow_misia.libyuv.PlanePrimitive +import io.github.crow_misia.libyuv.RotateMode +import io.github.crow_misia.libyuv.RowStride +import org.webrtc.VideoFrame + +object YuvFrame { + private const val TAG = "YuvFrame" + + private lateinit var webRtcI420Buffer: VideoFrame.I420Buffer + private lateinit var libYuvI420Buffer: I420Buffer + private var libYuvRotatedI420Buffer: I420Buffer? = null + private var libYuvAbgrBuffer: AbgrBuffer? = null + + /** + * Converts VideoFrame.Buffer YUV frame to an ARGB_8888 Bitmap. Applies stored rotation. + * @return A new Bitmap containing the converted frame. + */ + fun bitmapFromVideoFrame(videoFrame: VideoFrame?): Bitmap? { + if (videoFrame == null) { + return null + } + + return try { + webRtcI420Buffer = videoFrame.buffer.toI420()!! + createLibYuvI420Buffer() + rotateLibYuvI420Buffer(videoFrame.rotation) + createLibYuvAbgrBuffer() + cleanUp() + libYuvAbgrBuffer!!.asBitmap() + } catch (t: Throwable) { + Log.e(TAG, "Failed to convert a VideoFrame", t) + null + } + } + + private fun createLibYuvI420Buffer() { + val width = webRtcI420Buffer.width + val height = webRtcI420Buffer.height + + libYuvI420Buffer = I420Buffer.wrap( + planeY = PlanePrimitive(RowStride(webRtcI420Buffer.strideY), webRtcI420Buffer.dataY), + planeU = PlanePrimitive(RowStride(webRtcI420Buffer.strideU), webRtcI420Buffer.dataU), + planeV = PlanePrimitive(RowStride(webRtcI420Buffer.strideV), webRtcI420Buffer.dataV), + width = width, + height = height, + ) + } + + private fun rotateLibYuvI420Buffer(rotationDegrees: Int) { + val width = webRtcI420Buffer.width + val height = webRtcI420Buffer.height + + when (rotationDegrees) { + 90, -270 -> changeOrientation(width, height, RotateMode.ROTATE_90) // upside down, 90 + 180, -180 -> keepOrientation(width, height, RotateMode.ROTATE_180) // right, 180 + 270, -90 -> changeOrientation(width, height, RotateMode.ROTATE_270) // upright, 270 + else -> keepOrientation(width, height, RotateMode.ROTATE_0) // left, 0, default + } + } + + private fun changeOrientation(width: Int, height: Int, rotateMode: RotateMode) { + libYuvRotatedI420Buffer?.close() + libYuvRotatedI420Buffer = I420Buffer.allocate(height, width) // swapped width and height + libYuvI420Buffer.rotate(libYuvRotatedI420Buffer!!, rotateMode) + } + + private fun keepOrientation(width: Int, height: Int, rotateMode: RotateMode) { + if (width != libYuvRotatedI420Buffer?.width || height != libYuvRotatedI420Buffer?.height) { + libYuvRotatedI420Buffer?.close() + libYuvRotatedI420Buffer = I420Buffer.allocate(width, height) + } + libYuvI420Buffer.rotate(libYuvRotatedI420Buffer!!, rotateMode) + } + + private fun createLibYuvAbgrBuffer() { + val width = libYuvRotatedI420Buffer!!.width + val height = libYuvRotatedI420Buffer!!.height + + if (width != libYuvAbgrBuffer?.width || height != libYuvAbgrBuffer?.height) { + libYuvAbgrBuffer?.close() + libYuvAbgrBuffer = AbgrBuffer.allocate(width, height) + } + libYuvRotatedI420Buffer!!.convertTo(libYuvAbgrBuffer!!) + } + + private fun cleanUp() { + libYuvI420Buffer.close() + webRtcI420Buffer.release() + // Rest of buffers are closed in the methods above + } +} diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/BackgroundBlurFactory.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/BackgroundBlurFactory.kt new file mode 100644 index 00000000..cea5c7fd --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/BackgroundBlurFactory.kt @@ -0,0 +1,95 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.factories + +import android.graphics.Bitmap +import android.graphics.Canvas +import com.google.android.gms.tasks.Tasks +import com.google.android.renderscript.Toolkit +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.segmentation.Segmentation +import com.google.mlkit.vision.segmentation.SegmentationMask +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessor +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessorFactoryInterface +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.BitmapVideoFilter +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.Segment +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.VideoFrameProcessorWithBitmapFilter +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.copySegment +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.newSegmentationMaskMatrix + + +// Original Sources +// https://github.com/GetStream/stream-video-android/blob/develop/stream-video-android-filters-video/src/main/kotlin/io/getstream/video/android/filters/video/BlurredBackgroundVideoFilter.kt +/** + * Applies a blur effect to the background of a video call. + * + * @param blurIntensity The intensity of the blur effect. See [BlurIntensity] for options. Defaults to [BlurIntensity.MEDIUM]. + * @param foregroundThreshold The confidence threshold for the foreground. Pixels with a confidence value greater than or equal to this threshold are considered to be in the foreground. Value is coerced between 0 and 1, inclusive. + */ +class BackgroundBlurFactory( + private val blurIntensity: BlurIntensity = BlurIntensity.MEDIUM, + private val foregroundThreshold: Double = DEFAULT_FOREGROUND_THRESHOLD, +) : VideoFrameProcessorFactoryInterface { + override fun build(): VideoFrameProcessor { + return VideoFrameProcessorWithBitmapFilter { + BlurredBackgroundVideoFilter(blurIntensity, foregroundThreshold) + } + } +} + +private class BlurredBackgroundVideoFilter( + private val blurIntensity: BlurIntensity, + foregroundThreshold: Double, +) : BitmapVideoFilter() { + private val options = + SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .enableRawSizeMask() + .build() + private val segmenter = Segmentation.getClient(options) + private lateinit var segmentationMask: SegmentationMask + private var foregroundThreshold: Double = foregroundThreshold.coerceIn(0.0, 1.0) + private val backgroundBitmap by lazy { + Bitmap.createBitmap( + segmentationMask.width, + segmentationMask.height, + Bitmap.Config.ARGB_8888, + ) + } + + override fun applyFilter(videoFrameBitmap: Bitmap) { + // Apply segmentation + val mlImage = InputImage.fromBitmap(videoFrameBitmap, 0) + val task = segmenter.process(mlImage) + segmentationMask = Tasks.await(task) + + // Copy the background segment to a new bitmap - backgroundBitmap + copySegment( + segment = Segment.BACKGROUND, + source = videoFrameBitmap, + destination = backgroundBitmap, + segmentationMask = segmentationMask, + confidenceThreshold = foregroundThreshold, + ) + + // Blur the background bitmap + val blurredBackgroundBitmap = Toolkit.blur(backgroundBitmap, blurIntensity.radius) + + // Draw the blurred background bitmap on the original bitmap + val canvas = Canvas(videoFrameBitmap) + val matrix = newSegmentationMaskMatrix(videoFrameBitmap, segmentationMask) + canvas.drawBitmap(blurredBackgroundBitmap, matrix, null) + } +} + +/** + * The intensity of the background blur effect. Used in [BlurredBackgroundVideoFilter]. + * Range is 1 to 25 + */ +enum class BlurIntensity(val radius: Int) { + LIGHT(5), + MEDIUM(10), + HEAVY(15), +} + +private const val DEFAULT_FOREGROUND_THRESHOLD: Double = + 0.999 // 1 is max confidence that pixel is in the foreground diff --git a/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/VirtualBackgroundFactory.kt b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/VirtualBackgroundFactory.kt new file mode 100644 index 00000000..e36b96ca --- /dev/null +++ b/packages/stream_video_flutter/android/src/main/kotlin/io/getstream/video/flutter/stream_video_flutter/videoFilters/factories/VirtualBackgroundFactory.kt @@ -0,0 +1,231 @@ +package io.getstream.video.flutter.stream_video_flutter.videoFilters.factories + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Canvas +import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.PorterDuff +import android.graphics.PorterDuffXfermode +import android.net.Uri +import android.util.Log +import androidx.annotation.Keep +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.segmentation.Segmentation +import com.google.mlkit.vision.segmentation.SegmentationMask +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessor +import com.cloudwebrtc.webrtc.videoEffects.VideoFrameProcessorFactoryInterface +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.BitmapVideoFilter +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.Segment +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.VideoFrameProcessorWithBitmapFilter +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.copySegment +import io.getstream.video.flutter.stream_video_flutter.videoFilters.common.newSegmentationMaskMatrix +import java.io.IOException +import java.net.URL +import io.flutter.embedding.engine.loader.FlutterLoader +import java.io.InputStream +import android.os.Handler +import android.os.Looper +import android.os.Build; +import io.flutter.FlutterInjector; + +/** + * original source: https://github.com/GetStream/stream-video-android/blob/develop/stream-video-android-filters-video/src/main/kotlin/io/getstream/video/android/filters/video/VirtualBackgroundVideoFilter.kt + * + * Applies a virtual background (custom image) to a video call. + * + * @param backgroundImageUrlString The image url of the custom background image. + * @param foregroundThreshold The confidence threshold for the foreground. Pixels with a confidence value greater than or equal to this threshold are considered to be in the foreground. Value is coerced between 0 and 1, inclusive. + */ +class VirtualBackgroundFactory( + private val appContext: Context, + private val backgroundImageUrlString: String, + private val foregroundThreshold: Double = DEFAULT_FOREGROUND_THRESHOLD, +) : VideoFrameProcessorFactoryInterface { + + override fun build(): VideoFrameProcessor { + return VideoFrameProcessorWithBitmapFilter { + VirtualBackgroundVideoFilter(appContext, backgroundImageUrlString, foregroundThreshold) + } + } + + companion object { + private const val TAG = "VirtualBackgroundFactory" + } +} + +/** + * Applies a virtual background (custom image) to a video call. + * + * @param backgroundImageUrlString The image url of the custom background image. + * @param foregroundThreshold The confidence threshold for the foreground. Pixels with a confidence value greater than or equal to this threshold are considered to be in the foreground. Value is coerced between 0 and 1, inclusive. + */ +@Keep +private class VirtualBackgroundVideoFilter( + appContext: Context, + backgroundImageUrlString: String, + foregroundThreshold: Double = DEFAULT_FOREGROUND_THRESHOLD, +) : BitmapVideoFilter() { + private val options = + SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .enableRawSizeMask() + .build() + private val segmenter = Segmentation.getClient(options) + private lateinit var segmentationMask: SegmentationMask + private lateinit var segmentationMatrix: Matrix + + private var foregroundThreshold: Double = foregroundThreshold.coerceIn(0.0, 1.0) + private val foregroundBitmap by lazy { + Bitmap.createBitmap( + segmentationMask.width, + segmentationMask.height, + Bitmap.Config.ARGB_8888, + ) + } + + private val virtualBackgroundBitmap by lazy { + Log.d(TAG, "getBitmapFromUrl - $backgroundImageUrlString") + try { + val uri = Uri.parse(backgroundImageUrlString) + if (uri.scheme == null) { // this is a local image + loadImageAssetAsBitmap(appContext, backgroundImageUrlString) + } else { + val url = URL(backgroundImageUrlString) + BitmapFactory.decodeStream(url.openConnection().getInputStream()) + } + } catch (e: IOException) { + Log.e(TAG, "cant get bitmap for image url: $backgroundImageUrlString", e) + null + } + } + + private val foregroundPaint by lazy { + Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_OUT) } + } + private var scaledVirtualBackgroundBitmap: Bitmap? = null + private var scaledVirtualBackgroundBitmapCopy: Bitmap? = null + + private var latestFrameWidth: Int? = null + private var latestFrameHeight: Int? = null + + override fun applyFilter(videoFrameBitmap: Bitmap) { + // Apply segmentation + val mlImage = InputImage.fromBitmap(videoFrameBitmap, 0) + val task = segmenter.process(mlImage) + segmentationMask = Tasks.await(task) + + // Copy the foreground segment (the person) to a new bitmap - foregroundBitmap + copySegment( + segment = Segment.FOREGROUND, + source = videoFrameBitmap, + destination = foregroundBitmap, + segmentationMask = segmentationMask, + confidenceThreshold = foregroundThreshold, + ) + + virtualBackgroundBitmap?.let { virtualBackgroundBitmap -> + val videoFrameCanvas = Canvas(videoFrameBitmap) + + // Scale the virtual background bitmap to the height of the video frame, if needed + if (scaledVirtualBackgroundBitmap == null || + videoFrameCanvas.width != latestFrameWidth || + videoFrameCanvas.height != latestFrameHeight + ) { + scaledVirtualBackgroundBitmap = scaleVirtualBackgroundBitmap( + bitmap = virtualBackgroundBitmap, + targetHeight = videoFrameCanvas.height, + ) + // Make a copy of the scaled virtual background bitmap. Used when processing each frame. + scaledVirtualBackgroundBitmapCopy = scaledVirtualBackgroundBitmap!!.copy( + /* config = */ + scaledVirtualBackgroundBitmap!!.config!!, + /* isMutable = */ + true, + ) + + latestFrameWidth = videoFrameBitmap.width + latestFrameHeight = videoFrameBitmap.height + + segmentationMatrix = newSegmentationMaskMatrix(videoFrameBitmap, segmentationMask) + } + + // Restore the virtual background after cutting-out the person in the previous frame + val backgroundCanvas = Canvas(scaledVirtualBackgroundBitmapCopy!!) + backgroundCanvas.drawBitmap(scaledVirtualBackgroundBitmap!!, 0f, 0f, null) + + // Cut out the person from the virtual background + backgroundCanvas.drawBitmap(foregroundBitmap, segmentationMatrix, foregroundPaint) + + // Draw the virtual background (with the cutout) on the video frame bitmap + videoFrameCanvas.drawBitmap(scaledVirtualBackgroundBitmapCopy!!, 0f, 0f, null) + } + } + + private fun scaleVirtualBackgroundBitmap(bitmap: Bitmap, targetHeight: Int): Bitmap { + val scale = targetHeight.toFloat() / bitmap.height + return ensureAlpha( + Bitmap.createScaledBitmap( + /* src = */ + bitmap, + /* dstWidth = */ + (bitmap.width * scale).toInt(), + /* dstHeight = */ + targetHeight, + /* filter = */ + true, + ), + ) + } + + private fun ensureAlpha(original: Bitmap): Bitmap { + return if (original.hasAlpha()) { + original + } else { + val bitmapWithAlpha = Bitmap.createBitmap( + original.width, + original.height, + Bitmap.Config.ARGB_8888, + ) + val canvas = Canvas(bitmapWithAlpha) + canvas.drawBitmap(original, 0f, 0f, null) + bitmapWithAlpha + } + } + + fun loadImageAssetAsBitmap(context: Context, assetPath: String): Bitmap? { + var bitmap: Bitmap? = null + var inputStream: InputStream? = null + + try { + inputStream = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + context.assets.open("flutter_assets/$assetPath") + } else { + val assetLookupKey = FlutterInjector.instance() + .flutterLoader() + .getLookupKeyForAsset(assetPath) + val assetManager = context.assets + val assetFileDescriptor = assetManager.openFd(assetLookupKey) + assetFileDescriptor.createInputStream() + } + + bitmap = BitmapFactory.decodeStream(inputStream) + return bitmap + } catch (e: Exception) { + e.printStackTrace() + } finally { + inputStream?.close() + } + return null + } + + companion object { + private const val TAG = "VirtualBackgroundVideoFilter" + } +} + +private const val DEFAULT_FOREGROUND_THRESHOLD: Double = + 0.7 // 1 is max confidence that pixel is in the foreground diff --git a/packages/stream_video_flutter/ios/Classes/StreamVideoFlutterPlugin.swift b/packages/stream_video_flutter/ios/Classes/StreamVideoFlutterPlugin.swift index 74c88f43..0017d14c 100644 --- a/packages/stream_video_flutter/ios/Classes/StreamVideoFlutterPlugin.swift +++ b/packages/stream_video_flutter/ios/Classes/StreamVideoFlutterPlugin.swift @@ -4,20 +4,65 @@ import flutter_webrtc public class StreamVideoFlutterPlugin: NSObject, FlutterPlugin { public static func register(with registrar: FlutterPluginRegistrar) { - let channel = FlutterMethodChannel(name: "stream_video_flutter", binaryMessenger: registrar.messenger()) + let channel = FlutterMethodChannel( + name: "stream_video_flutter", binaryMessenger: registrar.messenger()) let instance = StreamVideoFlutterPlugin() registrar.addMethodCallDelegate(instance, channel: channel) - + let factory = StreamPictureInPictureNativeViewFactory(messenger: registrar.messenger()) registrar.register( factory, withId: "stream-pip-view") } - + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { switch call.method { - case "getPlatformVersion": - result("iOS " + UIDevice.current.systemVersion) + case "isBackgroundEffectSupported": + if #available(iOS 15.0, *) { + result(true) + } else { + result(false) + } + case "registerBlurEffectProcessors": + if #available(iOS 15.0, *) { + ProcessorProvider.addProcessor( + BlurBackgroundVideoFrameProcessor(blurIntensity: BlurIntensity.light), + forName: "BackgroundBlurLight") + ProcessorProvider.addProcessor( + BlurBackgroundVideoFrameProcessor(blurIntensity: BlurIntensity.medium), + forName: "BackgroundBlurMedium") + ProcessorProvider.addProcessor( + BlurBackgroundVideoFrameProcessor(blurIntensity: BlurIntensity.heavy), + forName: "BackgroundBlurHeavy") + } else { + print("Background blur effects are not supported on iOS versions earlier than 15.0") + } + + result(nil) + case "registerImageEffectProcessors": + if #available(iOS 15.0, *) { + if let arguments = call.arguments as? [String: Any] { + guard let backgroundImageUrl = arguments["backgroundImageUrl"] as? String + else { + result( + FlutterError( + code: "INVALID_ARGUMENT", message: "Invalid argument", details: nil) + ) + return + } + + ProcessorProvider.addProcessor( + ImageBackgroundVideoFrameProcessor(backgroundImageUrl), + forName: "VirtualBackground-\(backgroundImageUrl)") + + result(nil) + } + + result(nil) + + } else { + print("Image overlay effects are not supported on iOS versions earlier than 15.0") + } default: result(FlutterMethodNotImplemented) } @@ -26,12 +71,12 @@ public class StreamVideoFlutterPlugin: NSObject, FlutterPlugin { class StreamPictureInPictureNativeViewFactory: NSObject, FlutterPlatformViewFactory { private var messenger: FlutterBinaryMessenger - + init(messenger: FlutterBinaryMessenger) { self.messenger = messenger super.init() } - + func create( withFrame frame: CGRect, viewIdentifier viewId: Int64, @@ -43,7 +88,7 @@ class StreamPictureInPictureNativeViewFactory: NSObject, FlutterPlatformViewFact arguments: args, binaryMessenger: messenger) } - + public func createArgsCodec() -> FlutterMessageCodec & NSObjectProtocol { return FlutterStandardMessageCodec.sharedInstance() } @@ -53,7 +98,7 @@ class StreamPictureInPictureNativeView: NSObject, FlutterPlatformView { private var _view: UIView private var methodChannel: FlutterMethodChannel? private lazy var pictureInPictureController = StreamPictureInPictureController() - + init( frame: CGRect, viewIdentifier viewId: Int64, @@ -62,24 +107,26 @@ class StreamPictureInPictureNativeView: NSObject, FlutterPlatformView { ) { _view = UIView() super.init() - - self.methodChannel = FlutterMethodChannel(name: "stream_video_flutter_pip" ,binaryMessenger: messenger) + + self.methodChannel = FlutterMethodChannel( + name: "stream_video_flutter_pip", binaryMessenger: messenger) methodChannel?.setMethodCallHandler(onMethodCall) - + pictureInPictureController?.sourceView = _view - + createNativeView(view: _view) } - + private func onMethodCall(call: FlutterMethodCall, result: FlutterResult) { - switch(call.method){ + switch call.method { case "setTrack": - let argumentsDictionary = call.arguments as? Dictionary + let argumentsDictionary = call.arguments as? [String: Any] let trackId = argumentsDictionary?["trackId"] as? String - + DispatchQueue.main.async { if let unwrappedTrackId = trackId { - let track = FlutterWebRTCPlugin.sharedSingleton()?.track(forId: unwrappedTrackId, peerConnectionId: nil); + let track = FlutterWebRTCPlugin.sharedSingleton()?.track( + forId: unwrappedTrackId, peerConnectionId: nil) if let videoTrack = track as? RTCVideoTrack { self.pictureInPictureController?.track = videoTrack } @@ -92,12 +139,12 @@ class StreamPictureInPictureNativeView: NSObject, FlutterPlatformView { result(FlutterMethodNotImplemented) } } - + func view() -> UIView { return _view } - - func createNativeView(view _view: UIView){ + + func createNativeView(view _view: UIView) { _view.backgroundColor = UIColor.clear } } diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/BlurBackgroundVideoFrameProcessor.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/BlurBackgroundVideoFrameProcessor.swift new file mode 100644 index 00000000..f838de6d --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/BlurBackgroundVideoFrameProcessor.swift @@ -0,0 +1,39 @@ +import Foundation + +@available(iOS 15.0, *) +final class BlurBackgroundVideoFrameProcessor: VideoFilter { + + @available(*, unavailable) + override public init( + filter: @escaping (Input) -> CIImage + ) { fatalError() } + + private lazy var backgroundImageFilterProcessor = { return BackgroundImageFilterProcessor() }() + + private let blurParameters: [String : Float] + + init(blurIntensity: BlurIntensity = BlurIntensity.medium) { + blurParameters = ["inputRadius": blurIntensity.rawValue] + + super.init( + filter: { input in input.originalImage } + ) + + self.filter = { input in + // https://developer.apple.com/library/archive/documentation/GraphicsImaging/Reference/CoreImageFilterReference/index.html#//apple_ref/doc/filter/ci/CIGaussianBlur + let backgroundImage = input.originalImage.applyingFilter("CIGaussianBlur", parameters: self.blurParameters) + + return self.backgroundImageFilterProcessor + .applyFilter( + input.originalPixelBuffer, + backgroundImage: backgroundImage + ) ?? input.originalImage + } + } +} + +enum BlurIntensity: Float { + case light = 5.0 + case medium = 10.0 + case heavy = 15.0 +} diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/ImageBackgroundVideoFrameProcessor.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/ImageBackgroundVideoFrameProcessor.swift new file mode 100644 index 00000000..6791c38d --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/ImageBackgroundVideoFrameProcessor.swift @@ -0,0 +1,109 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import CoreImage +import Foundation +import flutter_webrtc +import ios_platform_images + +/// A video filter that applies a custom image as the background. +/// +/// This filter uses a provided image taken from `backgroundImageUrl` as the background and combines it with +/// the foreground objects using a filter processor. It caches processed background images to optimize +/// performance for matching input sizes and orientations. +@available(iOS 15.0, *) +final class ImageBackgroundVideoFrameProcessor: VideoFilter { + + private struct CacheValue: Hashable { + var originalImageSize: CGSize + var originalImageOrientation: CGImagePropertyOrientation + var result: CIImage + + func hash(into hasher: inout Hasher) { + hasher.combine(originalImageSize.width) + hasher.combine(originalImageSize.height) + hasher.combine(originalImageOrientation) + } + } + + private var cachedValue: CacheValue? + private var backgroundImageUrl: String + + private lazy var backgroundImageFilterProcessor = { return BackgroundImageFilterProcessor() }() + + private lazy var backgroundCIImage: CIImage? = { + var bgUIImage: UIImage? + // if let url = URL(string: backgroundImageUrl) { + // check if its a local asset + bgUIImage = UIImage.flutterImageWithName(backgroundImageUrl) //RCTImageFromLocalAssetURL(url) + if bgUIImage == nil { + // if its not a local asset, then try to get it as a remote asset + if let url = URL(string: backgroundImageUrl), let data = try? Data(contentsOf: url) { + bgUIImage = UIImage(data: data) + } else { + NSLog("Failed to convert uri to image: -\(backgroundImageUrl)") + } + } + // } + if bgUIImage != nil { + return CIImage.init(image: bgUIImage!) + } + return nil + }() + + @available(*, unavailable) + override public init( + filter: @escaping (Input) -> CIImage + ) { fatalError() } + + init(_ backgroundImageUrl: String) { + self.backgroundImageUrl = backgroundImageUrl + super.init( + filter: { input in input.originalImage } + ) + + self.filter = { input in + guard let bgImage = self.backgroundCIImage else { return input.originalImage } + let cachedBackgroundImage = self.backgroundImage( + image: bgImage, originalImage: input.originalImage, + originalImageOrientation: input.originalImageOrientation) + + let outputImage: CIImage = + self.backgroundImageFilterProcessor + .applyFilter( + input.originalPixelBuffer, + backgroundImage: cachedBackgroundImage + ) ?? input.originalImage + + return outputImage + } + } + + /// Returns the cached or processed background image for a given original image (frame image). + private func backgroundImage( + image: CIImage, originalImage: CIImage, originalImageOrientation: CGImagePropertyOrientation + ) -> CIImage { + if let cachedValue = cachedValue, + cachedValue.originalImageSize == originalImage.extent.size, + cachedValue.originalImageOrientation == originalImageOrientation + { + return cachedValue.result + } else { + var cachedBackgroundImage = image.oriented(originalImageOrientation) + + if cachedBackgroundImage.extent.size != originalImage.extent.size { + cachedBackgroundImage = + cachedBackgroundImage + .resize(originalImage.extent.size) ?? cachedBackgroundImage + } + + cachedValue = .init( + originalImageSize: originalImage.extent.size, + originalImageOrientation: originalImageOrientation, + result: cachedBackgroundImage + ) + return cachedBackgroundImage + } + } +} diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/BackgroundImageFilterProcessor.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/BackgroundImageFilterProcessor.swift new file mode 100644 index 00000000..d74c8851 --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/BackgroundImageFilterProcessor.swift @@ -0,0 +1,74 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import CoreImage +import CoreImage.CIFilterBuiltins +import Foundation +import Vision + +/// Processes a video frame to create a new image with a custom background. +/// +/// This class generates a person segmentation mask using Vision, scales the mask +/// to match the video frame size, and blends the original image with a provided +/// background image using the mask. This allows for effects like background +/// replacement or blurring. +@available(iOS 15.0, *) +final class BackgroundImageFilterProcessor { + private let requestHandler = VNSequenceRequestHandler() + private let request: VNGeneratePersonSegmentationRequest + + + /// Initializes a new `BackgroundImageFilterProcessor` instance. + /// + /// - Parameters: + /// - qualityLevel: The quality level for segmentation, defaults to + /// `.balanced` if a neural engine is available, otherwise `.fast` for + /// performance. + init( + _ qualityLevel: VNGeneratePersonSegmentationRequest.QualityLevel = neuralEngineExists ? .balanced : .fast + ) { + let request = VNGeneratePersonSegmentationRequest() + request.qualityLevel = qualityLevel + request.outputPixelFormat = kCVPixelFormatType_OneComponent8 + self.request = request + } + + /// Applies the filter to a video frame using a background image. + /// + /// - Parameters: + /// - buffer: The video frame to process as a `CVPixelBuffer`. + /// - backgroundImage: The background image to blend with the foreground. + /// - Returns: A new `CIImage` with the processed frame, or `nil` if an error occurs. + func applyFilter( + _ buffer: CVPixelBuffer, + backgroundImage: CIImage + ) -> CIImage? { + do { + try requestHandler.perform([request], on: buffer) + + if let maskPixelBuffer = request.results?.first?.pixelBuffer { + let originalImage = CIImage(cvPixelBuffer: buffer) + var maskImage = CIImage(cvPixelBuffer: maskPixelBuffer) + + // Scale the mask image to fit the bounds of the video frame. + let scaleX = originalImage.extent.width / maskImage.extent.width + let scaleY = originalImage.extent.height / maskImage.extent.height + maskImage = maskImage.transformed(by: .init(scaleX: scaleX, y: scaleY)) + + // Blend the original, background, and mask images. + let blendFilter = CIFilter.blendWithMask() + blendFilter.inputImage = originalImage + blendFilter.backgroundImage = backgroundImage + blendFilter.maskImage = maskImage + + let result = blendFilter.outputImage + return result + } else { + return nil + } + } catch { + return nil + } + } +} diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/CIImage+Resize.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/CIImage+Resize.swift new file mode 100644 index 00000000..b68329be --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/CIImage+Resize.swift @@ -0,0 +1,32 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import CoreImage +import Foundation + +extension CIImage { + + /// Resizes the image to a specified target size while maintaining aspect ratio. + /// + /// This method creates a new `CIImage` instance resized to the provided `targetSize` + /// while preserving the original image's aspect ratio. It uses the Lanczos resampling filter + /// for high-quality scaling. + /// + /// - Parameters: + /// - targetSize: The desired size for the resized image. + /// + /// - Returns: A new `CIImage` instance resized to the target size, or nil if an error occurs. + func resize(_ targetSize: CGSize) -> CIImage? { + // Compute scale and corrective aspect ratio + let scale = targetSize.height / (extent.height) + let aspectRatio = targetSize.width / ((extent.width) * scale) + + // Apply resizing + let filter = CIFilter(name: "CILanczosScaleTransform")! + filter.setValue(self, forKey: kCIInputImageKey) + filter.setValue(NSNumber(value: scale), forKey: kCIInputScaleKey) + filter.setValue(NSNumber(value: aspectRatio), forKey: kCIInputAspectRatioKey) + return filter.outputImage + } +} diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/UIDevice+NeuralEngine.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/UIDevice+NeuralEngine.swift new file mode 100644 index 00000000..688e2e02 --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/UIDevice+NeuralEngine.swift @@ -0,0 +1,16 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +#if canImport(MLCompute) +import MLCompute +let neuralEngineExists = { + if #available(iOS 15.0, *) { + return MLCDevice.ane() != nil + } else { + return false + } +}() +#else +let neuralEngineExists = false +#endif diff --git a/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/VideoFilters.swift b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/VideoFilters.swift new file mode 100644 index 00000000..dcb6de5a --- /dev/null +++ b/packages/stream_video_flutter/ios/Classes/VideoFrameProcessors/Utils/VideoFilters.swift @@ -0,0 +1,112 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import Foundation +import flutter_webrtc + +#if canImport(UIKit) + import Foundation + import UIKit + + extension UIInterfaceOrientation { + /// Values of `CGImagePropertyOrientation` define the position of the pixel coordinate origin + /// point (0,0) and the directions of the coordinate axes relative to the intended display orientation of + /// the image. While `UIInterfaceOrientation` uses a different point as its (0,0), this extension + /// provides a simple way of mapping device orientation to image orientation. + var cgOrientation: CGImagePropertyOrientation { + switch self { + /// Handle known portrait orientations + case .portrait: + return .left + + case .portraitUpsideDown: + return .right + + /// Handle known landscape orientations + case .landscapeLeft: + return .up + + case .landscapeRight: + return .down + + /// Unknown case, return `up` for consistency + case .unknown: + return .up + + /// Default case for unknown orientations or future additions + /// Returns `up` for consistency. + @unknown default: + return .up + } + } + } + +#endif // #if canImport(UIKit) + +open class VideoFilter: NSObject, VideoFrameProcessorDelegate { + + /// An object which encapsulates the required input for a Video filter. + public struct Input { + /// The image (video frame) that the filter should be applied on. + public var originalImage: CIImage + + /// The pixelBuffer that produces the image (video frame) that the filter should be applied on. + public var originalPixelBuffer: CVPixelBuffer + + /// The orientation on which the image (video frame) was generated from. + public var originalImageOrientation: CGImagePropertyOrientation + } + /// Filter closure that takes a CIImage as input and returns a filtered CIImage as output. + public var filter: (Input) -> CIImage + + private let context: CIContext + + var sceneOrientation: UIInterfaceOrientation = .unknown + + /// Initializes a new VideoFilter instance with the provided parameters. + public init( + filter: @escaping (Input) -> CIImage + ) { + self.filter = filter + self.context = CIContext(options: [CIContextOption.useSoftwareRenderer: false]) + super.init() + // listen to when the device's orientation changes + NotificationCenter.default.addObserver( + self, + selector: #selector(updateRotation), + name: UIDevice.orientationDidChangeNotification, + object: nil + ) + updateRotation() + } + + @objc private func updateRotation() { + DispatchQueue.main.async { + self.sceneOrientation = + UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? .unknown + } + } + + public func capturer(_ capturer: RTCVideoCapturer!, didCapture frame: RTCVideoFrame!) + -> RTCVideoFrame! + { + if let rtcCVPixelBuffer = frame.buffer as? RTCCVPixelBuffer { + let pixelBuffer = rtcCVPixelBuffer.pixelBuffer + + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) + let outputImage: CIImage = self.filter( + Input( + originalImage: CIImage(cvPixelBuffer: pixelBuffer), + originalPixelBuffer: pixelBuffer, + originalImageOrientation: self.sceneOrientation.cgOrientation + ) + ) + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + context.render(outputImage, to: pixelBuffer) + return RTCVideoFrame.init( + buffer: rtcCVPixelBuffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs) + } + return frame + } +} diff --git a/packages/stream_video_flutter/ios/stream_video_flutter.podspec b/packages/stream_video_flutter/ios/stream_video_flutter.podspec index ea4e3617..af38b609 100644 --- a/packages/stream_video_flutter/ios/stream_video_flutter.podspec +++ b/packages/stream_video_flutter/ios/stream_video_flutter.podspec @@ -16,6 +16,7 @@ Official Flutter Plugin for Stream Video.. s.source_files = 'Classes/**/*' s.dependency 'Flutter' s.dependency 'flutter_webrtc' + s.dependency 'ios_platform_images' s.static_framework = true s.platform = :ios, '11.0' diff --git a/packages/stream_video_flutter/lib/src/video_effects/video_effects_manager.dart b/packages/stream_video_flutter/lib/src/video_effects/video_effects_manager.dart new file mode 100644 index 00000000..d290f53b --- /dev/null +++ b/packages/stream_video_flutter/lib/src/video_effects/video_effects_manager.dart @@ -0,0 +1,129 @@ +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:stream_video/stream_video.dart'; + +import '../../stream_video_flutter_platform_interface.dart'; + +const _tag = 'SVF:BackgrounFilters'; + +enum BlurIntensity { + light('BackgroundBlurLight'), + medium('BackgroundBlurMedium'), + heavy('BackgroundBlurHeavy'); + + const BlurIntensity(this.name); + final String name; +} + +class StreamVideoEffectsManager { + StreamVideoEffectsManager(this.call); + + static bool isBlurRegistered = false; + static Map isImageRegistered = {}; + + final Call call; + final _logger = taggedLogger(tag: _tag); + + String? currentEffect; + + Future isSupported() async { + return await StreamVideoFlutterPlatform.instance + .isBackgroundEffectSupported() ?? + false; + } + + Future applyBackgroundBlurFilter(BlurIntensity blurIntensity) async { + if (!(await isSupported())) { + return; + } + + if (!isBlurRegistered) { + await StreamVideoFlutterPlatform.instance.registerBlurEffectProcessors(); + isBlurRegistered = true; + } + + final trackId = await _getTrackId(); + if (trackId == null) { + return; + } + + await rtc.setVideoEffects( + trackId, + names: [ + blurIntensity.name, + ], + ); + + currentEffect = blurIntensity.name; + } + + Future applyBackgroundImageFilter(String imageUrl) async { + if (!(await isSupported())) { + return; + } + + if (!isImageRegistered.containsKey(imageUrl)) { + await StreamVideoFlutterPlatform.instance.registerImageEffectProcessors( + backgroundImageUrl: imageUrl, + ); + isImageRegistered[imageUrl] = true; + } + + final trackId = await _getTrackId(); + if (trackId == null) { + return; + } + + final effectName = 'VirtualBackground-$imageUrl'; + + await rtc.setVideoEffects( + trackId, + names: [ + effectName, + ], + ); + + currentEffect = effectName; + } + + Future disableAllFilters() async { + if (!(await isSupported())) { + return; + } + + final trackId = await _getTrackId(); + if (trackId == null) { + return; + } + + await rtc.setVideoEffects( + trackId, + names: [], + ); + + currentEffect = null; + } + + Future _getTrackId() async { + final trackPrefix = call.state.value.localParticipant?.trackIdPrefix; + if (trackPrefix == null) { + _logger.e( + () => + 'Could not apply background image filter, trackPrefix is null for localParticipant', + ); + return null; + } + + final track = call.getTrack(trackPrefix, SfuTrackType.video); + final trackId = track?.mediaTrack.id; + + if (trackId == null) { + _logger.e( + () => + 'Could not apply background image filter, could not find video track for localParticipant', + ); + return null; + } + + return trackId; + } +} diff --git a/packages/stream_video_flutter/lib/stream_video_flutter.dart b/packages/stream_video_flutter/lib/stream_video_flutter.dart index 98cc5704..04997d60 100644 --- a/packages/stream_video_flutter/lib/stream_video_flutter.dart +++ b/packages/stream_video_flutter/lib/stream_video_flutter.dart @@ -41,6 +41,7 @@ export 'src/models/stream_icon_toggle.dart'; export 'src/renderer/video_renderer.dart'; export 'src/theme/themes.dart'; export 'src/utils/device_segmentation.dart'; +export 'src/video_effects/video_effects_manager.dart'; export 'src/widgets/floating_view/floating_view_alignment.dart'; export 'src/widgets/floating_view/floating_view_container.dart'; export 'src/widgets/size_change_listener.dart'; diff --git a/packages/stream_video_flutter/lib/stream_video_flutter_method_channel.dart b/packages/stream_video_flutter/lib/stream_video_flutter_method_channel.dart index 0714124e..b3b73907 100644 --- a/packages/stream_video_flutter/lib/stream_video_flutter_method_channel.dart +++ b/packages/stream_video_flutter/lib/stream_video_flutter_method_channel.dart @@ -116,4 +116,27 @@ class MethodChannelStreamVideoFlutter extends StreamVideoFlutterPlatform { 'disablePictureInPictureMode', ); } + + @override + Future isBackgroundEffectSupported() async { + return methodChannel.invokeMethod( + 'isBackgroundEffectSupported', + ); + } + + @override + Future registerBlurEffectProcessors() { + return methodChannel.invokeMethod( + 'registerBlurEffectProcessors', + ); + } + + @override + Future registerImageEffectProcessors({ + required String backgroundImageUrl, + }) { + return methodChannel.invokeMethod('registerImageEffectProcessors', { + 'backgroundImageUrl': backgroundImageUrl, + }); + } } diff --git a/packages/stream_video_flutter/lib/stream_video_flutter_platform_interface.dart b/packages/stream_video_flutter/lib/stream_video_flutter_platform_interface.dart index 079ccea7..f8518ff8 100644 --- a/packages/stream_video_flutter/lib/stream_video_flutter_platform_interface.dart +++ b/packages/stream_video_flutter/lib/stream_video_flutter_platform_interface.dart @@ -62,4 +62,24 @@ abstract class StreamVideoFlutterPlatform extends PlatformInterface { Future setPictureInPictureEnabled({required bool enable}) { throw UnimplementedError('showPictureInPicture has not been implemented.'); } + + Future isBackgroundEffectSupported() { + throw UnimplementedError( + 'isBackgroundEffectSupported has not been implemented.', + ); + } + + Future registerBlurEffectProcessors() { + throw UnimplementedError( + 'registerBlurEffectProcessors has not been implemented.', + ); + } + + Future registerImageEffectProcessors({ + required String backgroundImageUrl, + }) { + throw UnimplementedError( + 'registerImageEffectProcessors has not been implemented.', + ); + } }