Merge branch 'develop'

GetStream · Sep 21, 2023 · 6a22bad · 6a22bad
2 parents 8c67f3e + f8cae48
commit 6a22bad
Show file tree

Hide file tree

Showing 107 changed files with 17,893 additions and 7,165 deletions.
diff --git a/README.md b/README.md
@@ -39,6 +39,13 @@ If you're interested in customizing the UI components for the Video SDK, check o
 <img src="https://github.com/GetStream/stream-video-android/assets/24237865/3cc08121-c8c8-4b71-8a96-0cf33b9f2c68" width="32%"/>
 </p>
 
+## 👔 Sample Projects
+
+You can find sample projects below that demonstrates use cases of Stream Video SDK for Android:
+
+- [Dogfooding](https://github.com/GetStream/stream-video-android/tree/develop/dogfooding): Dogfooding demonstrates Stream Video SDK for Android with modern Android tech stacks, such as Compose, Hilt, and Coroutines.
+- [Meeting Room Compose](https://github.com/GetStream/meeting-room-compose): A real-time meeting room app built with Jetpack Compose to demonstrate video communications.
+
 ## 👩‍💻 Free for Makers 👨‍💻
 
 Stream is free for most side and hobby projects. To qualify, your project/company needs to have < 5 team members and < $10k in monthly revenue. Makers get $100 in monthly credit for video for free.
@@ -116,6 +123,9 @@ Video roadmap and changelog is available [here](https://github.com/GetStream/pro
 ### 0.4.0 milestone
 
 - [X] Screensharing from mobile
+- [ ] Implement Chat overlay for Dogfooding
+- [ ] Add Dogfooding instructions + directs Google Play
+- [ ] Reaction dialog API for Compose
 - [ ] Complete Livestreaming APIs and Tutorials for hosting & watching
 - [ ] Android SDK development.md cleanup (Daniel)
 - [ ] Upgrade to more recent versions of webrtc (Kanat)

diff --git a/app/src/main/kotlin/io/getstream/video/android/app/ui/CallScreen.kt b/app/src/main/kotlin/io/getstream/video/android/app/ui/CallScreen.kt
@@ -21,6 +21,7 @@ import androidx.compose.runtime.Composable
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.tooling.preview.Preview
+import io.getstream.video.android.app.BuildConfig
 import io.getstream.video.android.compose.theme.VideoTheme
 import io.getstream.video.android.compose.ui.components.call.activecall.CallContent
 import io.getstream.video.android.core.Call
@@ -50,6 +51,7 @@ fun CallScreen(
                 else -> Unit
             }
         },
+        enableDiagnostics = BuildConfig.DEBUG,
     )
 }
 

diff --git a/docusaurus/docs/Android/05-ui-cookbook/01-overview.mdx b/docusaurus/docs/Android/05-ui-cookbook/01-overview.mdx
@@ -3,98 +3,125 @@ title: Overview
 description: Overview of the UI cookbook
 ---
 
-Stream UI components are highly customizable and allow you to fully customize styles to your taste. This UI Cookbook will walk you through how to customize each component in your video call.
-
-## UI Cookbook
-
-export const CookbookCardGrid = ({ children }) => (
-  <ol
+export const CookbookCard = ({ title, link, img }) => (
+  <div
     style={{
-      display: "flex",
-      flexWrap: "wrap",
-      gap: "0.5rem",
-      padding: 0,
+      display: 'flex',
+      flexDirection: 'column',
+      justifyContent: 'start',
+      alignItems: 'center',
+      gap: '0.5rem',
     }}
   >
-    {children}
-  </ol>
+    <h4 style={{ margin: '0px' }}>{title}</h4>
+    <a href={link}>
+      <p style={{ margin: '0', padding: '0' }}>
+        <img src={img} />
+      </p>
+    </a>
+  </div>
 );
 
-export const CookbookCard = ({ title, children }) => (
-  <li
-    style={{
-      display: "flex",
-      flexDirection: "column",
-      margin: "0 0.5rem",
-      listStyleType: "none",
-      color: "#d3d3d3",
-      width: "250px",
-      height: "200px",
-    }}
-  >
-    <h4>{title}</h4>
-    {children}
-  </li>
-);
+import CallControls from '../assets/cookbook/replacing-call-controls.png';
+import Label from '../assets/cookbook/removing-label-and-indicators.png';
+import VideoLayout from '../assets/cookbook/custom-video-layout.png';
+import IncomingCall from '../assets/cookbook/incoming-call.png';
+import LobbyPreview from '../assets/cookbook/lobby-preview.png';
+import VideoFallback from '../assets/cookbook/no-video-fallback-avatar.png';
+
+import PermissionRequests from '../assets/cookbook/permission-requests.png';
+import VolumeIndicator from '../assets/cookbook/audio-volume-indicator.png';
+import Reactions from '../assets/cookbook/reactions.png';
+import WatchingLivestream from '../assets/cookbook/livestream-live-label.png';
 
-This cookbook aims to show you how to build your own UI elements for video calling.
+import ConnectionQuality from '../assets/cookbook/network-quality.png';
+import SpeakingWhileMuted from '../assets/cookbook/speaking-while-muted.png';
+import ConnectionWarning from '../assets/cookbook/connection-unstable.png';
+
+Stream UI components are highly customizable and allow you to fully customize styles to your taste. This UI Cookbook will walk you through how to customize each component in your video call.
 
 ### Video Calls & Ringing
 
-<div>
-  <CookbookCardGrid>
-    <CookbookCard title="Replacing Call Controls">
-      <img src={require("../assets/cookbook/replacing-call-controls.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Custom label &amp; indicators">
-      <img src={require("../assets/cookbook/removing-label-and-indicators.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Custom Video Layout">
-      <img src={require("../assets/cookbook/custom-video-layout.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Incoming Call">
-      <img src={require("../assets/cookbook/incoming-call.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Lobby/Preview">
-      <img src={require("../assets/cookbook/lobby-preview.png").default} />
-    </CookbookCard>
-    <CookbookCard title="No-video fallback">
-      <img src={require("../assets/cookbook/no-video-fallback-avatar.png").default} />
-    </CookbookCard>
-  </CookbookCardGrid>
+<div
+  style={{ display: 'grid', gridTemplateColumns: '1fr 1fr 1fr', gap: '1.2rem' }}
+>
+  <CookbookCard
+    title="Replacing Control Actions"
+    link="../control-actions"
+    img={CallControls}
+  ></CookbookCard>
+  <CookbookCard
+    title="Custom label &amp; indicators"
+    link="../participant-label"
+    img={Label}
+  ></CookbookCard>
+  <CookbookCard
+    title="Video Renderer"
+    link="../video-renderer"
+    img={VideoLayout}
+  ></CookbookCard>
+  <CookbookCard
+    title="Incoming Call &amp; Outging Call"
+    link="../incoming-and-outgoing-call"
+    img={VideoLayout}
+  ></CookbookCard>
+  <CookbookCard
+    title="Call Lobby"
+    link="../lobby-preview"
+    img={LobbyPreview}
+  ></CookbookCard>
+  <CookbookCard
+    title="Video Fallback"
+    link="../video-fallback"
+    img={VideoFallback}
+  ></CookbookCard>
 </div>
 
 ### Audio rooms & Livestreams
 
-<div>
-  <CookbookCardGrid>
-    <CookbookCard title="Permission Requests">
-      <img src={require("../assets/cookbook/permission-requests.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Audio volume Indicator">
-      <img src={require("../assets/cookbook/audio-volume-indicator.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Reactions">
-      <img src={require("../assets/cookbook/reactions.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Watching a Livestream">
-      <img src={require("../assets/cookbook/reactions.png").default} />
-    </CookbookCard>
-  </CookbookCardGrid>
+<div
+  style={{ display: 'grid', gridTemplateColumns: '1fr 1fr 1fr', gap: '1.2rem' }}
+>
+  <CookbookCard
+    title="Permission Requests"
+    link="../permission-requests"
+    img={PermissionRequests}
+  ></CookbookCard>
+  <CookbookCard
+    title="Audio volume Indicator"
+    link="../audio-volume-indicator"
+    img={VolumeIndicator}
+  ></CookbookCard>
+  <CookbookCard
+    title="Reactions"
+    link="../reactions"
+    img={Reactions}
+  ></CookbookCard>
+  <CookbookCard
+    title="Watching a Livestream"
+    link="../watching-livestream"
+    img={WatchingLivestream}
+  ></CookbookCard>
 </div>
 
 ### Small Components
 
-<div>
-  <CookbookCardGrid>
-    <CookbookCard title="Network Quality Indicator">
-      <img src={require("../assets/cookbook/network-quality.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Speaking While Muted">
-      <img src={require("../assets/cookbook/speaking-while-muted.png").default} />
-    </CookbookCard>
-    <CookbookCard title="Connection Unstable">
-      <img src={require("../assets/cookbook/connection-unstable.png").default} />
-    </CookbookCard>
-  </CookbookCardGrid>
-</div>
+<div
+  style={{ display: 'grid', gridTemplateColumns: '1fr 1fr 1fr', gap: '1.2rem' }}
+>
+  <CookbookCard
+    title="Network Quality Indicator"
+    link="../network-quality-indicator"
+    img={PermissionRequests}
+  ></CookbookCard>
+  <CookbookCard
+    title="Speaking While Muted"
+    link="../speaking-while-muted"
+    img={VolumeIndicator}
+  ></CookbookCard>
+  <CookbookCard
+    title="Connection Unstable"
+    link="../network-quality-indicator"
+    img={Reactions}
+  ></CookbookCard>
+</div>
diff --git a/docusaurus/docs/Android/06-advanced/05-apply-video-filters.mdx b/docusaurus/docs/Android/06-advanced/05-apply-video-filters.mdx
@@ -3,9 +3,132 @@ title: Video & Audio filters
 description: How to build video or audio filters
 ---
 
-## Apply Custom Video Filters
+## Video Filters
 
-// TODO - cover how to apply custom filters, where they live/exist and some common examples
+Some calling apps allow filters to be applied to the current user's video, such as blurring the background, adding AR elements (glasses, moustaches, etc) or applying image filters (such as sepia, bloom etc). StreamVideo's Android SDK has support for injecting your custom filter into the calling experience.
 
-// sepia
-// grayscale
+How does this work? You can inject a filter through `Call.videoFilter`, you will receive each frame of the user's local video as `Bitmap`, allowing you to apply the filters (by mutating the `Bitmap`). This way you have complete freedom over the processing pipeline.
+
+## Adding a Video Filter
+
+Create a `BitmapVideoFilter` or `RawVideoFilter` instance in your project. Here is how the abstract classes look like:
+
+```kotlin
+abstract class BitmapVideoFilter : VideoFilter() {
+    fun filter(bitmap: Bitmap)
+}
+
+abstract class RawVideoFilter : VideoFilter() {
+    abstract fun filter(videoFrame: VideoFrame, surfaceTextureHelper: SurfaceTextureHelper): VideoFrame
+}
+```
+The `BitmapVideoFilter` is a simpler filter that gives you a `Bitmap` which you can then manipulate directly. But it's less performant than using the `RawVideoFilter` which gives you direct access to `VideoFrame` from WebRTC and there is no overhead compared to `BitmapVideoFilter` (like YUV <-> ARGB conversions).
+
+And then set the video filter into `Call.videoFilter`.
+
+We can create and set a simple black and white filter like this:
+```kotlin
+call.videoFilter = object: BitmapVideoFilter() {
+    override fun filter(bitmap: Bitmap) {
+        val c = Canvas(bitmap)
+        val paint = Paint()
+        val cm = ColorMatrix()
+        cm.setSaturation(0f)
+        val f = ColorMatrixColorFilter(cm)
+        paint.colorFilter = f
+        c.drawBitmap(bitmap, 0f, 0f, paint)
+    }
+}
+```
+
+:::note
+You need to manipulate the original bitmap instance to apply the filters. You can of course create a new bitmap in the process, but you need to then draw it on the `bitmap` instance you get in the `filter` callback
+:::
+
+## Adding AI Filters
+
+In some cases, you might also want to apply AI filters. That can be an addition to the user's face (glasses, moustaches, etc), or an ML filter. In this section this use-case will be covered. Specifically, you will use the [Selfie Segmentation](https://developers.google.com/ml-kit/vision/selfie-segmentation/android) from Google's ML kit to change the background behind you.
+
+First include the necessary dependency (check for latest version [here](https://developers.google.com/ml-kit/vision/selfie-segmentation/android#before_you_begin)
+
+```gradle
+dependencies {
+  implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta4'
+}
+```
+
+Create a class that will hold your custom filter. The initialisation of the `Segmentation` class is done according to the [official docs](https://developers.google.com/ml-kit/vision/selfie-segmentation/android#enable_raw_size_mask).
+
+```kotlin
+class SelfieSegmentation {
+
+    private val options =
+        SelfieSegmenterOptions.Builder()
+            .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
+            .enableRawSizeMask()
+            .build()
+    private val segmenter = Segmentation.getClient(options)
+
+    fun applyFilter(bitmap: Bitmap) {
+        // Send the bitmap into ML Kit for processing
+        val mlImage = InputImage.fromBitmap(bitmap, 0)
+        val task = segmenter.process(mlImage)
+        // Wait for result synchronously on same thread
+        val mask = Tasks.await(task)
+
+        val isRawSizeMaskEnabled =  mask.width != bitmap.width || mask.height != bitmap.height
+        val scaleX = bitmap.width * 1f / mask.width
+        val scaleY = bitmap.height * 1f / mask.height
+
+        // Create a bitmap mask to cover the background
+        val maskBitmap = Bitmap.createBitmap(
+            maskColorsFromByteBuffer(mask), mask.width, mask.height, Bitmap.Config.ARGB_8888
+        )
+        // Create a canvas from the frame bitmap
+        val canvas = Canvas(bitmap)
+        val matrix = Matrix()
+        if (isRawSizeMaskEnabled) {
+            matrix.preScale(scaleX, scaleY)
+        }
+        // And now draw the bitmap mask onto the original bitmap
+        canvas.drawBitmap(maskBitmap, matrix, null)
+
+        maskBitmap.recycle()
+    }
+
+    private fun maskColorsFromByteBuffer(mask: SegmentationMask): IntArray {
+        val colors = IntArray(mask.width * mask.height)
+        for (i in 0 until mask.width * mask.height) {
+            val backgroundLikelihood = 1 - mask.buffer.float
+            if (backgroundLikelihood > 0.9) {
+                colors[i] = Color.argb(128, 0, 0, 255)
+            } else if (backgroundLikelihood > 0.2) {
+                val alpha = (182.9 * backgroundLikelihood - 36.6 + 0.5).toInt()
+                colors[i] = Color.argb(alpha, 0, 0, 255)
+            }
+        }
+        return colors
+    }
+}
+```
+
+And now set the custom filter into our SDK:
+
+```kotlin
+call.videoFilter = object: BitmapVideoFilter() {
+
+val selfieFilter = SelfieSegmentation()
+
+    override fun filter(bitmap: Bitmap) {
+        selfieFilter.applyFilter(bitmap)
+    }
+}
+```
+
+The result:
+
+![Stream Filter](../assets/screenshot_video_filter.png)
+
+## Audio Filters
+
+TODO
diff --git a/docusaurus/docs/Android/assets/screenshot_video_filter.png b/docusaurus/docs/Android/assets/screenshot_video_filter.png