Unify LiveKit and Matrix connection states

package.json

@@ -54,7 +54,7 @@
     "@opentelemetry/sdk-trace-base": "^2.0.0",
     "@opentelemetry/sdk-trace-web": "^2.0.0",
     "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@playwright/test": "^1.56.1",
+    "@playwright/test": "^1.57.0",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-slider": "^1.1.2",
     "@radix-ui/react-visually-hidden": "^1.0.3",

playwright/fixtures/widget-user.ts

@@ -111,19 +111,27 @@ async function registerUser(
   await page.getByRole("textbox", { name: "Confirm password" }).click();
   await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD);
   await page.getByRole("button", { name: "Register" }).click();
-  const continueButton = page.getByRole("button", { name: "Continue" });
-  try {
-    await expect(continueButton).toBeVisible({ timeout: 5000 });
-    await page
-      .getByRole("textbox", { name: "Password", exact: true })
-      .fill(PASSWORD);
-    await continueButton.click();
-  } catch {
-    // continueButton not visible, continue as normal
-  }
+
   await expect(
     page.getByRole("heading", { name: `Welcome ${username}` }),
   ).toBeVisible();
+
+  const browserUnsupportedToast = page
+    .getByText("Element does not support this browser")
+    .locator("..")
+    .locator("..");
+
+  // Dismiss incompatible browser toast
+  const dismissButton = browserUnsupportedToast.getByRole("button", {
+    name: "Dismiss",
+  });
+  try {
+    await expect(dismissButton).toBeVisible({ timeout: 700 });
+    await dismissButton.click();
+  } catch {
+    // dismissButton not visible, continue as normal
+  }
+
   await setDevToolElementCallDevUrl(page);
 
   const clientHandle = await page.evaluateHandle(() =>

src/room/GroupCallView.tsx

@@ -160,6 +160,7 @@ export const GroupCallView: FC<Props> = ({
   }, [rtcSession]);
 
   // TODO move this into the callViewModel LocalMembership.ts
+  // We might actually not need this at all. Since we get into fatalError on those errors already?
   useTypedEventEmitter(
     rtcSession,
     MatrixRTCSessionEvent.MembershipManagerError,
@@ -313,6 +314,7 @@ export const GroupCallView: FC<Props> = ({
 
   const navigate = useNavigate();
 
+  // TODO split this into leave and onDisconnect
   const onLeft = useCallback(
     (
       reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",

src/room/InCallView.tsx

@@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
 import classNames from "classnames";
 import { BehaviorSubject, map } from "rxjs";
 import { useObservable } from "observable-hooks";
-import { logger } from "matrix-js-sdk/lib/logger";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import {
   VoiceCallSolidIcon,
   VolumeOnSolidIcon,
@@ -109,6 +109,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t
 import { type Layout } from "../state/layout-types.ts";
 import { ObservableScope } from "../state/ObservableScope.ts";
 
+const logger = rootLogger.getChild("[InCallView]");
+
 const maxTapDurationMs = 400;
 
 export interface ActiveCallProps
@@ -127,6 +129,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const mediaDevices = useMediaDevices();
   const trackProcessorState$ = useTrackProcessorObservable$();
   useEffect(() => {
+    logger.info("START CALL VIEW SCOPE");
     const scope = new ObservableScope();
     const reactionsReader = new ReactionsReader(scope, props.rtcSession);
     const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
@@ -151,7 +154,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     setVm(vm);
 
     vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
+
     return (): void => {
+      logger.info("END CALL VIEW SCOPE");
       scope.end();
     };
   }, [
@@ -270,7 +275,10 @@ export const InCallView: FC<InCallViewProps> = ({
   const ringOverlay = useBehavior(vm.ringOverlay$);
   const fatalCallError = useBehavior(vm.fatalError$);
   // Stop the rendering and throw for the error boundary
-  if (fatalCallError) throw fatalCallError;
+  if (fatalCallError) {
+    logger.debug("fatalCallError stop rendering", fatalCallError);
+    throw fatalCallError;
+  }
 
   // We need to set the proper timings on the animation based upon the sound length.
   const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;

src/room/LobbyView.tsx

@@ -79,9 +79,9 @@ export const LobbyView: FC<Props> = ({
   waitingForInvite,
 }) => {
   useEffect(() => {
-    logger.info("[Lifecycle] GroupCallView Component mounted");
+    logger.info("[Lifecycle] LobbyView Component mounted");
     return (): void => {
-      logger.info("[Lifecycle] GroupCallView Component unmounted");
+      logger.info("[Lifecycle] LobbyView Component unmounted");
     };
   }, []);
 

src/state/CallViewModel/CallViewModel.ts

@@ -15,6 +15,7 @@ import {
 } from "livekit-client";
 import { type Room as MatrixRoom } from "matrix-js-sdk";
 import {
+  catchError,
   combineLatest,
   distinctUntilChanged,
   filter,
@@ -93,14 +94,14 @@ import {
   type SpotlightLandscapeLayoutMedia,
   type SpotlightPortraitLayoutMedia,
 } from "../layout-types.ts";
-import { type ElementCallError } from "../../utils/errors.ts";
+import { ElementCallError } from "../../utils/errors.ts";
 import { type ObservableScope } from "../ObservableScope.ts";
 import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
 import {
   createLocalMembership$,
   enterRTCSession,
-  RTCBackendState,
-} from "./localMember/LocalMembership.ts";
+  TransportState,
+} from "./localMember/LocalMember.ts";
 import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
 import {
   createMemberships$,
@@ -425,7 +426,18 @@ export function createCallViewModel$(
     connectionFactory: connectionFactory,
     inputTransports$: scope.behavior(
       combineLatest(
-        [localTransport$, membershipsAndTransports.transports$],
+        [
+          localTransport$.pipe(
+            catchError((e: unknown) => {
+              logger.info(
+                "dont pass local transport to createConnectionManager$. localTransport$ threw an error",
+                e,
+              );
+              return of(null);
+            }),
+          ),
+          membershipsAndTransports.transports$,
+        ],
         (localTransport, transports) => {
           const localTransportAsArray = localTransport ? [localTransport] : [];
           return transports.mapInner((transports) => [
@@ -457,13 +469,13 @@ export function createCallViewModel$(
 
   const localMembership = createLocalMembership$({
     scope: scope,
-    homeserverConnected$: createHomeserverConnected$(
+    homeserverConnected: createHomeserverConnected$(
       scope,
       client,
       matrixRTCSession,
     ),
     muteStates: muteStates,
-    joinMatrixRTC: async (transport: LivekitTransport) => {
+    joinMatrixRTC: (transport: LivekitTransport) => {
       return enterRTCSession(
         matrixRTCSession,
         transport,
@@ -578,17 +590,6 @@ export function createCallViewModel$(
     ),
   );
 
-  /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
-   */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
-  const reconnecting$ = localMembership.reconnecting$;
-
   const audioParticipants$ = scope.behavior(
     matrixLivekitMembers$.pipe(
       switchMap((membersWithEpoch) => {
@@ -636,7 +637,7 @@ export function createCallViewModel$(
   );
 
   const handsRaised$ = scope.behavior(
-    handsRaisedSubject$.pipe(pauseWhen(reconnecting$)),
+    handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)),
   );
 
   const reactions$ = scope.behavior(
@@ -649,7 +650,7 @@ export function createCallViewModel$(
           ]),
         ),
       ),
-      pauseWhen(reconnecting$),
+      pauseWhen(localMembership.reconnecting$),
     ),
   );
 
@@ -740,7 +741,7 @@ export function createCallViewModel$(
             livekitRoom$,
             focusUrl$,
             mediaDevices,
-            reconnecting$,
+            localMembership.reconnecting$,
             displayName$,
             matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
             handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
@@ -1423,13 +1424,44 @@ export function createCallViewModel$(
   // reassigned here to make it publicly accessible
   const toggleScreenSharing = localMembership.toggleScreenSharing;
 
+  const errors$ = scope.behavior<{
+    transportError?: ElementCallError;
+    matrixError?: ElementCallError;
+    connectionError?: ElementCallError;
+    publishError?: ElementCallError;
+  } | null>(
+    localMembership.localMemberState$.pipe(
+      map((value) => {
+        const returnObject: {
+          transportError?: ElementCallError;
+          matrixError?: ElementCallError;
+          connectionError?: ElementCallError;
+          publishError?: ElementCallError;
+        } = {};
+        if (value instanceof ElementCallError) return { transportError: value };
+        if (value === TransportState.Waiting) return null;
+        if (value.matrix instanceof ElementCallError)
+          returnObject.matrixError = value.matrix;
+        if (value.media instanceof ElementCallError)
+          returnObject.publishError = value.media;
+        else if (
+          typeof value.media === "object" &&
+          value.media.connection instanceof ElementCallError
+        )
+          returnObject.connectionError = value.media.connection;
+        return returnObject;
+      }),
+    ),
+    null,
+  );
+
   return {
     autoLeave$: autoLeave$,
     callPickupState$: callPickupState$,
     ringOverlay$: ringOverlay$,
     leave$: leave$,
     hangup: (): void => userHangup$.next(),
-    join: localMembership.requestConnect,
+    join: localMembership.requestJoinAndPublish,
     toggleScreenSharing: toggleScreenSharing,
     sharingScreen$: sharingScreen$,
 
@@ -1439,9 +1471,17 @@ export function createCallViewModel$(
     unhoverScreen: (): void => screenUnhover$.next(),
 
     fatalError$: scope.behavior(
-      localMembership.connectionState.livekit$.pipe(
-        filter((v) => v.state === RTCBackendState.Error),
-        map((s) => s.error),
+      errors$.pipe(
+        map((errors) => {
+          logger.debug("errors$ to compute any fatal errors:", errors);
+          return (
+            errors?.transportError ??
+            errors?.matrixError ??
+            errors?.connectionError ??
+            null
+          );
+        }),
+        filter((error) => error !== null),
       ),
       null,
     ),
@@ -1474,7 +1514,7 @@ export function createCallViewModel$(
     showFooter$: showFooter$,
     earpieceMode$: earpieceMode$,
     audioOutputSwitcher$: audioOutputSwitcher$,
-    reconnecting$: reconnecting$,
+    reconnecting$: localMembership.reconnecting$,
   };
 }