Merge pull request #6070 from mozilla/audio-image

Add media audio image

src/bit-components.js

@@ -183,6 +183,10 @@ export const MediaVideo = defineComponent({
   autoPlay: Types.ui8,
   ratio: Types.f32
 });
+/**
+ * @type {Map<EntityId, HTMLVideoElement}>}
+ */
+export const MediaVideoData = new Map();
 export const MixerAnimatableInitialize = defineComponent({});
 export const MixerAnimatable = defineComponent({});
 /**

src/bit-systems/audio-emitter-system.ts

@@ -1,13 +1,7 @@
 import { addComponent, addEntity, defineQuery, removeComponent } from "bitecs";
-import {
-  PositionalAudio,
-  Audio as StereoAudio,
-  AudioListener as ThreeAudioListener,
-  MeshStandardMaterial,
-  Mesh
-} from "three";
+import { PositionalAudio, Audio as StereoAudio, AudioListener as ThreeAudioListener } from "three";
 import { HubsWorld } from "../app";
-import { AudioEmitter, AudioSettingsChanged } from "../bit-components";
+import { AudioEmitter, AudioSettingsChanged, MediaVideoData } from "../bit-components";
 import { AudioType, SourceType } from "../components/audio-params";
 import { AudioSystem } from "../systems/audio-system";
 import { applySettings, getCurrentAudioSettings, updateAudioSettings } from "../update-audio-settings";
@@ -71,8 +65,7 @@ export function makeAudioEntity(world: HubsWorld, source: number, sourceType: So
   }
 
   if (sourceType === SourceType.MEDIA_VIDEO) {
-    const videoObj = world.eid2obj.get(source) as Mesh;
-    const video = (videoObj.material as MeshStandardMaterial).map!.image as HTMLVideoElement;
+    const video = MediaVideoData.get(source)!;
     if (video.paused) {
       APP.isAudioPaused.add(eid);
     } else {

src/bit-systems/video-menu-system.ts

@@ -10,6 +10,7 @@ import {
   HeldRemoteRight,
   HoveredRemoteRight,
   MediaVideo,
+  MediaVideoData,
   NetworkedVideo,
   VideoMenu,
   VideoMenuItem
@@ -82,7 +83,7 @@ export function videoMenuSystem(world: HubsWorld, userinput: any) {
     const videoEid = VideoMenu.videoRef[eid];
     if (!videoEid) return;
     const menuObj = world.eid2obj.get(eid)!;
-    const video = (world.eid2obj.get(videoEid) as any).material.map.image as HTMLVideoElement;
+    const video = MediaVideoData.get(videoEid)!;
     const togglePlayVideo = userinput.get(paths.actions.cursor.right.togglePlayVideo);
     if (togglePlayVideo) {
       if (hasComponent(world, NetworkedVideo, videoEid)) {

src/bit-systems/video-system.ts

@@ -1,11 +1,12 @@
 import { addComponent, defineQuery, enterQuery, exitQuery, hasComponent } from "bitecs";
-import { Mesh, MeshStandardMaterial } from "three";
+import { Mesh } from "three";
 import { HubsWorld } from "../app";
 import {
   AudioParams,
   AudioSettingsChanged,
   MediaLoaded,
   MediaVideo,
+  MediaVideoData,
   Networked,
   NetworkedVideo,
   Owned
@@ -31,7 +32,7 @@ const mediaLoadedQuery = enterQuery(mediaLoadStatusQuery);
 export function videoSystem(world: HubsWorld, audioSystem: AudioSystem) {
   mediaVideoEnterQuery(world).forEach(function (videoEid) {
     const videoObj = world.eid2obj.get(videoEid) as Mesh;
-    const video = (videoObj.material as MeshStandardMaterial).map!.image as HTMLVideoElement;
+    const video = MediaVideoData.get(videoEid)!;
     if (MediaVideo.autoPlay[videoEid]) {
       video.play().catch(() => {
         // Need to deal with the fact play() may fail if user has not interacted with browser yet.
@@ -60,6 +61,7 @@ export function videoSystem(world: HubsWorld, audioSystem: AudioSystem) {
     audioParamsEid && APP.audioOverrides.delete(audioParamsEid);
     Emitter2Params.delete(videoEid);
     Emitter2Audio.delete(videoEid);
+    MediaVideoData.delete(videoEid);
   });
 
   networkedVideoEnterQuery(world).forEach(function (eid) {
@@ -69,7 +71,7 @@ export function videoSystem(world: HubsWorld, audioSystem: AudioSystem) {
   });
 
   networkedVideoQuery(world).forEach(function (eid) {
-    const video = (world.eid2obj.get(eid) as any).material.map.image as HTMLVideoElement;
+    const video = MediaVideoData.get(eid)!;
     if (hasComponent(world, Owned, eid)) {
       NetworkedVideo.time[eid] = video.currentTime;
       let flags = 0;

src/inflators/video.js

@@ -2,9 +2,9 @@ import { create360ImageMesh, createImageMesh } from "../utils/create-image-mesh"
 import { addComponent } from "bitecs";
 import { addObject3DComponent } from "../utils/jsx-entity";
 import { ProjectionMode } from "../utils/projection-mode";
-import { MediaVideo } from "../bit-components";
+import { MediaVideo, MediaVideoData } from "../bit-components";
 
-export function inflateVideo(world, eid, { texture, ratio, projection, autoPlay }) {
+export function inflateVideo(world, eid, { texture, ratio, projection, autoPlay, video }) {
   const mesh =
     projection === ProjectionMode.SPHERE_EQUIRECTANGULAR
       ? create360ImageMesh(texture, ratio)
@@ -13,5 +13,6 @@ export function inflateVideo(world, eid, { texture, ratio, projection, autoPlay
   addComponent(world, MediaVideo, eid);
   MediaVideo.autoPlay[eid] = autoPlay ? 1 : 0;
   MediaVideo.ratio[eid] = ratio;
+  MediaVideoData.set(eid, video);
   return eid;
 }

src/textures/HubsVideoTexture.ts

@@ -0,0 +1,69 @@
+import { LinearFilter, Texture } from "three";
+import { Mapping, TextureDataType, TextureFilter, PixelFormat, Wrapping } from "three";
+
+export class HubsVideoTexture extends Texture {
+  isVideoTexture: boolean;
+  wasPaused: boolean;
+  video: HTMLVideoElement;
+
+  constructor(
+    video: HTMLVideoElement,
+    image?: HTMLImageElement,
+    mapping?: Mapping,
+    wrapS?: Wrapping,
+    wrapT?: Wrapping,
+    magFilter?: TextureFilter,
+    minFilter?: TextureFilter,
+    format?: PixelFormat,
+    type?: TextureDataType,
+    anisotropy?: number
+  ) {
+    super(image ? image : video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy);
+
+    this.video = video;
+    if (!image) {
+      this.isVideoTexture = true;
+
+      this.minFilter = minFilter !== undefined ? minFilter : LinearFilter;
+      this.magFilter = magFilter !== undefined ? magFilter : LinearFilter;
+
+      this.generateMipmaps = false;
+
+      const scope = this;
+
+      function updateVideo() {
+        scope.needsUpdate = true;
+        video.requestVideoFrameCallback(updateVideo);
+      }
+
+      if ("requestVideoFrameCallback" in video) {
+        video.requestVideoFrameCallback(updateVideo);
+      }
+    }
+  }
+
+  clone(): any {
+    return new HubsVideoTexture(this.video, !this.isVideoTexture && this.image.data).copy(this);
+  }
+
+  update() {
+    if (this.isVideoTexture) {
+      const video = this.image;
+      const paused = video.paused;
+      const hasVideoFrameCallback = "requestVideoFrameCallback" in video;
+
+      // Don't transfer textures from paused videos.
+      if (paused && this.wasPaused) return;
+
+      if (hasVideoFrameCallback === false && video.readyState >= video.HAVE_CURRENT_DATA) {
+        if (paused) {
+          this.wasPaused = true;
+        } else if (this.wasPaused) {
+          this.wasPaused = false;
+        }
+
+        this.needsUpdate = true;
+      }
+    }
+  }
+}

src/utils/jsx-entity.ts

@@ -66,7 +66,7 @@ import {
 import { inflateSpawnpoint, inflateWaypoint, WaypointParams } from "../inflators/waypoint";
 import { inflateReflectionProbe, ReflectionProbeParams } from "../inflators/reflection-probe";
 import { HubsWorld } from "../app";
-import { Group, Material, Object3D, Texture, VideoTexture } from "three";
+import { Group, Material, Object3D, Texture } from "three";
 import { AlphaMode } from "./create-image-mesh";
 import { MediaLoaderParams } from "../inflators/media-loader";
 import { preload } from "./preload";
@@ -96,6 +96,7 @@ import { BoxColliderParams, inflateBoxCollider } from "../inflators/box-collider
 import { inflateTrimesh } from "../inflators/trimesh";
 import { HeightFieldParams, inflateHeightField } from "../inflators/heightfield";
 import { inflateAudioSettings } from "../inflators/audio-settings";
+import { HubsVideoTexture } from "../textures/HubsVideoTexture";
 
 preload(
   new Promise(resolve => {
@@ -275,10 +276,11 @@ export interface JSXComponentData extends ComponentData {
     cacheKey: string;
   };
   video?: {
-    texture: VideoTexture;
+    texture: HubsVideoTexture;
     ratio: number;
     projection: ProjectionMode;
     autoPlay: boolean;
+    video: HTMLVideoElement;
   };
   networkedVideo?: true;
   videoMenu?: {

src/utils/load-audio-texture.ts

@@ -1,14 +1,26 @@
-import { VideoTexture } from "three";
 import { createVideoOrAudioEl } from "../utils/media-utils";
+import audioIcon from "../assets/images/audio.png";
+import { HubsVideoTexture } from "../textures/HubsVideoTexture";
 
 // TODO: Replace async with function*?
-// TODO: Integrate with loadVideoTexture in load-audio-texture?
-export async function loadAudioTexture(src: string) : Promise<{texture: VideoTexture, ratio: number}> {
+export async function loadAudioTexture(
+  src: string
+): Promise<{ texture: HubsVideoTexture; ratio: number; video: HTMLVideoElement }> {
   const videoEl = createVideoOrAudioEl("video") as HTMLVideoElement;
-  const texture = new VideoTexture(videoEl);
+  const imageEl = new Image();
+  imageEl.src = audioIcon;
+  imageEl.crossOrigin = "anonymous";
+  const texture = new HubsVideoTexture(videoEl, imageEl);
+  imageEl.onload = () => {
+    texture.needsUpdate = true;
+  };
 
   const isReady = () => {
-    return videoEl.readyState > 0;
+    return (
+      videoEl.readyState > 0 &&
+      (texture.image.videoHeight || texture.image.height) &&
+      (texture.image.videoWidth || texture.image.width)
+    );
   };
 
   return new Promise((resolve, reject) => {
@@ -28,10 +40,9 @@ export async function loadAudioTexture(src: string) : Promise<{texture: VideoTex
     const poll = () => {
       if (isReady()) {
         videoEl.onerror = null;
-        // TODO: AudioIcon image must be used to render and
-	//       ratio must be of the AudioIcon. Fix this.
-	//       Also see the comment in utils/load-audio.
-        resolve({ texture, ratio: 3.0 / 4.0 });
+        const height = texture.image.videoHeight || texture.image.height;
+        const width = texture.image.videoWidth || texture.image.width;
+        resolve({ texture, ratio: height / width, video: videoEl });
       } else {
         pollTimeout = setTimeout(poll, 500);
       }

src/utils/load-audio.tsx

@@ -1,18 +1,14 @@
 /** @jsx createElementEntity */
 import { createElementEntity } from "../utils/jsx-entity";
 import { ProjectionMode } from "./projection-mode";
-import { VideoTexture } from "three";
 import { renderAsEntity } from "../utils/jsx-entity";
 import { loadAudioTexture } from "../utils/load-audio-texture";
 import { HubsWorld } from "../app";
+import { HubsVideoTexture } from "../textures/HubsVideoTexture";
 
 export function* loadAudio(world: HubsWorld, url: string) {
-  const { texture, ratio }: { texture: VideoTexture; ratio: number } = yield loadAudioTexture(url);
-
-  // TODO: VideoTexture.image must be content that be played
-  //       in video-system. And it is also used to render.
-  //       It is audio here so the object will be rendered as
-  //       black. Audio icon must be rendered. Fix this.
+  const { texture, ratio, video }: { texture: HubsVideoTexture; ratio: number; video: HTMLVideoElement } =
+    yield loadAudioTexture(url);
 
   return renderAsEntity(
     world,
@@ -27,7 +23,8 @@ export function* loadAudio(world: HubsWorld, url: string) {
         texture,
         ratio,
         autoPlay: true,
-        projection: ProjectionMode.FLAT
+        projection: ProjectionMode.FLAT,
+        video
       }}
     ></entity>
   );

src/utils/load-video-texture.js

@@ -1,8 +1,9 @@
-import { LinearFilter, VideoTexture, sRGBEncoding } from "three";
+import { LinearFilter, sRGBEncoding } from "three";
 import HLS from "hls.js";
 import { DashVideoTexture } from "../textures/DashVideoTexture";
 import { HLSVideoTexture } from "../textures/HLSVideoTexture";
 import { createDashPlayer, createHLSPlayer, createVideoOrAudioEl } from "./media-utils";
+import { HubsVideoTexture } from "../textures/HubsVideoTexture";
 
 export async function loadVideoTexture(src, contentType) {
   const videoEl = createVideoOrAudioEl("video");
@@ -44,7 +45,7 @@ export async function loadVideoTexture(src, contentType) {
     }
 
     if (texture === null) {
-      texture = new VideoTexture(videoEl);
+      texture = new HubsVideoTexture(videoEl);
       videoEl.src = src;
       videoEl.onerror = failLoad;
     }
@@ -61,7 +62,7 @@ export async function loadVideoTexture(src, contentType) {
 
         const height = texture.image.videoHeight || texture.image.height;
         const width = texture.image.videoWidth || texture.image.width;
-        resolve({ texture, audioSourceEl: texture.image, ratio: height / width });
+        resolve({ texture, audioSourceEl: texture.image, ratio: height / width, video: videoEl });
       } else {
         pollTimeout = setTimeout(poll, 500);
       }

src/utils/load-video.tsx

@@ -1,13 +1,14 @@
 /** @jsx createElementEntity */
 import { createElementEntity } from "../utils/jsx-entity";
 import { ProjectionMode } from "./projection-mode";
-import { VideoTexture } from "three";
 import { renderAsEntity } from "../utils/jsx-entity";
 import { loadVideoTexture } from "../utils/load-video-texture";
 import { HubsWorld } from "../app";
+import { HubsVideoTexture } from "../textures/HubsVideoTexture";
 
 export function* loadVideo(world: HubsWorld, url: string, contentType: string) {
-  const { texture, ratio }: { texture: VideoTexture; ratio: number } = yield loadVideoTexture(url, contentType);
+  const { texture, ratio, video }: { texture: HubsVideoTexture; ratio: number; video: HTMLVideoElement } =
+    yield loadVideoTexture(url, contentType);
 
   return renderAsEntity(
     world,
@@ -20,7 +21,8 @@ export function* loadVideo(world: HubsWorld, url: string, contentType: string) {
         texture,
         ratio,
         autoPlay: true,
-        projection: ProjectionMode.FLAT
+        projection: ProjectionMode.FLAT,
+        video
       }}
     ></entity>
   );