Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[wip] Iphone video fix #596

Open
wants to merge 3 commits into
base: 2024.2.x
Choose a base branch
from
Open

[wip] Iphone video fix #596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4165d81
Fix for showing videos on iphones. ServerManager downloads videos and…
lightest Feb 21, 2023
328cb54
merge with 2024.2.0
lightest Apr 3, 2024
f42d806
texture instantiation moved back to update method.
lightest Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 57 additions & 7 deletions src/core/ServerManager.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { PsychObject } from "../util/PsychObject.js";
import * as util from "../util/Util.js";
import { Scheduler } from "../util/Scheduler.js";
import { PsychoJS } from "./PsychoJS.js";
import * as PIXI from "pixi.js-legacy";

/**
* <p>This manager handles all communications between the experiment running in the participant's browser and the
Expand Down Expand Up @@ -1263,6 +1264,27 @@ export class ServerManager extends PsychObject
});
}

/**
* Check if all the resources were loaded and if so set the READY status and emit the DOWNLOAD_COMPLETED event.
*
* @protected
* @returns boolean - if downloading is done or not.
*/
_checkIfDownloadingIsDone (resourcesTotal)
{
if (this._nbLoadedResources === resourcesTotal)
{
this.setStatus(ServerManager.Status.READY);
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.DOWNLOAD_COMPLETED,
});

return true;
}

return false;
}

/**
* Download the specified resources.
*
Expand Down Expand Up @@ -1295,16 +1317,16 @@ export class ServerManager extends PsychObject
const surveyModelResources = [];
for (const name of resources)
{
const nameParts = name.toLowerCase().split(".");
const extension = (nameParts.length > 1) ? nameParts.pop() : undefined;
const pathStatusData = this._resources.get(name);
const pathParts = pathStatusData.path.toLowerCase().split(".");
const extension = (pathParts.length > 1) ? pathParts.pop() : undefined;

// warn the user if the resource does not have any extension:
if (typeof extension === "undefined")
{
this.psychoJS.logger.warn(`"${name}" does not appear to have an extension, which may negatively impact its loading. We highly recommend you add an extension.`);
}

const pathStatusData = this._resources.get(name);
if (typeof pathStatusData === "undefined")
{
throw Object.assign(response, { error: name + " has not been previously registered" });
Expand All @@ -1314,9 +1336,6 @@ export class ServerManager extends PsychObject
throw Object.assign(response, { error: name + " is already downloaded or is currently already downloading" });
}

const pathParts = pathStatusData.path.toLowerCase().split(".");
const pathExtension = (pathParts.length > 1) ? pathParts.pop() : undefined;

// preload.js with forced binary:
if (["csv", "odp", "xls", "xlsx", "json", "gif"].indexOf(extension) > -1)
{
Expand Down Expand Up @@ -1346,7 +1365,7 @@ export class ServerManager extends PsychObject
}

// font files:
else if (["ttf", "otf", "woff", "woff2","eot"].indexOf(pathExtension) > -1)
else if (["ttf", "otf", "woff", "woff2", "eot"].indexOf(extension) > -1)
{
fontResources.push(name);
}
Expand All @@ -1357,6 +1376,37 @@ export class ServerManager extends PsychObject
surveyModelResources.push(name);
}

// Videos (compatible with PIXI):
else if (["mp4", "m4v", "webm", "ogv", "h264", "avi", "mov"].indexOf(extension) > -1)
{
pathStatusData.data = PIXI.Texture.from(
pathStatusData.path,
{
resourceOptions: { autoPlay: false }
}
);

pathStatusData.data.baseTexture.resource.source.addEventListener(
"loadeddata",
() =>
{
pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADED;
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.RESOURCE_DOWNLOADED,
resource: name,
});

this._nbLoadedResources++;
this._checkIfDownloadingIsDone(resources.size);
});

pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADING;
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.DOWNLOADING_RESOURCE,
resource: name,
});
}

// all other extensions handled by preload.js (download type decided by preload.js):
else
{
Expand Down
97 changes: 65 additions & 32 deletions src/visual/MovieStim.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ export class MovieStim extends VisualStim

this.psychoJS.logger.debug("create a new MovieStim with name: ", name);

this._pixiTextureResource = undefined;

// Used in case when youtubeUrl parameter is set to a proper youtube url.
this._youtubePlayer = undefined;
this._ytPlayerIsReady = false;
Expand Down Expand Up @@ -196,6 +198,9 @@ export class MovieStim extends VisualStim

try
{
let htmlVideo = undefined;
this._pixiTextureResource = undefined;

// movie is undefined: that's fine but we raise a warning in case this is
// a symptom of an actual problem
if (typeof movie === "undefined")
Expand All @@ -206,47 +211,74 @@ export class MovieStim extends VisualStim
}
else
{
// if movie is a string, then it should be the name of a resource, which we get:
let videoResource;

// If movie is a string, then it should be the name of a resource, which we get:
if (typeof movie === "string")
{
movie = this.psychoJS.serverManager.getResource(movie);
videoResource = this.psychoJS.serverManager.getResource(movie);
}

// if movie is an instance of camera, get a video element from it:
// If movie is a HTMLVideoElement, pass it as is:
else if (movie instanceof HTMLVideoElement)
{
videoResource = movie;
}
// If movie is an instance of camera, get a video element from it:
else if (movie instanceof Camera)
{
// old behaviour: feeding a Camera to MovieStim plays the live stream:
const video = movie.getVideo();
// TODO remove previous one if there is one
movie = video;
videoResource = movie.getVideo();
// TODO remove previous movie if there is one

/*
// new behaviour: feeding a Camera to MovieStim replays the video previously recorded by the Camera:
const video = movie.getRecording();
movie = video;
*/
*/
}

// check that movie is now an HTMLVideoElement
if (!(movie instanceof HTMLVideoElement))
if (videoResource instanceof HTMLVideoElement)
{
throw `${movie.toString()} is not a video`;
htmlVideo = videoResource;
htmlVideo.playsInline = true;
}
else if (videoResource instanceof PIXI.Texture)
{
htmlVideo = videoResource.baseTexture.resource.source;
}
else
{
throw `${videoResource.toString()} is not a HTMLVideoElement nor PIXI.Texture!`;
}

this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${movie.src}, size= ${movie.videoWidth}x${movie.videoHeight}, duration= ${movie.duration}s`);
// Not using PIXI.Texture.from() on purpose, as it caches both PIXI.Texture and PIXI.BaseTexture.
// As a result of that we can have multiple MovieStim instances using same PIXI.BaseTexture,
// thus changing texture related properties like interpolation, or calling _pixi.destroy(true)
// will affect all MovieStims which happen to share that BaseTexture.
this._pixiTextureResource = new PIXI.Texture(
new PIXI.BaseTexture(
htmlVideo,
{
resourceOptions: { autoPlay: this.autoPlay }
}
)
);

this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${htmlVideo.src}, size= ${htmlVideo.videoWidth}x${htmlVideo.videoHeight}, duration= ${htmlVideo.duration}s`);

// ensure we have only one onended listener per HTMLVideoElement, since we can have several
// MovieStim with the same underlying HTMLVideoElement
// https://stackoverflow.com/questions/11455515
if (!movie.onended)
// TODO: make it actually work!
if (!htmlVideo.onended)
{
movie.onended = () =>
htmlVideo.onended = () =>
{
this.status = PsychoJS.Status.FINISHED;
};
}

// Resize the stim when video is loaded. Otherwise this._texture.width is 1.
// Resize the stim when video is loaded. Otherwise this._pixiTextureResource.width is 1.
const loadedDataCb = () =>
{
this.size = this._size;
Expand All @@ -261,7 +293,7 @@ export class MovieStim extends VisualStim
this.hideYoutubePlayer();
}

this._setAttribute("movie", movie, log);
this._setAttribute("movie", htmlVideo, log);
this._needUpdate = true;
this._needPixiUpdate = true;
}
Expand Down Expand Up @@ -308,7 +340,7 @@ export class MovieStim extends VisualStim
size = this._ensureNaNSizeConversion(size, this._movie);
}

if (this._texture !== undefined)
if (this._pixiTextureResource !== undefined)
{
this._applySizeToPixi(size);
}
Expand Down Expand Up @@ -779,23 +811,24 @@ export class MovieStim extends VisualStim
return;
}

// Not using PIXI.Texture.from() on purpose, as it caches both PIXI.Texture and PIXI.BaseTexture.
// As a result of that we can have multiple MovieStim instances using same PIXI.BaseTexture,
// thus changing texture related properties like interpolation, or calling _pixi.destroy(true)
// will affect all MovieStims which happen to share that BaseTexture.
this._texture = new PIXI.Texture(new PIXI.BaseTexture(
this._movie,
{
resourceOptions: { autoPlay: this.autoPlay }
}
));
// No PIXI.Texture, also return immediately.
if (this._pixiTextureResource === undefined)
{
return;
}

// create a PixiJS video sprite:
this._pixi = new PIXI.Sprite(this._texture);
this._pixiTextureResource.baseTexture.resource.autoPlay = this._autoPlay;
this._pixi = new PIXI.Sprite(this._pixiTextureResource);

if (this._autoPlay)
{
this._pixiTextureResource.baseTexture.resource.source.play();
}

// since _texture.width may not be immedialy available but the rest of the code needs its value
// since _pixiTextureResource.width may not be immedialy available but the rest of the code needs its value
// we arrange for repeated calls to _updateIfNeeded until we have a width:
if (this._texture.width === 0)
if (this._pixiTextureResource.width === 0)
{
this._needUpdate = true;
this._needPixiUpdate = true;
Expand Down Expand Up @@ -843,9 +876,9 @@ export class MovieStim extends VisualStim
if (typeof displaySize === "undefined")
{
// use the size of the texture, if we have access to it:
if (typeof this._texture !== "undefined" && this._texture.width > 0)
if (typeof this._pixiTextureResource !== "undefined" && this._pixiTextureResource.width > 0)
{
const textureSize = [this._texture.width, this._texture.height];
const textureSize = [this._pixiTextureResource.width, this._pixiTextureResource.height];
displaySize = util.to_unit(textureSize, "pix", this.win, this.units);
}
}
Expand Down