HoshinoKoji/psychojs
1
0
Fork 0
mirror of https://github.com/psychopy/psychojs.git synced 2025-05-10 10:40:54 +00:00
Code Actions Packages Releases Activity

Fix for showing videos on iphones. ServerManager downloads videos and saves them as PIXI.Texture. MovieStim prepared to work with texture resource.

Browse Source
This commit is contained in:
lgtst 2023-02-21 18:06:16 +00:00
parent ec3283bb7b
commit 4165d81e0c
2 changed files with 115 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/core/ServerManager.js
View File

@ -16,6 +16,7 @@ import { PsychObject } from "../util/PsychObject.js";
import * as util from "../util/Util.js";
import { Scheduler } from "../util/Scheduler.js";
import { PsychoJS } from "./PsychoJS.js";
import * as PIXI from "pixi.js-legacy";
/**
* <p>This manager handles all communications between the experiment running in the participant's browser and the
@ -1182,6 +1183,27 @@ export class ServerManager extends PsychObject
});
}
/**
* Check if all the resources were loaded and if so set the READY status and emit the DOWNLOAD_COMPLETED event.
*
* @protected
* @returns boolean - if downloading is done or not.
*/
_checkIfDownloadingIsDone (resourcesTotal)
{
if (this._nbLoadedResources === resourcesTotal)
{
this.setStatus(ServerManager.Status.READY);
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.DOWNLOAD_COMPLETED,
});
return true;
}
return false;
}
/**
* Download the specified resources.
*
@ -1214,8 +1236,9 @@ export class ServerManager extends PsychObject
const surveyModelResources = [];
for (const name of resources)
{
const nameParts = name.toLowerCase().split(".");
const extension = (nameParts.length > 1) ? nameParts.pop() : undefined;
const pathStatusData = this._resources.get(name);
const pathParts = pathStatusData.path.toLowerCase().split(".");
const extension = (pathParts.length > 1) ? pathParts.pop() : undefined;
// warn the user if the resource does not have any extension:
if (typeof extension === "undefined")
@ -1223,7 +1246,6 @@ export class ServerManager extends PsychObject
this.psychoJS.logger.warn(`"${name}" does not appear to have an extension, which may negatively impact its loading. We highly recommend you add an extension.`);
}
const pathStatusData = this._resources.get(name);
if (typeof pathStatusData === "undefined")
{
throw Object.assign(response, { error: name + " has not been previously registered" });
@ -1233,9 +1255,6 @@ export class ServerManager extends PsychObject
throw Object.assign(response, { error: name + " is already downloaded or is currently already downloading" });
}
const pathParts = pathStatusData.path.toLowerCase().split(".");
const pathExtension = (pathParts.length > 1) ? pathParts.pop() : undefined;
// preload.js with forced binary:
if (["csv", "odp", "xls", "xlsx", "json"].indexOf(extension) > -1)
{
@ -1265,7 +1284,7 @@ export class ServerManager extends PsychObject
}
// font files:
else if (["ttf", "otf", "woff", "woff2"].indexOf(pathExtension) > -1)
else if (["ttf", "otf", "woff", "woff2"].indexOf(extension) > -1)
{
fontResources.push(name);
}
@ -1276,6 +1295,37 @@ export class ServerManager extends PsychObject
surveyModelResources.push(name);
}
// Videos (compatible with PIXI):
else if (["mp4", "m4v", "webm", "ogv", "h264", "avi", "mov"].indexOf(extension) > -1)
{
pathStatusData.data = PIXI.Texture.from(
pathStatusData.path,
{
resourceOptions: { autoPlay: false }
}
);
pathStatusData.data.baseTexture.resource.source.addEventListener(
"loadeddata",
() =>
{
pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADED;
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.RESOURCE_DOWNLOADED,
resource: name,
});
this._nbLoadedResources++;
this._checkIfDownloadingIsDone(resources.size);
});
pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADING;
this.emit(ServerManager.Event.RESOURCE, {
message: ServerManager.Event.DOWNLOADING_RESOURCE,
resource: name,
});
}
// all other extensions handled by preload.js (download type decided by preload.js):
else
{

81
src/visual/MovieStim.js
View File

@ -57,6 +57,8 @@ export class MovieStim extends VisualStim
this.psychoJS.logger.debug("create a new MovieStim with name: ", name);
this._pixiTextureResource = undefined;
// movie and movie control:
this._addAttribute(
"movie",
@ -149,6 +151,9 @@ export class MovieStim extends VisualStim
try
{
let htmlVideo = undefined;
this._pixiTextureResource = undefined;
// movie is undefined: that's fine but we raise a warning in case this is
// a symptom of an actual problem
if (typeof movie === "undefined")
@ -160,19 +165,24 @@ export class MovieStim extends VisualStim
else
{
// if movie is a string, then it should be the name of a resource, which we get:
let videoResource;
// If movie is a string, then it should be the name of a resource, which we get:
if (typeof movie === "string")
{
movie = this.psychoJS.serverManager.getResource(movie);
videoResource = this.psychoJS.serverManager.getResource(movie);
}
// if movie is an instance of camera, get a video element from it:
// If movie is a HTMLVideoElement, pass it as is:
else if (movie instanceof HTMLVideoElement)
{
videoResource = movie;
}
// If movie is an instance of camera, get a video element from it:
else if (movie instanceof Camera)
{
// old behaviour: feeding a Camera to MovieStim plays the live stream:
const video = movie.getVideo();
// TODO remove previous one if there is one
movie = video;
videoResource = movie.getVideo();
// TODO remove previous movie one if there is one
/*
// new behaviour: feeding a Camera to MovieStim replays the video previously recorded by the Camera:
@ -181,27 +191,38 @@ export class MovieStim extends VisualStim
*/
}
// check that movie is now an HTMLVideoElement
if (!(movie instanceof HTMLVideoElement))
if (videoResource instanceof HTMLVideoElement)
{
throw `${movie.toString()} is not a video`;
htmlVideo = videoResource;
htmlVideo.playsInline = true;
this._pixiTextureResource = PIXI.Texture.from(htmlVideo, { resourceOptions: { autoPlay: false } });
}
else if (videoResource instanceof PIXI.Texture)
{
htmlVideo = videoResource.baseTexture.resource.source;
this._pixiTextureResource = videoResource;
}
else
{
throw `${videoResource.toString()} is not a HTMLVideoElement nor PIXI.Texture!`;
}
this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${movie.src}, size= ${movie.videoWidth}x${movie.videoHeight}, duration= ${movie.duration}s`);
this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${htmlVideo.src}, size= ${htmlVideo.videoWidth}x${htmlVideo.videoHeight}, duration= ${htmlVideo.duration}s`);
// ensure we have only one onended listener per HTMLVideoElement, since we can have several
// MovieStim with the same underlying HTMLVideoElement
// https://stackoverflow.com/questions/11455515
if (!movie.onended)
// TODO: make it actually work!
if (!htmlVideo.onended)
{
movie.onended = () =>
htmlVideo.onended = () =>
{
this.status = PsychoJS.Status.FINISHED;
};
}
}
this._setAttribute("movie", movie, log);
this._setAttribute("movie", htmlVideo, log);
this._needUpdate = true;
this._needPixiUpdate = true;
}
@ -369,13 +390,24 @@ export class MovieStim extends VisualStim
return;
}
// create a PixiJS video sprite:
this._texture = PIXI.Texture.from(this._movie, { resourceOptions: { autoPlay: this.autoPlay } });
this._pixi = new PIXI.Sprite(this._texture);
// No PIXI.Texture, also return immediately.
if (this._pixiTextureResource === undefined)
{
return;
}
// since _texture.width may not be immedialy available but the rest of the code needs its value
// create a PixiJS video sprite:
this._pixiTextureResource.baseTexture.resource.autoPlay = this._autoPlay;
this._pixi = new PIXI.Sprite(this._pixiTextureResource);
if (this._autoPlay)
{
this._pixiTextureResource.baseTexture.resource.source.play();
}
// since _pixiTextureResource.width may not be immedialy available but the rest of the code needs its value
// we arrange for repeated calls to _updateIfNeeded until we have a width:
if (this._texture.width === 0)
if (this._pixiTextureResource.width === 0)
{
this._needUpdate = true;
this._needPixiUpdate = true;
@ -396,11 +428,14 @@ export class MovieStim extends VisualStim
// set the scale:
const displaySize = this._getDisplaySize();
const size_px = util.to_px(displaySize, this.units, this.win);
const scaleX = size_px[0] / this._texture.width;
const scaleY = size_px[1] / this._texture.height;
const scaleX = size_px[0] / this._pixiTextureResource.width;
const scaleY = size_px[1] / this._pixiTextureResource.height;
this._pixi.scale.x = this.flipHoriz ? -scaleX : scaleX;
this._pixi.scale.y = this.flipVert ? scaleY : -scaleY;
this._pixi.width = size_px[0];
this._pixi.height = size_px[1];
// set the position, rotation, and anchor (movie centered on pos):
this._pixi.position = to_pixiPoint(this.pos, this.units, this.win);
this._pixi.rotation = -this.ori * Math.PI / 180;
@ -424,9 +459,9 @@ export class MovieStim extends VisualStim
if (typeof displaySize === "undefined")
{
// use the size of the texture, if we have access to it:
if (typeof this._texture !== "undefined" && this._texture.width > 0)
if (typeof this._pixiTextureResource !== "undefined" && this._pixiTextureResource.width > 0)
{
const textureSize = [this._texture.width, this._texture.height];
const textureSize = [this._pixiTextureResource.width, this._pixiTextureResource.height];
displaySize = util.to_unit(textureSize, "pix", this.win, this.units);
}
}