HoshinoKoji/psychojs
1
0
Fork 0
mirror of https://github.com/psychopy/psychojs.git synced 2025-05-10 10:40:54 +00:00
Code Actions Packages Releases Activity

Merge branch 'master' into main

Browse Source
This commit is contained in:
Alain Pitiot 2021-07-23 07:42:29 +02:00 committed by GitHub
commit 7975dd8df0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1257 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "psychojs",
"version": "2021.2.0",
"version": "2021.2.x",
"private": true,
"description": "Helps run in-browser neuroscience, psychology, and psychophysics experiments",
"license": "MIT",

19
src/core/ServerManager.js
View File

@ -506,8 +506,21 @@ export class ServerManager extends PsychObject
}
}
// download those registered resources for which download = true:
/*await*/ this._downloadResources(resourcesToDownload);
// download those registered resources for which download = true
// note: we return a Promise that will be resolved when all the resources are downloaded
return new Promise((resolve, reject) =>
{
const uuid = this.on(ServerManager.Event.RESOURCE, (signal) =>
{
if (signal.message === ServerManager.Event.DOWNLOAD_COMPLETED)
{
this.off(ServerManager.Event.RESOURCE, uuid);
resolve();
}
});
this._downloadResources(resourcesToDownload);
});
}
catch (error)
{
@ -898,7 +911,7 @@ export class ServerManager extends PsychObject
* @protected
* @param {Set} resources - a set of names of previously registered resources
*/
_downloadResources(resources)
async _downloadResources(resources)
{
const response = {
origin: "ServerManager._downloadResources",

315
src/data/QuestHandler.js Normal file
View File

@ -0,0 +1,315 @@
/** @module data */
/**
* Quest Trial Handler
*
* @author Alain Pitiot & Thomas Pronk
* @version 2021.2.0
* @copyright (c) 2017-2020 Ilixa Ltd. (http://ilixa.com) (c) 2020-2021 Open Science Tools Ltd. (https://opensciencetools.org)
* @license Distributed under the terms of the MIT License
*/
import {TrialHandler} from "./TrialHandler";
/**
* <p>A Trial Handler that implements the Quest algorithm for quick measurement of
psychophysical thresholds.</p>
*
* @class
* @extends PsychObject
* @param {Object} options
* @param {module:core.PsychoJS} options.psychoJS - the PsychoJS instance
* @param {number} options.nTrials - maximum number of trials
* @param {module:data.QuestHandler.Method} options.method - the QUEST method
* @param {boolean} [options.autoLog= false] - whether or not to log
*/
export class QuestHandler extends TrialHandler
{
/**
* @constructor
* @public
*/
constructor({
psychoJS,
varName,
startVal,
startValSd,
minVal,
maxVal,
pThreshold,
nTrials,
stopInterval,
method,
beta,
delta,
gamma,
grain,
name,
autoLog
} = {})
{
super({
psychoJS,
name,
autoLog,
method: TrialHandler.Method.SEQUENTIAL,
trialList: Array(nTrials),
nReps: 1
});
this._addAttribute('varName', varName);
this._addAttribute('startVal', startVal);
this._addAttribute('minVal', minVal, Number.MIN_VALUE);
this._addAttribute('maxVal', maxVal, Number.MAX_VALUE);
this._addAttribute('startValSd', startValSd);
this._addAttribute('pThreshold', pThreshold, 0.82);
this._addAttribute('nTrials', nTrials);
this._addAttribute('stopInterval', stopInterval, Number.MIN_VALUE);
this._addAttribute('beta', beta, 3.5);
this._addAttribute('delta', delta, 0.01);
this._addAttribute('gamma', gamma, 0.5);
this._addAttribute('grain', grain, 0.01);
this._addAttribute('method', method, QuestHandler.Method.QUANTILE);
// setup jsQuest:
this._setupJsQuest();
}
/**
* Add a response and update the PDF.
*
* @public
* @param{number} response - the response to the trial, must be either 0 (incorrect,
* non-detected) or 1 (correct, detected).
*/
addResponse(response)
{
// check that response is either 0 or 1:
if (response !== 0 && response !== 1)
{
throw {
origin: 'QuestHandler.addResponse',
context: 'when adding a trial response',
error: `the response must be either 0 or 1, got: ${JSON.stringify(response)}`
};
}
// update the QUEST pdf:
this._jsQuest = jsQUEST.QuestUpdate(this._jsQuest, this._questValue, response);
if (!this._finished)
{
// estimate the next value of the QUEST variable (and update the trial list and snapshots):
this._estimateQuestValue();
}
}
/**
* Simulate a response.
*
* @param{number} trueValue
*/
simulate(trueValue)
{
const response = jsQUEST.QuestSimulate(this._jsQuest, this._questValue, trueValue);
// restrict to limits:
this._questValue = Math.max(this._minVal, Math.min(this._maxVal, this._questValue));
this._psychoJS.logger.debug(`simulated response: ${response}`);
return response;
}
/**
* Get the mean of the Quest posterior PDF.
*
* @returns {number} the mean
*/
mean()
{
return jsQUEST.QuestMean(this._jsQuest);
}
/**
* Get the standard deviation of the Quest posterior PDF.
*
* @returns {number} the standard deviation
*/
sd()
{
return jsQUEST.QuestSd(this._jsQuest);
}
/**
* Get the mode of the Quest posterior PDF.
*
* @returns {number} the mode
*/
mode()
{
const [mode, pdf] = jsQUEST.QuestMode(this._jsQuest);
return mode;
}
/**
* Get the standard deviation of the Quest posterior PDF.
*
* @param{number} quantileOrder the quantile order
* @returns {number} the quantile
*/
quantile(quantileOrder)
{
return jsQUEST.QuestQuantile(this._jsQuest, quantileOrder);
}
/**
* Get an estimate of the 5%-95% confidence interval (CI).
*
* @public
* @param{boolean} [getDifference=false] if true, return the width of the CI instead of the CI
*/
confInterval(getDifference = false)
{
const CI = [
jsQUEST.QuestQuantile(this._jsQuest, 0.05),
jsQUEST.QuestQuantile(this._jsQuest, 0.95)
];
if (getDifference)
{
return Math.abs(CI[0] - CI[1]);
}
else
{
return CI;
}
}
/**
* Setup the JS Quest object.
*
* @protected
*/
_setupJsQuest()
{
this._jsQuest = jsQUEST.QuestCreate(
this._startVal,
this._startValSd,
this._pThreshold,
this._beta,
this._delta,
this._gamma,
this._grain);
this._estimateQuestValue();
}
/**
* Estimate the next value of the QUEST variable, based on the current value
* and on the selected QUEST method.
*
* @protected
*/
_estimateQuestValue()
{
// estimate the value based on the chosen QUEST method:
if (this._method === QuestHandler.Method.QUANTILE)
{
this._questValue = jsQUEST.QuestQuantile(this._jsQuest);
}
else if (this._method === QuestHandler.Method.MEAN)
{
this._questValue = jsQUEST.QuestMean(this._jsQuest);
}
else if (this._method === QuestHandler.Method.MODE)
{
const [mode, pdf] = jsQUEST.QuestMode(this._jsQuest);
this._questValue = mode;
}
else
{
throw {
origin: 'QuestHandler._estimateQuestValue',
context: 'when estimating the next value of the QUEST variable',
error: `unknown method: ${this._method}, please use: mean, mode, or quantile`
};
}
this._psychoJS.logger.debug(`estimated value for QUEST variable ${this._varName}: ${this._questValue}`);
// check whether we should finish the trial:
if (this.thisN > 0 &&
(this.nRemaining === 0 || this.confInterval(true) < this._stopInterval))
{
this._finished = true;
// update the snapshots associated with the current trial in the trial list:
for (let t = 0; t < this._trialList.length-1; ++t)
{
// the current trial is the last defined one:
if (typeof this._trialList[t+1] === 'undefined')
{
this._snapshots[t].finished = true;
break;
}
}
return;
}
// update the next undefined trial in the trial list, and the associated snapshot:
for (let t = 0; t < this._trialList.length; ++t)
{
if (typeof this._trialList[t] === 'undefined')
{
this._trialList[t] = { [this._varName]: this._questValue };
if (typeof this._snapshots[t] !== 'undefined')
{
this._snapshots[t][this._varName] = this._questValue;
this._snapshots[t].trialAttributes.push(this._varName);
}
break;
}
}
}
}
/**
* QuestHandler method
*
* @enum {Symbol}
* @readonly
* @public
*/
QuestHandler.Method = {
/**
* Quantile threshold estimate.
*/
QUANTILE: Symbol.for('QUANTILE'),
/**
* Mean threshold estimate.
*/
MEAN: Symbol.for('MEAN'),
/**
* Mode threshold estimate.
*/
MODE: Symbol.for('MODE')
};

7
src/data/index.js
View File

@ -1,3 +1,4 @@
export * from "./ExperimentHandler.js";
export * from "./TrialHandler.js";
// export * from './Shelf.js';
export * from './ExperimentHandler.js';
export * from './TrialHandler.js';
export * from './QuestHandler';
//export * from './Shelf.js';

2
src/sound/Microphone.js
View File

@ -38,7 +38,7 @@ export class Microphone extends PsychObject
this._addAttribute("format", format, "audio/webm;codecs=opus", this._onChange);
this._addAttribute("sampleRateHz", sampleRateHz, 48000, this._onChange);
this._addAttribute("clock", clock, new Clock());
this._addAttribute("autoLog", false, autoLog);
this._addAttribute("autoLog", autoLog, autoLog);
this._addAttribute("status", PsychoJS.Status.NOT_STARTED);
// prepare the recording:

585
src/visual/Camera.js Normal file
View File

@ -0,0 +1,585 @@
/**
* Manager handling the recording of video signal.
*
* @author Alain Pitiot
* @version 2021.2.0
* @copyright (c) 2021 Open Science Tools Ltd. (https://opensciencetools.org)
* @license Distributed under the terms of the MIT License
*/
import {Clock} from "../util/Clock";
import {PsychObject} from "../util/PsychObject";
import {PsychoJS} from "../core/PsychoJS";
import * as util from '../util/Util';
import {ExperimentHandler} from "../data/ExperimentHandler";
// import {VideoClip} from "./VideoClip";
/**
* <p>This manager handles the recording of video signal.</p>
*
* @name module:visual.Camera
* @class
* @param {Object} options
* @param @param {module:core.Window} options.win - the associated Window
* @param {string} [options.format='video/webm;codecs=vp9'] the video format
* @param {Clock} [options.clock= undefined] - an optional clock
* @param {boolean} [options.autoLog= false] - whether or not to log
*/
export class Camera extends PsychObject
{
constructor({win, name, format, clock, autoLog} = {})
{
super(win._psychoJS);
this._addAttribute('win', win, undefined);
this._addAttribute('name', name, 'camera');
this._addAttribute('format', format, 'video/webm;codecs=vp9', this._onChange);
this._addAttribute('clock', clock, new Clock());
this._addAttribute('autoLog', autoLog, false);
this._addAttribute('status', PsychoJS.Status.NOT_STARTED);
// prepare the recording:
this._prepareRecording();
if (this._autoLog)
{
this._psychoJS.experimentLogger.exp(`Created ${this.name} = ${this.toString()}`);
}
}
/**
* Get the underlying video stream.
*
* @name module:visual.Camera#getStream
* @public
* @returns {MediaStream} the video stream
*/
getStream()
{
return this._stream;
}
/**
* Get a video element pointing to the Camera stream.
*
* @name module:visual.Camera#getVideo
* @public
* @returns {HTMLVideoElement} a video element
*/
getVideo()
{
// note: we need to return a new video each time, since the camera feed can be used by
// several stimuli and one of them might pause the feed
// create a video with the appropriate size:
const video = document.createElement('video');
this._videos.push(video);
video.width = this._streamSettings.width;
video.height = this._streamSettings.height;
video.autoplay = true;
// prevent clicking:
video.onclick = (mouseEvent) =>
{
mouseEvent.preventDefault();
return false;
};
// use the camera stream as source for the video:
video.srcObject = this._stream;
return video;
}
/**
* Submit a request to start the recording.
*
* @name module:visual.Camera#start
* @public
* @return {Promise} promise fulfilled when the recording actually started
*/
start()
{
// if the camera is currently paused, a call to start resumes it
// with a new recording:
if (this._status === PsychoJS.Status.PAUSED)
{
return this.resume({clear: true});
}
if (this._status !== PsychoJS.Status.STARTED)
{
this._psychoJS.logger.debug('request to start video recording');
try
{
if (!this._recorder)
{
throw 'the recorder has not been created yet, possibly because the participant has not given the authorisation to record video';
}
this._recorder.start();
// return a promise, which will be satisfied when the recording actually starts, which
// is also when the reset of the clock and the change of status takes place
const self = this;
return new Promise((resolve, reject) =>
{
self._startCallback = resolve;
self._errorCallback = reject;
});
}
catch (error)
{
this._psychoJS.logger.error('unable to start the video recording: ' + JSON.stringify(error));
this._status = PsychoJS.Status.ERROR;
throw {
origin: 'Camera.start',
context: 'when starting the video recording for camera: ' + this._name,
error
};
}
}
}
/**
* Submit a request to stop the recording.
*
* @name module:visual.Camera#stop
* @public
* @param {Object} options
* @param {string} [options.filename] the name of the file to which the video recording
* will be saved
* @return {Promise} promise fulfilled when the recording actually stopped, and the recorded
* data was made available
*/
stop({filename} = {})
{
if (this._status === PsychoJS.Status.STARTED || this._status === PsychoJS.Status.PAUSED)
{
this._psychoJS.logger.debug('request to stop video recording');
// stop the videos:
for (const video of this._videos)
{
video.pause();
}
this._stopOptions = {
filename
};
// note: calling the stop method of the MediaRecorder will first raise
// a dataavailable event, and then a stop event
// ref: https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder/stop
this._recorder.stop();
// return a promise, which will be satisfied when the recording actually stops and the data
// has been made available:
const self = this;
return new Promise((resolve, reject) =>
{
self._stopCallback = resolve;
self._errorCallback = reject;
});
}
}
/**
* Submit a request to pause the recording.
*
* @name module:visual.Camera#pause
* @public
* @return {Promise} promise fulfilled when the recording actually paused
*/
pause()
{
if (this._status === PsychoJS.Status.STARTED)
{
this._psychoJS.logger.debug('request to pause video recording');
try
{
if (!this._recorder)
{
throw 'the recorder has not been created yet, possibly because the participant has not given the authorisation to record video';
}
// note: calling the pause method of the MediaRecorder raises a pause event
this._recorder.pause();
// return a promise, which will be satisfied when the recording actually pauses:
const self = this;
return new Promise((resolve, reject) =>
{
self._pauseCallback = resolve;
self._errorCallback = reject;
});
}
catch (error)
{
self._psychoJS.logger.error('unable to pause the video recording: ' + JSON.stringify(error));
this._status = PsychoJS.Status.ERROR;
throw {
origin: 'Camera.pause',
context: 'when pausing the video recording for camera: ' + this._name,
error
};
}
}
}
/**
* Submit a request to resume the recording.
*
* <p>resume has no effect if the recording was not previously paused.</p>
*
* @name module:visual.Camera#resume
* @param {Object} options
* @param {boolean} [options.clear= false] whether or not to empty the video buffer before
* resuming the recording
* @return {Promise} promise fulfilled when the recording actually resumed
*/
resume({clear = false } = {})
{
if (this._status === PsychoJS.Status.PAUSED)
{
this._psychoJS.logger.debug('request to resume video recording');
try
{
if (!this._recorder)
{
throw 'the recorder has not been created yet, possibly because the participant has not given the authorisation to record video';
}
// empty the audio buffer is needed:
if (clear)
{
this._audioBuffer = [];
this._videoBuffer.length = 0;
}
this._recorder.resume();
// return a promise, which will be satisfied when the recording actually resumes:
const self = this;
return new Promise((resolve, reject) =>
{
self._resumeCallback = resolve;
self._errorCallback = reject;
});
}
catch (error)
{
self._psychoJS.logger.error('unable to resume the video recording: ' + JSON.stringify(error));
this._status = PsychoJS.Status.ERROR;
throw {
origin: 'Camera.resume',
context: 'when resuming the video recording for camera: ' + this._name,
error
};
}
}
}
/**
* Submit a request to flush the recording.
*
* @name module:visual.Camera#flush
* @public
* @return {Promise} promise fulfilled when the data has actually been made available
*/
flush()
{
if (this._status === PsychoJS.Status.STARTED || this._status === PsychoJS.Status.PAUSED)
{
this._psychoJS.logger.debug('request to flush video recording');
// note: calling the requestData method of the MediaRecorder will raise a
// dataavailable event
// ref: https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder/requestData
this._recorder.requestData();
// return a promise, which will be satisfied when the data has been made available:
const self = this;
return new Promise((resolve, reject) =>
{
self._dataAvailableCallback = resolve;
self._errorCallback = reject;
});
}
}
/**
* Offer the audio recording to the participant as a video file to download.
*
* @name module:visual.Camera#download
* @function
* @public
* @param {string} filename the filename
*/
download(filename = 'video.webm')
{
const videoBlob = new Blob(this._videoBuffer);
const anchor = document.createElement('a');
anchor.href = window.URL.createObjectURL(videoBlob);
anchor.download = filename;
document.body.appendChild(anchor);
anchor.click();
document.body.removeChild(anchor);
}
/**
* Upload the video recording to the pavlovia server.
*
* @name module:visual.Camera#upload
* @function
* @public
* @param {string} tag an optional tag for the audio file
*/
async upload({tag} = {})
{
// default tag: the name of this Camera object
if (typeof tag === 'undefined')
{
tag = this._name;
}
// add a format-dependent video extension to the tag:
tag += util.extensionFromMimeType(this._format);
// if the video recording cannot be uploaded, e.g. the experiment is running locally, or
// if it is piloting mode, then we offer the video recording as a file for download:
if (this._psychoJS.getEnvironment() !== ExperimentHandler.Environment.SERVER ||
this._psychoJS.config.experiment.status !== 'RUNNING' ||
this._psychoJS._serverMsg.has('__pilotToken'))
{
return this.download(tag);
}
// upload the blob:
// TODO uploadAudio -> uploadAudioVideo
const videoBlob = new Blob(this._videoBuffer);
return this._psychoJS.serverManager.uploadAudio(videoBlob, tag);
}
/**
* Get the current video recording as a VideoClip in the given format.
*
* @name module:visual.Camera#getRecording
* @function
* @public
* @param {string} tag an optional tag for the video clip
* @param {boolean} [flush=false] whether or not to first flush the recording
*/
async getRecording({tag, flush = false} = {})
{
// default tag: the name of this Microphone object
if (typeof tag === 'undefined')
{
tag = this._name;
}
// TODO
/*
const videoClip = new VideoClip({
psychoJS: this._psychoJS,
name: tag,
format: this._format,
data: new Blob(this._videoBuffer)
});
return videoClip;
*/
}
/**
* Callback for changes to the recording settings.
*
* <p>Changes to the settings require the recording to stop and be re-started.</p>
*
* @name module:visual.Camera#_onChange
* @function
* @protected
*/
_onChange()
{
if (this._status === PsychoJS.Status.STARTED)
{
this.stop();
}
this._prepareRecording();
this.start();
}
/**
* Prepare the recording.
*
* @name module:visual.Camera#_prepareRecording
* @function
* @protected
*/
async _prepareRecording()
{
// empty the video buffer:
this._videoBuffer = [];
this._recorder = null;
this._videos = [];
// create a new stream with ideal dimensions:
this._stream = await navigator.mediaDevices.getUserMedia({
video: {
width: {
ideal: 1920
},
height: {
ideal: 1080
}
}
});
// check the actual width and height:
this._streamSettings = this._stream.getVideoTracks()[0].getSettings();
this._psychoJS.logger.debug(`camera stream settings: ${JSON.stringify(this._streamSettings)}`);
// check that the specified format is supported, use default if it is not:
let options;
if (typeof this._format === 'string' && MediaRecorder.isTypeSupported(this._format))
{
options = { type: this._format };
}
else
{
this._psychoJS.logger.warn(`The specified video format, ${this._format}, is not supported by this browser, using the default format instead`);
}
// create a video recorder:
this._recorder = new MediaRecorder(this._stream, options);
// setup the callbacks:
const self = this;
// called upon Camera.start(), at which point the audio data starts being gathered
// into a blob:
this._recorder.onstart = () =>
{
self._videoBuffer = [];
self._videoBuffer.length = 0;
self._clock.reset();
self._status = PsychoJS.Status.STARTED;
self._psychoJS.logger.debug('video recording started');
// resolve the Microphone.start promise:
if (self._startCallback)
{
self._startCallback(self._psychoJS.monotonicClock.getTime());
}
};
// called upon Camera.pause():
this._recorder.onpause = () =>
{
self._status = PsychoJS.Status.PAUSED;
self._psychoJS.logger.debug('video recording paused');
// resolve the Microphone.pause promise:
if (self._pauseCallback)
{
self._pauseCallback(self._psychoJS.monotonicClock.getTime());
}
};
// called upon Camera.resume():
this._recorder.onresume = () =>
{
self._status = PsychoJS.Status.STARTED;
self._psychoJS.logger.debug('video recording resumed');
// resolve the Microphone.resume promise:
if (self._resumeCallback)
{
self._resumeCallback(self._psychoJS.monotonicClock.getTime());
}
};
// called when video data is available, typically upon Camera.stop() or Camera.flush():
this._recorder.ondataavailable = (event) =>
{
const data = event.data;
// add data to the buffer:
self._videoBuffer.push(data);
self._psychoJS.logger.debug('video data added to the buffer');
// resolve the data available promise, if needed:
if (self._dataAvailableCallback)
{
self._dataAvailableCallback(self._psychoJS.monotonicClock.getTime());
}
};
// called upon Camera.stop(), after data has been made available:
this._recorder.onstop = () =>
{
self._psychoJS.logger.debug('video recording stopped');
self._status = PsychoJS.Status.NOT_STARTED;
// resolve the Microphone.stop promise:
if (self._stopCallback)
{
self._stopCallback(self._psychoJS.monotonicClock.getTime());
}
// treat stop options if there are any:
// download to a file, immediately offered to the participant:
if (typeof self._stopOptions.filename === 'string')
{
self.download(self._stopOptions.filename);
}
};
// called upon recording errors:
this._recorder.onerror = (event) =>
{
// TODO
self._psychoJS.logger.error('video recording error: ' + JSON.stringify(event));
self._status = PsychoJS.Status.ERROR;
};
}
}

306
src/visual/FaceDetector.js Normal file
View File

@ -0,0 +1,306 @@
/**
* Manager handling the detecting of faces in video streams.
*
* @author Alain Pitiot
* @version 2021.2.0
* @copyright (c) 2021 Open Science Tools Ltd. (https://opensciencetools.org)
* @license Distributed under the terms of the MIT License
*/
import {PsychoJS} from "../core/PsychoJS";
import * as util from '../util/Util';
import {Color} from '../util/Color';
import {Camera} from "./Camera";
import {VisualStim} from "./VisualStim";
import * as PIXI from "pixi.js-legacy";
/**
* <p>This manager handles the detecting of faces in video streams.</p>
* <p>The detection is performed using the Face-API library: https://github.com/justadudewhohacks/face-api.js</p>
*
* @name module:visual.FaceDetector
* @class
* @param {Object} options
* @param {String} options.name - the name used when logging messages from the detector
* @param @param {module:core.Window} options.win - the associated Window
* @param @param {string | HTMLVideoElement | module:visual.Camera} input - the name of a
* movie resource or of a HTMLVideoElement or of a Camera component
* @param {string} [options.faceApiUrl= 'face-api.js'] - the Url of the face-api library
* @param {string} [options.modelDir= 'models'] - the directory where to find the face detection models
* @param {string} [options.units= "norm"] - the units of the stimulus (e.g. for size, position, vertices)
* @param {Array.<number>} [options.pos= [0, 0]] - the position of the center of the stimulus
* @param {string} [options.units= 'norm'] - the units of the stimulus vertices, size and position
* @param {number} [options.ori= 0.0] - the orientation (in degrees)
* @param {number} [options.size] - the size of the rendered image (the size of the image will be used if size is not specified)
* @param {number} [options.opacity= 1.0] - the opacity
* @param {boolean} [options.autoDraw= false] - whether or not the stimulus should be automatically drawn on every frame flip
* @param {boolean} [options.autoLog= false] - whether or not to log
*/
export class FaceDetector extends VisualStim
{
constructor({name, win, input, modelDir, faceApiUrl, units, ori, opacity, pos, size, autoDraw, autoLog} = {})
{
super({name, win, units, ori, opacity, pos, size, autoDraw, autoLog});
// TODO deal with onChange (see MovieStim and Camera)
this._addAttribute('input', input, undefined);
this._addAttribute('faceApiUrl', faceApiUrl, 'face-api.js');
this._addAttribute('modelDir', modelDir, 'models');
this._addAttribute('autoLog', autoLog, false);
this._addAttribute('status', PsychoJS.Status.NOT_STARTED);
// init face-api:
this._initFaceApi();
if (this._autoLog)
{
this._psychoJS.experimentLogger.exp(`Created ${this.name} = ${this.toString()}`);
}
}
/**
* Setter for the video attribute.
*
* @name module:visual.FaceDetector#setCamera
* @public
* @param {string | HTMLVideoElement | module:visual.Camera} input - the name of a
* movie resource or a HTMLVideoElement or a Camera component
* @param {boolean} [log= false] - whether of not to log
*/
setInput(input, log = false)
{
const response = {
origin: 'FaceDetector.setInput',
context: 'when setting the video of FaceDetector: ' + this._name
};
try
{
// movie is undefined: that's fine but we raise a warning in case this is
// a symptom of an actual problem
if (typeof input === 'undefined')
{
this.psychoJS.logger.warn('setting the movie of MovieStim: ' + this._name + ' with argument: undefined.');
this.psychoJS.logger.debug('set the movie of MovieStim: ' + this._name + ' as: undefined');
}
else
{
// if movie is a string, then it should be the name of a resource, which we get:
if (typeof input === 'string')
{
// TODO create a movie with that resource, and use the movie as input
}
// if movie is an instance of camera, get a video element from it:
else if (input instanceof Camera)
{
const video = input.getVideo();
// TODO remove previous one if there is one
// document.body.appendChild(video);
input = video;
}
// check that video is now an HTMLVideoElement
if (!(input instanceof HTMLVideoElement))
{
throw input.toString() + ' is not a video';
}
this.psychoJS.logger.debug(`set the video of FaceDetector: ${this._name} as: src= ${input.src}, size= ${input.videoWidth}x${input.videoHeight}, duration= ${input.duration}s`);
// ensure we have only one onended listener per HTMLVideoElement, since we can have several
// MovieStim with the same underlying HTMLVideoElement
// https://stackoverflow.com/questions/11455515
if (!input.onended)
{
input.onended = () =>
{
this.status = PsychoJS.Status.FINISHED;
};
}
}
this._setAttribute('input', input, log);
this._needUpdate = true;
this._needPixiUpdate = true;
}
catch (error)
{
throw Object.assign(response, {error});
}
}
/**
* Start detecting faces.
*
* @name module:visual.FaceDetector#start
* @public
* @param {number} period - the detection period, in ms (e.g. 100 ms for 10Hz)
* @param detectionCallback - the callback triggered when detection results are available
* @param {boolean} [log= false] - whether of not to log
*/
start(period, detectionCallback, log = false)
{
this.status = PsychoJS.Status.STARTED;
if (typeof this._detectionId !== 'undefined')
{
clearInterval(this._detectionId);
this._detectionId = undefined;
}
this._detectionId = setInterval(
async () =>
{
this._detections = await faceapi.detectAllFaces(
this._input,
new faceapi.TinyFaceDetectorOptions()
)
.withFaceLandmarks()
.withFaceExpressions();
this._needUpdate = true;
this._needPixiUpdate = true;
detectionCallback(this._detections);
},
period);
}
/**
* Stop detecting faces.
*
* @name module:visual.FaceDetector#stop
* @public
* @param {boolean} [log= false] - whether of not to log
*/
stop(log = false)
{
this.status = PsychoJS.Status.NOT_STARTED;
if (typeof this._detectionId !== 'undefined')
{
clearInterval(this._detectionId);
this._detectionId = undefined;
}
}
/**
* Init the Face-API library.
*
* @name module:visual.FaceDetector#_initFaceApi
* @private
*/
async _initFaceApi()
{/*
// load the library:
await this._psychoJS.serverManager.prepareResources([
{
'name': 'face-api.js',
'path': this.faceApiUrl,
'download': true
}
]);*/
// load the models:
faceapi.nets.tinyFaceDetector.loadFromUri(this._modelDir);
faceapi.nets.faceLandmark68Net.loadFromUri(this._modelDir);
faceapi.nets.faceRecognitionNet.loadFromUri(this._modelDir);
faceapi.nets.faceExpressionNet.loadFromUri(this._modelDir);
}
/**
* Update the visual representation of the detected faces, if necessary.
*
* @name module:visual.FaceDetector#_updateIfNeeded
* @private
*/
_updateIfNeeded()
{
if (!this._needUpdate)
{
return;
}
this._needUpdate = false;
if (this._needPixiUpdate)
{
this._needPixiUpdate = false;
if (typeof this._pixi !== 'undefined')
{
this._pixi.destroy(true);
}
this._pixi = new PIXI.Container();
this._pixi.interactive = true;
this._body = new PIXI.Graphics();
this._body.interactive = true;
this._pixi.addChild(this._body);
const size_px = util.to_px(this.size, this.units, this.win);
if (typeof this._detections !== 'undefined')
{
for (const detection of this._detections)
{
const landmarks = detection.landmarks;
const imageWidth = detection.alignedRect.imageWidth;
const imageHeight = detection.alignedRect.imageHeight;
for (const position of landmarks.positions)
{
this._body.beginFill(new Color('red').int, this._opacity);
this._body.drawCircle(
position._x / imageWidth * size_px[0] - size_px[0] / 2,
position._y / imageHeight * size_px[1] - size_px[1] / 2,
2);
this._body.endFill();
}
}
}
}
this._pixi.scale.x = 1;
this._pixi.scale.y = -1;
this._pixi.rotation = this.ori * Math.PI / 180;
this._pixi.position = util.to_pixiPoint(this.pos, this.units, this.win);
this._pixi.alpha = this._opacity;
}
/**
* Estimate the bounding box.
*
* @name module:visual.FaceDetector#_estimateBoundingBox
* @function
* @override
* @protected
*/
_estimateBoundingBox()
{
// TODO
/*this._boundingBox = new PIXI.Rectangle(
this._pos[0] + this._getLengthUnits(limits_px[0]),
this._pos[1] + this._getLengthUnits(limits_px[1]),
this._getLengthUnits(limits_px[2] - limits_px[0]),
this._getLengthUnits(limits_px[3] - limits_px[1])
);*/
// TODO take the orientation into account
}
}

39
src/visual/MovieStim.js
View File

@ -14,6 +14,8 @@ import { ColorMixin } from "../util/ColorMixin.js";
import { to_pixiPoint } from "../util/Pixi.js";
import * as util from "../util/Util.js";
import { VisualStim } from "./VisualStim.js";
import {Camera} from "./Camera.js";
/**
* Movie Stimulus.
@ -24,7 +26,8 @@ import { VisualStim } from "./VisualStim.js";
* @param {Object} options
* @param {String} options.name - the name used when logging messages from this stimulus
* @param {module:core.Window} options.win - the associated Window
* @param {string | HTMLVideoElement} options.movie - the name of the movie resource or the HTMLVideoElement corresponding to the movie
* @param {string | HTMLVideoElement | module:visual.Camera} movie - the name of a
* movie resource or of a HTMLVideoElement or of a Camera component
* @param {string} [options.units= "norm"] - the units of the stimulus (e.g. for size, position, vertices)
* @param {Array.<number>} [options.pos= [0, 0]] - the position of the center of the stimulus
* @param {string} [options.units= 'norm'] - the units of the stimulus vertices, size and position
@ -134,8 +137,8 @@ export class MovieStim extends VisualStim
*
* @name module:visual.MovieStim#setMovie
* @public
* @param {string | HTMLVideoElement} movie - the name of the movie resource or a
* HTMLVideoElement
* @param {string | HTMLVideoElement | module:visual.Camera} movie - the name of a
* movie resource or of a HTMLVideoElement or of a Camera component
* @param {boolean} [log= false] - whether of not to log
*/
setMovie(movie, log = false)
@ -147,30 +150,42 @@ export class MovieStim extends VisualStim
try
{
// movie is undefined: that's fine but we raise a warning in case this is a symptom of an actual problem
if (typeof movie === "undefined")
// movie is undefined: that's fine but we raise a warning in case this is
// a symptom of an actual problem
if (typeof movie === 'undefined')
{
this.psychoJS.logger.warn("setting the movie of MovieStim: " + this._name + " with argument: undefined.");
this.psychoJS.logger.debug("set the movie of MovieStim: " + this._name + " as: undefined");
this.psychoJS.logger.warn(
`setting the movie of MovieStim: ${this._name} with argument: undefined.`);
this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: undefined`);
}
else
{
// movie is a string: it should be the name of a resource, which we load
// if movie is a string, then it should be the name of a resource, which we get:
if (typeof movie === "string")
{
movie = this.psychoJS.serverManager.getResource(movie);
}
// movie should now be an actual HTMLVideoElement: we raise an error if it is not
// if movie is an instance of camera, get a video element from it:
else if (movie instanceof Camera)
{
const video = movie.getVideo();
// TODO remove previous one if there is one
// document.body.appendChild(video);
movie = video;
}
// check that movie is now an HTMLVideoElement
if (!(movie instanceof HTMLVideoElement))
{
throw "the argument: " + movie.toString() + ' is not a video" }';
throw movie.toString() + " is not a video";
}
this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${movie.src}, size= ${movie.videoWidth}x${movie.videoHeight}, duration= ${movie.duration}s`);
// ensure we have only one onended listener per HTMLVideoElement (we can have several
// MovieStim with the same underlying HTMLVideoElement)
// ensure we have only one onended listener per HTMLVideoElement, since we can have several
// MovieStim with the same underlying HTMLVideoElement
// https://stackoverflow.com/questions/11455515
if (!movie.onended)
{

2
src/visual/index.js
View File

@ -10,3 +10,5 @@ export * from "./TextBox.js";
export * from "./TextInput.js";
export * from "./TextStim.js";
export * from "./VisualStim.js";
export * from "./Camera.js";
export * from "./FaceDetector.js";