mirror of
https://github.com/jspsych/jsPsych.git
synced 2025-05-12 08:38:11 +00:00
Merge pull request #2098 from bjoluc/preloading
Rework preloading in MediaAPI
This commit is contained in:
Josh de Leeuw
GitHub
commit
f4d04e5056
@ -170,13 +170,6 @@ export class JsPsych {
|
||||
|
||||
document.documentElement.setAttribute("jspsych", "present");
|
||||
|
||||
// Register preloading for the plugins referenced in the timeline
|
||||
for (const [pluginName, parameters] of this.timeline.extractPreloadParameters()) {
|
||||
for (const [parameter, type] of Object.entries(parameters)) {
|
||||
this.pluginAPI.registerPreload(pluginName, parameter, type);
|
||||
}
|
||||
}
|
||||
|
||||
this.startExperiment();
|
||||
await this.finished;
|
||||
}
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import { JsPsych } from "./JsPsych";
|
||||
import { ParameterType } from "./modules/plugins";
|
||||
import {
|
||||
repeat,
|
||||
sampleWithReplacement,
|
||||
@ -534,54 +533,4 @@ export class TimelineNode {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a map that, for each of the timeline's (nested) plugins, maps the plugin name to an
|
||||
* object that maps media parameters to their corresponding `preload` type, if not prevented by a
|
||||
* `preload: false` flag in a parameter's description.
|
||||
*/
|
||||
extractPreloadParameters() {
|
||||
type PreloadType = "audio" | "image" | "video";
|
||||
const preloadMap = new Map<string, Record<string, PreloadType>>();
|
||||
|
||||
/** Maps parameter types to their corresponding preload type */
|
||||
const parameterTypeMap = new Map<number, PreloadType>([
|
||||
[ParameterType.AUDIO, "audio"],
|
||||
[ParameterType.IMAGE, "image"],
|
||||
[ParameterType.VIDEO, "video"],
|
||||
]);
|
||||
|
||||
function recurseTimeline(node: TimelineNode) {
|
||||
const isTimeline = typeof node.timeline_parameters !== "undefined";
|
||||
|
||||
if (isTimeline) {
|
||||
for (const childNode of node.timeline_parameters.timeline) {
|
||||
recurseTimeline(childNode);
|
||||
}
|
||||
} else if (node.trial_parameters.type.info) {
|
||||
// node is a trial with type.info set
|
||||
|
||||
// Get the plugin name and parameters object from the info object
|
||||
const { name: pluginName, parameters } = node.trial_parameters.type.info;
|
||||
|
||||
if (!preloadMap.has(pluginName)) {
|
||||
preloadMap.set(
|
||||
pluginName,
|
||||
Object.fromEntries(
|
||||
Object.entries<any>(parameters)
|
||||
// Filter out parameter entries with media types and a non-false `preload` option
|
||||
.filter(
|
||||
([_name, { type, preload }]) => parameterTypeMap.has(type) && (preload ?? true)
|
||||
)
|
||||
// Map each entry's value to its preload type
|
||||
.map(([name, { type }]) => [name, parameterTypeMap.get(type)])
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recurseTimeline(this);
|
||||
return preloadMap;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,15 @@
|
||||
import { ParameterType } from "../../modules/plugins";
|
||||
import { flatten, unique } from "../utils";
|
||||
|
||||
const preloadParameterTypes = <const>[
|
||||
ParameterType.AUDIO,
|
||||
ParameterType.IMAGE,
|
||||
ParameterType.VIDEO,
|
||||
];
|
||||
type PreloadType = typeof preloadParameterTypes[number];
|
||||
|
||||
export class MediaAPI {
|
||||
constructor(private useWebaudio: boolean, private webaudioContext: AudioContext) {}
|
||||
constructor(private useWebaudio: boolean, private webaudioContext?: AudioContext) {}
|
||||
|
||||
// video //
|
||||
private video_buffers = {};
|
||||
@ -52,8 +60,6 @@ export class MediaAPI {
|
||||
}
|
||||
|
||||
// preloading stimuli //
|
||||
|
||||
private preloads = [];
|
||||
private preload_requests = [];
|
||||
|
||||
private img_cache = {};
|
||||
@ -238,95 +244,71 @@ export class MediaAPI {
|
||||
}
|
||||
}
|
||||
|
||||
registerPreload(plugin_name, parameter, media_type) {
|
||||
if (["audio", "image", "video"].indexOf(media_type) === -1) {
|
||||
console.error(
|
||||
"Invalid media_type parameter for jsPsych.pluginAPI.registerPreload. Please check the plugin file."
|
||||
);
|
||||
}
|
||||
|
||||
var preload = {
|
||||
plugin: plugin_name,
|
||||
parameter: parameter,
|
||||
media_type: media_type,
|
||||
};
|
||||
|
||||
this.preloads.push(preload);
|
||||
}
|
||||
private preloadMap = new Map<string, Record<string, PreloadType>>();
|
||||
|
||||
getAutoPreloadList(timeline_description: any[]) {
|
||||
function getTrialsOfTypeFromTimelineDescription(td, target_type, inherited_type?) {
|
||||
var trials = [];
|
||||
/** Map each preload parameter type to a set of paths to be preloaded */
|
||||
const preloadPaths = Object.fromEntries(
|
||||
preloadParameterTypes.map((type) => [type, new Set<string>()])
|
||||
);
|
||||
|
||||
for (var i = 0; i < td.length; i++) {
|
||||
var node = td[i];
|
||||
if (Array.isArray(node.timeline)) {
|
||||
if (typeof node.type !== "undefined") {
|
||||
inherited_type = node.type;
|
||||
}
|
||||
trials = trials.concat(
|
||||
getTrialsOfTypeFromTimelineDescription(node.timeline, target_type, inherited_type)
|
||||
const traverseTimeline = (node, inheritedTrialType?) => {
|
||||
const isTimeline = typeof node.timeline !== "undefined";
|
||||
|
||||
if (isTimeline) {
|
||||
for (const childNode of node.timeline) {
|
||||
traverseTimeline(childNode, node.type ?? inheritedTrialType);
|
||||
}
|
||||
} else if ((node.type ?? inheritedTrialType)?.info) {
|
||||
// node is a trial with type.info set
|
||||
|
||||
// Get the plugin name and parameters object from the info object
|
||||
const { name: pluginName, parameters } = (node.type ?? inheritedTrialType).info;
|
||||
|
||||
// Extract parameters to be preloaded and their types from parameter info if this has not
|
||||
// yet been done for `pluginName`
|
||||
if (!this.preloadMap.has(pluginName)) {
|
||||
this.preloadMap.set(
|
||||
pluginName,
|
||||
Object.fromEntries(
|
||||
Object.entries<any>(parameters)
|
||||
// Filter out parameter entries with media types and a non-false `preload` option
|
||||
.filter(
|
||||
([_name, { type, preload }]) =>
|
||||
preloadParameterTypes.includes(type) && (preload ?? true)
|
||||
)
|
||||
// Map each entry's value to its parameter type
|
||||
.map(([name, { type }]) => [name, type])
|
||||
)
|
||||
);
|
||||
} else {
|
||||
if (typeof node.type !== "undefined" && node.type.info.name == target_type) {
|
||||
trials.push(node);
|
||||
}
|
||||
if (typeof node.type == "undefined" && inherited_type.info.name == target_type) {
|
||||
trials.push(Object.assign({}, { type: target_type }, node));
|
||||
}
|
||||
|
||||
// Add preload paths from this trial
|
||||
for (const [parameterName, parameterType] of Object.entries(
|
||||
this.preloadMap.get(pluginName)
|
||||
)) {
|
||||
const parameterValue = node[parameterName];
|
||||
const elements = preloadPaths[parameterType];
|
||||
|
||||
if (typeof parameterValue === "string") {
|
||||
elements.add(parameterValue);
|
||||
} else if (Array.isArray(parameterValue)) {
|
||||
for (const element of flatten(parameterValue)) {
|
||||
if (typeof element === "string") {
|
||||
elements.add(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return trials;
|
||||
}
|
||||
|
||||
// list of items to preload
|
||||
var images = [];
|
||||
var audio = [];
|
||||
var video = [];
|
||||
|
||||
// construct list
|
||||
for (var i = 0; i < this.preloads.length; i++) {
|
||||
var type = this.preloads[i].plugin;
|
||||
var param = this.preloads[i].parameter;
|
||||
var media = this.preloads[i].media_type;
|
||||
|
||||
var trials = getTrialsOfTypeFromTimelineDescription(timeline_description, type);
|
||||
for (var j = 0; j < trials.length; j++) {
|
||||
if (typeof trials[j][param] == "undefined") {
|
||||
console.warn("jsPsych failed to auto preload one or more files:");
|
||||
console.warn("no parameter called " + param + " in plugin " + type);
|
||||
} else if (typeof trials[j][param] !== "function") {
|
||||
if (media === "image") {
|
||||
images = images.concat(flatten([trials[j][param]]));
|
||||
} else if (media === "audio") {
|
||||
audio = audio.concat(flatten([trials[j][param]]));
|
||||
} else if (media === "video") {
|
||||
video = video.concat(flatten([trials[j][param]]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
images = unique(flatten(images));
|
||||
audio = unique(flatten(audio));
|
||||
video = unique(flatten(video));
|
||||
|
||||
// remove any nulls false values
|
||||
images = images.filter(function (x) {
|
||||
return x != false && x != null;
|
||||
});
|
||||
audio = audio.filter(function (x) {
|
||||
return x != false && x != null;
|
||||
});
|
||||
video = video.filter(function (x) {
|
||||
return x != false && x != null;
|
||||
});
|
||||
traverseTimeline({ timeline: timeline_description });
|
||||
|
||||
return {
|
||||
images,
|
||||
audio,
|
||||
video,
|
||||
images: [...preloadPaths[ParameterType.IMAGE]],
|
||||
audio: [...preloadPaths[ParameterType.AUDIO]],
|
||||
video: [...preloadPaths[ParameterType.VIDEO]],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import htmlKeyboardResponse from "@jspsych/plugin-html-keyboard-response";
|
||||
|
||||
import { ParameterType, initJsPsych } from "../../src";
|
||||
import { TimelineNode } from "../../src/TimelineNode";
|
||||
import { initJsPsych } from "../../src";
|
||||
import { pressKey, startTimeline } from "../utils";
|
||||
|
||||
describe("loop function", () => {
|
||||
@ -509,89 +508,3 @@ describe("add node to end of timeline", () => {
|
||||
pressKey("a");
|
||||
});
|
||||
});
|
||||
|
||||
describe("TimelineNode", () => {
|
||||
const createTimelineNode = (parameters) => new TimelineNode(initJsPsych(), parameters);
|
||||
|
||||
describe("extractPreloadParameters", () => {
|
||||
it("works for a single trial", () => {
|
||||
const preloadMap = createTimelineNode({
|
||||
type: {
|
||||
info: {
|
||||
name: "my-plugin",
|
||||
parameters: {
|
||||
one: { type: ParameterType.IMAGE },
|
||||
two: { type: ParameterType.VIDEO },
|
||||
three: { type: ParameterType.AUDIO },
|
||||
four: { type: ParameterType.IMAGE, preload: true },
|
||||
five: { type: ParameterType.IMAGE, preload: false },
|
||||
six: {
|
||||
type: ParameterType.STRING,
|
||||
// This is illegal! But it should still not be added
|
||||
preload: true,
|
||||
},
|
||||
seven: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}).extractPreloadParameters();
|
||||
|
||||
expect(preloadMap.get("my-plugin")).toEqual({
|
||||
one: "image",
|
||||
two: "video",
|
||||
three: "audio",
|
||||
four: "image",
|
||||
});
|
||||
});
|
||||
|
||||
it("works for a nested timeline", () => {
|
||||
const preloadMap = createTimelineNode({
|
||||
timeline: [
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin1",
|
||||
parameters: { one: { type: ParameterType.STRING } },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin2",
|
||||
parameters: { one: { type: ParameterType.AUDIO } },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
timeline: [
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin3",
|
||||
parameters: {
|
||||
one: { type: ParameterType.VIDEO },
|
||||
two: { type: ParameterType.IMAGE },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}).extractPreloadParameters();
|
||||
|
||||
expect(preloadMap.get("plugin1")).toEqual({});
|
||||
expect(preloadMap.get("plugin2")).toEqual({ one: "audio" });
|
||||
expect(preloadMap.get("plugin3")).toEqual({ one: "video", two: "image" });
|
||||
});
|
||||
|
||||
it("ignores trials with a function type", () => {
|
||||
const preloadMap = createTimelineNode({
|
||||
type: () => ({ info: { name: "my-dynamic-trial-type" } }),
|
||||
}).extractPreloadParameters();
|
||||
|
||||
expect(preloadMap.size).toEqual(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,32 +1,115 @@
|
||||
// import imageKeyboardResponse from "@jspsych/plugin-image-keyboard-response";
|
||||
import { ParameterType } from "../../src";
|
||||
import { MediaAPI } from "../../src/modules/plugin-api/MediaAPI";
|
||||
|
||||
import { initJsPsych } from "../../src";
|
||||
|
||||
describe("getAutoPreloadList", () => {
|
||||
test.skip("gets whole timeline when no argument provided", async () => {
|
||||
const timeline = [
|
||||
describe("getAutoPreloadList()", () => {
|
||||
it("works for a single trial", () => {
|
||||
const preloads = new MediaAPI(false).getAutoPreloadList([
|
||||
{
|
||||
// @ts-ignore TODO enable this test once the plugin is a class
|
||||
type: imageKeyboardResponse,
|
||||
stimulus: "img/foo.png",
|
||||
render_on_canvas: false,
|
||||
type: {
|
||||
info: {
|
||||
name: "my-plugin",
|
||||
parameters: {
|
||||
one: { type: ParameterType.IMAGE },
|
||||
two: { type: ParameterType.VIDEO },
|
||||
three: { type: ParameterType.AUDIO },
|
||||
four: { type: ParameterType.IMAGE, preload: true },
|
||||
five: { type: ParameterType.IMAGE, preload: false },
|
||||
six: {
|
||||
type: ParameterType.STRING,
|
||||
// This is illegal! But it should still not do anything
|
||||
preload: true,
|
||||
},
|
||||
seven: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
one: "i1",
|
||||
two: ["v1", ["v2", "v3"]], // arrays should be flattened
|
||||
// three is undefined, should not be a problem
|
||||
four: [null, false, 10], // non-string values should be ignored
|
||||
five: "i2", // ignored: preload=false
|
||||
six: "six", // ignored: ParameterType.STRING
|
||||
},
|
||||
];
|
||||
]);
|
||||
|
||||
const jsPsych = initJsPsych();
|
||||
expect(jsPsych.pluginAPI.getAutoPreloadList(timeline).images).toBe("img/foo.png");
|
||||
expect(preloads).toEqual({
|
||||
images: ["i1"],
|
||||
audio: [],
|
||||
video: ["v1", "v2", "v3"],
|
||||
});
|
||||
});
|
||||
|
||||
test.skip("works with images", async () => {
|
||||
const timeline = [
|
||||
it("works for a nested timeline", () => {
|
||||
const preloads = new MediaAPI(false).getAutoPreloadList([
|
||||
{
|
||||
// @ts-ignore TODO enable this test once the plugin is a class
|
||||
type: imageKeyboardResponse,
|
||||
stimulus: "img/foo.png",
|
||||
timeline: [
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin1",
|
||||
parameters: { one: { type: ParameterType.STRING } },
|
||||
},
|
||||
},
|
||||
one: "plugin1-one",
|
||||
},
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin2",
|
||||
parameters: { one: { type: ParameterType.AUDIO } },
|
||||
},
|
||||
},
|
||||
one: "plugin2-one",
|
||||
},
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin3",
|
||||
parameters: {
|
||||
one: { type: ParameterType.VIDEO },
|
||||
two: { type: ParameterType.IMAGE },
|
||||
},
|
||||
},
|
||||
},
|
||||
timeline: [
|
||||
{ one: "plugin3-one", two: "plugin3-two" },
|
||||
{
|
||||
type: {
|
||||
info: {
|
||||
name: "plugin4",
|
||||
parameters: {
|
||||
one: { type: ParameterType.VIDEO },
|
||||
two: { type: ParameterType.STRING },
|
||||
},
|
||||
},
|
||||
},
|
||||
one: "plugin4-one",
|
||||
two: "plugin4-two",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
]);
|
||||
|
||||
const jsPsych = initJsPsych();
|
||||
expect(jsPsych.pluginAPI.getAutoPreloadList(timeline).images[0]).toBe("img/foo.png");
|
||||
expect(preloads).toEqual({
|
||||
images: ["plugin3-two"],
|
||||
audio: ["plugin2-one"],
|
||||
video: ["plugin3-one", "plugin4-one"],
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores trials with a function type", () => {
|
||||
const preloads = new MediaAPI(false).getAutoPreloadList([
|
||||
{
|
||||
type: () => ({ info: { name: "my-dynamic-trial-type" } }),
|
||||
},
|
||||
]);
|
||||
|
||||
expect(preloads).toEqual({
|
||||
images: [],
|
||||
audio: [],
|
||||
video: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user