Merge pull request #2098 from bjoluc/preloading

Rework preloading in MediaAPI

packages/jspsych/src/JsPsych.ts

@@ -170,13 +170,6 @@ export class JsPsych {
 
     document.documentElement.setAttribute("jspsych", "present");
 
-    // Register preloading for the plugins referenced in the timeline
-    for (const [pluginName, parameters] of this.timeline.extractPreloadParameters()) {
-      for (const [parameter, type] of Object.entries(parameters)) {
-        this.pluginAPI.registerPreload(pluginName, parameter, type);
-      }
-    }
-
     this.startExperiment();
     await this.finished;
   }

packages/jspsych/src/TimelineNode.ts

@@ -1,5 +1,4 @@
 import { JsPsych } from "./JsPsych";
-import { ParameterType } from "./modules/plugins";
 import {
   repeat,
   sampleWithReplacement,
@@ -534,54 +533,4 @@ export class TimelineNode {
       );
     }
   }
-
-  /**
-   * Extracts a map that, for each of the timeline's (nested) plugins, maps the plugin name to an
-   * object that maps media parameters to their corresponding `preload` type, if not prevented by a
-   * `preload: false` flag in a parameter's description.
-   */
-  extractPreloadParameters() {
-    type PreloadType = "audio" | "image" | "video";
-    const preloadMap = new Map<string, Record<string, PreloadType>>();
-
-    /** Maps parameter types to their corresponding preload type */
-    const parameterTypeMap = new Map<number, PreloadType>([
-      [ParameterType.AUDIO, "audio"],
-      [ParameterType.IMAGE, "image"],
-      [ParameterType.VIDEO, "video"],
-    ]);
-
-    function recurseTimeline(node: TimelineNode) {
-      const isTimeline = typeof node.timeline_parameters !== "undefined";
-
-      if (isTimeline) {
-        for (const childNode of node.timeline_parameters.timeline) {
-          recurseTimeline(childNode);
-        }
-      } else if (node.trial_parameters.type.info) {
-        // node is a trial with type.info set
-
-        // Get the plugin name and parameters object from the info object
-        const { name: pluginName, parameters } = node.trial_parameters.type.info;
-
-        if (!preloadMap.has(pluginName)) {
-          preloadMap.set(
-            pluginName,
-            Object.fromEntries(
-              Object.entries<any>(parameters)
-                // Filter out parameter entries with media types and a non-false `preload` option
-                .filter(
-                  ([_name, { type, preload }]) => parameterTypeMap.has(type) && (preload ?? true)
-                )
-                // Map each entry's value to its preload type
-                .map(([name, { type }]) => [name, parameterTypeMap.get(type)])
-            )
-          );
-        }
-      }
-    }
-
-    recurseTimeline(this);
-    return preloadMap;
-  }
 }

packages/jspsych/src/modules/plugin-api/MediaAPI.ts

@@ -1,7 +1,15 @@
+import { ParameterType } from "../../modules/plugins";
 import { flatten, unique } from "../utils";
 
+const preloadParameterTypes = <const>[
+  ParameterType.AUDIO,
+  ParameterType.IMAGE,
+  ParameterType.VIDEO,
+];
+type PreloadType = typeof preloadParameterTypes[number];
+
 export class MediaAPI {
-  constructor(private useWebaudio: boolean, private webaudioContext: AudioContext) {}
+  constructor(private useWebaudio: boolean, private webaudioContext?: AudioContext) {}
 
   // video //
   private video_buffers = {};
@@ -52,8 +60,6 @@ export class MediaAPI {
   }
 
   // preloading stimuli //
-
-  private preloads = [];
   private preload_requests = [];
 
   private img_cache = {};
@@ -238,95 +244,71 @@ export class MediaAPI {
     }
   }
 
-  registerPreload(plugin_name, parameter, media_type) {
+  private preloadMap = new Map<string, Record<string, PreloadType>>();
-    if (["audio", "image", "video"].indexOf(media_type) === -1) {
-      console.error(
-        "Invalid media_type parameter for jsPsych.pluginAPI.registerPreload. Please check the plugin file."
-      );
-    }
-
-    var preload = {
-      plugin: plugin_name,
-      parameter: parameter,
-      media_type: media_type,
-    };
-
-    this.preloads.push(preload);
-  }
 
   getAutoPreloadList(timeline_description: any[]) {
-    function getTrialsOfTypeFromTimelineDescription(td, target_type, inherited_type?) {
+    /** Map each preload parameter type to a set of paths to be preloaded */
-      var trials = [];
+    const preloadPaths = Object.fromEntries(
+      preloadParameterTypes.map((type) => [type, new Set<string>()])
+    );
 
-      for (var i = 0; i < td.length; i++) {
+    const traverseTimeline = (node, inheritedTrialType?) => {
-        var node = td[i];
+      const isTimeline = typeof node.timeline !== "undefined";
-        if (Array.isArray(node.timeline)) {
+
-          if (typeof node.type !== "undefined") {
+      if (isTimeline) {
-            inherited_type = node.type;
+        for (const childNode of node.timeline) {
-          }
+          traverseTimeline(childNode, node.type ?? inheritedTrialType);
-          trials = trials.concat(
+        }
-            getTrialsOfTypeFromTimelineDescription(node.timeline, target_type, inherited_type)
+      } else if ((node.type ?? inheritedTrialType)?.info) {
+        // node is a trial with type.info set
+
+        // Get the plugin name and parameters object from the info object
+        const { name: pluginName, parameters } = (node.type ?? inheritedTrialType).info;
+
+        // Extract parameters to be preloaded and their types from parameter info if this has not
+        // yet been done for `pluginName`
+        if (!this.preloadMap.has(pluginName)) {
+          this.preloadMap.set(
+            pluginName,
+            Object.fromEntries(
+              Object.entries<any>(parameters)
+                // Filter out parameter entries with media types and a non-false `preload` option
+                .filter(
+                  ([_name, { type, preload }]) =>
+                    preloadParameterTypes.includes(type) && (preload ?? true)
+                )
+                // Map each entry's value to its parameter type
+                .map(([name, { type }]) => [name, type])
+            )
           );
-        } else {
+        }
-          if (typeof node.type !== "undefined" && node.type.info.name == target_type) {
+
-            trials.push(node);
+        // Add preload paths from this trial
-          }
+        for (const [parameterName, parameterType] of Object.entries(
-          if (typeof node.type == "undefined" && inherited_type.info.name == target_type) {
+          this.preloadMap.get(pluginName)
-            trials.push(Object.assign({}, { type: target_type }, node));
+        )) {
+          const parameterValue = node[parameterName];
+          const elements = preloadPaths[parameterType];
+
+          if (typeof parameterValue === "string") {
+            elements.add(parameterValue);
+          } else if (Array.isArray(parameterValue)) {
+            for (const element of flatten(parameterValue)) {
+              if (typeof element === "string") {
+                elements.add(element);
+              }
+            }
           }
         }
       }
+    };
 
-      return trials;
+    traverseTimeline({ timeline: timeline_description });
-    }
-
-    // list of items to preload
-    var images = [];
-    var audio = [];
-    var video = [];
-
-    // construct list
-    for (var i = 0; i < this.preloads.length; i++) {
-      var type = this.preloads[i].plugin;
-      var param = this.preloads[i].parameter;
-      var media = this.preloads[i].media_type;
-
-      var trials = getTrialsOfTypeFromTimelineDescription(timeline_description, type);
-      for (var j = 0; j < trials.length; j++) {
-        if (typeof trials[j][param] == "undefined") {
-          console.warn("jsPsych failed to auto preload one or more files:");
-          console.warn("no parameter called " + param + " in plugin " + type);
-        } else if (typeof trials[j][param] !== "function") {
-          if (media === "image") {
-            images = images.concat(flatten([trials[j][param]]));
-          } else if (media === "audio") {
-            audio = audio.concat(flatten([trials[j][param]]));
-          } else if (media === "video") {
-            video = video.concat(flatten([trials[j][param]]));
-          }
-        }
-      }
-    }
-
-    images = unique(flatten(images));
-    audio = unique(flatten(audio));
-    video = unique(flatten(video));
-
-    // remove any nulls false values
-    images = images.filter(function (x) {
-      return x != false && x != null;
-    });
-    audio = audio.filter(function (x) {
-      return x != false && x != null;
-    });
-    video = video.filter(function (x) {
-      return x != false && x != null;
-    });
 
     return {
-      images,
+      images: [...preloadPaths[ParameterType.IMAGE]],
-      audio,
+      audio: [...preloadPaths[ParameterType.AUDIO]],
-      video,
+      video: [...preloadPaths[ParameterType.VIDEO]],
     };
   }
 

packages/jspsych/tests/core/timelines.test.ts

@@ -1,7 +1,6 @@
 import htmlKeyboardResponse from "@jspsych/plugin-html-keyboard-response";
 
-import { ParameterType, initJsPsych } from "../../src";
+import { initJsPsych } from "../../src";
-import { TimelineNode } from "../../src/TimelineNode";
 import { pressKey, startTimeline } from "../utils";
 
 describe("loop function", () => {
@@ -509,89 +508,3 @@ describe("add node to end of timeline", () => {
     pressKey("a");
   });
 });
-
-describe("TimelineNode", () => {
-  const createTimelineNode = (parameters) => new TimelineNode(initJsPsych(), parameters);
-
-  describe("extractPreloadParameters", () => {
-    it("works for a single trial", () => {
-      const preloadMap = createTimelineNode({
-        type: {
-          info: {
-            name: "my-plugin",
-            parameters: {
-              one: { type: ParameterType.IMAGE },
-              two: { type: ParameterType.VIDEO },
-              three: { type: ParameterType.AUDIO },
-              four: { type: ParameterType.IMAGE, preload: true },
-              five: { type: ParameterType.IMAGE, preload: false },
-              six: {
-                type: ParameterType.STRING,
-                // This is illegal! But it should still not be added
-                preload: true,
-              },
-              seven: {},
-            },
-          },
-        },
-      }).extractPreloadParameters();
-
-      expect(preloadMap.get("my-plugin")).toEqual({
-        one: "image",
-        two: "video",
-        three: "audio",
-        four: "image",
-      });
-    });
-
-    it("works for a nested timeline", () => {
-      const preloadMap = createTimelineNode({
-        timeline: [
-          {
-            type: {
-              info: {
-                name: "plugin1",
-                parameters: { one: { type: ParameterType.STRING } },
-              },
-            },
-          },
-          {
-            type: {
-              info: {
-                name: "plugin2",
-                parameters: { one: { type: ParameterType.AUDIO } },
-              },
-            },
-          },
-          {
-            timeline: [
-              {
-                type: {
-                  info: {
-                    name: "plugin3",
-                    parameters: {
-                      one: { type: ParameterType.VIDEO },
-                      two: { type: ParameterType.IMAGE },
-                    },
-                  },
-                },
-              },
-            ],
-          },
-        ],
-      }).extractPreloadParameters();
-
-      expect(preloadMap.get("plugin1")).toEqual({});
-      expect(preloadMap.get("plugin2")).toEqual({ one: "audio" });
-      expect(preloadMap.get("plugin3")).toEqual({ one: "video", two: "image" });
-    });
-
-    it("ignores trials with a function type", () => {
-      const preloadMap = createTimelineNode({
-        type: () => ({ info: { name: "my-dynamic-trial-type" } }),
-      }).extractPreloadParameters();
-
-      expect(preloadMap.size).toEqual(0);
-    });
-  });
-});

packages/jspsych/tests/pluginAPI/preloads.test.ts

@@ -1,32 +1,115 @@
-// import imageKeyboardResponse from "@jspsych/plugin-image-keyboard-response";
+import { ParameterType } from "../../src";
+import { MediaAPI } from "../../src/modules/plugin-api/MediaAPI";
 
-import { initJsPsych } from "../../src";
+describe("getAutoPreloadList()", () => {
-
+  it("works for a single trial", () => {
-describe("getAutoPreloadList", () => {
+    const preloads = new MediaAPI(false).getAutoPreloadList([
-  test.skip("gets whole timeline when no argument provided", async () => {
-    const timeline = [
       {
-        // @ts-ignore TODO enable this test once the plugin is a class
+        type: {
-        type: imageKeyboardResponse,
+          info: {
-        stimulus: "img/foo.png",
+            name: "my-plugin",
-        render_on_canvas: false,
+            parameters: {
+              one: { type: ParameterType.IMAGE },
+              two: { type: ParameterType.VIDEO },
+              three: { type: ParameterType.AUDIO },
+              four: { type: ParameterType.IMAGE, preload: true },
+              five: { type: ParameterType.IMAGE, preload: false },
+              six: {
+                type: ParameterType.STRING,
+                // This is illegal! But it should still not do anything
+                preload: true,
+              },
+              seven: {},
+            },
+          },
+        },
+        one: "i1",
+        two: ["v1", ["v2", "v3"]], // arrays should be flattened
+        // three is undefined, should not be a problem
+        four: [null, false, 10], // non-string values should be ignored
+        five: "i2", // ignored: preload=false
+        six: "six", // ignored: ParameterType.STRING
       },
-    ];
+    ]);
 
-    const jsPsych = initJsPsych();
+    expect(preloads).toEqual({
-    expect(jsPsych.pluginAPI.getAutoPreloadList(timeline).images).toBe("img/foo.png");
+      images: ["i1"],
+      audio: [],
+      video: ["v1", "v2", "v3"],
+    });
   });
 
-  test.skip("works with images", async () => {
+  it("works for a nested timeline", () => {
-    const timeline = [
+    const preloads = new MediaAPI(false).getAutoPreloadList([
       {
-        // @ts-ignore TODO enable this test once the plugin is a class
+        timeline: [
-        type: imageKeyboardResponse,
+          {
-        stimulus: "img/foo.png",
+            type: {
+              info: {
+                name: "plugin1",
+                parameters: { one: { type: ParameterType.STRING } },
+              },
+            },
+            one: "plugin1-one",
+          },
+          {
+            type: {
+              info: {
+                name: "plugin2",
+                parameters: { one: { type: ParameterType.AUDIO } },
+              },
+            },
+            one: "plugin2-one",
+          },
+          {
+            type: {
+              info: {
+                name: "plugin3",
+                parameters: {
+                  one: { type: ParameterType.VIDEO },
+                  two: { type: ParameterType.IMAGE },
+                },
+              },
+            },
+            timeline: [
+              { one: "plugin3-one", two: "plugin3-two" },
+              {
+                type: {
+                  info: {
+                    name: "plugin4",
+                    parameters: {
+                      one: { type: ParameterType.VIDEO },
+                      two: { type: ParameterType.STRING },
+                    },
+                  },
+                },
+                one: "plugin4-one",
+                two: "plugin4-two",
+              },
+            ],
+          },
+        ],
       },
-    ];
+    ]);
 
-    const jsPsych = initJsPsych();
+    expect(preloads).toEqual({
-    expect(jsPsych.pluginAPI.getAutoPreloadList(timeline).images[0]).toBe("img/foo.png");
+      images: ["plugin3-two"],
+      audio: ["plugin2-one"],
+      video: ["plugin3-one", "plugin4-one"],
+    });
+  });
+
+  it("ignores trials with a function type", () => {
+    const preloads = new MediaAPI(false).getAutoPreloadList([
+      {
+        type: () => ({ info: { name: "my-dynamic-trial-type" } }),
+      },
+    ]);
+
+    expect(preloads).toEqual({
+      images: [],
+      audio: [],
+      video: [],
+    });
   });
 });