initial attempt at record-video extension

packages/extension-record-video/jest.config.cjs

@@ -0,0 +1 @@
+module.exports = require("@jspsych/config/jest").makePackageConfig(__dirname);

packages/extension-record-video/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@jspsych/extension-record-video",
+  "version": "0.0.1",
+  "description": "jsPsych extension for recording video",
+  "type": "module",
+  "main": "dist/index.cjs",
+  "exports": {
+    "import": "./dist/index.js",
+    "require": "./dist/index.cjs"
+  },
+  "typings": "dist/index.d.ts",
+  "unpkg": "dist/index.browser.min.js",
+  "files": [
+    "src",
+    "dist"
+  ],
+  "source": "src/index.ts",
+  "scripts": {
+    "test": "jest",
+    "test:watch": "npm test -- --watch",
+    "tsc": "tsc",
+    "build": "rollup --config",
+    "build:watch": "npm run build -- --watch"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jspsych/jsPsych.git",
+    "directory": "packages/extension-record-video"
+  },
+  "author": "Josh de Leeuw",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/jspsych/jsPsych/issues"
+  },
+  "homepage": "https://www.jspsych.org/latest/extensions/record-video",
+  "peerDependencies": {
+    "jspsych": ">=7.0.0"
+  },
+  "devDependencies": {
+    "@jspsych/config": "^1.3.0",
+    "@jspsych/test-utils": "^1.1.0"
+  }
+}

packages/extension-record-video/rollup.config.mjs

@@ -0,0 +1,3 @@
+import { makeRollupConfig } from "@jspsych/config/rollup";
+
+export default makeRollupConfig("jsPsychExtensionRecordVideo");

packages/extension-record-video/src/index.spec.ts

@@ -0,0 +1,276 @@
+import htmlKeyboardResponse from "@jspsych/plugin-html-keyboard-response";
+import { mouseDown, mouseMove, mouseUp, pressKey, startTimeline } from "@jspsych/test-utils";
+import { initJsPsych } from "jspsych";
+
+import MouseTrackingExtension from ".";
+
+jest.useFakeTimers();
+
+describe("Mouse Tracking Extension", () => {
+  test("adds mouse move data to trial", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: "<div id='target' style='width:500px; height: 500px;'></div>",
+        extensions: [{ type: MouseTrackingExtension }],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+
+    mouseMove(50, 50, displayElement.querySelector("#target"));
+    mouseMove(55, 50, displayElement.querySelector("#target"));
+    mouseMove(60, 50, displayElement.querySelector("#target"));
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_data[0]).toMatchObject({
+      x: targetRect.x + 50,
+      y: targetRect.y + 50,
+      event: "mousemove",
+    });
+    expect(getData().values()[0].mouse_tracking_data[1]).toMatchObject({
+      x: targetRect.x + 55,
+      y: targetRect.y + 50,
+      event: "mousemove",
+    });
+    expect(getData().values()[0].mouse_tracking_data[2]).toMatchObject({
+      x: targetRect.x + 60,
+      y: targetRect.y + 50,
+      event: "mousemove",
+    });
+  });
+
+  test("adds mouse down data to trial", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: "<div id='target' style='width:500px; height: 500px;'></div>",
+        extensions: [
+          {
+            type: MouseTrackingExtension,
+            params: { events: ["mousedown"] },
+          },
+        ],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+
+    mouseDown(50, 50, displayElement.querySelector("#target"));
+    mouseDown(55, 50, displayElement.querySelector("#target"));
+    mouseDown(60, 50, displayElement.querySelector("#target"));
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_data[0]).toMatchObject({
+      x: targetRect.x + 50,
+      y: targetRect.y + 50,
+      event: "mousedown",
+    });
+    expect(getData().values()[0].mouse_tracking_data[1]).toMatchObject({
+      x: targetRect.x + 55,
+      y: targetRect.y + 50,
+      event: "mousedown",
+    });
+    expect(getData().values()[0].mouse_tracking_data[2]).toMatchObject({
+      x: targetRect.x + 60,
+      y: targetRect.y + 50,
+      event: "mousedown",
+    });
+  });
+
+  test("adds mouse up data to trial", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: "<div id='target' style='width:500px; height: 500px;'></div>",
+        extensions: [
+          {
+            type: MouseTrackingExtension,
+            params: { events: ["mouseup"] },
+          },
+        ],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+
+    mouseUp(50, 50, displayElement.querySelector("#target"));
+    mouseUp(55, 50, displayElement.querySelector("#target"));
+    mouseUp(60, 50, displayElement.querySelector("#target"));
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_data[0]).toMatchObject({
+      x: targetRect.x + 50,
+      y: targetRect.y + 50,
+      event: "mouseup",
+    });
+    expect(getData().values()[0].mouse_tracking_data[1]).toMatchObject({
+      x: targetRect.x + 55,
+      y: targetRect.y + 50,
+      event: "mouseup",
+    });
+    expect(getData().values()[0].mouse_tracking_data[2]).toMatchObject({
+      x: targetRect.x + 60,
+      y: targetRect.y + 50,
+      event: "mouseup",
+    });
+  });
+
+  test("ignores mousemove when not in events", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: "<div id='target' style='width:500px; height: 500px;'></div>",
+        extensions: [
+          {
+            type: MouseTrackingExtension,
+            params: { events: ["mousedown"] },
+          },
+        ],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+
+    mouseMove(50, 50, displayElement.querySelector("#target"));
+    mouseMove(55, 50, displayElement.querySelector("#target"));
+    mouseDown(60, 50, displayElement.querySelector("#target"));
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_data.length).toBe(1);
+
+    expect(getData().values()[0].mouse_tracking_data[0]).toMatchObject({
+      x: targetRect.x + 60,
+      y: targetRect.y + 50,
+      event: "mousedown",
+    });
+  });
+
+  test("records bounding rect of targets in data", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: `
+          <div id='target' style='width:500px; height: 500px;'></div>
+          <div id='target2' style='width:200px; height: 200px;'></div>
+        `,
+        extensions: [
+          { type: MouseTrackingExtension, params: { targets: ["#target", "#target2"] } },
+        ],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+    const target2Rect = displayElement.querySelector("#target2").getBoundingClientRect();
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_targets["#target"]).toEqual(targetRect);
+    expect(getData().values()[0].mouse_tracking_targets["#target2"]).toEqual(target2Rect);
+  });
+
+  test("ignores mousemove events that are faster than minimum_sample_time", async () => {
+    const jsPsych = initJsPsych({
+      extensions: [{ type: MouseTrackingExtension, params: { minimum_sample_time: 100 } }],
+    });
+
+    const timeline = [
+      {
+        type: htmlKeyboardResponse,
+        stimulus: "<div id='target' style='width:500px; height: 500px;'></div>",
+        extensions: [{ type: MouseTrackingExtension }],
+      },
+    ];
+
+    const { displayElement, getHTML, getData, expectFinished } = await startTimeline(
+      timeline,
+      jsPsych
+    );
+
+    const targetRect = displayElement.querySelector("#target").getBoundingClientRect();
+
+    mouseMove(50, 50, displayElement.querySelector("#target"));
+    jest.advanceTimersByTime(50);
+
+    // this one should be ignored
+    mouseMove(55, 50, displayElement.querySelector("#target"));
+    jest.advanceTimersByTime(50);
+
+    // this one should register
+    mouseMove(60, 50, displayElement.querySelector("#target"));
+
+    pressKey("a");
+
+    await expectFinished();
+
+    expect(getData().values()[0].mouse_tracking_data[0]).toMatchObject({
+      x: targetRect.x + 50,
+      y: targetRect.y + 50,
+      event: "mousemove",
+    });
+    expect(getData().values()[0].mouse_tracking_data[1]).toMatchObject({
+      x: targetRect.x + 60,
+      y: targetRect.y + 50,
+      event: "mousemove",
+    });
+  });
+});

packages/extension-record-video/src/index.ts

@@ -0,0 +1,48 @@
+import { JsPsych, JsPsychExtension, JsPsychExtensionInfo } from "jspsych";
+
+class RecordVideoExtension implements JsPsychExtension {
+  static info: JsPsychExtensionInfo = {
+    name: "record-video",
+  };
+
+  constructor(private jsPsych: JsPsych) {}
+
+  private recordedChunks = [];
+  private recorder = null;
+
+  initialize = async () => {};
+
+  on_start = (): void => {
+    this.recorder = this.jsPsych.pluginAPI.getCameraRecorder();
+    this.recordedChunks = [];
+
+    if (!this.recorder) {
+      console.log("Camera not initialized. Do you need to run the initialize-camera plugin?");
+      return;
+    }
+
+    this.recorder.ondataavailable = this.handleOnDataAvailable;
+  };
+
+  on_load = () => {
+    this.recorder.start();
+  };
+
+  on_finish = () => {
+    this.recorder.stop();
+
+    this.recorder.ondataavailable = null;
+
+    return {
+      record_video_data: new Blob(this.recordedChunks),
+    };
+  };
+
+  private handleOnDataAvailable(event) {
+    if (event.data.size > 0) {
+      this.recordedChunks.push(event.data);
+    }
+  }
+}
+
+export default RecordVideoExtension;

packages/extension-record-video/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "@jspsych/config/tsconfig.core.json",
+  "compilerOptions": {
+    "baseUrl": "."
+  },
+  "include": ["src"]
+}