From 3a3d32971cbffa76c113d480505dbf77473a4a04 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 19 Nov 2021 17:00:10 -0500 Subject: [PATCH 01/19] start work on audio-input plugins --- examples/jspsych-initialize-microphone.html | 26 +++ .../src/modules/plugin-api/MediaAPI.ts | 11 ++ .../jest.config.cjs | 1 + .../plugin-html-audio-response/package.json | 43 +++++ .../rollup.config.mjs | 3 + .../src/index.spec.ts | 156 ++++++++++++++++++ .../plugin-html-audio-response/src/index.ts | 141 ++++++++++++++++ .../plugin-html-audio-response/tsconfig.json | 7 + .../jest.config.cjs | 1 + .../plugin-initialize-microphone/package.json | 43 +++++ .../rollup.config.mjs | 3 + .../src/index.spec.ts | 32 ++++ .../plugin-initialize-microphone/src/index.ts | 89 ++++++++++ .../tsconfig.json | 7 + 14 files changed, 563 insertions(+) create mode 100644 examples/jspsych-initialize-microphone.html create mode 100644 packages/plugin-html-audio-response/jest.config.cjs create mode 100644 packages/plugin-html-audio-response/package.json create mode 100644 packages/plugin-html-audio-response/rollup.config.mjs create mode 100644 packages/plugin-html-audio-response/src/index.spec.ts create mode 100644 packages/plugin-html-audio-response/src/index.ts create mode 100644 packages/plugin-html-audio-response/tsconfig.json create mode 100644 packages/plugin-initialize-microphone/jest.config.cjs create mode 100644 packages/plugin-initialize-microphone/package.json create mode 100644 packages/plugin-initialize-microphone/rollup.config.mjs create mode 100644 packages/plugin-initialize-microphone/src/index.spec.ts create mode 100644 packages/plugin-initialize-microphone/src/index.ts create mode 100644 packages/plugin-initialize-microphone/tsconfig.json diff --git a/examples/jspsych-initialize-microphone.html b/examples/jspsych-initialize-microphone.html new file mode 100644 index 00000000..dc581781 --- /dev/null +++ b/examples/jspsych-initialize-microphone.html @@ -0,0 +1,26 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts index d8e7441f..27868fe1 100644 --- a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts +++ b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts @@ -323,4 +323,15 @@ export class MediaAPI { } this.preload_requests = []; } + + private microphone_recorder: MediaRecorder; + + initializeMicrophoneRecorder(stream: MediaStream) { + const recorder = new MediaRecorder(stream); + this.microphone_recorder = recorder; + } + + getMicrophoneRecorder(): MediaRecorder { + return this.microphone_recorder; + } } diff --git a/packages/plugin-html-audio-response/jest.config.cjs b/packages/plugin-html-audio-response/jest.config.cjs new file mode 100644 index 00000000..6ac19d5c --- /dev/null +++ b/packages/plugin-html-audio-response/jest.config.cjs @@ -0,0 +1 @@ +module.exports = require("@jspsych/config/jest").makePackageConfig(__dirname); diff --git a/packages/plugin-html-audio-response/package.json b/packages/plugin-html-audio-response/package.json new file mode 100644 index 00000000..0ab38603 --- /dev/null +++ b/packages/plugin-html-audio-response/package.json @@ -0,0 +1,43 @@ +{ + "name": "@jspsych/plugin-html-audio-response", + "version": "0.1.0", + "description": "jsPsych plugin for displaying a stimulus and recording an audio response through the microphone", + "type": "module", + "main": "dist/index.cjs", + "exports": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "typings": "dist/index.d.ts", + "unpkg": "dist/index.browser.min.js", + "files": [ + "src", + "dist" + ], + "source": "src/index.ts", + "scripts": { + "test": "jest", + "test:watch": "npm test -- --watch", + "tsc": "tsc", + "build": "rollup --config", + "build:watch": "npm run build -- --watch" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/jspsych/jsPsych.git", + "directory": "packages/plugin-html-audio-response" + }, + "author": "Josh de Leeuw", + "license": "MIT", + "bugs": { + "url": "https://github.com/jspsych/jsPsych/issues" + }, + "homepage": "https://www.jspsych.org/latest/plugins/html-audio-response", + "peerDependencies": { + "jspsych": ">=7.0.0" + }, + "devDependencies": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0" + } +} diff --git a/packages/plugin-html-audio-response/rollup.config.mjs b/packages/plugin-html-audio-response/rollup.config.mjs new file mode 100644 index 00000000..353f92ad --- /dev/null +++ b/packages/plugin-html-audio-response/rollup.config.mjs @@ -0,0 +1,3 @@ +import { makeRollupConfig } from "@jspsych/config/rollup"; + +export default makeRollupConfig("jsPsychHtmlAudioResponse"); diff --git a/packages/plugin-html-audio-response/src/index.spec.ts b/packages/plugin-html-audio-response/src/index.spec.ts new file mode 100644 index 00000000..f768522a --- /dev/null +++ b/packages/plugin-html-audio-response/src/index.spec.ts @@ -0,0 +1,156 @@ +import { clickTarget, startTimeline } from "@jspsych/test-utils"; + +import htmlButtonResponse from "."; + +jest.useFakeTimers(); + +describe("html-button-response", () => { + test("displays html stimulus", async () => { + const { getHTML } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + }, + ]); + + expect(getHTML()).toContain( + '
this is html
' + ); + }); + + test("display button labels", async () => { + const { getHTML } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice1", "button-choice2"], + }, + ]); + + expect(getHTML()).toContain(''); + expect(getHTML()).toContain(''); + }); + + test("display button html", async () => { + const { getHTML } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["buttonChoice"], + button_html: '', + }, + ]); + + expect(getHTML()).toContain(''); + }); + + test("display should clear after button click", async () => { + const { getHTML, expectFinished } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + }, + ]); + + expect(getHTML()).toContain( + '
this is html
' + ); + + clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); + + await expectFinished(); + }); + + test("prompt should append below button", async () => { + const { getHTML } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + prompt: "

this is a prompt

", + }, + ]); + + expect(getHTML()).toContain( + '

this is a prompt

' + ); + }); + + test("should hide stimulus if stimulus-duration is set", async () => { + const { displayElement } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + stimulus_duration: 500, + }, + ]); + + const stimulusElement = displayElement.querySelector( + "#jspsych-html-button-response-stimulus" + ); + + expect(stimulusElement.style.visibility).toBe(""); + + jest.advanceTimersByTime(500); + expect(stimulusElement.style.visibility).toBe("hidden"); + }); + + test("should end trial when trial duration is reached", async () => { + const { getHTML, expectFinished } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + trial_duration: 500, + }, + ]); + + expect(getHTML()).toContain( + '
this is html
' + ); + + jest.advanceTimersByTime(500); + await expectFinished(); + }); + + test("should end trial when button is clicked", async () => { + const { getHTML, expectFinished } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + response_ends_trial: true, + }, + ]); + + expect(getHTML()).toContain( + '
this is html
' + ); + + clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); + await expectFinished(); + }); + + test("class should have responded when button is clicked", async () => { + const { getHTML } = await startTimeline([ + { + type: htmlButtonResponse, + stimulus: "this is html", + choices: ["button-choice"], + response_ends_trial: false, + }, + ]); + + expect(getHTML()).toContain( + '
this is html
' + ); + + clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); + expect(document.querySelector("#jspsych-html-button-response-stimulus").className).toBe( + " responded" + ); + }); +}); diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts new file mode 100644 index 00000000..b0bbe480 --- /dev/null +++ b/packages/plugin-html-audio-response/src/index.ts @@ -0,0 +1,141 @@ +import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych"; + +const info = { + name: "html-audio-response", + parameters: { + /** The HTML string to be displayed */ + stimulus: { + type: ParameterType.HTML_STRING, + pretty_name: "Stimulus", + default: undefined, + }, + + /** How long to show the stimulus. */ + stimulus_duration: { + type: ParameterType.INT, + pretty_name: "Stimulus duration", + default: null, + }, + /** How long to show the trial. */ + trial_duration: { + type: ParameterType.INT, + pretty_name: "Trial duration", + default: null, + }, + }, +}; + +type Info = typeof info; + +/** + * html-audio-response + * jsPsych plugin for displaying a stimulus and recording an audio response through a microphone + * @author Josh de Leeuw + * @see {@link https://www.jspsych.org/plugins/jspsych-html-audio-response/ html-audio-response plugin documentation on jspsych.org} + */ +class HtmlAudioResponsePlugin implements JsPsychPlugin { + static info = info; + private stimulus_start_time; + private recorder_start_time; + + constructor(private jsPsych: JsPsych) {} + + trial(display_element: HTMLElement, trial: TrialType) { + const ro = new ResizeObserver((entries) => { + console.log(`ro perf = ${performance.now()}`); + }); + + ro.observe(display_element); + + this.showDisplay(display_element, trial); + this.addButtonEvent(display_element, trial); + + this.stimulus_start_time = performance.now(); + + this.startRecording(); + + // hide image if timing is set + if (trial.stimulus_duration !== null) { + this.jsPsych.pluginAPI.setTimeout(() => { + display_element.querySelector( + "#jspsych-html-audio-response-stimulus" + ).style.visibility = "hidden"; + }, trial.stimulus_duration); + } + + // end trial if time limit is set + if (trial.trial_duration !== null) { + this.jsPsych.pluginAPI.setTimeout(() => { + this.endTrial(display_element, trial); + }, trial.trial_duration); + } + } + + private showDisplay(display_element, trial) { + let html = `
${trial.stimulus}
`; + + html += `

`; + + display_element.innerHTML = html; + } + + private addButtonEvent(display_element, trial) { + display_element.querySelector("#finish-trial").addEventListener("click", () => { + const end_time = performance.now(); + const rt = Math.round(end_time - this.stimulus_start_time); + this.endTrial(display_element, trial, rt); + }); + } + + private startRecording() { + const recorder: MediaRecorder = this.jsPsych.pluginAPI.getMicrophoneRecorder(); + const recorded_data_chunks = []; + + recorder.addEventListener("dataavailable", (e) => { + if (e.data.size > 0) { + recorded_data_chunks.push(e.data); + } + console.log("dataavail event"); + }); + + recorder.addEventListener("stop", () => { + const url = new Blob(recorded_data_chunks, { type: "audio/webm" }); + const reader = new FileReader(); + reader.addEventListener("load", () => { + const base64 = (reader.result as string).split(",")[1]; + console.log(base64); + }); + reader.readAsDataURL(url); + + console.log(this.stimulus_start_time, this.recorder_start_time); + }); + + recorder.addEventListener("start", (e) => { + this.recorder_start_time = e.timeStamp; + }); + + this.jsPsych.pluginAPI.setTimeout(() => { + recorder.stop(); + }, 2000); + recorder.start(); + } + + private endTrial(display_element, trial, rt: number = null) { + // kill any remaining setTimeout handlers + this.jsPsych.pluginAPI.clearAllTimeouts(); + + // gather the data to store for the trial + var trial_data = { + rt: rt, + stimulus: trial.stimulus, + }; + + // clear the display + display_element.innerHTML = ""; + + // move on to the next trial + this.jsPsych.finishTrial(trial_data); + } +} + +export default HtmlAudioResponsePlugin; diff --git a/packages/plugin-html-audio-response/tsconfig.json b/packages/plugin-html-audio-response/tsconfig.json new file mode 100644 index 00000000..588f0448 --- /dev/null +++ b/packages/plugin-html-audio-response/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@jspsych/config/tsconfig.core.json", + "compilerOptions": { + "baseUrl": "." + }, + "include": ["src"] +} diff --git a/packages/plugin-initialize-microphone/jest.config.cjs b/packages/plugin-initialize-microphone/jest.config.cjs new file mode 100644 index 00000000..6ac19d5c --- /dev/null +++ b/packages/plugin-initialize-microphone/jest.config.cjs @@ -0,0 +1 @@ +module.exports = require("@jspsych/config/jest").makePackageConfig(__dirname); diff --git a/packages/plugin-initialize-microphone/package.json b/packages/plugin-initialize-microphone/package.json new file mode 100644 index 00000000..e33b41e6 --- /dev/null +++ b/packages/plugin-initialize-microphone/package.json @@ -0,0 +1,43 @@ +{ + "name": "@jspsych/plugin-initiliaze-microphone", + "version": "0.1.0", + "description": "jsPsych plugin for getting permission to initialize the user's microphone", + "type": "module", + "main": "dist/index.cjs", + "exports": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "typings": "dist/index.d.ts", + "unpkg": "dist/index.browser.min.js", + "files": [ + "src", + "dist" + ], + "source": "src/index.ts", + "scripts": { + "test": "jest", + "test:watch": "npm test -- --watch", + "tsc": "tsc", + "build": "rollup --config", + "build:watch": "npm run build -- --watch" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/jspsych/jsPsych.git", + "directory": "packages/plugin-initialize-microphone" + }, + "author": "Josh de Leeuw", + "license": "MIT", + "bugs": { + "url": "https://github.com/jspsych/jsPsych/issues" + }, + "homepage": "https://www.jspsych.org/latest/plugins/initialize-microphone", + "peerDependencies": { + "jspsych": ">=7.0.0" + }, + "devDependencies": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0" + } +} diff --git a/packages/plugin-initialize-microphone/rollup.config.mjs b/packages/plugin-initialize-microphone/rollup.config.mjs new file mode 100644 index 00000000..d3d56cbf --- /dev/null +++ b/packages/plugin-initialize-microphone/rollup.config.mjs @@ -0,0 +1,3 @@ +import { makeRollupConfig } from "@jspsych/config/rollup"; + +export default makeRollupConfig("jsPsychInitializeMicrophone"); diff --git a/packages/plugin-initialize-microphone/src/index.spec.ts b/packages/plugin-initialize-microphone/src/index.spec.ts new file mode 100644 index 00000000..33df9bb4 --- /dev/null +++ b/packages/plugin-initialize-microphone/src/index.spec.ts @@ -0,0 +1,32 @@ +import { startTimeline } from "@jspsych/test-utils"; + +import callFunction from "."; + +describe("call-function plugin", () => { + test("calls function", async () => { + const { getData, expectFinished } = await startTimeline([ + { + type: callFunction, + func: () => 1, + }, + ]); + + await expectFinished(); + expect(getData().values()[0].value).toBe(1); + }); + + test("async function works", async () => { + const { getData, expectFinished } = await startTimeline([ + { + type: callFunction, + async: true, + func: (done) => { + done(10); + }, + }, + ]); + + await expectFinished(); + expect(getData().values()[0].value).toBe(10); + }); +}); diff --git a/packages/plugin-initialize-microphone/src/index.ts b/packages/plugin-initialize-microphone/src/index.ts new file mode 100644 index 00000000..8a78e524 --- /dev/null +++ b/packages/plugin-initialize-microphone/src/index.ts @@ -0,0 +1,89 @@ +import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych"; + +const info = { + name: "initialize-microphone", + parameters: { + /** Function to call */ + func: { + type: ParameterType.BOOL, + default: true, + }, + /** Is the function call asynchronous? */ + async: { + type: ParameterType.BOOL, + default: false, + }, + }, +}; + +type Info = typeof info; + +/** + * **initialize-microphone** + * + * jsPsych plugin for getting permission to initialize a microphone + * + * @author Josh de Leeuw + * @see {@link https://www.jspsych.org/plugins/jspsych-initialize-microphone/ initialize-microphone plugin documentation on jspsych.org} + */ +class InitializeMicrophonePlugin implements JsPsychPlugin { + static info = info; + + constructor(private jsPsych: JsPsych) {} + + trial(display_element: HTMLElement, trial: TrialType) { + this.run_trial(display_element).then((id) => { + this.jsPsych.finishTrial({ + device_id: id, + }); + }); + } + + private async run_trial(display_element) { + await this.askForPermission(); + + const devices = await navigator.mediaDevices.enumerateDevices(); + const mics = devices.filter( + (d) => d.kind === "audioinput" && d.deviceId !== "default" && d.deviceId !== "communications" + ); + + // remove entries with duplicate groupID + const unique_mics = mics.filter( + (mic, index, arr) => arr.findIndex((v) => v.groupId == mic.groupId) == index + ); + + const mic_id = await this.showMicrophoneSelection(display_element, unique_mics); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: mic_id } }); + + this.jsPsych.pluginAPI.initializeMicrophoneRecorder(stream); + + return mic_id; + } + + private async askForPermission() { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + return stream; + } + + private showMicrophoneSelection(display_element, devices: MediaDeviceInfo[]) { + let html = ` +

Please select the microphone you would like to use.

+ "; + html += '

'; + display_element.innerHTML = html; + + return new Promise((resolve) => { + display_element.querySelector("#btn-select-mic").addEventListener("click", () => { + const mic = display_element.querySelector("#which-mic").value; + resolve(mic); + }); + }); + } +} + +export default InitializeMicrophonePlugin; diff --git a/packages/plugin-initialize-microphone/tsconfig.json b/packages/plugin-initialize-microphone/tsconfig.json new file mode 100644 index 00000000..588f0448 --- /dev/null +++ b/packages/plugin-initialize-microphone/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@jspsych/config/tsconfig.core.json", + "compilerOptions": { + "baseUrl": "." + }, + "include": ["src"] +} From c12f05bd8689f885ee1d56caaacb72ec8106518b Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 19 Nov 2021 17:25:23 -0500 Subject: [PATCH 02/19] try moving stim display inside `start` event for media recorder --- packages/plugin-html-audio-response/src/index.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index b0bbe480..d8130906 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -42,17 +42,16 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { trial(display_element: HTMLElement, trial: TrialType) { const ro = new ResizeObserver((entries) => { - console.log(`ro perf = ${performance.now()}`); + console.log("ro event"); + this.stimulus_start_time = performance.now(); }); ro.observe(display_element); - this.showDisplay(display_element, trial); - this.addButtonEvent(display_element, trial); + // this.showDisplay(display_element, trial); + // this.addButtonEvent(display_element, trial); - this.stimulus_start_time = performance.now(); - - this.startRecording(); + this.startRecording(display_element, trial); // hide image if timing is set if (trial.stimulus_duration !== null) { @@ -87,7 +86,7 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { }); } - private startRecording() { + private startRecording(display_element, trial) { const recorder: MediaRecorder = this.jsPsych.pluginAPI.getMicrophoneRecorder(); const recorded_data_chunks = []; @@ -112,6 +111,8 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { recorder.addEventListener("start", (e) => { this.recorder_start_time = e.timeStamp; + this.showDisplay(display_element, trial); + this.addButtonEvent(display_element, trial); }); this.jsPsych.pluginAPI.setTimeout(() => { From 49f5cf4b02f06a391c0df03c4483f5b34976c6ac Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 19 Nov 2021 17:26:44 -0500 Subject: [PATCH 03/19] remove incorrectly duplicated testing files --- .../plugin-html-audio-response/package.json | 2 +- .../src/index.spec.ts | 156 ------------------ .../plugin-initialize-microphone/package.json | 2 +- .../src/index.spec.ts | 32 ---- 4 files changed, 2 insertions(+), 190 deletions(-) delete mode 100644 packages/plugin-html-audio-response/src/index.spec.ts delete mode 100644 packages/plugin-initialize-microphone/src/index.spec.ts diff --git a/packages/plugin-html-audio-response/package.json b/packages/plugin-html-audio-response/package.json index 0ab38603..d0a9cef5 100644 --- a/packages/plugin-html-audio-response/package.json +++ b/packages/plugin-html-audio-response/package.json @@ -16,7 +16,7 @@ ], "source": "src/index.ts", "scripts": { - "test": "jest", + "test": "jest --passWithNoTests", "test:watch": "npm test -- --watch", "tsc": "tsc", "build": "rollup --config", diff --git a/packages/plugin-html-audio-response/src/index.spec.ts b/packages/plugin-html-audio-response/src/index.spec.ts deleted file mode 100644 index f768522a..00000000 --- a/packages/plugin-html-audio-response/src/index.spec.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { clickTarget, startTimeline } from "@jspsych/test-utils"; - -import htmlButtonResponse from "."; - -jest.useFakeTimers(); - -describe("html-button-response", () => { - test("displays html stimulus", async () => { - const { getHTML } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - }, - ]); - - expect(getHTML()).toContain( - '
this is html
' - ); - }); - - test("display button labels", async () => { - const { getHTML } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice1", "button-choice2"], - }, - ]); - - expect(getHTML()).toContain(''); - expect(getHTML()).toContain(''); - }); - - test("display button html", async () => { - const { getHTML } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["buttonChoice"], - button_html: '', - }, - ]); - - expect(getHTML()).toContain(''); - }); - - test("display should clear after button click", async () => { - const { getHTML, expectFinished } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - }, - ]); - - expect(getHTML()).toContain( - '
this is html
' - ); - - clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); - - await expectFinished(); - }); - - test("prompt should append below button", async () => { - const { getHTML } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - prompt: "

this is a prompt

", - }, - ]); - - expect(getHTML()).toContain( - '

this is a prompt

' - ); - }); - - test("should hide stimulus if stimulus-duration is set", async () => { - const { displayElement } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - stimulus_duration: 500, - }, - ]); - - const stimulusElement = displayElement.querySelector( - "#jspsych-html-button-response-stimulus" - ); - - expect(stimulusElement.style.visibility).toBe(""); - - jest.advanceTimersByTime(500); - expect(stimulusElement.style.visibility).toBe("hidden"); - }); - - test("should end trial when trial duration is reached", async () => { - const { getHTML, expectFinished } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - trial_duration: 500, - }, - ]); - - expect(getHTML()).toContain( - '
this is html
' - ); - - jest.advanceTimersByTime(500); - await expectFinished(); - }); - - test("should end trial when button is clicked", async () => { - const { getHTML, expectFinished } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - response_ends_trial: true, - }, - ]); - - expect(getHTML()).toContain( - '
this is html
' - ); - - clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); - await expectFinished(); - }); - - test("class should have responded when button is clicked", async () => { - const { getHTML } = await startTimeline([ - { - type: htmlButtonResponse, - stimulus: "this is html", - choices: ["button-choice"], - response_ends_trial: false, - }, - ]); - - expect(getHTML()).toContain( - '
this is html
' - ); - - clickTarget(document.querySelector("#jspsych-html-button-response-button-0")); - expect(document.querySelector("#jspsych-html-button-response-stimulus").className).toBe( - " responded" - ); - }); -}); diff --git a/packages/plugin-initialize-microphone/package.json b/packages/plugin-initialize-microphone/package.json index e33b41e6..f20ba5b1 100644 --- a/packages/plugin-initialize-microphone/package.json +++ b/packages/plugin-initialize-microphone/package.json @@ -16,7 +16,7 @@ ], "source": "src/index.ts", "scripts": { - "test": "jest", + "test": "jest --passWithNoTests", "test:watch": "npm test -- --watch", "tsc": "tsc", "build": "rollup --config", diff --git a/packages/plugin-initialize-microphone/src/index.spec.ts b/packages/plugin-initialize-microphone/src/index.spec.ts deleted file mode 100644 index 33df9bb4..00000000 --- a/packages/plugin-initialize-microphone/src/index.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { startTimeline } from "@jspsych/test-utils"; - -import callFunction from "."; - -describe("call-function plugin", () => { - test("calls function", async () => { - const { getData, expectFinished } = await startTimeline([ - { - type: callFunction, - func: () => 1, - }, - ]); - - await expectFinished(); - expect(getData().values()[0].value).toBe(1); - }); - - test("async function works", async () => { - const { getData, expectFinished } = await startTimeline([ - { - type: callFunction, - async: true, - func: (done) => { - done(10); - }, - }, - ]); - - await expectFinished(); - expect(getData().values()[0].value).toBe(10); - }); -}); From aa660ad597c364d7423a90f3db2719c1a10bd8fd Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Mon, 22 Nov 2021 10:38:05 -0500 Subject: [PATCH 04/19] add customization of labels; refresh list of mics based on live availability --- .../plugin-initialize-microphone/src/index.ts | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/packages/plugin-initialize-microphone/src/index.ts b/packages/plugin-initialize-microphone/src/index.ts index 8a78e524..9a08c97b 100644 --- a/packages/plugin-initialize-microphone/src/index.ts +++ b/packages/plugin-initialize-microphone/src/index.ts @@ -4,14 +4,14 @@ const info = { name: "initialize-microphone", parameters: { /** Function to call */ - func: { - type: ParameterType.BOOL, - default: true, + device_select_message: { + type: ParameterType.HTML_STRING, + default: `

Please select the microphone you would like to use.

`, }, /** Is the function call asynchronous? */ - async: { - type: ParameterType.BOOL, - default: false, + button_label: { + type: ParameterType.STRING, + default: "Use this microphone", }, }, }; @@ -32,27 +32,25 @@ class InitializeMicrophonePlugin implements JsPsychPlugin { constructor(private jsPsych: JsPsych) {} trial(display_element: HTMLElement, trial: TrialType) { - this.run_trial(display_element).then((id) => { + this.run_trial(display_element, trial).then((id) => { this.jsPsych.finishTrial({ device_id: id, }); }); } - private async run_trial(display_element) { + private async run_trial(display_element: HTMLElement, trial: TrialType) { await this.askForPermission(); - const devices = await navigator.mediaDevices.enumerateDevices(); - const mics = devices.filter( - (d) => d.kind === "audioinput" && d.deviceId !== "default" && d.deviceId !== "communications" - ); + this.showMicrophoneSelection(display_element, trial); - // remove entries with duplicate groupID - const unique_mics = mics.filter( - (mic, index, arr) => arr.findIndex((v) => v.groupId == mic.groupId) == index - ); + this.updateDeviceList(display_element); - const mic_id = await this.showMicrophoneSelection(display_element, unique_mics); + navigator.mediaDevices.ondevicechange = (e) => { + this.updateDeviceList(display_element); + }; + + const mic_id = await this.waitForSelection(display_element); const stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: mic_id } }); @@ -66,17 +64,16 @@ class InitializeMicrophonePlugin implements JsPsychPlugin { return stream; } - private showMicrophoneSelection(display_element, devices: MediaDeviceInfo[]) { + private showMicrophoneSelection(display_element, trial: TrialType) { let html = ` -

Please select the microphone you would like to use.

- "; - html += '

'; + ${trial.device_select_message} + +

`; display_element.innerHTML = html; + } + private waitForSelection(display_element) { return new Promise((resolve) => { display_element.querySelector("#btn-select-mic").addEventListener("click", () => { const mic = display_element.querySelector("#which-mic").value; @@ -84,6 +81,31 @@ class InitializeMicrophonePlugin implements JsPsychPlugin { }); }); } + + private updateDeviceList(display_element) { + navigator.mediaDevices.enumerateDevices().then((devices) => { + const mics = devices.filter( + (d) => + d.kind === "audioinput" && d.deviceId !== "default" && d.deviceId !== "communications" + ); + + // remove entries with duplicate groupID + const unique_mics = mics.filter( + (mic, index, arr) => arr.findIndex((v) => v.groupId == mic.groupId) == index + ); + + // reset the list by clearing all current options + display_element.querySelector("#which-mic").innerHTML = ""; + + unique_mics.forEach((d) => { + let el = document.createElement("option"); + el.value = d.deviceId; + el.innerHTML = d.label; + + display_element.querySelector("#which-mic").appendChild(el); + }); + }); + } } export default InitializeMicrophonePlugin; From 827df6c95d57961940016f7c8f9d7bc8b2d84a60 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Mon, 22 Nov 2021 11:48:19 -0500 Subject: [PATCH 05/19] reorganization and parameterization --- examples/jspsych-initialize-microphone.html | 5 +- .../plugin-html-audio-response/src/index.ts | 85 ++++++++++++------- 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/examples/jspsych-initialize-microphone.html b/examples/jspsych-initialize-microphone.html index dc581781..f0a859c1 100644 --- a/examples/jspsych-initialize-microphone.html +++ b/examples/jspsych-initialize-microphone.html @@ -17,7 +17,10 @@ let ar = { type: jsPsychHtmlAudioResponse, - stimulus: '

Speak!

' + stimulus: '

Speak!

', + on_finish: (data) => { + console.log(data); + } } jsPsych.run([init_mic, ar]); diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index d8130906..d4c556c3 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -9,7 +9,6 @@ const info = { pretty_name: "Stimulus", default: undefined, }, - /** How long to show the stimulus. */ stimulus_duration: { type: ParameterType.INT, @@ -20,7 +19,11 @@ const info = { trial_duration: { type: ParameterType.INT, pretty_name: "Trial duration", - default: null, + default: 2000, + }, + button_label: { + type: ParameterType.STRING, + default: "Continue", }, }, }; @@ -37,88 +40,104 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { static info = info; private stimulus_start_time; private recorder_start_time; + private recorder: MediaRecorder; + private response; + private load_resolver; constructor(private jsPsych: JsPsych) {} trial(display_element: HTMLElement, trial: TrialType) { - const ro = new ResizeObserver((entries) => { - console.log("ro event"); - this.stimulus_start_time = performance.now(); - }); + this.recorder = this.jsPsych.pluginAPI.getMicrophoneRecorder(); - ro.observe(display_element); + this.setupRecordingEvents(display_element, trial); - // this.showDisplay(display_element, trial); - // this.addButtonEvent(display_element, trial); + this.startRecording(); - this.startRecording(display_element, trial); - - // hide image if timing is set + // setup timer for hiding the stimulus if (trial.stimulus_duration !== null) { this.jsPsych.pluginAPI.setTimeout(() => { - display_element.querySelector( - "#jspsych-html-audio-response-stimulus" - ).style.visibility = "hidden"; + this.hideStimulus(display_element); }, trial.stimulus_duration); } - // end trial if time limit is set + // setup timer for ending the trial if (trial.trial_duration !== null) { this.jsPsych.pluginAPI.setTimeout(() => { - this.endTrial(display_element, trial); + this.stopRecording().then(() => { + this.endTrial(display_element, trial); + }); }, trial.trial_duration); } } private showDisplay(display_element, trial) { + const ro = new ResizeObserver((entries) => { + this.stimulus_start_time = performance.now(); + console.log("ro event"); + ro.disconnect(); + }); + + ro.observe(display_element); + let html = `
${trial.stimulus}
`; - html += `

`; + html += `

`; display_element.innerHTML = html; } + private hideStimulus(display_element: HTMLElement) { + display_element.querySelector( + "#jspsych-html-audio-response-stimulus" + ).style.visibility = "hidden"; + } + private addButtonEvent(display_element, trial) { display_element.querySelector("#finish-trial").addEventListener("click", () => { const end_time = performance.now(); const rt = Math.round(end_time - this.stimulus_start_time); - this.endTrial(display_element, trial, rt); + this.stopRecording().then(() => { + this.endTrial(display_element, trial, rt); + }); }); } - private startRecording(display_element, trial) { - const recorder: MediaRecorder = this.jsPsych.pluginAPI.getMicrophoneRecorder(); + private setupRecordingEvents(display_element, trial) { const recorded_data_chunks = []; - recorder.addEventListener("dataavailable", (e) => { + this.recorder.addEventListener("dataavailable", (e) => { if (e.data.size > 0) { recorded_data_chunks.push(e.data); } - console.log("dataavail event"); }); - recorder.addEventListener("stop", () => { + this.recorder.addEventListener("stop", () => { const url = new Blob(recorded_data_chunks, { type: "audio/webm" }); const reader = new FileReader(); reader.addEventListener("load", () => { const base64 = (reader.result as string).split(",")[1]; - console.log(base64); + this.response = base64; + this.load_resolver(); }); reader.readAsDataURL(url); - - console.log(this.stimulus_start_time, this.recorder_start_time); }); - recorder.addEventListener("start", (e) => { + this.recorder.addEventListener("start", (e) => { this.recorder_start_time = e.timeStamp; this.showDisplay(display_element, trial); this.addButtonEvent(display_element, trial); }); + } - this.jsPsych.pluginAPI.setTimeout(() => { - recorder.stop(); - }, 2000); - recorder.start(); + private startRecording() { + this.recorder.start(); + } + + private stopRecording() { + this.recorder.stop(); + return new Promise((resolve) => { + this.load_resolver = resolve; + }); } private endTrial(display_element, trial, rt: number = null) { @@ -129,6 +148,8 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { var trial_data = { rt: rt, stimulus: trial.stimulus, + response: this.response, + estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time), }; // clear the display From ffd7e1e45a3a2f55a1f7910b4b60bfc92a2e6181 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Mon, 22 Nov 2021 12:21:10 -0500 Subject: [PATCH 06/19] add `allow_playback` option for rerecording audio --- examples/jspsych-initialize-microphone.html | 1 + .../plugin-html-audio-response/src/index.ts | 80 +++++++++++++------ 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/examples/jspsych-initialize-microphone.html b/examples/jspsych-initialize-microphone.html index f0a859c1..fd8d5507 100644 --- a/examples/jspsych-initialize-microphone.html +++ b/examples/jspsych-initialize-microphone.html @@ -18,6 +18,7 @@ let ar = { type: jsPsychHtmlAudioResponse, stimulus: '

Speak!

', + allow_playback: true, on_finish: (data) => { console.log(data); } diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index d4c556c3..31f389a2 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -16,7 +16,7 @@ const info = { default: null, }, /** How long to show the trial. */ - trial_duration: { + recording_duration: { type: ParameterType.INT, pretty_name: "Trial duration", default: 2000, @@ -25,6 +25,10 @@ const info = { type: ParameterType.STRING, default: "Continue", }, + allow_playback: { + type: ParameterType.BOOL, + default: false, + }, }, }; @@ -41,8 +45,10 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { private stimulus_start_time; private recorder_start_time; private recorder: MediaRecorder; + private audio_url; private response; private load_resolver; + private rt: number = null; constructor(private jsPsych: JsPsych) {} @@ -52,22 +58,6 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { this.setupRecordingEvents(display_element, trial); this.startRecording(); - - // setup timer for hiding the stimulus - if (trial.stimulus_duration !== null) { - this.jsPsych.pluginAPI.setTimeout(() => { - this.hideStimulus(display_element); - }, trial.stimulus_duration); - } - - // setup timer for ending the trial - if (trial.trial_duration !== null) { - this.jsPsych.pluginAPI.setTimeout(() => { - this.stopRecording().then(() => { - this.endTrial(display_element, trial); - }); - }, trial.trial_duration); - } } private showDisplay(display_element, trial) { @@ -95,9 +85,13 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { private addButtonEvent(display_element, trial) { display_element.querySelector("#finish-trial").addEventListener("click", () => { const end_time = performance.now(); - const rt = Math.round(end_time - this.stimulus_start_time); + this.rt = Math.round(end_time - this.stimulus_start_time); this.stopRecording().then(() => { - this.endTrial(display_element, trial, rt); + if (trial.allow_playback) { + this.showPlaybackControls(display_element, trial); + } else { + this.endTrial(display_element, trial); + } }); }); } @@ -112,20 +106,44 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { }); this.recorder.addEventListener("stop", () => { - const url = new Blob(recorded_data_chunks, { type: "audio/webm" }); + const data = new Blob(recorded_data_chunks, { type: "audio/webm" }); + this.audio_url = URL.createObjectURL(data); const reader = new FileReader(); reader.addEventListener("load", () => { const base64 = (reader.result as string).split(",")[1]; this.response = base64; this.load_resolver(); }); - reader.readAsDataURL(url); + reader.readAsDataURL(data); }); this.recorder.addEventListener("start", (e) => { + // resets the recorded data + recorded_data_chunks.length = 0; + this.recorder_start_time = e.timeStamp; this.showDisplay(display_element, trial); this.addButtonEvent(display_element, trial); + + // setup timer for hiding the stimulus + if (trial.stimulus_duration !== null) { + this.jsPsych.pluginAPI.setTimeout(() => { + this.hideStimulus(display_element); + }, trial.stimulus_duration); + } + + // setup timer for ending the trial + if (trial.recording_duration !== null) { + this.jsPsych.pluginAPI.setTimeout(() => { + this.stopRecording().then(() => { + if (trial.allow_playback) { + this.showPlaybackControls(display_element, trial); + } else { + this.endTrial(display_element, trial); + } + }); + }, trial.recording_duration); + } }); } @@ -140,13 +158,29 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { }); } - private endTrial(display_element, trial, rt: number = null) { + private showPlaybackControls(display_element, trial) { + display_element.innerHTML = ` +

+ + + `; + + display_element.querySelector("#record-again").addEventListener("click", this.startRecording); + display_element.querySelector("#continue").addEventListener("click", () => { + this.endTrial(display_element, trial); + }); + + // const audio = display_element.querySelector('#playback'); + // audio.src = + } + + private endTrial(display_element, trial) { // kill any remaining setTimeout handlers this.jsPsych.pluginAPI.clearAllTimeouts(); // gather the data to store for the trial var trial_data = { - rt: rt, + rt: this.rt, stimulus: trial.stimulus, response: this.response, estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time), From 0cc69e0854eb59fe2de3131e42d9b45d88fb870f Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Mon, 22 Nov 2021 14:39:16 -0500 Subject: [PATCH 07/19] add parameter to store audio url; release it if not stored --- .../plugin-html-audio-response/src/index.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index 31f389a2..c08ccefb 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -29,6 +29,10 @@ const info = { type: ParameterType.BOOL, default: false, }, + store_audio_url: { + type: ParameterType.BOOL, + default: false, + }, }, }; @@ -165,7 +169,11 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { `; - display_element.querySelector("#record-again").addEventListener("click", this.startRecording); + display_element.querySelector("#record-again").addEventListener("click", () => { + // release object url to save memory + URL.revokeObjectURL(this.audio_url); + this.startRecording(); + }); display_element.querySelector("#continue").addEventListener("click", () => { this.endTrial(display_element, trial); }); @@ -179,13 +187,19 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { this.jsPsych.pluginAPI.clearAllTimeouts(); // gather the data to store for the trial - var trial_data = { + var trial_data: any = { rt: this.rt, stimulus: trial.stimulus, response: this.response, estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time), }; + if (trial.store_audio_url) { + trial_data.audio_url = this.audio_url; + } else { + URL.revokeObjectURL(this.audio_url); + } + // clear the display display_element.innerHTML = ""; From 5568ca3b26bd68f27307fba17039acb008db35da Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Tue, 23 Nov 2021 14:51:55 -0500 Subject: [PATCH 08/19] add docs for new audio response plugins --- .../jspsych-html-audio-response-demo1.html | 62 ++++++++ .../jspsych-html-audio-response-demo2.html | 70 +++++++++ .../jspsych-html-audio-response-demo3.html | 82 ++++++++++ .../jspsych-initialize-microphone-demo1.html | 53 +++++++ docs/plugins/html-audio-response.md | 144 ++++++++++++++++++ docs/plugins/initialize-microphone.md | 45 ++++++ docs/plugins/list-of-plugins.md | 2 + examples/jspsych-initialize-microphone.html | 2 +- mkdocs.yml | 2 + .../plugin-html-audio-response/src/index.ts | 52 ++++--- 10 files changed, 494 insertions(+), 20 deletions(-) create mode 100644 docs/demos/jspsych-html-audio-response-demo1.html create mode 100644 docs/demos/jspsych-html-audio-response-demo2.html create mode 100644 docs/demos/jspsych-html-audio-response-demo3.html create mode 100644 docs/demos/jspsych-initialize-microphone-demo1.html create mode 100644 docs/plugins/html-audio-response.md create mode 100644 docs/plugins/initialize-microphone.md diff --git a/docs/demos/jspsych-html-audio-response-demo1.html b/docs/demos/jspsych-html-audio-response-demo1.html new file mode 100644 index 00000000..f72278ec --- /dev/null +++ b/docs/demos/jspsych-html-audio-response-demo1.html @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + diff --git a/docs/demos/jspsych-html-audio-response-demo2.html b/docs/demos/jspsych-html-audio-response-demo2.html new file mode 100644 index 00000000..90242ff4 --- /dev/null +++ b/docs/demos/jspsych-html-audio-response-demo2.html @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + diff --git a/docs/demos/jspsych-html-audio-response-demo3.html b/docs/demos/jspsych-html-audio-response-demo3.html new file mode 100644 index 00000000..8af68a30 --- /dev/null +++ b/docs/demos/jspsych-html-audio-response-demo3.html @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + diff --git a/docs/demos/jspsych-initialize-microphone-demo1.html b/docs/demos/jspsych-initialize-microphone-demo1.html new file mode 100644 index 00000000..022b540d --- /dev/null +++ b/docs/demos/jspsych-initialize-microphone-demo1.html @@ -0,0 +1,53 @@ + + + + + + --> + + + + + + + diff --git a/docs/plugins/html-audio-response.md b/docs/plugins/html-audio-response.md new file mode 100644 index 00000000..c32470c0 --- /dev/null +++ b/docs/plugins/html-audio-response.md @@ -0,0 +1,144 @@ +# html-audio-response + +This plugin displays HTML content and records audio from the participant via a microphone. + +In order to get access to the microphone, you need to use the [initialize-microphone plugin](initialize-microphone.md) on your timeline prior to using this plugin. +Once access is granted for an experiment you do not need to get permission again. + +This plugin records audio data in [base 64 format](https://developer.mozilla.org/en-US/docs/Glossary/Base64). +This is a text-based representation of the audio which can be coverted to various audio formats using a variety of [online tools](https://www.google.com/search?q=base64+audio+decoder) as well as in languages like python and R. + +**This plugin will generate a large amount of data, and you will need to be careful about how you handle this data.** +Even a few seconds of audio recording will add 10s of kB to jsPsych's data. +Multiply this by a handful (or more) of trials, and the data objects will quickly get large. +If you need to record a lot of audio, either many trials worth or just a few trials with longer responses, we recommend that you save the data to your server immediately after the trial and delete the data in jsPsych's data object. +See below for an example of how to do this. + +This plugin also provides the option to store the recorded audio files as [Object URLs](https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL) via `save_audio_url: true`. +This will generate a URL that is storing a copy of the recorded audio, which can be used for subsequent playback. +See below for an example where the recorded audio is used as the stimulus in a subsequent trial. +This feature is turned off by default because it uses a relatively large amount of memory compared to most jsPsych features. +If you are running an experiment where you need this feature and you are recording lots of audio snippets, you may want to manually revoke the URLs when you no longer need them using [`URL.revokeObjectURL(objectURL)`](https://developer.mozilla.org/en-US/docs/Web/API/URL/revokeObjectURL). + +!!! warning + When recording from a microphone your experiment will need to be running over `https://` protocol. If you try to run the experiment locally using the `file://` protocol or over `http://` protocol you will not be able to access the microphone because of [potential security problems](https://blog.mozilla.org/webrtc/camera-microphone-require-https-in-firefox-68/). + +## Parameters + +In addition to the [parameters available in all plugins](../overview/plugins.md#parameters-available-in-all-plugins), this plugin accepts the following parameters. Parameters with a default value of *undefined* must be specified. Other parameters can be left unspecified if the default value is acceptable. + +Parameter | Type | Default Value | Description +----------|------|---------------|------------ +stimulus | HTML string | undefined | The HTML content to be displayed. +recording_duration | numeric | 2000 | The maximum length of the recording, in milliseconds. The default value is intentionally set low because of the potential to accidentally record very large data files if left too high. You can set this to `null` to allow the participant to control the length of the recording via the done button, but be careful with this option as it can lead to crashing the browser if the participant waits too long to stop the recording. +stimulus_duration | numeric | null | How long to display the stimulus in milliseconds. The visibility CSS property of the stimulus will be set to `hidden` after this time has elapsed. If this is null, then the stimulus will remain visible until the trial ends. +show_done_button | bool | true | Whether to show a button on the screen that the participant can click to finish the recording. +done_button_label | string | 'Continue' | The label for the done button. +allow_playback | bool | false | Whether to allow the participant to listen to their recording and decide whether to rerecord or not. If `true`, then the participant will be shown an interface to play their recorded audio and click one of two buttons to either accept the recording or rerecord. If rerecord is selected, then stimulus will be shown again, as if the trial is starting again from the beginning. +record_again_button_label | string | 'Record again' | The label for the record again button enabled when `allow_playback: true`. +accept_button_label | string | 'Continue' | The label for the accept button enabled when `allow_playback: true`. +save_audio_url | bool | false | If `true`, then an [Object URL](https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL) will be generated and stored for the recorded audio. Only set this to `true` if you plan to reuse the recorded audio later in the experiment, as it is a potentially memory-intensive feature. + + +## Data Generated + +In addition to the [default data collected by all plugins](../overview/plugins.md#data-collected-by-all-plugins), this plugin collects the following data for each trial. + +Name | Type | Value +-----|------|------ +rt | numeric | The time, since the onset of the stimulus, for the participant to click the done button. If the button is not clicked (or not enabled), then `rt` will be `null`. +response | base64 string | The base64-encoded audio data. +stimulus | string | The HTML content that was displayed on the screen. +estimated_stimulus_onset | number | This is an estimate of when the stimulus appeared relative to the start of the audio recording. The plugin is configured so that the recording should start prior to the display of the stimulus. We have not yet been able to verify the accuracy of this estimate with external measurement devices. +audio_url | string | A URL to a copy of the audio data. + +## Examples + +???+ example "Simple spoken response to a stimulus" + === "Code" + ```javascript + var trial = { + type: jsPsychHtmlAudioResponse, + stimulus: ` +

GREEN

+

Speak the color of the ink.

`, + recording_duration: 3500 + }; + ``` + + === "Demo" +
+ +
+ + Open demo in new tab + +???+ example "Allow playback and rerecording; save data to server immediately" + === "Code" + ```javascript + var trial = { + type: jsPsychHtmlAudioResponse, + stimulus: ` +

Please sing the first few seconds of a song and click the button when you are done.

+ `, + recording_duration: 15000, + allow_playback: true, + on_finish: function(data){ + fetch('/save-my-data.php', { audio_base64: data.response }) + .then((audio_id){ + data.response = audio_id; + }); + } + }; + ``` + + This example assumes that there is a script on your experiment server that accepts the data called `save-my-data.php`. It also assumes that the script will generate a response with an ID for the stored audio file (`audio_id`). In the example, we replace the very long base64 representation of the audio file with the generated ID, which could be just a handful of characters. This would let you link files to responses in data analysis, without having to store long audio files in memory during the experiment. + + === "Demo" +
+ +
+ + Open demo in new tab + +???+ example "Use recorded audio as a subsequent stimulus" + === "Code" + ```javascript + var instruction = { + type: jsPsychHtmlButtonResponse, + stimulus: ` + +

Make up a name for this shape. When you have one in mind, click the button and then say the name aloud.

+ `, + choices: ['I am ready.'] + } + + var record = { + type: jsPsychHtmlAudioResponse, + stimulus: ` + +

Recording...

+ `, + recording_duration: 1500, + save_audio_url: true + }; + + var playback = { + type: jsPsychAudioButtonResponse, + stimulus: ()=>{ + return jsPsych.data.get().last(1).values()[0].audio_url; + }, + prompt: '

Click the object the matches the spoken name.

', + choices: ['img/9.gif','img/10.gif','img/11.gif','img/12.gif'], + button_html: '' + } + ``` + + === "Demo" +
+ +
+ + Open demo in new tab + + diff --git a/docs/plugins/initialize-microphone.md b/docs/plugins/initialize-microphone.md new file mode 100644 index 00000000..c85e45ac --- /dev/null +++ b/docs/plugins/initialize-microphone.md @@ -0,0 +1,45 @@ +# initialize-microphone + +This plugin asks the participant to grant permission to access a microphone. +If multiple microphones are connected to the participant's device, then it allows the participant to pick which device to use. +Once access is granted for an experiment you do not need to get permission again. + +Once the microphone is selected with this plugin it can be accessed with [`jsPsych.pluginAPI.getMicrophoneRecorder()`](dead-link.md). + +!!! warning + When recording from a microphone your experiment will need to be running over `https://` protocol. If you try to run the experiment locally using the `file://` protocol or over `http://` protocol you will not be able to access the microphone because of [potential security problems](https://blog.mozilla.org/webrtc/camera-microphone-require-https-in-firefox-68/). + +## Parameters + +In addition to the [parameters available in all plugins](../overview/plugins.md#parameters-available-in-all-plugins), this plugin accepts the following parameters. Parameters with a default value of *undefined* must be specified. Other parameters can be left unspecified if the default value is acceptable. + +Parameter | Type | Default Value | Description +----------|------|---------------|------------ +device_select_message | html string | `

Please select the microphone you would like to use.

` | The message to display when the user is presented with a dropdown list of available devices. +button_label | sting | 'Use this microphone.' | The label for the select button. + + +## Data Generated + +In addition to the [default data collected by all plugins](../overview/plugins.md#data-collected-by-all-plugins), this plugin collects the following data for each trial. + +Name | Type | Value +-----|------|------ +device_id | string | The [device ID](https://developer.mozilla.org/en-US/docs/Web/API/MediaDeviceInfo/deviceId) of the selected microphone. + +## Examples + +???+ example "Ask for microphone permission" + === "Code" + ```javascript + var trial = { + type: jsPsychInitializeMicrophone + }; + ``` + + === "Demo" +
+ +
+ + Open demo in new tab \ No newline at end of file diff --git a/docs/plugins/list-of-plugins.md b/docs/plugins/list-of-plugins.md index 43c8f7ce..273d4541 100644 --- a/docs/plugins/list-of-plugins.md +++ b/docs/plugins/list-of-plugins.md @@ -24,6 +24,7 @@ Plugin | Description [external‑html](external-html.md) | Displays an external HTML page (such as a consent form) and lets the subject respond by clicking a button or pressing a key. Plugin can validate their response, which is useful for making sure that a subject has granted consent before starting the experiment. [free‑sort](free-sort.md) | Displays a set of images on the screen in random locations. Subjects can click and drag the images to move them around the screen. Records all the moves made by the subject, so the sequence of moves can be recovered from the data. [fullscreen](fullscreen.md) | Toggles the experiment in and out of fullscreen mode. +[html‑audio‑response](html-audio-response.md) | Display an HTML-formatted stimulus and records an audio response via a microphone. [html‑button‑response](html-button-response.md) | Display an HTML-formatted stimulus and allow the subject to respond by choosing a button to click. The button can be customized extensively, e.g., using images in place of standard buttons. [html‑keyboard‑response](html-keyboard-response.md) | Display an HTML-formatted stimulus and allow the subject to respond by pressing a key. [html‑slider‑response](html-slider-response.md) | Display an HTML-formatted stimulus and allow the subject to respond by moving a slider to indicate a value. @@ -32,6 +33,7 @@ Plugin | Description [image‑button‑response](image-button-response.md) | Display an image and allow the subject to respond by choosing a button to click. The button can be customized extensively, e.g., using images in place of standard buttons. [image‑keyboard‑response](image-keyboard-response.md) | Display an image and allow the subject to respond by pressing a key. [image‑slider‑response](image-slider-response.md) | Display an image and allow the subject to respond by moving a slider to indicate a value. +[initialize‑microphone](initialize-microphone.md) | Request permission to use the subject's microphone to record audio and allows the subject to choose which microphone to use if multiple devices are enabled. [instructions](instructions.md) | For displaying instructions to the subject. Allows the subject to navigate between pages of instructions using keys or buttons. [maxdiff](maxdiff.md) | Displays rows of alternatives to be selected for two mutually-exclusive categories, typically as 'most' or 'least' on a particular criteria (e.g. importance, preference, similarity). The participant responds by selecting one radio button corresponding to an alternative in both the left and right response columns. [preload](preload.md) | This plugin loads images, audio, and video files into the browser's memory before they are needed in the experiment, in order to improve stimulus and response timing, and to avoid disrupting the flow of the experiment. diff --git a/examples/jspsych-initialize-microphone.html b/examples/jspsych-initialize-microphone.html index fd8d5507..b4014e91 100644 --- a/examples/jspsych-initialize-microphone.html +++ b/examples/jspsych-initialize-microphone.html @@ -17,7 +17,7 @@ let ar = { type: jsPsychHtmlAudioResponse, - stimulus: '

Speak!

', + stimulus: '
', allow_playback: true, on_finish: (data) => { console.log(data); diff --git a/mkdocs.yml b/mkdocs.yml index e2bb334e..c7a44962 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,6 +86,7 @@ nav: - 'external-html': 'plugins/external-html.md' - 'free-sort': 'plugins/free-sort.md' - 'fullscreen': 'plugins/fullscreen.md' + - 'html-audio-response': 'plugins/html-audio-response.md' - 'html-button-response': 'plugins/html-button-response.md' - 'html-keyboard-response': 'plugins/html-keyboard-response.md' - 'html-slider-response': 'plugins/html-slider-response.md' @@ -94,6 +95,7 @@ nav: - 'image-button-response': 'plugins/image-button-response.md' - 'image-keyboard-response': 'plugins/image-keyboard-response.md' - 'image-slider-response': 'plugins/image-slider-response.md' + - 'initialize-microphone': 'plugins/initialize-microphone.md' - 'instructions': 'plugins/instructions.md' - 'maxdiff': 'plugins/maxdiff.md' - 'preload': 'plugins/preload.md' diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index c08ccefb..37e768c0 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -6,22 +6,31 @@ const info = { /** The HTML string to be displayed */ stimulus: { type: ParameterType.HTML_STRING, - pretty_name: "Stimulus", default: undefined, }, /** How long to show the stimulus. */ stimulus_duration: { type: ParameterType.INT, - pretty_name: "Stimulus duration", default: null, }, /** How long to show the trial. */ recording_duration: { type: ParameterType.INT, - pretty_name: "Trial duration", default: 2000, }, - button_label: { + show_done_button: { + type: ParameterType.BOOL, + default: true, + }, + done_button_label: { + type: ParameterType.STRING, + default: "Continue", + }, + record_again_button_label: { + type: ParameterType.STRING, + default: "Record again", + }, + accept_button_label: { type: ParameterType.STRING, default: "Continue", }, @@ -29,7 +38,7 @@ const info = { type: ParameterType.BOOL, default: false, }, - store_audio_url: { + save_audio_url: { type: ParameterType.BOOL, default: false, }, @@ -75,7 +84,9 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { let html = `
${trial.stimulus}
`; - html += `

`; + if (trial.show_done_button) { + html += `

`; + } display_element.innerHTML = html; } @@ -87,17 +98,20 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { } private addButtonEvent(display_element, trial) { - display_element.querySelector("#finish-trial").addEventListener("click", () => { - const end_time = performance.now(); - this.rt = Math.round(end_time - this.stimulus_start_time); - this.stopRecording().then(() => { - if (trial.allow_playback) { - this.showPlaybackControls(display_element, trial); - } else { - this.endTrial(display_element, trial); - } + const btn = display_element.querySelector("#finish-trial"); + if (btn) { + btn.addEventListener("click", () => { + const end_time = performance.now(); + this.rt = Math.round(end_time - this.stimulus_start_time); + this.stopRecording().then(() => { + if (trial.allow_playback) { + this.showPlaybackControls(display_element, trial); + } else { + this.endTrial(display_element, trial); + } + }); }); - }); + } } private setupRecordingEvents(display_element, trial) { @@ -165,8 +179,8 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { private showPlaybackControls(display_element, trial) { display_element.innerHTML = `

- - + + `; display_element.querySelector("#record-again").addEventListener("click", () => { @@ -194,7 +208,7 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time), }; - if (trial.store_audio_url) { + if (trial.save_audio_url) { trial_data.audio_url = this.audio_url; } else { URL.revokeObjectURL(this.audio_url); From c81b500771ce449ac8145925170a63a08ab0465e Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Tue, 23 Nov 2021 15:06:03 -0500 Subject: [PATCH 09/19] add changesets --- .changeset/calm-books-wink.md | 5 +++++ .changeset/green-buttons-suffer.md | 5 +++++ .changeset/rich-parents-relax.md | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 .changeset/calm-books-wink.md create mode 100644 .changeset/green-buttons-suffer.md create mode 100644 .changeset/rich-parents-relax.md diff --git a/.changeset/calm-books-wink.md b/.changeset/calm-books-wink.md new file mode 100644 index 00000000..8a27d739 --- /dev/null +++ b/.changeset/calm-books-wink.md @@ -0,0 +1,5 @@ +--- +"jspsych": minor +--- + +Added microphone related features to the `pluginAPI` module: `initializeMicrophoneRecorder()` and `getMicrophoneRecorder()`. These allow sharing of the `MediaRecorder` object attached to the microphone's `MediaStream` across trials. diff --git a/.changeset/green-buttons-suffer.md b/.changeset/green-buttons-suffer.md new file mode 100644 index 00000000..3fc5ec60 --- /dev/null +++ b/.changeset/green-buttons-suffer.md @@ -0,0 +1,5 @@ +--- +"@jspsych/plugin-initiliaze-microphone": major +--- + +Initial release of the `initialize-microphone` plugin. This plugin handles getting permission to use the microphone and selecting which microphone to use. See [the plugin's documentation](https://www.jspsych.org/latest/plugins/initialize-microphone) for details. diff --git a/.changeset/rich-parents-relax.md b/.changeset/rich-parents-relax.md new file mode 100644 index 00000000..3473dbe3 --- /dev/null +++ b/.changeset/rich-parents-relax.md @@ -0,0 +1,5 @@ +--- +"@jspsych/plugin-html-audio-response": major +--- + +Initial release of the `html-audio-response` plugin. Allows recording audio responses from the participant via a microphone. See [the plugin's documentation](https://www.jspsych.org/latest/plugins/html-audio-response) for details. From aa0d28f990da1676da9c1fb02c07597c62f7d910 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Tue, 23 Nov 2021 15:24:17 -0500 Subject: [PATCH 10/19] make default `null` --- packages/jspsych/src/modules/plugin-api/MediaAPI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts index 27868fe1..c94a8f2b 100644 --- a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts +++ b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts @@ -324,7 +324,7 @@ export class MediaAPI { this.preload_requests = []; } - private microphone_recorder: MediaRecorder; + private microphone_recorder: MediaRecorder = null; initializeMicrophoneRecorder(stream: MediaStream) { const recorder = new MediaRecorder(stream); From e2f843b27c045936f17b58557b0d718b2ed2b0ed Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Tue, 23 Nov 2021 15:24:31 -0500 Subject: [PATCH 11/19] add docs for new pluginAPI features --- docs/reference/jspsych-pluginAPI.md | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/reference/jspsych-pluginAPI.md b/docs/reference/jspsych-pluginAPI.md index 32e56fc7..e152fa9d 100644 --- a/docs/reference/jspsych-pluginAPI.md +++ b/docs/reference/jspsych-pluginAPI.md @@ -325,6 +325,63 @@ jsPsych.pluginAPI.getAutoPreloadList(timeline); --- +### getMicrophoneRecorder + +```javascript +jsPsych.pluginAPI.getMicrophoneRecorder() +``` + +#### Parameters + +None + +#### Return value + +A `MediaRecorder` object connected to the `MediaStream` for the active microphone. + +#### Description + +Provides access to the `MediaRecorder` created by [initializeMicrophoneRecorder()](#initializemicrophonerecorder). +If no microphone recorder exists, it returns `null`. + +#### Example + +```javascript +const recorder = jsPsych.pluginAPI.getMicrophoneRecorder(); +``` + +--- + +### initializeMicrophoneRecorder + +```javascript +jsPsych.pluginAPI.initializeMicrophoneRecorder(stream) +``` + +#### Parameters + +Parameter | Type | Description +----------|------|------------ +stream | `MediaStream` | The `MediaStream` object from an active microphone device. + +#### Return value + +None. + +#### Description + +Generates a `MediaRecorder` object from provided `MediaStream` and stores this for access via [getMicrophoneRecorder()](#getmicrophonerecorder). + +#### Example + +```javascript +const stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: mic_id } }); + +jsPsych.pluginAPI.initializeMicrophoneRecorder(stream); +``` + +--- + ### preloadAudio ```javascript From 7dbc303f90c2133a9cd815ac7bf06f13ed28c02e Mon Sep 17 00:00:00 2001 From: bjoluc Date: Wed, 24 Nov 2021 16:51:57 +0100 Subject: [PATCH 12/19] add MediaRecorder types, polyfill ResizeObserver --- package-lock.json | 76 ++++++++++++++++++- packages/jspsych/package.json | 3 +- .../plugin-html-audio-response/package.json | 3 + .../plugin-html-audio-response/src/index.ts | 1 + 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index cfcab522..e4b86170 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2502,6 +2502,10 @@ "resolved": "packages/plugin-fullscreen", "link": true }, + "node_modules/@jspsych/plugin-html-audio-response": { + "resolved": "packages/plugin-html-audio-response", + "link": true + }, "node_modules/@jspsych/plugin-html-button-response": { "resolved": "packages/plugin-html-button-response", "link": true @@ -2534,6 +2538,10 @@ "resolved": "packages/plugin-image-slider-response", "link": true }, + "node_modules/@jspsych/plugin-initiliaze-microphone": { + "resolved": "packages/plugin-initialize-microphone", + "link": true + }, "node_modules/@jspsych/plugin-instructions": { "resolved": "packages/plugin-instructions", "link": true @@ -2913,6 +2921,12 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/dom-mediacapture-record": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.11.tgz", + "integrity": "sha512-ODVOH95x08arZhbQOjH3no7Iye64akdO+55nM+IGtTzpu2ACKr9CQTrI//CCVieIjlI/eL+rK1hQjMycxIgylQ==", + "dev": true + }, "node_modules/@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", @@ -12608,6 +12622,11 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" }, + "node_modules/resize-observer-polyfill": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", + "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" + }, "node_modules/resolve": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", @@ -15580,7 +15599,8 @@ }, "devDependencies": { "@jspsych/config": "^1.0.0", - "@jspsych/test-utils": "^1.0.0" + "@jspsych/test-utils": "^1.0.0", + "@types/dom-mediacapture-record": "^1.0.11" } }, "packages/plugin-animation": { @@ -15779,6 +15799,21 @@ "jspsych": ">=7.0.0" } }, + "packages/plugin-html-audio-response": { + "name": "@jspsych/plugin-html-audio-response", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "resize-observer-polyfill": "^1.5.1" + }, + "devDependencies": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0" + }, + "peerDependencies": { + "jspsych": ">=7.0.0" + } + }, "packages/plugin-html-button-response": { "name": "@jspsych/plugin-html-button-response", "version": "1.0.0", @@ -15875,6 +15910,18 @@ "jspsych": ">=7.0.0" } }, + "packages/plugin-initialize-microphone": { + "name": "@jspsych/plugin-initiliaze-microphone", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0" + }, + "peerDependencies": { + "jspsych": ">=7.0.0" + } + }, "packages/plugin-instructions": { "name": "@jspsych/plugin-instructions", "version": "1.0.0", @@ -18083,6 +18130,14 @@ "@jspsych/test-utils": "^1.0.0" } }, + "@jspsych/plugin-html-audio-response": { + "version": "file:packages/plugin-html-audio-response", + "requires": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0", + "resize-observer-polyfill": "*" + } + }, "@jspsych/plugin-html-button-response": { "version": "file:packages/plugin-html-button-response", "requires": { @@ -18139,6 +18194,13 @@ "@jspsych/test-utils": "^1.0.0" } }, + "@jspsych/plugin-initiliaze-microphone": { + "version": "file:packages/plugin-initialize-microphone", + "requires": { + "@jspsych/config": "^1.0.0", + "@jspsych/test-utils": "^1.0.0" + } + }, "@jspsych/plugin-instructions": { "version": "file:packages/plugin-instructions", "requires": { @@ -18537,6 +18599,12 @@ "@babel/types": "^7.3.0" } }, + "@types/dom-mediacapture-record": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.11.tgz", + "integrity": "sha512-ODVOH95x08arZhbQOjH3no7Iye64akdO+55nM+IGtTzpu2ACKr9CQTrI//CCVieIjlI/eL+rK1hQjMycxIgylQ==", + "dev": true + }, "@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", @@ -23999,6 +24067,7 @@ "requires": { "@jspsych/config": "^1.0.0", "@jspsych/test-utils": "^1.0.0", + "@types/dom-mediacapture-record": "^1.0.11", "auto-bind": "^4.0.0", "random-words": "^1.1.1" } @@ -26002,6 +26071,11 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" }, + "resize-observer-polyfill": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", + "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" + }, "resolve": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", diff --git a/packages/jspsych/package.json b/packages/jspsych/package.json index 7e7418ee..356dca01 100644 --- a/packages/jspsych/package.json +++ b/packages/jspsych/package.json @@ -44,6 +44,7 @@ }, "devDependencies": { "@jspsych/config": "^1.0.0", - "@jspsych/test-utils": "^1.0.0" + "@jspsych/test-utils": "^1.0.0", + "@types/dom-mediacapture-record": "^1.0.11" } } diff --git a/packages/plugin-html-audio-response/package.json b/packages/plugin-html-audio-response/package.json index d0a9cef5..9dcf218a 100644 --- a/packages/plugin-html-audio-response/package.json +++ b/packages/plugin-html-audio-response/package.json @@ -39,5 +39,8 @@ "devDependencies": { "@jspsych/config": "^1.0.0", "@jspsych/test-utils": "^1.0.0" + }, + "dependencies": { + "resize-observer-polyfill": "^1.5.1" } } diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index 37e768c0..8767e128 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -1,4 +1,5 @@ import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych"; +import ResizeObserver from "resize-observer-polyfill"; const info = { name: "html-audio-response", From 76e755a18edf3a285988d6535725a285094f03b8 Mon Sep 17 00:00:00 2001 From: bjoluc Date: Wed, 24 Nov 2021 19:46:28 +0100 Subject: [PATCH 13/19] replace `resize-observer-polyfill` with `@types/resize-observer-browser` --- package-lock.json | 30 +++++++++---------- .../plugin-html-audio-response/package.json | 6 ++-- .../plugin-html-audio-response/src/index.ts | 1 - 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index e4b86170..fa2edcf1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3036,6 +3036,12 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.4.1.tgz", "integrity": "sha512-Fo79ojj3vdEZOHg3wR9ksAMRz4P3S5fDB5e/YWZiFnyFQI1WY2Vftu9XoXVVtJfxB7Bpce/QTqWSSntkz2Znrw==" }, + "node_modules/@types/resize-observer-browser": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@types/resize-observer-browser/-/resize-observer-browser-0.1.6.tgz", + "integrity": "sha512-61IfTac0s9jvNtBCpyo86QeaN8qqpMGHdK0uGKCCIy2dt5/Yk84VduHIdWAcmkC5QvdkPL0p5eWYgUZtHKKUVg==", + "dev": true + }, "node_modules/@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -12622,11 +12628,6 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" }, - "node_modules/resize-observer-polyfill": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", - "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" - }, "node_modules/resolve": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", @@ -15803,12 +15804,10 @@ "name": "@jspsych/plugin-html-audio-response", "version": "0.1.0", "license": "MIT", - "dependencies": { - "resize-observer-polyfill": "^1.5.1" - }, "devDependencies": { "@jspsych/config": "^1.0.0", - "@jspsych/test-utils": "^1.0.0" + "@jspsych/test-utils": "^1.0.0", + "@types/resize-observer-browser": "^0.1.6" }, "peerDependencies": { "jspsych": ">=7.0.0" @@ -18135,7 +18134,7 @@ "requires": { "@jspsych/config": "^1.0.0", "@jspsych/test-utils": "^1.0.0", - "resize-observer-polyfill": "*" + "@types/resize-observer-browser": "^0.1.6" } }, "@jspsych/plugin-html-button-response": { @@ -18714,6 +18713,12 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.4.1.tgz", "integrity": "sha512-Fo79ojj3vdEZOHg3wR9ksAMRz4P3S5fDB5e/YWZiFnyFQI1WY2Vftu9XoXVVtJfxB7Bpce/QTqWSSntkz2Znrw==" }, + "@types/resize-observer-browser": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@types/resize-observer-browser/-/resize-observer-browser-0.1.6.tgz", + "integrity": "sha512-61IfTac0s9jvNtBCpyo86QeaN8qqpMGHdK0uGKCCIy2dt5/Yk84VduHIdWAcmkC5QvdkPL0p5eWYgUZtHKKUVg==", + "dev": true + }, "@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -26071,11 +26076,6 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" }, - "resize-observer-polyfill": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", - "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" - }, "resolve": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", diff --git a/packages/plugin-html-audio-response/package.json b/packages/plugin-html-audio-response/package.json index 9dcf218a..df42b79b 100644 --- a/packages/plugin-html-audio-response/package.json +++ b/packages/plugin-html-audio-response/package.json @@ -38,9 +38,7 @@ }, "devDependencies": { "@jspsych/config": "^1.0.0", - "@jspsych/test-utils": "^1.0.0" - }, - "dependencies": { - "resize-observer-polyfill": "^1.5.1" + "@jspsych/test-utils": "^1.0.0", + "@types/resize-observer-browser": "^0.1.6" } } diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index 8767e128..37e768c0 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -1,5 +1,4 @@ import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych"; -import ResizeObserver from "resize-observer-polyfill"; const info = { name: "html-audio-response", From e69165804a44bfaed5e5fff6145f7dc6afe217bc Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Sat, 27 Nov 2021 16:29:42 -0500 Subject: [PATCH 14/19] teardown recording events when trial ends --- .../plugin-html-audio-response/src/index.ts | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index 37e768c0..90de2b5f 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -62,6 +62,10 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { private response; private load_resolver; private rt: number = null; + private start_event_handler; + private stop_event_handler; + private data_available_handler; + private recorded_data_chunks = []; constructor(private jsPsych: JsPsych) {} @@ -74,10 +78,10 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { } private showDisplay(display_element, trial) { - const ro = new ResizeObserver((entries) => { + const ro = new ResizeObserver((entries, observer) => { this.stimulus_start_time = performance.now(); - console.log("ro event"); - ro.disconnect(); + observer.unobserve(display_element); + //observer.disconnect(); }); ro.observe(display_element); @@ -115,16 +119,14 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { } private setupRecordingEvents(display_element, trial) { - const recorded_data_chunks = []; - - this.recorder.addEventListener("dataavailable", (e) => { + this.data_available_handler = (e) => { if (e.data.size > 0) { - recorded_data_chunks.push(e.data); + this.recorded_data_chunks.push(e.data); } - }); + }; - this.recorder.addEventListener("stop", () => { - const data = new Blob(recorded_data_chunks, { type: "audio/webm" }); + this.stop_event_handler = () => { + const data = new Blob(this.recorded_data_chunks, { type: "audio/webm" }); this.audio_url = URL.createObjectURL(data); const reader = new FileReader(); reader.addEventListener("load", () => { @@ -133,11 +135,11 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { this.load_resolver(); }); reader.readAsDataURL(data); - }); + }; - this.recorder.addEventListener("start", (e) => { + this.start_event_handler = (e) => { // resets the recorded data - recorded_data_chunks.length = 0; + this.recorded_data_chunks.length = 0; this.recorder_start_time = e.timeStamp; this.showDisplay(display_element, trial); @@ -162,7 +164,13 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { }); }, trial.recording_duration); } - }); + }; + + this.recorder.addEventListener("dataavailable", this.data_available_handler); + + this.recorder.addEventListener("stop", this.stop_event_handler); + + this.recorder.addEventListener("start", this.start_event_handler); } private startRecording() { @@ -197,6 +205,12 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { } private endTrial(display_element, trial) { + // clear recordering event handler + + this.recorder.removeEventListener("dataavailable", this.data_available_handler); + this.recorder.removeEventListener("start", this.start_event_handler); + this.recorder.removeEventListener("stop", this.stop_event_handler); + // kill any remaining setTimeout handlers this.jsPsych.pluginAPI.clearAllTimeouts(); From 308a791a8aa98f915fa344ad5bedfa4d326a400e Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Sat, 27 Nov 2021 16:30:01 -0500 Subject: [PATCH 15/19] add timing test for audio input --- .../tests/timing-tests/audio-input.html | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 packages/jspsych/tests/timing-tests/audio-input.html diff --git a/packages/jspsych/tests/timing-tests/audio-input.html b/packages/jspsych/tests/timing-tests/audio-input.html new file mode 100644 index 00000000..acbc3c98 --- /dev/null +++ b/packages/jspsych/tests/timing-tests/audio-input.html @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + From e8f1bd6d0819c342c8f618ac99241b29320836a0 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Sat, 27 Nov 2021 16:39:38 -0500 Subject: [PATCH 16/19] fix cases where timeouts fire after expected event --- .../plugin-html-audio-response/src/index.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/plugin-html-audio-response/src/index.ts b/packages/plugin-html-audio-response/src/index.ts index 90de2b5f..7125cbe4 100644 --- a/packages/plugin-html-audio-response/src/index.ts +++ b/packages/plugin-html-audio-response/src/index.ts @@ -96,9 +96,10 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { } private hideStimulus(display_element: HTMLElement) { - display_element.querySelector( - "#jspsych-html-audio-response-stimulus" - ).style.visibility = "hidden"; + const el: HTMLElement = display_element.querySelector("#jspsych-html-audio-response-stimulus"); + if (el) { + el.style.visibility = "hidden"; + } } private addButtonEvent(display_element, trial) { @@ -155,13 +156,17 @@ class HtmlAudioResponsePlugin implements JsPsychPlugin { // setup timer for ending the trial if (trial.recording_duration !== null) { this.jsPsych.pluginAPI.setTimeout(() => { - this.stopRecording().then(() => { - if (trial.allow_playback) { - this.showPlaybackControls(display_element, trial); - } else { - this.endTrial(display_element, trial); - } - }); + // this check is necessary for cases where the + // done_button is clicked before the timer expires + if (this.recorder.state !== "inactive") { + this.stopRecording().then(() => { + if (trial.allow_playback) { + this.showPlaybackControls(display_element, trial); + } else { + this.endTrial(display_element, trial); + } + }); + } }, trial.recording_duration); } }; From 37749e8df8dbd98bb41b34305b83bcc36769362e Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Sun, 28 Nov 2021 22:47:57 -0500 Subject: [PATCH 17/19] update versions --- docs/demos/jspsych-html-audio-response-demo1.html | 8 ++++---- docs/demos/jspsych-html-audio-response-demo2.html | 8 ++++---- docs/demos/jspsych-html-audio-response-demo3.html | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/demos/jspsych-html-audio-response-demo1.html b/docs/demos/jspsych-html-audio-response-demo1.html index f72278ec..f88b69f8 100644 --- a/docs/demos/jspsych-html-audio-response-demo1.html +++ b/docs/demos/jspsych-html-audio-response-demo1.html @@ -1,11 +1,11 @@ - - - + + + - + diff --git a/docs/demos/jspsych-html-audio-response-demo2.html b/docs/demos/jspsych-html-audio-response-demo2.html index 90242ff4..edb8af5c 100644 --- a/docs/demos/jspsych-html-audio-response-demo2.html +++ b/docs/demos/jspsych-html-audio-response-demo2.html @@ -1,11 +1,11 @@ - - - + + + - + diff --git a/docs/demos/jspsych-html-audio-response-demo3.html b/docs/demos/jspsych-html-audio-response-demo3.html index 8af68a30..c561f376 100644 --- a/docs/demos/jspsych-html-audio-response-demo3.html +++ b/docs/demos/jspsych-html-audio-response-demo3.html @@ -1,12 +1,12 @@ - - - - + + + + - + From cd134fb1b779b0ad69a2112933a8702d3066a8bb Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Sun, 28 Nov 2021 22:48:36 -0500 Subject: [PATCH 18/19] fix version number for audio-response --- docs/demos/jspsych-html-audio-response-demo1.html | 2 +- docs/demos/jspsych-html-audio-response-demo2.html | 2 +- docs/demos/jspsych-html-audio-response-demo3.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/demos/jspsych-html-audio-response-demo1.html b/docs/demos/jspsych-html-audio-response-demo1.html index f88b69f8..e8b829d0 100644 --- a/docs/demos/jspsych-html-audio-response-demo1.html +++ b/docs/demos/jspsych-html-audio-response-demo1.html @@ -3,7 +3,7 @@ - +