HoshinoKoji/jsPsych
1
0
Fork 0
mirror of https://github.com/jspsych/jsPsych.git synced 2025-05-10 11:10:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

start work on audio-input plugins

Browse Source
This commit is contained in:
Josh de Leeuw 2021-11-19 17:00:10 -05:00
parent 2922bc5dad
commit 3a3d32971c
14 changed files with 563 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
examples/jspsych-initialize-microphone.html Normal file
View File

@ -0,0 +1,26 @@
<!DOCTYPE html>
<html>
<head>
<script src="../packages/jspsych/dist/index.browser.js"></script>
<script src="../packages/plugin-initialize-microphone/dist/index.browser.js"></script>
<script src="../packages/plugin-html-audio-response/dist/index.browser.js"></script>
<link rel="stylesheet" href="../packages/jspsych/css/jspsych.css">
</head>
<body></body>
<script>
var jsPsych = initJsPsych();
let init_mic = {
type: jsPsychInitializeMicrophone,
}
let ar = {
type: jsPsychHtmlAudioResponse,
stimulus: '<p>Speak!</p>'
}
jsPsych.run([init_mic, ar]);
</script>
</html>

11
packages/jspsych/src/modules/plugin-api/MediaAPI.ts
View File

@ -323,4 +323,15 @@ export class MediaAPI {
} }
this.preload_requests = []; this.preload_requests = [];
} }
private microphone_recorder: MediaRecorder;
initializeMicrophoneRecorder(stream: MediaStream) {
const recorder = new MediaRecorder(stream);
this.microphone_recorder = recorder;
}
getMicrophoneRecorder(): MediaRecorder {
return this.microphone_recorder;
}
} }

1
packages/plugin-html-audio-response/jest.config.cjs Normal file
View File

@ -0,0 +1 @@
module.exports = require("@jspsych/config/jest").makePackageConfig(__dirname);

43
packages/plugin-html-audio-response/package.json Normal file
View File

@ -0,0 +1,43 @@
{
"name": "@jspsych/plugin-html-audio-response",
"version": "0.1.0",
"description": "jsPsych plugin for displaying a stimulus and recording an audio response through the microphone",
"type": "module",
"main": "dist/index.cjs",
"exports": {
"import": "./dist/index.js",
"require": "./dist/index.cjs"
},
"typings": "dist/index.d.ts",
"unpkg": "dist/index.browser.min.js",
"files": [
"src",
"dist"
],
"source": "src/index.ts",
"scripts": {
"test": "jest",
"test:watch": "npm test -- --watch",
"tsc": "tsc",
"build": "rollup --config",
"build:watch": "npm run build -- --watch"
},
"repository": {
"type": "git",
"url": "git+https://github.com/jspsych/jsPsych.git",
"directory": "packages/plugin-html-audio-response"
},
"author": "Josh de Leeuw",
"license": "MIT",
"bugs": {
"url": "https://github.com/jspsych/jsPsych/issues"
},
"homepage": "https://www.jspsych.org/latest/plugins/html-audio-response",
"peerDependencies": {
"jspsych": ">=7.0.0"
},
"devDependencies": {
"@jspsych/config": "^1.0.0",
"@jspsych/test-utils": "^1.0.0"
}
}

3
packages/plugin-html-audio-response/rollup.config.mjs Normal file
View File

@ -0,0 +1,3 @@
import { makeRollupConfig } from "@jspsych/config/rollup";
export default makeRollupConfig("jsPsychHtmlAudioResponse");

156
packages/plugin-html-audio-response/src/index.spec.ts Normal file
View File

@ -0,0 +1,156 @@
import { clickTarget, startTimeline } from "@jspsych/test-utils";
import htmlButtonResponse from ".";
jest.useFakeTimers();
describe("html-button-response", () => {
test("displays html stimulus", async () => {
const { getHTML } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
},
]);
expect(getHTML()).toContain(
'<div id="jspsych-html-button-response-stimulus">this is html</div>'
);
});
test("display button labels", async () => {
const { getHTML } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice1", "button-choice2"],
},
]);
expect(getHTML()).toContain('<button class="jspsych-btn">button-choice1</button>');
expect(getHTML()).toContain('<button class="jspsych-btn">button-choice2</button>');
});
test("display button html", async () => {
const { getHTML } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["buttonChoice"],
button_html: '<button class="jspsych-custom-button">%choice%</button>',
},
]);
expect(getHTML()).toContain('<button class="jspsych-custom-button">buttonChoice</button>');
});
test("display should clear after button click", async () => {
const { getHTML, expectFinished } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
},
]);
expect(getHTML()).toContain(
'<div id="jspsych-html-button-response-stimulus">this is html</div>'
);
clickTarget(document.querySelector("#jspsych-html-button-response-button-0"));
await expectFinished();
});
test("prompt should append below button", async () => {
const { getHTML } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
prompt: "<p>this is a prompt</p>",
},
]);
expect(getHTML()).toContain(
'<button class="jspsych-btn">button-choice</button></div></div><p>this is a prompt</p>'
);
});
test("should hide stimulus if stimulus-duration is set", async () => {
const { displayElement } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
stimulus_duration: 500,
},
]);
const stimulusElement = displayElement.querySelector<HTMLElement>(
"#jspsych-html-button-response-stimulus"
);
expect(stimulusElement.style.visibility).toBe("");
jest.advanceTimersByTime(500);
expect(stimulusElement.style.visibility).toBe("hidden");
});
test("should end trial when trial duration is reached", async () => {
const { getHTML, expectFinished } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
trial_duration: 500,
},
]);
expect(getHTML()).toContain(
'<div id="jspsych-html-button-response-stimulus">this is html</div>'
);
jest.advanceTimersByTime(500);
await expectFinished();
});
test("should end trial when button is clicked", async () => {
const { getHTML, expectFinished } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
response_ends_trial: true,
},
]);
expect(getHTML()).toContain(
'<div id="jspsych-html-button-response-stimulus">this is html</div>'
);
clickTarget(document.querySelector("#jspsych-html-button-response-button-0"));
await expectFinished();
});
test("class should have responded when button is clicked", async () => {
const { getHTML } = await startTimeline([
{
type: htmlButtonResponse,
stimulus: "this is html",
choices: ["button-choice"],
response_ends_trial: false,
},
]);
expect(getHTML()).toContain(
'<div id="jspsych-html-button-response-stimulus">this is html</div>'
);
clickTarget(document.querySelector("#jspsych-html-button-response-button-0"));
expect(document.querySelector("#jspsych-html-button-response-stimulus").className).toBe(
" responded"
);
});
});

141
packages/plugin-html-audio-response/src/index.ts Normal file
View File

@ -0,0 +1,141 @@
import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych";
const info = <const>{
name: "html-audio-response",
parameters: {
/** The HTML string to be displayed */
stimulus: {
type: ParameterType.HTML_STRING,
pretty_name: "Stimulus",
default: undefined,
},
/** How long to show the stimulus. */
stimulus_duration: {
type: ParameterType.INT,
pretty_name: "Stimulus duration",
default: null,
},
/** How long to show the trial. */
trial_duration: {
type: ParameterType.INT,
pretty_name: "Trial duration",
default: null,
},
},
};
type Info = typeof info;
/**
* html-audio-response
* jsPsych plugin for displaying a stimulus and recording an audio response through a microphone
* @author Josh de Leeuw
* @see {@link https://www.jspsych.org/plugins/jspsych-html-audio-response/ html-audio-response plugin documentation on jspsych.org}
*/
class HtmlAudioResponsePlugin implements JsPsychPlugin<Info> {
static info = info;
private stimulus_start_time;
private recorder_start_time;
constructor(private jsPsych: JsPsych) {}
trial(display_element: HTMLElement, trial: TrialType<Info>) {
const ro = new ResizeObserver((entries) => {
console.log(`ro perf = ${performance.now()}`);
});
ro.observe(display_element);
this.showDisplay(display_element, trial);
this.addButtonEvent(display_element, trial);
this.stimulus_start_time = performance.now();
this.startRecording();
// hide image if timing is set
if (trial.stimulus_duration !== null) {
this.jsPsych.pluginAPI.setTimeout(() => {
display_element.querySelector<HTMLElement>(
"#jspsych-html-audio-response-stimulus"
).style.visibility = "hidden";
}, trial.stimulus_duration);
}
// end trial if time limit is set
if (trial.trial_duration !== null) {
this.jsPsych.pluginAPI.setTimeout(() => {
this.endTrial(display_element, trial);
}, trial.trial_duration);
}
}
private showDisplay(display_element, trial) {
let html = `<div id="jspsych-html-audio-response-stimulus">${trial.stimulus}</div>`;
html += `<p><button class="jspsych-btn" id="finish-trial">Done</button></p>`;
display_element.innerHTML = html;
}
private addButtonEvent(display_element, trial) {
display_element.querySelector("#finish-trial").addEventListener("click", () => {
const end_time = performance.now();
const rt = Math.round(end_time - this.stimulus_start_time);
this.endTrial(display_element, trial, rt);
});
}
private startRecording() {
const recorder: MediaRecorder = this.jsPsych.pluginAPI.getMicrophoneRecorder();
const recorded_data_chunks = [];
recorder.addEventListener("dataavailable", (e) => {
if (e.data.size > 0) {
recorded_data_chunks.push(e.data);
}
console.log("dataavail event");
});
recorder.addEventListener("stop", () => {
const url = new Blob(recorded_data_chunks, { type: "audio/webm" });
const reader = new FileReader();
reader.addEventListener("load", () => {
const base64 = (reader.result as string).split(",")[1];
console.log(base64);
});
reader.readAsDataURL(url);
console.log(this.stimulus_start_time, this.recorder_start_time);
});
recorder.addEventListener("start", (e) => {
this.recorder_start_time = e.timeStamp;
});
this.jsPsych.pluginAPI.setTimeout(() => {
recorder.stop();
}, 2000);
recorder.start();
}
private endTrial(display_element, trial, rt: number = null) {
// kill any remaining setTimeout handlers
this.jsPsych.pluginAPI.clearAllTimeouts();
// gather the data to store for the trial
var trial_data = {
rt: rt,
stimulus: trial.stimulus,
};
// clear the display
display_element.innerHTML = "";
// move on to the next trial
this.jsPsych.finishTrial(trial_data);
}
}
export default HtmlAudioResponsePlugin;

7
packages/plugin-html-audio-response/tsconfig.json Normal file
View File

@ -0,0 +1,7 @@
{
"extends": "@jspsych/config/tsconfig.core.json",
"compilerOptions": {
"baseUrl": "."
},
"include": ["src"]
}

1
packages/plugin-initialize-microphone/jest.config.cjs Normal file
View File

@ -0,0 +1 @@
module.exports = require("@jspsych/config/jest").makePackageConfig(__dirname);

43
packages/plugin-initialize-microphone/package.json Normal file
View File

@ -0,0 +1,43 @@
{
"name": "@jspsych/plugin-initiliaze-microphone",
"version": "0.1.0",
"description": "jsPsych plugin for getting permission to initialize the user's microphone",
"type": "module",
"main": "dist/index.cjs",
"exports": {
"import": "./dist/index.js",
"require": "./dist/index.cjs"
},
"typings": "dist/index.d.ts",
"unpkg": "dist/index.browser.min.js",
"files": [
"src",
"dist"
],
"source": "src/index.ts",
"scripts": {
"test": "jest",
"test:watch": "npm test -- --watch",
"tsc": "tsc",
"build": "rollup --config",
"build:watch": "npm run build -- --watch"
},
"repository": {
"type": "git",
"url": "git+https://github.com/jspsych/jsPsych.git",
"directory": "packages/plugin-initialize-microphone"
},
"author": "Josh de Leeuw",
"license": "MIT",
"bugs": {
"url": "https://github.com/jspsych/jsPsych/issues"
},
"homepage": "https://www.jspsych.org/latest/plugins/initialize-microphone",
"peerDependencies": {
"jspsych": ">=7.0.0"
},
"devDependencies": {
"@jspsych/config": "^1.0.0",
"@jspsych/test-utils": "^1.0.0"
}
}

3
packages/plugin-initialize-microphone/rollup.config.mjs Normal file
View File

@ -0,0 +1,3 @@
import { makeRollupConfig } from "@jspsych/config/rollup";
export default makeRollupConfig("jsPsychInitializeMicrophone");

32
packages/plugin-initialize-microphone/src/index.spec.ts Normal file
View File

@ -0,0 +1,32 @@
import { startTimeline } from "@jspsych/test-utils";
import callFunction from ".";
describe("call-function plugin", () => {
test("calls function", async () => {
const { getData, expectFinished } = await startTimeline([
{
type: callFunction,
func: () => 1,
},
]);
await expectFinished();
expect(getData().values()[0].value).toBe(1);
});
test("async function works", async () => {
const { getData, expectFinished } = await startTimeline([
{
type: callFunction,
async: true,
func: (done) => {
done(10);
},
},
]);
await expectFinished();
expect(getData().values()[0].value).toBe(10);
});
});

89
packages/plugin-initialize-microphone/src/index.ts Normal file
View File

@ -0,0 +1,89 @@
import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych";
const info = <const>{
name: "initialize-microphone",
parameters: {
/** Function to call */
func: {
type: ParameterType.BOOL,
default: true,
},
/** Is the function call asynchronous? */
async: {
type: ParameterType.BOOL,
default: false,
},
},
};
type Info = typeof info;
/**
* **initialize-microphone**
*
* jsPsych plugin for getting permission to initialize a microphone
*
* @author Josh de Leeuw
* @see {@link https://www.jspsych.org/plugins/jspsych-initialize-microphone/ initialize-microphone plugin documentation on jspsych.org}
*/
class InitializeMicrophonePlugin implements JsPsychPlugin<Info> {
static info = info;
constructor(private jsPsych: JsPsych) {}
trial(display_element: HTMLElement, trial: TrialType<Info>) {
this.run_trial(display_element).then((id) => {
this.jsPsych.finishTrial({
device_id: id,
});
});
}
private async run_trial(display_element) {
await this.askForPermission();
const devices = await navigator.mediaDevices.enumerateDevices();
const mics = devices.filter(
(d) => d.kind === "audioinput" && d.deviceId !== "default" && d.deviceId !== "communications"
);
// remove entries with duplicate groupID
const unique_mics = mics.filter(
(mic, index, arr) => arr.findIndex((v) => v.groupId == mic.groupId) == index
);
const mic_id = await this.showMicrophoneSelection(display_element, unique_mics);
const stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: mic_id } });
this.jsPsych.pluginAPI.initializeMicrophoneRecorder(stream);
return mic_id;
}
private async askForPermission() {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
return stream;
}
private showMicrophoneSelection(display_element, devices: MediaDeviceInfo[]) {
let html = `
<p>Please select the microphone you would like to use.</p>
<select name="mic" id="which-mic" style="font-size:14px; font-family: 'Open Sans', 'Arial', sans-serif; padding: 4px;">`;
for (const d of devices) {
html += `<option value="${d.deviceId}">${d.label}</option>`;
}
html += "</select>";
html += '<p><button class="jspsych-btn" id="btn-select-mic">Use this microphone</button></p>';
display_element.innerHTML = html;
return new Promise((resolve) => {
display_element.querySelector("#btn-select-mic").addEventListener("click", () => {
const mic = display_element.querySelector("#which-mic").value;
resolve(mic);
});
});
}
}
export default InitializeMicrophonePlugin;

7
packages/plugin-initialize-microphone/tsconfig.json Normal file
View File

@ -0,0 +1,7 @@
{
"extends": "@jspsych/config/tsconfig.core.json",
"compilerOptions": {
"baseUrl": "."
},
"include": ["src"]
}