mirror of
https://github.com/psychopy/psychojs.git
synced 2025-05-10 10:40:54 +00:00
346 lines
11 KiB
HTML
346 lines
11 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>JSDoc: Source: sound/AudioClip.js</title>
|
|
|
|
<script src="scripts/prettify/prettify.js"> </script>
|
|
<script src="scripts/prettify/lang-css.js"> </script>
|
|
<!--[if lt IE 9]>
|
|
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
|
|
<![endif]-->
|
|
<link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
|
|
<link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
|
|
</head>
|
|
|
|
<body>
|
|
|
|
<div id="main">
|
|
|
|
<h1 class="page-title">Source: sound/AudioClip.js</h1>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<article>
|
|
<pre class="prettyprint source linenums"><code>/**
|
|
* AudioClip encapsulate an audio recording.
|
|
*
|
|
* @author Alain Pitiot and Sotiri Bakagiannis
|
|
* @version 2021.x
|
|
* @copyright (c) 2021 Open Science Tools Ltd. (https://opensciencetools.org)
|
|
* @license Distributed under the terms of the MIT License
|
|
*/
|
|
|
|
import {PsychObject} from "../util/PsychObject";
|
|
import {PsychoJS} from "../core/PsychoJS";
|
|
|
|
|
|
/**
|
|
* <p>AudioClip encapsulate an audio recording.</p>
|
|
*
|
|
* @name module:sound.AudioClip
|
|
* @class
|
|
* @param {Object} options
|
|
* @param {module:core.PsychoJS} options.psychoJS - the PsychoJS instance
|
|
* @param {String} [options.name= 'audioclip'] - the name used when logging messages
|
|
* @param {string} options.format the format for the audio file
|
|
* @param {number} options.sampleRateHz - the sampling rate
|
|
* @param {Blob} options.data - the audio data, in the given format, at the given sampling rate
|
|
* @param {boolean} [options.autoLog= false] - whether or not to log
|
|
*/
|
|
export class AudioClip extends PsychObject
|
|
{
|
|
|
|
constructor({psychoJS, name, sampleRateHz, format, data, autoLog} = {})
|
|
{
|
|
super(psychoJS);
|
|
|
|
this._addAttribute('name', name, 'audioclip');
|
|
this._addAttribute('format', format);
|
|
this._addAttribute('sampleRateHz', sampleRateHz);
|
|
this._addAttribute('data', data);
|
|
this._addAttribute('autoLog', false, autoLog);
|
|
this._addAttribute('status', AudioClip.Status.CREATED);
|
|
|
|
if (this._autoLog)
|
|
{
|
|
this._psychoJS.experimentLogger.exp(`Created ${this.name} = ${this.toString()}`);
|
|
}
|
|
|
|
// decode the blob into an audio buffer:
|
|
this._decodeAudio();
|
|
}
|
|
|
|
|
|
play()
|
|
{
|
|
// TODO check that the audio buffer is ready
|
|
|
|
// play the audio buffer:
|
|
const source = this._audioContext.createBufferSource();
|
|
source.buffer = this._audioBuffer;
|
|
source.connect(this._audioContext.destination);
|
|
source.start();
|
|
}
|
|
|
|
|
|
upload()
|
|
{
|
|
// TODO
|
|
}
|
|
|
|
|
|
/**
|
|
* Transcribe the audio clip.
|
|
*
|
|
* ref: https://cloud.google.com/speech-to-text/docs/reference/rest/v1/speech/recognize
|
|
*
|
|
* @param {Object} options
|
|
* @param engine
|
|
* @param {String} options.languageCode - the BCP-47 language code for the recognition,
|
|
* e.g. 'en-gb'
|
|
* @return {Promise<void>}
|
|
*/
|
|
async transcribe({engine, languageCode} = {})
|
|
{
|
|
// wait for the decoding to complete:
|
|
await this._decodeAudio();
|
|
|
|
return new Promise(async (resolve, reject) =>
|
|
{
|
|
// convert the Float32 PCM audio data to UInt16:
|
|
const buffer = new ArrayBuffer(this._audioData.length * 2);
|
|
const uint16View = new Uint16Array(buffer);
|
|
for (let t = 0; t < this._audioData.length; ++t)
|
|
{
|
|
uint16View[t] = (this._audioData[t] < 0)
|
|
? this._audioData[t] * 0x8000
|
|
: this._audioData[t] * 0x7FFF;
|
|
}
|
|
|
|
// encode it to base64:
|
|
const base64Data = this._base64ArrayBuffer(new Uint8Array(buffer));
|
|
|
|
// query the Google speech-to-text service:
|
|
const body = {
|
|
config: {
|
|
encoding: 'LINEAR16',
|
|
sampleRateHertz: this._sampleRateHz,
|
|
languageCode
|
|
},
|
|
audio: {
|
|
content: base64Data
|
|
},
|
|
};
|
|
|
|
// TODO get the key from the designer's pavlovia account
|
|
const GOOGLE_API_KEY = 'AIzaSyDngTi-pJcVrm_Kr2yTKV8OYLtfRN180gY';
|
|
const url = `https://speech.googleapis.com/v1/speech:recognize?key=${GOOGLE_API_KEY}`;
|
|
|
|
const response = await fetch(url, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(body)
|
|
});
|
|
|
|
// convert the response to json:
|
|
const results = await response.json();
|
|
console.log('>>>> ', results);
|
|
|
|
resolve(results[0].alternatives[0]);
|
|
});
|
|
}
|
|
|
|
|
|
_decodeAudio()
|
|
{
|
|
// if the audio clip is ready, the PCM audio data is available in _audioData, a Float32Array:
|
|
if (this._status === AudioClip.Status.READY)
|
|
{
|
|
return;
|
|
}
|
|
|
|
|
|
// if we are already decoding, wait until the process completed:
|
|
if (this._status === AudioClip.Status.DECODING)
|
|
{
|
|
const self = this;
|
|
return new Promise(function (resolve, reject)
|
|
{
|
|
self._decodingCallbacks.push(resolve);
|
|
// TODO
|
|
// self._errorCallback = reject;
|
|
}.bind(this));
|
|
}
|
|
|
|
|
|
// otherwise, start decoding the input formatted audio data:
|
|
this._status = AudioClip.Status.DECODING;
|
|
this._audioData = null;
|
|
this._decodingCallbacks = [];
|
|
|
|
this._audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
|
sampleRate: this._sampleRateHz
|
|
});
|
|
|
|
const reader = new window.FileReader();
|
|
reader.onloadend = async () =>
|
|
{
|
|
// decode the ArrayBuffer containing the formatted audio data (e.g. webm)
|
|
// into an audio buffer:
|
|
this._audioBuffer = await this._audioContext.decodeAudioData(reader.result);
|
|
|
|
// get the Float32Array containing the PCM data:
|
|
this._audioData = this._audioBuffer.getChannelData(0);
|
|
|
|
// we are now ready to translate and play:
|
|
this._status = AudioClip.Status.READY;
|
|
|
|
// resolve all the promises waiting for the decoding to complete:
|
|
for (const callback of this._decodingCallbacks)
|
|
{
|
|
callback();
|
|
}
|
|
};
|
|
|
|
reader.onerror = (error) =>
|
|
{
|
|
// TODO
|
|
}
|
|
|
|
reader.readAsArrayBuffer(this._data);
|
|
}
|
|
|
|
|
|
/**
|
|
* Convert an array buffer to a base64 string.
|
|
*
|
|
* @note this is only very lightly adapted from the folowing post of @Grantlyk:
|
|
* https://gist.github.com/jonleighton/958841#gistcomment-1953137
|
|
*
|
|
* the following only works for small buffers:
|
|
* const dataAsString = String.fromCharCode.apply(null, new Uint8Array(buffer));
|
|
* base64Data = window.btoa(dataAsString);
|
|
*
|
|
* @param arrayBuffer
|
|
* @return {string} the base64 encoded input buffer
|
|
*/
|
|
_base64ArrayBuffer(arrayBuffer)
|
|
{
|
|
let base64 = '';
|
|
const encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
|
|
|
|
const bytes = new Uint8Array(arrayBuffer);
|
|
const byteLength = bytes.byteLength;
|
|
const byteRemainder = byteLength % 3;
|
|
const mainLength = byteLength - byteRemainder;
|
|
|
|
let a;
|
|
let b;
|
|
let c;
|
|
let d;
|
|
let chunk;
|
|
|
|
// Main loop deals with bytes in chunks of 3
|
|
for (let i = 0; i < mainLength; i += 3) {
|
|
// Combine the three bytes into a single integer
|
|
chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
|
|
|
|
// Use bitmasks to extract 6-bit segments from the triplet
|
|
a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
|
|
b = (chunk & 258048) >> 12; // 258048 = (2^6 - 1) << 12
|
|
c = (chunk & 4032) >> 6; // 4032 = (2^6 - 1) << 6
|
|
d = chunk & 63; // 63 = 2^6 - 1
|
|
|
|
// Convert the raw binary segments to the appropriate ASCII encoding
|
|
base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
|
|
}
|
|
|
|
// Deal with the remaining bytes and padding
|
|
if (byteRemainder === 1) {
|
|
chunk = bytes[mainLength];
|
|
|
|
a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2
|
|
|
|
// Set the 4 least significant bits to zero
|
|
b = (chunk & 3) << 4; // 3 = 2^2 - 1
|
|
|
|
base64 += `${encodings[a]}${encodings[b]}==`;
|
|
} else if (byteRemainder === 2) {
|
|
chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];
|
|
|
|
a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
|
|
b = (chunk & 1008) >> 4; // 1008 = (2^6 - 1) << 4
|
|
|
|
// Set the 2 least significant bits to zero
|
|
c = (chunk & 15) << 2; // 15 = 2^4 - 1
|
|
|
|
base64 += `${encodings[a]}${encodings[b]}${encodings[c]}=`;
|
|
}
|
|
|
|
return base64;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
/**
|
|
* Recognition engines.
|
|
*
|
|
* @name module:sound.AudioClip#Engine
|
|
* @enum {Symbol}
|
|
* @readonly
|
|
* @public
|
|
*/
|
|
AudioClip.Engine = {
|
|
/**
|
|
* Google Cloud Speech-to-Text.
|
|
*/
|
|
GOOGLE: Symbol.for('GOOGLE')
|
|
};
|
|
|
|
|
|
/**
|
|
* AudioClip status.
|
|
*
|
|
* @enum {Symbol}
|
|
* @readonly
|
|
* @public
|
|
*/
|
|
AudioClip.Status = {
|
|
CREATED: Symbol.for('CREATED'),
|
|
|
|
DECODING: Symbol.for('DECODING'),
|
|
|
|
READY: Symbol.for('READY')
|
|
};
|
|
</code></pre>
|
|
</article>
|
|
</section>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<nav>
|
|
<h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-core.html">core</a></li><li><a href="module-data.html">data</a></li><li><a href="module-sound.html">sound</a></li><li><a href="module-util.html">util</a></li><li><a href="module-visual.html">visual</a></li></ul><h3>Classes</h3><ul><li><a href="module-core.BuilderKeyResponse.html">BuilderKeyResponse</a></li><li><a href="module-core.EventManager.html">EventManager</a></li><li><a href="module-core.GUI.html">GUI</a></li><li><a href="module-core.Keyboard.html">Keyboard</a></li><li><a href="module-core.KeyPress.html">KeyPress</a></li><li><a href="module-core.Logger.html">Logger</a></li><li><a href="module-core.MinimalStim.html">MinimalStim</a></li><li><a href="module-core.Mouse.html">Mouse</a></li><li><a href="module-core.PsychoJS.html">PsychoJS</a></li><li><a href="module-core.ServerManager.html">ServerManager</a></li><li><a href="module-core.Window.html">Window</a></li><li><a href="module-data.ExperimentHandler.html">ExperimentHandler</a></li><li><a href="module-data.Shelf.html">Shelf</a></li><li><a href="module-data.TrialHandler.html">TrialHandler</a></li><li><a href="module-sound.AudioClip.html">AudioClip</a></li><li><a href="module-sound.Microphone.html">Microphone</a></li><li><a href="module-sound.Sound.html">Sound</a></li><li><a href="module-sound.TonePlayer.html">TonePlayer</a></li><li><a href="module-sound.TrackPlayer.html">TrackPlayer</a></li><li><a href="module-sound.Transcriber.html">Transcriber</a></li><li><a href="module-sound.Transcript.html">Transcript</a></li><li><a href="module-util.Clock.html">Clock</a></li><li><a href="module-util.Color.html">Color</a></li><li><a href="module-util.CountdownTimer.html">CountdownTimer</a></li><li><a href="module-util.EventEmitter.html">EventEmitter</a></li><li><a href="module-util.MixinBuilder.html">MixinBuilder</a></li><li><a href="module-util.MonotonicClock.html">MonotonicClock</a></li><li><a href="module-util.PsychObject.html">PsychObject</a></li><li><a href="module-util.Scheduler.html">Scheduler</a></li><li><a href="module-visual.Form.html">Form</a></li><li><a href="module-visual.ImageStim.html">ImageStim</a></li><li><a href="module-visual.MovieStim.html">MovieStim</a></li><li><a href="module-visual.Polygon.html">Polygon</a></li><li><a href="module-visual.Rect.html">Rect</a></li><li><a href="module-visual.ShapeStim.html">ShapeStim</a></li><li><a href="module-visual.Slider.html">Slider</a></li><li><a href="module-visual.TextBox.html">TextBox</a></li><li><a href="module-visual.TextStim.html">TextStim</a></li><li><a href="module-visual.VisualStim.html">VisualStim</a></li><li><a href="TextInput.html">TextInput</a></li></ul><h3>Interfaces</h3><ul><li><a href="module-sound.SoundPlayer.html">SoundPlayer</a></li></ul><h3>Mixins</h3><ul><li><a href="module-core.WindowMixin.html">WindowMixin</a></li><li><a href="module-util.ColorMixin.html">ColorMixin</a></li></ul>
|
|
</nav>
|
|
|
|
<br class="clear">
|
|
|
|
<footer>
|
|
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.6</a> on Fri May 28 2021 10:49:17 GMT+0200 (Central European Summer Time)
|
|
</footer>
|
|
|
|
<script> prettyPrint(); </script>
|
|
<script src="scripts/linenumber.js"> </script>
|
|
</body>
|
|
</html>
|