# sound/AudioClip.js

    /**
     * AudioClip encapsulates an audio recording.
     *
     * @author Alain Pitiot and Sotiri Bakagiannis
     * @version 2022.2.3
     * @copyright (c) 2021 Open Science Tools Ltd. (https://opensciencetools.org)
     * @license Distributed under the terms of the MIT License
     */

    import { PsychoJS } from "../core/PsychoJS.js";
    import { ExperimentHandler } from "../data/ExperimentHandler.js";
    import { PsychObject } from "../util/PsychObject.js";
    import * as util from "../util/Util.js";

    /**
     * <p>AudioClip encapsulates an audio recording.</p>
     *
     * @extends PsychObject
     */
    export class AudioClip extends PsychObject
    {
    	/**
    	 * @memberOf module:sound
    	 * @param {Object} options
    	 * @param {module:core.PsychoJS} options.psychoJS - the PsychoJS instance
    	 * @param {String} [options.name= 'audioclip'] - the name used when logging messages
    	 * @param {string} options.format the format for the audio file
    	 * @param {number} options.sampleRateHz - the sampling rate
    	 * @param {Blob} options.data - the audio data, in the given format, at the given sampling rate
    	 * @param {boolean} [options.autoLog= false] - whether or not to log
    	 */
    	constructor({ psychoJS, name, sampleRateHz, format, data, autoLog } = {})
    	{
    		super(psychoJS);

    		this._addAttribute("name", name, "audioclip");
    		this._addAttribute("format", format);
    		this._addAttribute("sampleRateHz", sampleRateHz);
    		this._addAttribute("data", data);
    		this._addAttribute("autoLog", false, autoLog);
    		this._addAttribute("status", AudioClip.Status.CREATED);

    		// add a volume attribute, for playback:
    		this._addAttribute("volume", 1.0);

    		if (this._autoLog)
    		{
    			this._psychoJS.experimentLogger.exp(`Created ${this.name} = ${this.toString()}`);
    		}

    		// decode the blob into an audio buffer:
    		this._decodeAudio();
    	}

    	/**
    	 * Set the volume of the playback.
    	 *
    	 * @param {number} volume - the volume of the playback (must be between 0.0 and 1.0)
    	 */
    	setVolume(volume)
    	{
    		this._volume = volume;
    	}

    	/**
    	 * Start playing the audio clip.
    	 *
    	 * @public
    	 */
    	async startPlayback()
    	{
    		this._psychoJS.logger.debug("request to play the audio clip");

    		// wait for the decoding to complete:
    		await this._decodeAudio();

    		// note: we need to prepare the audio graph anew each time since, for instance, an
    		// AudioBufferSourceNode can only be played once
    		// ref: https://developer.mozilla.org/en-US/docs/Web/API/AudioBufferSourceNode

    		// create a source node from the in-memory audio data in _audioBuffer:
    		this._source = this._audioContext.createBufferSource();
    		this._source.buffer = this._audioBuffer;

    		// create a gain node, so we can control the volume:
    		this._gainNode = this._audioContext.createGain();

    		// connect the nodes:
    		this._source.connect(this._gainNode);
    		this._gainNode.connect(this._audioContext.destination);

    		// set the volume:
    		this._gainNode.gain.value = this._volume;

    		// start the playback:
    		this._source.start();
    	}

    	/**
    	 * Stop playing the audio clip.
    	 *
    	 * @param {number} [fadeDuration = 17] - how long the fading out should last, in ms
    	 */
    	async stopPlayback(fadeDuration = 17)
    	{
    		// TODO deal with fade duration

    		// stop the playback:
    		this._source.stop();
    	}

    	/**
    	 * Get the duration of the audio clip, in seconds.
    	 *
    	 * @returns {Promise<number>} the duration of the audio clip
    	 */
    	async getDuration()
    	{
    		// wait for the decoding to complete:
    		await this._decodeAudio();

    		return this._audioBuffer.duration;
    	}

    	/**
    	 * Upload the audio clip to the pavlovia server.
    	 *
    	 * @public
    	 */
    	upload()
    	{
    		this._psychoJS.logger.debug("request to upload the audio clip to pavlovia.org");

    		// add a format-dependent audio extension to the name:
    		const filename = this._name + util.extensionFromMimeType(this._format);

    		// if the audio recording cannot be uploaded, e.g. the experiment is running locally, or
    		// if it is piloting mode, then we offer the audio clip as a file for download:
    		if (
    			this._psychoJS.getEnvironment() !== ExperimentHandler.Environment.SERVER
    			|| this._psychoJS.config.experiment.status !== "RUNNING"
    			|| this._psychoJS._serverMsg.has("__pilotToken")
    		)
    		{
    			return this.download(filename);
    		}

    		// upload the data:
    		return this._psychoJS.serverManager.uploadAudioVideo({
    			mediaBlob: this._data,
    			tag: filename
    		});
    	}

    	/**
    	 * Offer the audio clip to the participant as a sound file to download.
    	 */
    	download(filename = "audio.webm")
    	{
    		const anchor = document.createElement("a");
    		anchor.href = window.URL.createObjectURL(this._data);
    		anchor.download = filename;
    		document.body.appendChild(anchor);
    		anchor.click();
    		document.body.removeChild(anchor);
    	}

    	/**
    	 * Transcribe the audio clip.
    	 *
    	 * @param {Object} options
    	 * @param {Symbol} options.engine - the speech-to-text engine
    	 * @param {String} options.languageCode - the BCP-47 language code for the recognition,
    	 * 	e.g. 'en-GB'
    	 * @return {Promise} a promise resolving to the transcript and associated
    	 * 	transcription confidence
    	 */
    	async transcribe({ engine, languageCode } = {})
    	{
    		const response = {
    			origin: "AudioClip.transcribe",
    			context: `when transcribing audio clip: ${this._name}`,
    		};

    		this._psychoJS.logger.debug(response);

    		// get the secret key from the experiment configuration:
    		const fullEngineName = `sound.AudioClip.Engine.${Symbol.keyFor(engine)}`;
    		let transcriptionKey;
    		for (const key of this._psychoJS.config.experiment.keys)
    		{
    			if (key.name === fullEngineName)
    			{
    				transcriptionKey = key.value;
    			}
    		}
    		if (typeof transcriptionKey === "undefined")
    		{
    			throw {
    				...response,
    				error: `missing key for engine: ${fullEngineName}`,
    			};
    		}

    		// wait for the decoding to complete:
    		await this._decodeAudio();

    		// dispatch on engine:
    		if (engine === AudioClip.Engine.GOOGLE)
    		{
    			return this._GoogleTranscribe(transcriptionKey, languageCode);
    		}
    		else
    		{
    			throw {
    				...response,
    				error: `unsupported speech-to-text engine: ${engine}`,
    			};
    		}
    	}

    	/**
    	 * Transcribe the audio clip using the Google Cloud Speech-To-Text Engine.
    	 *
    	 * ref: https://cloud.google.com/speech-to-text/docs/reference/rest/v1/speech/recognize
    	 *
    	 * @protected
    	 * @param {String} transcriptionKey - the secret key to the Google service
    	 * @param {String} languageCode - the BCP-47 language code for the recognition, e.g. 'en-GB'
    	 * @return {Promise} a promise resolving to the transcript and associated
    	 * 	transcription confidence
    	 */
    	_GoogleTranscribe(transcriptionKey, languageCode)
    	{
    		return new Promise(async (resolve, reject) =>
    		{
    			// convert the Float32 PCM audio data to UInt16:
    			const buffer = new ArrayBuffer(this._audioData.length * 2);
    			const uint16View = new Uint16Array(buffer);
    			for (let t = 0; t < this._audioData.length; ++t)
    			{
    				uint16View[t] = (this._audioData[t] < 0)
    					? this._audioData[t] * 0x8000
    					: this._audioData[t] * 0x7FFF;
    			}

    			// encode it to base64:
    			const base64Data = this._base64ArrayBuffer(new Uint8Array(buffer));

    			// query the Google speech-to-text service:
    			const body = {
    				config: {
    					encoding: "LINEAR16",
    					sampleRateHertz: this._sampleRateHz,
    					languageCode,
    				},
    				audio: {
    					content: base64Data,
    				},
    			};

    			const url = `https://speech.googleapis.com/v1/speech:recognize?key=${transcriptionKey}`;

    			const response = await fetch(url, {
    				method: "POST",
    				headers: {
    					"Content-Type": "application/json",
    				},
    				body: JSON.stringify(body),
    			});

    			// convert the response to json:
    			const decodedResponse = await response.json();
    			this._psychoJS.logger.debug("speech.googleapis.com response:", JSON.stringify(decodedResponse));

    			// TODO deal with more than one results and/or alternatives
    			if (("results" in decodedResponse) && (decodedResponse.results.length > 0))
    			{
    				resolve(decodedResponse.results[0].alternatives[0]);
    			}
    			else
    			{
    				// no transcription available:
    				resolve({
    					transcript: "",
    					confidence: -1,
    				});
    			}
    		});
    	}

    	/**
    	 * Decode the formatted audio data (e.g. webm) into a 32bit float PCM audio buffer.
    	 *
    	 * @protected
    	 */
    	_decodeAudio()
    	{
    		this._psychoJS.logger.debug("request to decode the data of the audio clip");

    		// if the audio clip is ready, the PCM audio data is available in _audioData, a Float32Array:
    		if (this._status === AudioClip.Status.READY)
    		{
    			return;
    		}

    		// if we are already decoding, wait until the process completed:
    		if (this._status === AudioClip.Status.DECODING)
    		{
    			const self = this;
    			return new Promise(function(resolve, reject)
    			{
    				self._decodingCallbacks.push(resolve);

    				// self._errorCallback = reject; // TODO
    			}.bind(this));
    		}

    		// otherwise, start decoding the input formatted audio data:
    		this._status = AudioClip.Status.DECODING;
    		this._audioData = null;
    		this._source = null;
    		this._gainNode = null;
    		this._decodingCallbacks = [];

    		this._audioContext = new (window.AudioContext || window.webkitAudioContext)({
    			sampleRate: this._sampleRateHz,
    		});

    		const reader = new window.FileReader();
    		reader.onloadend = async () =>
    		{
    			try
    			{
    				// decode the ArrayBuffer containing the formatted audio data (e.g. webm)
    				// into an audio buffer:
    				this._audioBuffer = await this._audioContext.decodeAudioData(reader.result);

    				// get the Float32Array containing the PCM data:
    				this._audioData = this._audioBuffer.getChannelData(0);

    				// we are now ready to translate and play:
    				this._status = AudioClip.Status.READY;

    				// resolve all the promises waiting for the decoding to complete:
    				for (const callback of this._decodingCallbacks)
    				{
    					callback();
    				}
    			}
    			catch (error)
    			{
    				console.error(error);

    				// TODO
    			}
    		};

    		reader.onerror = (error) =>
    		{
    			// TODO
    		};

    		reader.readAsArrayBuffer(this._data);
    	}

    	/**
    	 * Convert an array buffer to a base64 string.
    	 *
    	 * @note this is heavily inspired by the following post by @Grantlyk:
    	 * https://gist.github.com/jonleighton/958841#gistcomment-1953137
    	 * It is necessary since the following approach only works for small buffers:
    	 * const dataAsString = String.fromCharCode.apply(null, new Uint8Array(buffer));
    	 * base64Data = window.btoa(dataAsString);
    	 *
    	 * @protected
    	 * @param arrayBuffer - the input buffer
    	 * @return {string} the base64 encoded input buffer
    	 */
    	_base64ArrayBuffer(arrayBuffer)
    	{
    		let base64 = "";
    		const encodings = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

    		const bytes = new Uint8Array(arrayBuffer);
    		const byteLength = bytes.byteLength;
    		const byteRemainder = byteLength % 3;
    		const mainLength = byteLength - byteRemainder;

    		let a;
    		let b;
    		let c;
    		let d;
    		let chunk;

    		// Main loop deals with bytes in chunks of 3
    		for (let i = 0; i < mainLength; i += 3)
    		{
    			// Combine the three bytes into a single integer
    			chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];

    			// Use bitmasks to extract 6-bit segments from the triplet
    			a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
    			b = (chunk & 258048) >> 12; // 258048   = (2^6 - 1) << 12
    			c = (chunk & 4032) >> 6; // 4032     = (2^6 - 1) << 6
    			d = chunk & 63; // 63       = 2^6 - 1

    			// Convert the raw binary segments to the appropriate ASCII encoding
    			base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
    		}

    		// Deal with the remaining bytes and padding
    		if (byteRemainder === 1)
    		{
    			chunk = bytes[mainLength];

    			a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2

    			// Set the 4 least significant bits to zero
    			b = (chunk & 3) << 4; // 3   = 2^2 - 1

    			base64 += `${encodings[a]}${encodings[b]}==`;
    		}
    		else if (byteRemainder === 2)
    		{
    			chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];

    			a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
    			b = (chunk & 1008) >> 4; // 1008  = (2^6 - 1) << 4

    			// Set the 2 least significant bits to zero
    			c = (chunk & 15) << 2; // 15    = 2^4 - 1

    			base64 += `${encodings[a]}${encodings[b]}${encodings[c]}=`;
    		}

    		return base64;
    	}
    }

    /**
     * Recognition engines.
     *
     * @enum {Symbol}
     * @readonly
     */
    AudioClip.Engine = {
    	/**
    	 * Google Cloud Speech-to-Text.
    	 */
    	GOOGLE: Symbol.for("GOOGLE"),
    };

    /**
     * AudioClip status.
     *
     * @enum {Symbol}
     * @readonly
     */
    AudioClip.Status = {
    	CREATED: Symbol.for("CREATED"),

    	DECODING: Symbol.for("DECODING"),

    	READY: Symbol.for("READY"),
    };