import { ParameterType } from "../../modules/plugins"; import { unique } from "../utils"; import { AudioPlayer } from "./AudioPlayer"; const preloadParameterTypes = [ ParameterType.AUDIO, ParameterType.IMAGE, ParameterType.VIDEO, ]; type PreloadType = typeof preloadParameterTypes[number]; export class MediaAPI { constructor(public useWebaudio: boolean) { if ( this.useWebaudio && typeof window !== "undefined" && typeof window.AudioContext !== "undefined" ) { this.context = new AudioContext(); } } // video // private video_buffers = {}; getVideoBuffer(videoID: string) { if (videoID.startsWith("blob:")) { this.video_buffers[videoID] = videoID; } return this.video_buffers[videoID]; } // audio // private context: AudioContext = null; private audio_buffers = []; audioContext(): AudioContext { if (this.context && this.context.state !== "running") { this.context.resume(); } return this.context; } async getAudioPlayer(audioID: string): Promise { if (this.audio_buffers[audioID] instanceof AudioPlayer) { return this.audio_buffers[audioID]; } else { this.audio_buffers[audioID] = new AudioPlayer(audioID, { useWebAudio: this.useWebaudio, audioContext: this.context, }); await this.audio_buffers[audioID].load(); return this.audio_buffers[audioID]; } } // preloading stimuli // private preload_requests = []; private img_cache = {}; preloadAudio( files, callback_complete = () => {}, callback_load = (filepath: string) => {}, callback_error = (error) => {} ) { files = unique(files.flat()); let n_loaded = 0; if (files.length == 0) { callback_complete(); return; } for (const file of files) { // check if file was already loaded if (this.audio_buffers[file] instanceof AudioPlayer) { n_loaded++; callback_load(file); if (n_loaded == files.length) { callback_complete(); } } else { this.audio_buffers[file] = new AudioPlayer(file, { useWebAudio: this.useWebaudio, audioContext: this.context, }); this.audio_buffers[file] .load() .then(() => { n_loaded++; callback_load(file); if (n_loaded == files.length) { callback_complete(); } }) .catch((e) => { callback_error(e); }); } } } preloadImages( images, callback_complete = () => {}, callback_load = (filepath) => {}, callback_error = (error_msg) => {} ) { // flatten the images array images = unique(images.flat()); var n_loaded = 0; if (images.length === 0) { callback_complete(); return; } for (let i = 0; i < images.length; i++) { const img = new Image(); const src = images[i]; img.onload = () => { n_loaded++; callback_load(src); if (n_loaded === images.length) { callback_complete(); } }; img.onerror = (e) => { callback_error({ source: src, error: e }); }; img.src = src; this.img_cache[src] = img; this.preload_requests.push(img); } } preloadVideo( videos, callback_complete = () => {}, callback_load = (filepath) => {}, callback_error = (error_msg) => {} ) { // flatten the video array videos = unique(videos.flat()); let n_loaded = 0; if (videos.length === 0) { callback_complete(); return; } for (const video of videos) { const video_buffers = this.video_buffers; //based on option 4 here: http://dinbror.dk/blog/how-to-preload-entire-html5-video-before-play-solved/ const request = new XMLHttpRequest(); request.open("GET", video, true); request.responseType = "blob"; request.onload = () => { if (request.status === 200 || request.status === 0) { const videoBlob = request.response; video_buffers[video] = URL.createObjectURL(videoBlob); // IE10+ n_loaded++; callback_load(video); if (n_loaded === videos.length) { callback_complete(); } } }; request.onerror = (e) => { let err: ProgressEvent | string = e; if (request.status == 404) { err = "404"; } callback_error({ source: video, error: err }); }; request.onloadend = (e) => { if (request.status == 404) { callback_error({ source: video, error: "404" }); } }; request.send(); this.preload_requests.push(request); } } private preloadMap = new Map>(); getAutoPreloadList(timeline_description: any[]) { /** Map each preload parameter type to a set of paths to be preloaded */ const preloadPaths = Object.fromEntries( preloadParameterTypes.map((type) => [type, new Set()]) ); const traverseTimeline = (node, inheritedTrialType?) => { const isTimeline = typeof node.timeline !== "undefined"; if (isTimeline) { for (const childNode of node.timeline) { traverseTimeline(childNode, node.type ?? inheritedTrialType); } } else if ((node.type ?? inheritedTrialType)?.info) { // node is a trial with type.info set // Get the plugin name and parameters object from the info object const { name: pluginName, parameters } = (node.type ?? inheritedTrialType).info; // Extract parameters to be preloaded and their types from parameter info if this has not // yet been done for `pluginName` if (!this.preloadMap.has(pluginName)) { this.preloadMap.set( pluginName, Object.fromEntries( Object.entries(parameters) // Filter out parameter entries with media types and a non-false `preload` option .filter( ([_name, { type, preload }]) => preloadParameterTypes.includes(type) && (preload ?? true) ) // Map each entry's value to its parameter type .map(([name, { type }]) => [name, type]) ) ); } // Add preload paths from this trial for (const [parameterName, parameterType] of Object.entries( this.preloadMap.get(pluginName) )) { const parameterValue = node[parameterName]; const elements = preloadPaths[parameterType]; if (typeof parameterValue === "string") { elements.add(parameterValue); } else if (Array.isArray(parameterValue)) { for (const element of parameterValue.flat()) { if (typeof element === "string") { elements.add(element); } } } } } }; traverseTimeline({ timeline: timeline_description }); return { images: [...preloadPaths[ParameterType.IMAGE]], audio: [...preloadPaths[ParameterType.AUDIO]], video: [...preloadPaths[ParameterType.VIDEO]], }; } cancelPreloads() { for (const request of this.preload_requests) { request.onload = () => {}; request.onerror = () => {}; request.oncanplaythrough = () => {}; request.onabort = () => {}; } this.preload_requests = []; } private microphone_recorder: MediaRecorder = null; initializeMicrophoneRecorder(stream: MediaStream) { const recorder = new MediaRecorder(stream); this.microphone_recorder = recorder; } getMicrophoneRecorder(): MediaRecorder { return this.microphone_recorder; } private camera_stream: MediaStream = null; private camera_recorder: MediaRecorder = null; initializeCameraRecorder(stream: MediaStream, opts?: MediaRecorderOptions) { this.camera_stream = stream; const recorder = new MediaRecorder(stream, opts); this.camera_recorder = recorder; } getCameraStream(): MediaStream { return this.camera_stream; } getCameraRecorder(): MediaRecorder { return this.camera_recorder; } }