From 42b04d4c06beb016468e1c36ba357d8c3ee68567 Mon Sep 17 00:00:00 2001 From: James Gentes Date: Sat, 27 Jan 2024 16:56:41 -0800 Subject: [PATCH] deploying ok --- app/api/audioEvents.client.ts | 590 ++++++++++++++++++++++++++++++ app/api/audioHandlers.client.ts | 304 +++++++++++++++ app/api/db/__dbSchema.ts | 179 +++++++++ app/api/db/dbHandlers.ts | 232 ++++++++++++ app/api/fileHandlers.ts | 245 +++++++++++++ app/api/renderWaveform.client.tsx | 227 ++++++++++++ app/api/stemHandler.ts | 143 ++++++++ 7 files changed, 1920 insertions(+) create mode 100644 app/api/audioEvents.client.ts create mode 100644 app/api/audioHandlers.client.ts create mode 100644 app/api/db/__dbSchema.ts create mode 100644 app/api/db/dbHandlers.ts create mode 100644 app/api/fileHandlers.ts create mode 100644 app/api/renderWaveform.client.tsx create mode 100644 app/api/stemHandler.ts diff --git a/app/api/audioEvents.client.ts b/app/api/audioEvents.client.ts new file mode 100644 index 0000000..889b4d6 --- /dev/null +++ b/app/api/audioEvents.client.ts @@ -0,0 +1,590 @@ +// This file allows events to be received which need access to the waveform, rather than passing waveform aroun' +import type WaveSurfer from 'wavesurfer.js' +import RegionsPlugin, { + type Region +} from 'wavesurfer.js/dist/plugins/regions.js' +import { calcMarkers } from '~/api/audioHandlers.client' +import { + getAppState, + getAudioState, + setAppState, + setAudioState +} from '~/api/db/appState.client' +import { + Stem, + Track, + TrackPrefs, + _removeFromMix, + db, + getPrefs, + getTrackPrefs, + setTrackPrefs, + updateTrack +} from '~/api/db/dbHandlers' +import { initAudioContext, initWaveform } from '~/api/renderWaveform.client' +import { convertToSecs } from '~/utils/tableOps' + +// audioEvent are emitted by controls (e.g. buttons) to signal changes in audio, such as Play, adjust BPM, etc and the listeners are attached to the waveform when it is rendered + +const _getAllWaveforms = (): WaveSurfer[] => { + const [audioState] = getAudioState() + + const waveforms: WaveSurfer[] = [] + + for (const { waveform } of Object.values(audioState)) { + if (!waveform) continue + waveforms.push(waveform) + } + + return waveforms +} + +const audioEvents = { + onReady: async (trackId: Track['id'], stem?: Stem) => { + const [waveform] = getAudioState[trackId].waveform() + if (!waveform) return + + if (!stem) { + // Generate beat markers (regions) and apply them to waveform + await calcMarkers(trackId) + + const plugins = waveform.getActivePlugins() + const regionsPlugin = plugins[0] as RegionsPlugin + + const { mixpointTime, beatResolution = 1 } = await getTrackPrefs(trackId) + + // Adjust zoom based on previous mixPrefs + waveform.zoom( + beatResolution === '1:1' ? 80 : beatResolution === '1:2' ? 40 : 20 + ) + + // Remove analyzing overlay + setAppState.analyzing(prev => { + prev.delete(trackId) + return prev + }) + + // Style scrollbar (this is a workaround for https://github.com/katspaugh/wavesurfer.js/issues/2933) + const style = document.createElement('style') + style.textContent = `::-webkit-scrollbar { + height: 18px; + } + + ::-webkit-scrollbar-corner, ::-webkit-scrollbar-track { + border-top: 1px solid rgba(128,128,128,.3); + } + + ::-webkit-scrollbar-thumb { + background-color: rgba(4, 146, 247, 0.5); + border-radius: 8px; + border: 6px solid transparent; + width: 15%; + background-clip: content-box; + }` + waveform.getWrapper().appendChild(style) + + // add classname value to waveform.getWrapper() + waveform.getWrapper().classList.add('wrapper') + + // Update time + let [time] = getAudioState[trackId].time() + if (!time) { + time = mixpointTime || regionsPlugin.getRegions()[0]?.start || 0 + setAudioState[trackId].time(time) + } + + // account for resize of browser window + waveform.on('redraw', () => audioEvents.seek(trackId)) + } else { + // Remove from stemsAnalyzing + setAppState.stemsAnalyzing(prev => { + prev.delete(trackId) + return prev + }) + } + + // Update BPM if adjusted + const { adjustedBpm } = await getTrackPrefs(trackId) + const { bpm = 1 } = (await db.tracks.get(trackId)) || {} + const playbackRate = (adjustedBpm || bpm) / bpm + waveform.setPlaybackRate(playbackRate) + }, + + clickToSeek: async ( + trackId: Track['id'], + e: React.MouseEvent, + parentElement: HTMLElement + ) => { + // get click position of parent element and convert to time + const shadowRoot = parentElement.shadowRoot as ShadowRoot + const wrapper = shadowRoot.querySelector('.wrapper') as HTMLElement + const scrollbar = shadowRoot.querySelector('.scroll') as HTMLElement + const boundary = wrapper.getBoundingClientRect() + const position = Math.min( + 1, + (e.clientX + + scrollbar.scrollLeft - + Math.abs(scrollbar.scrollLeft - Math.abs(boundary.x))) / + boundary.width + ) + + const { duration = 1 } = (await db.tracks.get(trackId)) || {} + + audioEvents.seek(trackId, duration * position) + }, + + ejectTrack: async (trackId: Track['id']) => { + if (!trackId) return + + audioEvents.pause(trackId) + + // Destroy waveform and stems before removing from audioState + audioEvents.destroy(trackId) + + // Remove track from mix state (dexie) + await _removeFromMix(trackId) + + // Remove track from audioState (teaful) + const [audioState] = getAudioState() + const { [trackId]: _, ...rest } = audioState + setAudioState(rest) + + // If this is not the last track in the mix, open drawer, otherwise the drawer will open automatically + const { tracks } = (await getPrefs('mix', 'tracks')) || {} + const mixViewVisible = !!tracks?.filter(t => t).length + + if (mixViewVisible) setAppState.openDrawer(true) + }, + + play: async (trackId?: Track['id']) => { + let tracks + if (!trackId) { + // pull players from audioState to play all + ;[tracks] = getAudioState() + tracks = Object.keys(tracks).map(Number) + } else tracks = [trackId] + + // synchronize playback of all tracks + audioEvents.multiSync(tracks.filter(id => !!id)) + }, + + multiSync: async (trackIds: Track['id'][]) => { + // Sync all waveforms to the same position + let [syncTimer] = getAppState.syncTimer() + if (syncTimer) cancelAnimationFrame(syncTimer) + + const dataArray = new Uint8Array(2048) // fftSize + + const getVolume = (analyserNode: AnalyserNode) => { + analyserNode.getByteTimeDomainData(dataArray) + return (Math.max(...dataArray) - 128) / 128 + } + + // setup sync loop + const syncLoop = () => { + syncTimer = requestAnimationFrame(syncLoop) + setAppState.syncTimer(syncTimer) + + // for volume meters + for (const trackId of trackIds) { + const [{ waveform, stems, analyserNode }] = getAudioState[trackId]() + + const volumes: number[] = [] // to aggregate for main volume meter + + if (stems) { + for (const [stem, { analyserNode }] of Object.entries(stems)) { + if (analyserNode) { + const vol = getVolume(analyserNode) + volumes.push(vol) + setAudioState[trackId].stems[stem as Stem].volumeMeter(vol) + } + } + } else volumes.push(getVolume(analyserNode)) + + // aggregate stem volumes for main volume meter + setAudioState[trackId].volumeMeter(Math.max(...volumes)) + setAudioState[trackId].time(waveform.getCurrentTime()) + } + } + + syncTimer = requestAnimationFrame(syncLoop) + + for (const trackId of trackIds) { + const [{ waveform, stems }] = getAudioState[trackId]() + if (!waveform) continue + + setAudioState[trackId as number].playing(true) + + initAudioContext({ + trackId, + media: waveform.getMediaElement() + }) + + waveform.play() + + if (stems) { + // if we have stems, mute the main waveforms + waveform.setVolume(0) + + for (const [stem, { waveform }] of Object.entries(stems)) { + if (waveform) { + initAudioContext({ + trackId, + stem: stem as Stem, + media: waveform.getMediaElement() + }) + waveform.play() + } + } + } + } + }, + + pause: async (trackId?: Track['id']) => { + // this needs to pause all stems so requires a bit of logic + let waveforms + let trackIds + + const [syncTimer] = getAppState.syncTimer() + if (syncTimer) cancelAnimationFrame(syncTimer) + + if (trackId) { + const [waveform] = getAudioState[trackId].waveform() + waveforms = [waveform] + trackIds = [trackId] + } else { + waveforms = _getAllWaveforms() + const [tracks] = getAudioState() + trackIds = Object.keys(tracks) + } + + const stopWaveform = (waveform: WaveSurfer) => waveform.pause() + + for (const waveform of waveforms) { + if (waveform) stopWaveform(waveform) + } + + for (const id of trackIds) { + const [stems] = getAudioState[Number(id)].stems() + if (stems) { + for (const [stem, { waveform }] of Object.entries(stems)) { + // set volume meter to zero for the stem + setAudioState[Number(id)].stems[stem as Stem].volumeMeter(0) + + if (waveform) stopWaveform(waveform) + } + } + setAudioState[Number(id)].playing(false) + setAudioState[Number(id)].volumeMeter(0) + } + }, + + // Scroll to previous/next beat marker + seek: async ( + trackId: Track['id'], + seconds?: number, // no default here, we want to be able to seek to 0 + direction?: 'previous' | 'next' + ) => { + if (!trackId) return + + const [{ waveform, playing, time }] = getAudioState[trackId]() + if (!waveform) return + + const currentTime = seconds ?? time + if (playing) await audioEvents.pause(trackId) + + const { duration = 1 } = (await db.tracks.get(trackId)) || {} + + const regionsPlugin = waveform.getActivePlugins()[0] as RegionsPlugin + + // find the closest marker to the current time + const regions = regionsPlugin.getRegions() + + const findClosestRegion = (time: number) => { + return regions.findIndex((region: Region) => { + if (region.start > time) return true + }) + } + + let currentIndex = findClosestRegion(!direction ? currentTime : time) + currentIndex = currentIndex === -1 ? regions.length - 1 : currentIndex + + const previous = regions[(currentIndex || 1) - 1] + const current = regions[currentIndex] + const next = regions[Math.min(currentIndex, regions.length - 2) + 1] + + const previousDiff = Math.abs(currentTime - previous.start) + const currentDiff = Math.abs(currentTime - current.start) + const nextDiff = Math.abs(currentTime - next.start) + + let closestTime = current.start // default current wins + if (direction) { + closestTime = + direction === 'previous' + ? regions[Math.max(currentIndex - 2, 0)].start + : current.start + } else if (previousDiff < currentDiff) { + // previous wins + if (currentDiff < nextDiff) { + // previous wins + closestTime = previous.start + } else { + if (previousDiff < nextDiff) { + // previous wins + closestTime = previous.start + } else { + // next wins + closestTime = next.start + } + } + } + + setAudioState[trackId].time(closestTime) + + const [stems] = getAudioState[trackId].stems() + if (stems) { + for (const [, { waveform: stemWave }] of Object.entries(stems)) { + stemWave?.seekTo(closestTime / duration) + } + } + + waveform.seekTo(closestTime / duration) + + // resume playing if not at the end of track + if (playing && closestTime < duration) audioEvents.play(trackId) + }, + + seekMixpoint: async (trackId: Track['id']) => { + const { mixpointTime = 0 } = (await getTrackPrefs(trackId)) || {} + audioEvents.seek(trackId, mixpointTime) + }, + + // crossfade handles the sliders that mix between stems or full track + crossfade: async (sliderVal: number, stemType?: Stem) => { + const { tracks } = await getPrefs('mix') + + const sliderPercent = sliderVal / 100 + + // Keep volumes at 100% when at 50% crossfade + // [left, right] @ 0% = [1, 0] 50% = [1, 1] 100% = [0, 1] + const volumes = [ + Math.min(1, 1 + Math.cos(sliderPercent * Math.PI)), + Math.min(1, 1 + Math.cos((1 - sliderPercent) * Math.PI)) + ] + + if (tracks) { + for (const [i, track] of tracks.entries()) { + if (track) audioEvents.updateVolume(Number(track), volumes[i], stemType) + } + } + }, + + updateVolume: (trackId: number, volume: number, stemType?: Stem) => { + const [{ volume: trackVol = 1, stems, gainNode, stemState }] = + getAudioState[trackId]() + + // if we have a stemType, this is a stem crossfader + if (stemType) { + if (!stems) return + + // adjust the gain of the stem as a percentage of the track volume + // (75% crossfader x 50% stem fader = 37.5% stem volume) + const stemGain = stems[stemType]?.gainNode + stemGain?.gain.setValueAtTime(trackVol * volume, 0) + setAudioState[trackId].stems[stemType].volume(volume) + return + } + + // otherwise this is main crossfader + if (stemState !== 'ready') { + gainNode?.gain.setValueAtTime(volume, 0) + } else if (stems) { + for (const stem of Object.keys(stems)) { + const [stemGain] = getAudioState[trackId].stems[stem as Stem].gainNode() + const [stemVol = 1] = + getAudioState[trackId].stems[stem as Stem].volume() + + // adjust the gain of the stem as a percentage of the track volume + // (75% crossfader x 50% stem fader = 37.5% stem volume) + stemGain?.gain.setValueAtTime(trackVol * stemVol, 0) + } + + setAudioState[trackId].volume(volume) + } + }, + + beatResolution: async ( + trackId: Track['id'], + beatResolution: TrackPrefs['beatResolution'] + ): Promise => { + const [waveform] = getAudioState[trackId].waveform() + if (!waveform || !beatResolution) return + + // Update mixPrefs + await setTrackPrefs(trackId, { beatResolution }) + + // Adjust zoom + switch (beatResolution) { + case '1:4': + waveform.zoom(20) + break + case '1:2': + waveform.zoom(40) + break + case '1:1': + waveform.zoom(80) + break + } + + calcMarkers(trackId) + }, + + bpm: async ( + trackId: Track['id'], + adjustedBpm: TrackPrefs['adjustedBpm'] + ): Promise => { + const [{ stems, waveform, playing }] = getAudioState[trackId]() + if (!adjustedBpm) return + + const { bpm } = (await db.tracks.get(trackId)) || {} + + const playbackRate = adjustedBpm / (bpm || adjustedBpm) + + if (playing) audioEvents.pause(trackId) + + const adjustPlaybackRate = (waveform: WaveSurfer) => + waveform.setPlaybackRate(playbackRate) + + // update stem playback rate in realtime + if (stems) { + for (const { waveform } of Object.values(stems)) { + if (!waveform) continue + + adjustPlaybackRate(waveform) + } + } else { + if (waveform) adjustPlaybackRate(waveform) + } + + if (playing) audioEvents.play(trackId) + + // Update mixPrefs + await setTrackPrefs(trackId, { adjustedBpm }) + }, + + offset: async ( + trackId: Track['id'], + adjustedOffset: Track['adjustedOffset'] + ): Promise => { + await updateTrack(trackId, { adjustedOffset }) + + calcMarkers(trackId) + }, + + setMixpoint: async ( + trackId: Track['id'], + mixpoint?: string + ): Promise => { + const [waveform] = getAudioState[trackId].waveform() + if (!waveform) return + + audioEvents.pause(trackId) + + const [time = 0] = getAudioState[trackId].time() + const { mixpointTime = 0 } = (await getTrackPrefs(trackId)) || {} + + const newMixpoint = mixpoint ? convertToSecs(mixpoint) : time + if (newMixpoint === mixpointTime) return + + setTrackPrefs(trackId, { mixpointTime: newMixpoint }) + + audioEvents.seek(trackId, newMixpoint) + }, + + stemVolume: (trackId: Track['id'], stemType: Stem, volume: number) => { + const [stems] = getAudioState[trackId].stems() + if (!stems) return + + const gainNode = stems[stemType as Stem]?.gainNode + if (gainNode) gainNode.gain.setValueAtTime(volume, 0) + + // set volume in state, which in turn will update components (volume sliders) + setAudioState[trackId].stems[stemType as Stem].volume(volume) + }, + + stemMuteToggle: (trackId: Track['id'], stemType: Stem, mute: boolean) => { + const [stems] = getAudioState[trackId].stems() + if (!stems) return + + const stem = stems[stemType as Stem] + const { gainNode, volume } = stem || {} + + gainNode?.gain.setValueAtTime(mute ? 0 : volume || 1, 0) + + setAudioState[trackId].stems[stemType as Stem].mute(mute) + }, + + stemSoloToggle: (trackId: Track['id'], stem: Stem, solo: boolean) => { + const [stems] = getAudioState[trackId].stems() + if (!stems) return + + for (const s of Object.keys(stems)) { + if (s !== stem) audioEvents.stemMuteToggle(trackId, s as Stem, solo) + } + }, + + stemZoom: async ( + trackId: Track['id'], + stem: TrackPrefs['stemZoom'] | 'all' + ) => { + // add track to analyzing state + setAppState.analyzing(prev => prev.add(trackId)) + + const [{ waveform }] = getAudioState[trackId]() + if (waveform) waveform.destroy() + + let file + + if (stem === 'all') { + ;({ file } = (await db.trackCache.get(trackId)) || {}) + } else { + const { stems } = (await db.trackCache.get(trackId)) || {} + if (!stems) return + + file = stems[stem as Stem]?.file + } + + if (!file) return + + await initWaveform({ trackId, file }) + + await setTrackPrefs(trackId, { + stemZoom: stem === 'all' ? undefined : stem + }) + }, + + destroy: (trackId: Track['id']) => { + const [waveform] = getAudioState[trackId].waveform() + + audioEvents.destroyStems(trackId) + if (waveform) waveform.destroy() + }, + + destroyStems: (trackId: Track['id']) => { + const [stems] = getAudioState[trackId].stems() + + if (stems) { + for (const stem of Object.values(stems)) { + stem?.waveform?.destroy() + } + } + + // Remove from stemsAnalyzing + setAppState.stemsAnalyzing(prev => { + prev.delete(trackId) + return prev + }) + } +} + +export { audioEvents } diff --git a/app/api/audioHandlers.client.ts b/app/api/audioHandlers.client.ts new file mode 100644 index 0000000..c06f21f --- /dev/null +++ b/app/api/audioHandlers.client.ts @@ -0,0 +1,304 @@ +import { H } from '@highlight-run/remix/client' +import type RegionsPlugin from 'wavesurfer.js/dist/plugins/regions.js' +import { guess as detectBPM } from 'web-audio-beat-detector' +import { + getAudioState, + setAppState, + setModalState +} from '~/api/db/appState.client' +import { + Track, + db, + getTrackPrefs, + putTracks, + setPrefs +} from '~/api/db/dbHandlers' +import { getPermission } from '~/api/fileHandlers' +import { errorHandler } from '~/utils/notifications' + +// This is the main track processing workflow when files are added to the app +const processTracks = async ( + handles: (FileSystemFileHandle | FileSystemDirectoryHandle)[] +) => { + const trackArray = await getTracksRecursively(handles) + return await analyzeTracks(trackArray) +} + +type partialTrack = Pick< + Track, + 'name' | 'size' | 'type' | 'fileHandle' | 'dirHandle' +> + +// The function iterates through file handles and collects the +// information needed to add them to the database, then hands off +// the array of track id's returned from the db for analysis. +async function getTracksRecursively( + handles: (FileSystemFileHandle | FileSystemDirectoryHandle)[] +): Promise { + const trackArray: partialTrack[] = [] + + // Change sort order to lastModified so new tracks are visible at the top + await setPrefs('user', { + sortColumn: 'lastModified', + sortDirection: 'descending' + }) + + const filesToTracks = async ( + fileOrDirectoryHandle: FileSystemFileHandle | FileSystemDirectoryHandle, + dirHandle?: FileSystemDirectoryHandle + ) => { + if (fileOrDirectoryHandle.kind === 'file') { + const { name, size, type } = await fileOrDirectoryHandle.getFile() + + if (!type || !type.startsWith('audio')) + return errorHandler(`${name} is not an audio file.`) + + if (name) + trackArray.push({ + name, + size, + type, + fileHandle: fileOrDirectoryHandle, + dirHandle + }) + } else if (fileOrDirectoryHandle.kind === 'directory') { + for await (const handle of fileOrDirectoryHandle.values()) { + await filesToTracks(handle, fileOrDirectoryHandle) + } + } + } + + for (const fileOrDirectoryHandle of handles) { + await filesToTracks(fileOrDirectoryHandle) + } + + const addTracksToDb = async () => { + // Ensure we have id's for our tracks, add them to the DB with updated lastModified dates + const updatedTracks = await putTracks(trackArray) + setAppState.processing(false) + H.track('Track Added', { trackQuantity: updatedTracks.length }) + return updatedTracks + } + + // Warn user if large number of tracks are added, this is due to memory leak in web audio api + if (trackArray.length > 100) { + // Show indicator inside empty table + setAppState.processing(true) + + setModalState({ + openState: true, + headerText: 'More than 100 tracks added', + bodyText: + 'Analyzing audio is memory intensive. If your browser runs out of memory, just refresh the page to release memory and continue analyzing tracks.', + confirmText: 'Continue', + confirmColor: 'success', + onConfirm: async () => { + setModalState.openState(false) + const updatedTracks = await addTracksToDb() + await analyzeTracks(updatedTracks) + }, + onCancel: () => { + setModalState.openState(false) + setAppState.processing(false) + } + }) + return [] + } + + return addTracksToDb() +} + +const analyzeTracks = async (tracks: Track[]): Promise => { + // Set analyzing state now to avoid tracks appearing with 'analyze' button + setAppState.analyzing( + prev => new Set([...prev, ...tracks.map(track => track.id)]) + ) + + // Return array of updated tracks + const updatedTracks: Track[] = [] + + let sorted + for (const track of tracks) { + if (!sorted) { + // Change sort order to lastModified so new tracks are visible at the top + await setPrefs('user', { + sortColumn: 'lastModified', + sortDirection: 'descending' + }) + setAppState.page(1) + sorted = true + } + + const { name, size, type, offset, bpm, duration, sampleRate, ...rest } = + await getAudioDetails(track) + + // adjust for miscalc tempo > 160bpm + const normalizedBpm = bpm > 160 ? bpm / 2 : bpm + + const updatedTrack = { + name, + size, + type, + duration, + bpm: normalizedBpm, + offset, + sampleRate, + ...rest + } + + const [trackWithId] = await putTracks([updatedTrack]) + updatedTracks.push(trackWithId) + + // Remove from analyzing state + setAppState.analyzing(prev => { + prev.delete(track.id) + return prev + }) + } + return updatedTracks +} + +const getAudioDetails = async ( + track: Track +): Promise<{ + name: string + size: number + type: string + offset: number + bpm: number + duration: number + sampleRate: number +}> => { + const file = await getPermission(track) + if (!file) { + setAppState.analyzing(new Set()) + throw errorHandler('Permission to the file or folder was denied.') + } + + const { name, size, type } = file + const arrayBuffer = await file.arrayBuffer() + + const audioCtx = new AudioContext() + const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer) + const { duration, sampleRate } = audioBuffer + + let offset = 0 + let bpm = 1 + + try { + ;({ offset, bpm } = await detectBPM(audioBuffer)) + } catch (e) { + errorHandler(`Unable to determine BPM for ${name}`) + } + + audioCtx.close() + + // Reduce offset to 2 decimal places + offset = Math.round(offset * 1e2) / 1e2 + + return { + name, + size, + type, + offset, + bpm, + duration, + sampleRate + } +} + +// CalcMarkers can be called independently for changes in beat offset or beat resolution +const calcMarkers = async (trackId: Track['id']): Promise => { + if (!trackId) return + + const [waveform] = getAudioState[trackId].waveform() + if (!waveform) return + + const regionsPlugin = waveform.getActivePlugins()[0] as RegionsPlugin + + if (!regionsPlugin) return + regionsPlugin.clearRegions() + + const track = await db.tracks.get(trackId) + if (!track) return + let { name, duration, offset, adjustedOffset, bpm } = track || {} + + // isNaN check here to allow for zero values + const valsMissing = + !duration || Number.isNaN(Number(bpm)) || Number.isNaN(Number(offset)) + + if (valsMissing) { + const analyzedTracks = await analyzeTracks([track]) + ;({ bpm, offset } = analyzedTracks[0]) + } + + if (!duration) return errorHandler(`Please try adding ${name} again.`) + + const { beatResolution = '1:4' } = await getTrackPrefs(trackId) + + const beatInterval = 60 / (bpm || 1) + const skipLength = beatInterval * Number(beatResolution.split(':')[1]) + + let startPoint = adjustedOffset || offset || 0 + + // Work backward from initialPeak to start of track (zerotime) based on bpm + while (startPoint - beatInterval > 0) startPoint -= beatInterval + + // Now that we have zerotime, move forward with markers based on the bpm + for (let time = startPoint; time < duration; time += skipLength) { + regionsPlugin.addRegion({ + start: time, + end: time, + color: 'rgba(4, 146, 247, 0.757)', + drag: false + }) + } +} + +// const createMix = async (TrackPrefsArray: TrackPrefs[]) => { +// // this is slow, also look at https://github.com/jackedgson/crunker and https://github.com/audiojs/audio-buffer-utils + +// const [wave0, wave1] = [...TrackPrefsArray].map(track => +// track.waveformData?.toJSON() +// ) + +// const track0Duration = +// (wave0 && (wave0.length / wave0.sample_rate) * wave0.samples_per_pixel) || 0 +// const track1Duration = +// (wave1 && +// (wave1.length / wave1.sample_rate) * wave1.samples_per_pixel - +// (TrackPrefsArray[0]?.mixPoint || 0) - +// (TrackPrefsArray[1]?.mixPoint || 0)) || +// 0 + +// const totalDuration = track0Duration + track1Duration + +// const arrayOfAudioBuffers = [] +// for (let t of TrackPrefsArray) +// arrayOfAudioBuffers.push(await getAudioBuffer(t.file!)) + +// var audioCtx = new AudioContext() + +// let finalMix = audioCtx.createBuffer( +// 2, +// totalDuration * 48000, +// arrayOfAudioBuffers[0].sampleRate +// ) + +// for (let i = 0; i < arrayOfAudioBuffers.length; i++) { +// // second loop for each channel ie. left and right +// for (let channel = 0; channel < 2; channel++) { +// //here we get a reference to the final mix buffer data +// let buffer = finalMix.getChannelData(channel) + +// //last is loop for updating/summing the track buffer with the final mix buffer +// for (let j = 0; j < arrayOfAudioBuffers[i].length; j++) { +// buffer[j] += arrayOfAudioBuffers[i].getChannelData(channel)[j] +// } +// } +// } + +// return finalMix +// } + +export { analyzeTracks, calcMarkers, getAudioDetails, processTracks } diff --git a/app/api/db/__dbSchema.ts b/app/api/db/__dbSchema.ts new file mode 100644 index 0000000..279c7bd --- /dev/null +++ b/app/api/db/__dbSchema.ts @@ -0,0 +1,179 @@ +// This file initializes Dexie (indexDB), defines the schema and creates tables +// Be sure to create MIGRATIONS for any changes to SCHEMA! +import Dexie from 'dexie' +import { Key } from 'react' + +// eventually allow the user to change these +const STATE_ROW_LIMIT = 100 + +// from https://dexie.org/docs/Typescript + +class MixpointDb extends Dexie { + tracks: Dexie.Table + mixes: Dexie.Table + sets: Dexie.Table + mixPrefs: Dexie.Table + setPrefs: Dexie.Table + userPrefs: Dexie.Table + trackCache: Dexie.Table + + constructor() { + super('MixpointDb') + this.version(1).stores({ + tracks: '++id, name, bpm, [name+size]', + mixes: '++id, tracks', + sets: '++id, mixes', + mixPrefs: 'date', + setPrefs: 'date', + userPrefs: 'date', + trackCache: 'id' + }) + this.version(2).upgrade((tx) => { + tx.table('userPrefs').toCollection().modify((userPref) => { + if (userPref.sortDirection === 'asc') userPref.sortDirection = 'ascending' + if (userPref.sortDirection === 'desc') userPref.sortDirection = 'descending' + }) + }) + + this.tracks = this.table('tracks') + this.mixes = this.table('mixes') + this.sets = this.table('sets') + this.mixPrefs = this.table('mixPrefs') + this.setPrefs = this.table('setPrefs') + this.userPrefs = this.table('userPrefs') + this.trackCache = this.table('trackCache') + } +} + +const db = new MixpointDb() + +// Core data models (tracks, mixes, sets) + +type Track = { + id: number + name: string + size: number + type: string // type of file as returned from fileHandle + fileHandle?: FileSystemFileHandle + dirHandle?: FileSystemDirectoryHandle + lastModified?: Date + duration?: number + bpm?: number + sampleRate?: number + offset?: number // first beat as determined by bpm analysis + adjustedOffset?: number + mixpoints?: Mixpoint[] + sets?: MixSet['id'][] +} + +// a mixpoint is a point in time where the To track begins to overlay the From track. +// a mixpoint is not the output of two tracks mixed together. + +type Mixpoint = { + timestamp: number + mixes: Mix['id'][] +} + +// a mix is a representation of the transition between tracks + +type Mix = { + id: number + status: string // Todo: define good | bad | unknown? + effects: { + timestamp: number + duration: number + }[] + lastState: MixPrefs +} + +type MixSet = { + id: number + mixIds: Mix['id'][] +} + +// The TrackCache provides a cache for file data. This allows the app to render +// waveforms without prompting the user for permission to read the file from +// disk, which cannot be done without interacting with the page first. +// Each file is a few megabytes, so the cache must be limited. +const STEMS = ['drums', 'bass', 'vocals', 'other'] as const +type Stem = (typeof STEMS)[number] + +type TrackCache = { + id: Track['id'] + file?: File + stems?: Partial<{ + [key in Stem]: { file?: File } + }> +} + +// Note TrackPrefs is not a table. Track states are contained in MixPrefs +type TrackPrefs = Partial<{ + id: Track['id'] + adjustedBpm: Track['bpm'] + beatResolution: '1:1' | '1:2' | '1:4' + stemZoom: Stem + mixpointTime: number // seconds +}> + +// State tables + +// Each row in a state table is a full representation of state at that point in time +// This allows easy undo/redo of state changes by using timestamps (primary key) +// State tables are limited to STATE_ROW_LIMIT rows (arbitrarily 100) + +type MixPrefs = Partial<{ + date: Date // current mix is most recent mixPrefs + tracks: Track['id'][] + trackPrefs: TrackPrefs[] +}> + +type SetPrefs = Partial<{ + date: Date + setId: MixSet['id'] +}> + +type UserPrefs = Partial<{ + date: Date + sortDirection: 'ascending' | 'descending' + sortColumn: Key + visibleColumns: Set // track table visible columns + stemsDirHandle: FileSystemDirectoryHandle // local folder on file system to store stems +}> + +// For state getter and setter +type StoreTypes = { + mix: MixPrefs + set: SetPrefs + user: UserPrefs +} + +// db hooks to limit the number of rows in a state table +const createHooks = (table: keyof StoreTypes) => { + db[`${table}Prefs`].hook('creating', async () => { + const count = await db[`${table}Prefs`].count() + if (count > STATE_ROW_LIMIT) { + const oldest = await db[`${table}Prefs`].orderBy('date').first() + if (oldest) db[`${table}Prefs`].delete(oldest.date) + } + }) +} + +const tables = ['mix', 'set', 'user'] as const +for (const table of tables) { + createHooks(table) +} + +// Avoid having two files export same type names +export type { + Track as __Track, + Mix as __Mix, + MixSet as __MixSet, + TrackPrefs as __TrackPrefs, + MixPrefs as __MixPrefs, + SetPrefs as __SetPrefs, + UserPrefs as __UserPrefs, + StoreTypes as __StoreTypes, + TrackCache as __TrackCache, + Stem as __Stem +} +export { db as __db, STEMS as __STEMS } diff --git a/app/api/db/dbHandlers.ts b/app/api/db/dbHandlers.ts new file mode 100644 index 0000000..159d81e --- /dev/null +++ b/app/api/db/dbHandlers.ts @@ -0,0 +1,232 @@ +// This file provides a few helper functions for interacting with the database +import { useLiveQuery } from 'dexie-react-hooks' +import { audioEvents } from '~/api/audioEvents.client' +import { + __Mix as Mix, + __MixPrefs as MixPrefs, + __MixSet as MixSet, + __STEMS as STEMS, + __SetPrefs as SetPrefs, + __Stem as Stem, + __StoreTypes as StoreTypes, + __Track as Track, + __TrackCache as TrackCache, + __TrackPrefs as TrackPrefs, + __UserPrefs as UserPrefs, + __db as db +} from '~/api/db/__dbSchema' +import { getPermission } from '~/api/fileHandlers' +import { errorHandler } from '~/utils/notifications' + +const CACHE_LIMIT = 25 + +const storeTrackCache = async ({ + id, + file, + stems +}: { + id: TrackCache['id'] + file?: TrackCache['file'] + stems?: TrackCache['stems'] +}) => { + // Retrieve any existing cache data + const cache = await db.trackCache.get(id) + if (!file) file = cache?.file + if (cache?.stems) stems = { ...cache.stems, ...stems } + + // Enforce database limit + const count = await db.trackCache.count() + if (count > CACHE_LIMIT) { + const oldest = await db.trackCache.orderBy('id').first() + if (oldest) db.trackCache.delete(oldest.id) + } + + await db.trackCache.put({ id, file, stems }) +} + +const updateTrack = async ( + trackId: Track['id'], + keyvals: Partial +): Promise => { + if (!trackId) throw errorHandler('No track id provided') + + const track = await db.tracks.get(trackId) + if (!track) throw errorHandler('No track found, try re-adding it.') + + const updatedTrack = { ...track, ...keyvals } + + await db.tracks.put(updatedTrack) + + return updatedTrack +} + +type TrackIdOptional = Omit & Partial> + +const putTracks = async (tracks: TrackIdOptional[]): Promise => { + const bulkTracks: Omit[] = [] + + for (let track of tracks) { + if (!track) continue + + // if this is a new file, check for existing track with same name and size + if (!track.id) { + // if below line changes, potentially remove [name+size] index + const dup = await db.tracks.get({ name: track.name, size: track.size }) + // if we found the track in the database already, set the primary key + if (dup) track = { ...dup, ...track } + } + + track.lastModified = new Date() + + // push into bulk array if it's not already there + if (!bulkTracks.some(t => t.name === track?.name && t.size === track?.size)) + bulkTracks.push(track) + } + + const updatedTracks = await db.tracks.bulkPut(bulkTracks as Track[], { + allKeys: true + }) + return (await db.tracks.bulkGet(updatedTracks)) as Track[] +} + +const removeTracks = async (ids: Track['id'][]): Promise => { + await db.tracks.bulkDelete(ids) + + // Ensure we delete the file cache when a track is deleted + await db.trackCache.bulkDelete(ids) +} +// const addMix = async ( +// trackIds: Track['id'][], +// mixPoints: MixPoint[] +// ): Promise => +// await db.mixes.add({ trackIds, mixPoints }) + +// Dirty tracks need analysis to determine bpm and duration +const getDirtyTracks = async (): Promise => + await db.tracks.filter(t => !t.bpm).toArray() + +const getMix = async (id: number): Promise => + await db.mixes.get(id) + +const removeMix = async (id: number): Promise => await db.mixes.delete(id) + +// state getter and setter + +// this function is a work of typescript wizardry +const getPrefs = async ( + table: T, + key?: keyof StoreTypes[T] +): Promise> => { + const state = + ((await db[`${table}Prefs`].orderBy('date').last()) as StoreTypes[T]) || {} + + return key ? ({ [key]: state[key] } as Partial) : state +} + +const setPrefs = async ( + table: keyof StoreTypes, + state: Partial +): Promise => { + const prevState = await getPrefs(table) + + await db[`${table}Prefs`].put({ + ...prevState, + ...state, + date: new Date() + }) +} + +const getTrackPrefs = async (trackId: Track['id']): Promise => { + const { tracks = [], trackPrefs = [] } = await getPrefs('mix') + const trackIndex = tracks.indexOf(trackId) + + return trackPrefs[trackIndex] || {} +} + +const getTrackName = async (trackId: Track['id']) => { + if (!trackId) return null + + const { name } = (await db.tracks.get(trackId)) || {} + + return name?.slice(0, -4) || 'Loading...' +} + +// Update the state for an individual track in the mix, such as when offset is adjusted +const setTrackPrefs = async ( + trackId: Track['id'], + state: Partial +): Promise => { + const { tracks = [], trackPrefs = [] } = await getPrefs('mix') + const trackIndex = tracks.indexOf(trackId) + + if (trackIndex === -1) return errorHandler('Track not found in mix state') + + const newState = { ...(trackPrefs[trackIndex] || {}), ...state } + trackPrefs[trackIndex] = newState + + await setPrefs('mix', { tracks, trackPrefs }) +} + +const addToMix = async (track: Track, trackSlot?: 0 | 1) => { + const file = await getPermission(track) + if (!file) return + + const { tracks = [], trackPrefs = [] } = await getPrefs('mix') + + // tracks should retain their position (ie. [0, 1]) + // is there a track in first position? if not, put this track there + const index = trackSlot ?? tracks[0] ? 1 : 0 + + // if there's already a track in this position, remove it first + if (tracks[index]) await audioEvents.ejectTrack(tracks[index]) + tracks[index] = track.id + trackPrefs[index] = { id: track.id } + + await setPrefs('mix', { tracks, trackPrefs }) +} + +const _removeFromMix = async (id: Track['id']) => { + // always use ejectTrack audioEvent to ensure track is removed from appState! + const { tracks = [], trackPrefs = [] } = await getPrefs('mix') + + const index = tracks.indexOf(id) + + if (index > -1) { + delete tracks[index] + delete trackPrefs[index] + } + + await setPrefs('mix', { tracks, trackPrefs }) +} + +export type { + Track, + Mix, + MixSet, + TrackPrefs, + MixPrefs, + SetPrefs, + UserPrefs, + StoreTypes, + TrackCache, + Stem +} +export { + db, + STEMS, + useLiveQuery, + updateTrack, + putTracks, + removeTracks, + getDirtyTracks, + getMix, + removeMix, + addToMix, + _removeFromMix, + getPrefs, + setPrefs, + getTrackPrefs, + getTrackName, + setTrackPrefs, + storeTrackCache +} diff --git a/app/api/fileHandlers.ts b/app/api/fileHandlers.ts new file mode 100644 index 0000000..5211a30 --- /dev/null +++ b/app/api/fileHandlers.ts @@ -0,0 +1,245 @@ +import { + StemState, + getAppState, + getAudioState, + setAppState, + setAudioState +} from '~/api/db/appState.client' +import { + STEMS, + Stem, + Track, + TrackCache, + addToMix, + db, + getPrefs, + setPrefs, + storeTrackCache +} from '~/api/db/dbHandlers' +import { errorHandler } from '~/utils/notifications' +import { processTracks } from './audioHandlers.client' + +function showOpenFilePickerPolyfill(options: OpenFilePickerOptions) { + return new Promise(resolve => { + const input = document.createElement('input') + input.type = 'file' + input.multiple = options.multiple || false + input.accept = (options.types || []) + .map(type => type.accept) + .flatMap(inst => Object.keys(inst).flatMap(key => inst[key])) + .join(',') + + input.addEventListener('change', () => { + resolve( + [...(input.files || [])].map(file => { + return { + getFile: async () => + new Promise(resolve => { + resolve(file) + }) + } + }) + ) + }) + + input.click() + }) +} + +const _getFile = async (track: Track): Promise => { + let handle = track.dirHandle || track.fileHandle + if (!handle) return null + + let file = null + const perms = await handle.queryPermission() + + if (perms === 'granted' && track.name) { + if (handle.kind === 'directory') { + handle = await handle.getFileHandle(track.name) + } + + if (handle) file = await handle.getFile() + } + + // Cache the file + if (file) await storeTrackCache({ id: track.id, file }) + + // In the case perms aren't granted, return null - we need to request permission + return file +} + +/** Returns the file if permission has been granted to a file. + * Will pull from cache or prompt the user if necessary + * (user must have interacted with the page first!) + * otherwise returns null + */ +const getPermission = async (track: Track): Promise => { + // First see if we have the file in the cache + const cache = await db.trackCache.get(track.id) + if (cache?.file) return cache.file + + // Check perms, directory handle is preferred over file handle + const file = await _getFile(track) + if (file) return file + + const handle = track.dirHandle || track.fileHandle + + try { + // Note: this will catch "DOMException: User activation is required + // to request permissions" if user hasn't interacted with the page yet + await handle?.requestPermission() + } catch (e) { + errorHandler('Permission to file or folder was not granted.') + } + + return await _getFile(track) +} + +const browseFile = async (trackSlot?: 0 | 1): Promise => { + // if the track drawer isn't open and we're in mix view, open it, otherwise show file picker + const { tracks } = (await getPrefs('mix', 'tracks')) || {} + const mixViewVisible = !!tracks?.filter(t => t).length + + const [openDrawer] = getAppState.openDrawer() + if (!openDrawer && mixViewVisible) return setAppState.openDrawer(true) + + if (typeof window.showOpenFilePicker !== 'function') { + window.showOpenFilePicker = showOpenFilePickerPolyfill as ( + options?: OpenFilePickerOptions | undefined + ) => Promise<[FileSystemFileHandle]> + } + + const files: FileSystemFileHandle[] | undefined = await window + .showOpenFilePicker({ multiple: true }) + .catch(e => { + if (e?.message?.includes('user aborted a request')) return [] + }) + + if (files?.length) { + const tracks = (await processTracks(files)) || [] + if (tracks.length === 1) addToMix(tracks[trackSlot || 0]) + } +} + +const getStemsDirHandle = async (): Promise< + FileSystemDirectoryHandle | undefined +> => { + const { stemsDirHandle } = await getPrefs('user') + + if (stemsDirHandle) { + // check if we have permission + if ( + (await stemsDirHandle.queryPermission({ mode: 'readwrite' })) === + 'granted' + ) { + return stemsDirHandle + } + + // no permission, so ask for it + if ( + (await stemsDirHandle.requestPermission({ mode: 'readwrite' })) === + 'granted' + ) { + return stemsDirHandle + } + } + + // no dirHandle, or permission was denied, so ask for a new one + const newStemsDirHandle = await window.showDirectoryPicker({ + startIn: stemsDirHandle, + id: 'stemsDir', + mode: 'readwrite' + }) + + if ( + (await newStemsDirHandle.queryPermission({ mode: 'readwrite' })) === + 'granted' + ) { + await setPrefs('user', { stemsDirHandle: newStemsDirHandle }) + return newStemsDirHandle + } +} + +const validateTrackStemAccess = async ( + trackId: Track['id'] +): Promise => { + if (!trackId) throw errorHandler('No Track id provided for stems') + + const [stemState] = getAudioState[trackId].stemState() + + const checkAccess = async () => { + // See if we have stems in cache + const { stems } = (await db.trackCache.get(trackId)) || {} + if (stems) return 'ready' + + // do we have a stem dir defined? + const { stemsDirHandle } = await getPrefs('user') + if (!stemsDirHandle) return 'selectStemDir' + + // do we have access to the stem dir? + try { + const stemDirAccess = await stemsDirHandle.queryPermission({ + mode: 'readwrite' + }) + if (stemDirAccess !== 'granted') return 'grantStemDirAccess' + } catch (e) { + // directory doesn't exist + return 'selectStemDir' + } + + if (stemState === 'processingStems') return stemState + + const { name } = (await db.tracks.get(trackId)) || {} + if (!name) return 'getStems' + + const FILENAME = name.substring(0, name.lastIndexOf('.')) + + // does the stem dir for this track exist? + let trackStemDirHandle + try { + trackStemDirHandle = await stemsDirHandle.getDirectoryHandle( + `${FILENAME} - stems` + ) + } catch (e) { + // directory doesn't exist + return 'getStems' + } + + // are there at least 4 files in the dir? + const localStems: TrackCache['stems'] = {} + try { + for await (const [name, fileHandle] of trackStemDirHandle.entries()) { + const file = (await fileHandle.getFile(name)) as File + const stemName = name.substring(0, name.lastIndexOf('.')) + if (stemName && STEMS.includes(stemName as Stem)) { + localStems[stemName as Stem] = { file } + } + } + } catch (e) { + throw errorHandler(e as Error) + } + + if (Object.keys(localStems).length < 4) return 'getStems' + + // cache the stems + await storeTrackCache({ id: trackId, stems: localStems }) + + // ready! + return 'ready' + } + + const state = await checkAccess() + if (state === 'ready') { + // remove analyzing + setAppState.stemsAnalyzing(prev => { + prev.delete(trackId) + return prev + }) + } + + if (stemState !== state) setAudioState[trackId].stemState(state) + + return state +} + +export { browseFile, getPermission, getStemsDirHandle, validateTrackStemAccess } diff --git a/app/api/renderWaveform.client.tsx b/app/api/renderWaveform.client.tsx new file mode 100644 index 0000000..aa2f042 --- /dev/null +++ b/app/api/renderWaveform.client.tsx @@ -0,0 +1,227 @@ +import { useEffect } from 'react' +import WaveSurfer, { type WaveSurferOptions } from 'wavesurfer.js' +import Minimap from 'wavesurfer.js/dist/plugins/minimap.js' +import RegionsPlugin from 'wavesurfer.js/dist/plugins/regions.js' +import { audioEvents } from '~/api/audioEvents.client' +import { + appState, + getAppState, + getAudioState, + setAppState, + setAudioState +} from '~/api/db/appState.client' +import { Stem, Track, db } from '~/api/db/dbHandlers' +import { getPermission } from '~/api/fileHandlers' +import { validateTrackStemAccess } from '~/api/fileHandlers' +import { ProgressBar } from '~/components/Loader' +import { errorHandler } from '~/utils/notifications' + +const PRIMARY_WAVEFORM_CONFIG = (trackId: Track['id']): WaveSurferOptions => ({ + container: `#zoomview-container_${trackId}`, + height: 60, + autoScroll: true, + autoCenter: true, + hideScrollbar: false, + barWidth: 2, + barHeight: 0.9, + barGap: 1, + plugins: [ + // Do not change the order of plugins! They are referenced by index :( + RegionsPlugin.create(), + Minimap.create({ + container: `#overview-container_${trackId}`, + height: 22, + waveColor: [ + 'rgba(117, 116, 116, 0.5)', + 'rgba(145, 145, 145, 0.8)', + 'rgba(145, 145, 145, 0.8)', + 'rgba(145, 145, 145, 0.8)' + ], + progressColor: 'rgba(0, 0, 0, 0.25)', + hideScrollbar: true + }) + + // Playhead.create({ + // moveOnSeek: true, + // returnOnPause: false, + // draw: true, + // }), + // CursorPlugin.create({ + // showTime: true, + // opacity: "1", + // customShowTimeStyle: { + // color: "#eee", + // padding: "0 4px", + // "font-size": "10px", + // backgroundColor: "rgba(0, 0, 0, 0.3)", + // }, + // }), + ] +}) + +// This function accepts either a full track (with no stem) or an individual stem ('bass', etc) +const initWaveform = async ({ + trackId, + file, + stem, + waveformConfig = PRIMARY_WAVEFORM_CONFIG(trackId) +}: { + trackId: Track['id'] + file: File + stem?: Stem + waveformConfig?: WaveSurferOptions +}): Promise => { + if (!trackId) throw errorHandler('No track ID provided to initWaveform') + + // an Audio object is required for Wavesurfer to use Web Audio + const media = new Audio(URL.createObjectURL(file)) + + const config: WaveSurferOptions = { + media, + cursorColor: '#555', + interact: false, + waveColor: [ + 'rgb(200, 165, 49)', + 'rgb(211, 194, 138)', + 'rgb(189, 60, 0)', + 'rgb(189, 60, 0)', + 'rgb(189, 60, 0)', + 'rgb(189, 60, 0)' + ], + progressColor: 'rgba(0, 0, 0, 0.45)', + ...waveformConfig + } + + const waveform = WaveSurfer.create(config) + + // Save waveform in audioState to track user interactions with the waveform and show progress + if (stem) { + setAudioState[trackId].stems[stem as Stem]({ + waveform, + volume: 1, + volumeMeter: 0, + mute: false + }) + } else { + setAudioState[trackId].waveform(waveform) + } + + waveform.once('ready', () => audioEvents.onReady(trackId, stem)) +} + +const initAudioContext = ({ + trackId, + stem, + media +}: { trackId: Track['id']; stem?: Stem; media: HTMLAudioElement }) => { + // audioContext cannot be initialized without user intervention, so this function is called when the audio is played for the first time per track or stem + + // Ensure this function is not called twice for the same track or stem + const [gainExists] = stem + ? getAudioState[trackId].stems[stem].gainNode() + : getAudioState[trackId].gainNode() + + if (gainExists) return + + let [audioContext] = getAppState.audioContext() + if (!audioContext) { + audioContext = new AudioContext() + setAppState.audioContext(audioContext) + } + + // gainNode is used to control volume of all stems at once + const gainNode = audioContext.createGain() + gainNode.connect(audioContext.destination) + + const analyserNode = audioContext.createAnalyser() + + // Connect the audio to the equalizer + media.addEventListener( + 'play', + async () => { + // Create a MediaElementSourceNode from the audio element + const mediaNode = audioContext?.createMediaElementSource(media) + + mediaNode?.connect(gainNode) + mediaNode?.connect(analyserNode) + }, + { once: true } + ) + + // Save waveform in audioState to track user interactions with the waveform and show progress + if (stem) { + setAudioState[trackId].stems[stem as Stem].gainNode(gainNode) + setAudioState[trackId].stems[stem as Stem].analyserNode(analyserNode) + } else { + setAudioState[trackId].analyserNode(analyserNode) + setAudioState[trackId].gainNode(gainNode) + } +} + +const TrackView = ({ trackId }: { trackId: Track['id'] }) => { + const [analyzingTracks] = appState.analyzing() + const analyzing = analyzingTracks.has(trackId) + + const containerClass = + 'p-0 border-1 border-divider rounded bg-default-50 overflow-hidden' + + return ( +
{ + const parent = e.currentTarget.firstElementChild as HTMLElement + audioEvents.clickToSeek(trackId, e, parent) + }} + onWheel={e => + audioEvents.seek(trackId, 0, e.deltaY > 0 ? 'next' : 'previous') + } + > + {!analyzing ? null : ( +
+
+ +
+
+ )} +
+ ) +} + +const Waveform = ({ + trackId +}: { + trackId: Track['id'] +}): JSX.Element | null => { + if (!trackId) throw errorHandler('No track to initialize.') + + useEffect(() => { + // Retrieve track, file and region data, then store waveform in audioState + const init = async () => { + const track = await db.tracks.get(trackId) + if (!track) throw errorHandler('Could not retrieve track from database.') + + const file = await getPermission(track) + if (!file) throw errorHandler(`Please try adding ${track.name} again.`) + + initWaveform({ trackId, file }) + } + + // prevent duplication on re-render while loading + const [analyzingTracks] = getAppState.analyzing() + const analyzing = analyzingTracks.has(trackId) + + if (!analyzing) init() + + // add track to analyzing state + setAppState.analyzing(prev => prev.add(trackId)) + + validateTrackStemAccess(trackId) + + return () => audioEvents.destroy(trackId) + }, [trackId]) + + return +} + +export { PRIMARY_WAVEFORM_CONFIG, Waveform, initAudioContext, initWaveform } diff --git a/app/api/stemHandler.ts b/app/api/stemHandler.ts new file mode 100644 index 0000000..d119cae --- /dev/null +++ b/app/api/stemHandler.ts @@ -0,0 +1,143 @@ +import { H } from '@highlight-run/remix/client' +import { setAudioState } from '~/api/db/appState.client' +import { STEMS, Stem, Track, db, storeTrackCache } from '~/api/db/dbHandlers' +import { getStemsDirHandle } from '~/api/fileHandlers' +import { errorHandler } from '~/utils/notifications' + +const STEMPROXY = 'https://stems.mixpoint.dev' +//const STEMPROXY = 'http://localhost:8787' + +type StemsArray = { + name: string + type: Stem + file: Blob +}[] + +const stemAudio = async (trackId: Track['id']) => { + // retrieve file from cache + const { file } = (await db.trackCache.get(trackId)) || {} + if (!file) throw errorHandler('No file found for track, try re-adding it.') + + const ENDPOINT_URL = `${STEMPROXY}/${encodeURIComponent(file.name)}` + const FILENAME = file.name.substring(0, file.name.lastIndexOf('.')) + const ENDPOINT_URL_NOEXT = `${STEMPROXY}/${FILENAME}` + + // ensure we have access to a directory to save the stems + const dirHandle = await getStemsDirHandle() + if (!dirHandle) { + // this would be due to denial of permission (ie. clicked cancel) + throw errorHandler('Permission to the file or folder was denied.') + } + + H.track('Track Stemmed') + + setAudioState[trackId].stemState('uploadingFile') + setAudioState[trackId].stemTimer(100) + + const handleErr = (msg?: string) => { + setAudioState[trackId].stemState('error') + // delete the file so retry will work + fetch(ENDPOINT_URL, { method: 'DELETE' }) + throw errorHandler(`Error generating stems: ${msg}`) + } + + const sendFile = async () => { + const formData = new FormData() + formData.append('file', file) + + const res = await fetch(ENDPOINT_URL, { + method: 'PUT', + body: formData + }) + + if (!res.ok) return handleErr(await res?.text()) + + // set timer for processing stems + const { size } = (await db.tracks.get(trackId)) || {} + // 0.0125 seconds per MB + setAudioState[trackId].stemTimer(((size || 1) / 100) * 0.013) + + return // started + } + + const checkForStems = async (): Promise => { + return new Promise((resolve, reject) => { + const waitForStems = async () => { + const res = await fetch(ENDPOINT_URL, { + method: 'HEAD' + }) + + if (res.status !== 404) { + setTimeout(waitForStems, 10000) // Retry after 10 seconds + return + } + + try { + const stems = await Promise.all( + STEMS.map(async stem => { + const res = await fetch(`${ENDPOINT_URL_NOEXT}/${stem}.mp3`) + + if (res.ok) + return { + name: `${FILENAME} - ${stem}.mp3`, + type: stem, + file: await res.blob() + } + throw new Error(await res?.text()) + }) + ) + + resolve(stems) + } catch (error) { + reject(error) + } + } + + waitForStems() + }) + } + + // send file to stemproxy and wait for stems + await sendFile() + + setAudioState[trackId].stemState('processingStems') + + // wait for stems to be generated + const stems: StemsArray = await checkForStems() + + setAudioState[trackId].stemState('downloadingStems') + + // create a new dir with name of audio file + let stemsDirHandle + try { + stemsDirHandle = await dirHandle.getDirectoryHandle(`${FILENAME} - stems`, { + create: true + }) + } catch (e) { + throw errorHandler('Error creating directory for stems.') + } + + for (const { name, type, file } of stems) { + const stemFile = await stemsDirHandle.getFileHandle(name, { + create: true + }) + + const writable = await stemFile.createWritable() + try { + await writable.write(file) + await writable.close() + } catch (error) { + throw errorHandler(`Error storing stem file: ${error}`) + } + + // store stem in cache + await storeTrackCache({ + id: trackId, + stems: { [type]: { file } } + }) + } + // give a couple of seconds before trying to render the stem waveform + setAudioState[trackId].stemState('ready') +} + +export { stemAudio }