From 1e95d82208ebebd5afda37fb8995027b081e33cf Mon Sep 17 00:00:00 2001 From: Casey Primozic Date: Thu, 2 Mar 2023 04:33:08 -0800 Subject: [PATCH] Get basic interactive wavetable editor working * Changing sliders for harmonic magnitudes and phases works, generates visible waveform, and plays via wavetable live * Move waveform and waveform image rendering to web worker --- .gitignore | 2 +- Justfile | 20 +- engine/Cargo.lock | 4 +- engine/common/Cargo.toml | 4 + engine/release.sh | 4 +- engine/waveform_renderer/Cargo.toml | 6 +- engine/waveform_renderer/src/lib.rs | 16 +- engine/wavegen/Cargo.toml | 6 +- engine/wavegen/src/bindings.rs | 49 +++- engine/wavegen/src/lib.rs | 4 +- engine/wavetable/src/fm/mod.rs | 2 +- engine/wavetable/src/lib.rs | 21 +- package.json | 1 + public/WaveTableNodeProcessor.js | 52 ++++- src/fmSynth/Wavetable/BuildWavetable.svelte | 67 +++++- .../Wavetable/BuildWavetableInstance.ts | 221 +++++++++++++----- .../WavetableConfiguratorWorker.worker.ts | 80 +++++++ .../nodes/CustomAudio/Sidechain.tsx | 4 +- .../nodes/CustomAudio/WaveTable/WaveTable.ts | 51 +++- webpack.base.js | 2 +- webpack.headless.js | 4 + yarn.lock | 5 + 22 files changed, 504 insertions(+), 121 deletions(-) create mode 100644 src/fmSynth/Wavetable/WavetableConfiguratorWorker.worker.ts diff --git a/.gitignore b/.gitignore index a5105c3c..0a11d5fc 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,4 @@ public/waveform_renderer.wasm public/compressor.wasm public/vocoder.wasm public/level_detector.wasm -src/wavegen* +public/wavegen.wasm diff --git a/Justfile b/Justfile index 86c904d7..46f06032 100644 --- a/Justfile +++ b/Justfile @@ -60,8 +60,7 @@ build-all: && wasm-bindgen ./target/wasm32-unknown-unknown/release/polysynth.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen ./target/wasm32-unknown-unknown/release/waveform_renderer.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen ./target/wasm32-unknown-unknown/release/note_container.wasm --browser --remove-producers-section --out-dir ./build \ - && wasm-bindgen ./target/wasm32-unknown-unknown/release/wav_decoder.wasm --browser --remove-producers-section --out-dir ./build \ - && wasm-bindgen ./target/wasm32-unknown-unknown/release/wavegen.wasm --browser --remove-producers-section --out-dir ./build + && wasm-bindgen ./target/wasm32-unknown-unknown/release/wav_decoder.wasm --browser --remove-producers-section --out-dir ./build cd - cp ./engine/target/wasm32-unknown-unknown/release/*.wasm ./public @@ -80,6 +79,8 @@ build-all: cp ./engine/target/wasm32-unknown-unknown/release/midi_quantizer.wasm ./public cp ./engine/target/wasm32-unknown-unknown/release/quantizer.wasm ./public cp ./engine/target/wasm32-unknown-unknown/release/compressor.wasm ./public + cp ./engine/target/wasm32-unknown-unknown/release/level_detector.wasm ./public + cp ./engine/target/wasm32-unknown-unknown/release/wavegen.wasm ./public cp ./engine/build/* ./src just build-sinsy @@ -109,15 +110,13 @@ run: && cp ./target/wasm32-unknown-unknown/release/waveform_renderer.wasm /tmp/wasm \ && cp ./target/wasm32-unknown-unknown/release/note_container.wasm /tmp/wasm \ && cp ./target/wasm32-unknown-unknown/release/wav_decoder.wasm /tmp/wasm \ - && cp ./target/wasm32-unknown-unknown/release/wavegen.wasm /tmp/wasm \ && wasm-bindgen /tmp/wasm/engine.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen /tmp/wasm/midi.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen /tmp/wasm/spectrum_viz.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen /tmp/wasm/polysynth.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen /tmp/wasm/waveform_renderer.wasm --browser --remove-producers-section --out-dir ./build \ && wasm-bindgen /tmp/wasm/note_container.wasm --browser --remove-producers-section --out-dir ./build \ - && wasm-bindgen /tmp/wasm/wav_decoder.wasm --browser --remove-producers-section --out-dir ./build \ - && wasm-bindgen /tmp/wasm/wavegen.wasm --browser --remove-producers-section --out-dir ./build + && wasm-bindgen /tmp/wasm/wav_decoder.wasm --browser --remove-producers-section --out-dir ./build cd - cp ./engine/build/* ./src/ @@ -137,6 +136,8 @@ run: cp ./engine/target/wasm32-unknown-unknown/release/midi_quantizer.wasm ./public cp ./engine/target/wasm32-unknown-unknown/release/quantizer.wasm ./public cp ./engine/target/wasm32-unknown-unknown/release/compressor.wasm ./public + cp ./engine/target/wasm32-unknown-unknown/release/level_detector.wasm ./public + cp ./engine/target/wasm32-unknown-unknown/release/wavegen.wasm ./public just debug-sinsy @@ -245,11 +246,6 @@ build-wav-decoder: cd - && wasm-bindgen ./engine/target/wasm32-unknown-unknown/debug/wav_decoder.wasm --browser --remove-producers-section --out-dir ./engine/build cp ./engine/build/wav_decoder* ./src/ -build-wavegen: - cd ./engine/wavegen && cargo build --release --target wasm32-unknown-unknown && \ - cd - && wasm-bindgen ./engine/target/wasm32-unknown-unknown/release/wavegen.wasm --browser --remove-producers-section --out-dir ./engine/build - cp ./engine/build/wavegen* ./src/ - build-event-scheduler: cd ./engine/event_scheduler && cargo build --release --target wasm32-unknown-unknown && \ cp ../target/wasm32-unknown-unknown/release/event_scheduler.wasm ../../public @@ -289,3 +285,7 @@ debug-vocoder: build-level-detector: cd ./engine/level_detector && cargo build --release --target wasm32-unknown-unknown && \ cp ../target/wasm32-unknown-unknown/release/level_detector.wasm ../../public + +build-wavegen: + cd ./engine/wavegen && cargo build --release --target wasm32-unknown-unknown && \ + cp ../target/wasm32-unknown-unknown/release/wavegen.wasm ../../public diff --git a/engine/Cargo.lock b/engine/Cargo.lock index bc95514c..1bf17839 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -113,6 +113,7 @@ dependencies = [ "rand", "rand_pcg 0.2.1", "uuid", + "wasm-bindgen", ] [[package]] @@ -1214,11 +1215,10 @@ dependencies = [ name = "wavegen" version = "0.1.0" dependencies = [ - "console_error_panic_hook", + "common", "fastapprox", "rustfft", "textplots", - "wasm-bindgen", "waveform_renderer", ] diff --git a/engine/common/Cargo.toml b/engine/common/Cargo.toml index d9fff902..3cb8fc3e 100644 --- a/engine/common/Cargo.toml +++ b/engine/common/Cargo.toml @@ -8,3 +8,7 @@ edition = "2021" uuid = { version = "1.2" } rand = "0.7.3" rand_pcg = "0.2.1" +wasm-bindgen = { version = "0.2.82", optional = true } + +[features] +bindgen = ["wasm-bindgen"] diff --git a/engine/release.sh b/engine/release.sh index 523325d7..4ffc164a 100755 --- a/engine/release.sh +++ b/engine/release.sh @@ -1,3 +1,5 @@ -cargo build --release --target wasm32-unknown-unknown --workspace --exclude common --exclude dsp --exclude wbg_logging && +cargo build --release --target wasm32-unknown-unknown --workspace --exclude common --exclude dsp --exclude wbg_logging --exclude wavegen && mv ./target/wasm32-unknown-unknown/release/wavetable.wasm ./target/wasm32-unknown-unknown/release/wavetable_no_simd.wasm && + cd wavegen && cargo build --target wasm32-unknown-unknown --release && + cd .. && cd wavetable && RUSTFLAGS="-Ctarget-feature=+simd128" cargo build --target wasm32-unknown-unknown --release --features=simd diff --git a/engine/waveform_renderer/Cargo.toml b/engine/waveform_renderer/Cargo.toml index 7d1876ba..0d71e4fd 100644 --- a/engine/waveform_renderer/Cargo.toml +++ b/engine/waveform_renderer/Cargo.toml @@ -9,10 +9,10 @@ crate-type = ["cdylib", "rlib"] [dependencies] wasm-bindgen = { version = "=0.2.82", optional = true } -common = { path = "../common" } -wbg_logging = { path = "../wbg_logging" } +common = { path = "../common", default_features = false, features = [] } +wbg_logging = { path = "../wbg_logging", optional = true } log = { version = "0.4", features = ["release_max_level_off"] } [features] -bindgen = ["wasm-bindgen"] +bindgen = ["wasm-bindgen", "common/bindgen", "wbg_logging"] default = ["bindgen"] diff --git a/engine/waveform_renderer/src/lib.rs b/engine/waveform_renderer/src/lib.rs index bf44e7e6..59b3f32e 100644 --- a/engine/waveform_renderer/src/lib.rs +++ b/engine/waveform_renderer/src/lib.rs @@ -36,7 +36,8 @@ impl WaveformRendererCtx { } } -#[cfg_attr(feature = "bindgen", wasm_bindgen)] +#[cfg(feature = "bindgen")] +#[wasm_bindgen] pub fn create_waveform_renderer_ctx( waveform_length_samples: u32, sample_rate: u32, @@ -44,6 +45,7 @@ pub fn create_waveform_renderer_ctx( height_px: u32, ) -> *mut WaveformRendererCtx { common::maybe_init(None); + wbg_logging::maybe_init(); let mut ctx = @@ -56,19 +58,22 @@ pub fn create_waveform_renderer_ctx( Box::into_raw(ctx) } -#[cfg_attr(feature = "bindgen", wasm_bindgen)] +#[cfg(feature = "bindgen")] +#[wasm_bindgen] pub fn append_samples_to_waveform(ctx: *mut WaveformRendererCtx, new_samples: &[f32]) -> usize { let ctx = unsafe { &mut *ctx }; ctx.waveform_buf.extend_from_slice(new_samples); ctx.waveform_buf.len() } -#[cfg_attr(feature = "bindgen", wasm_bindgen)] +#[cfg(feature = "bindgen")] +#[wasm_bindgen] pub fn free_waveform_renderer_ctx(ctx: *mut WaveformRendererCtx) { drop(unsafe { Box::from_raw(ctx) }) } -#[cfg_attr(feature = "bindgen", wasm_bindgen)] +#[cfg(feature = "bindgen")] +#[wasm_bindgen] pub fn get_waveform_buf_ptr(ctx: *mut WaveformRendererCtx) -> *mut f32 { unsafe { (*ctx).waveform_buf.as_mut_ptr() } } @@ -151,7 +156,8 @@ pub fn render_waveform(ctx: *mut WaveformRendererCtx, start_ms: u32, end_ms: u32 ctx.image_data_buf.as_mut_ptr() } -#[cfg_attr(feature = "bindgen", wasm_bindgen)] +#[cfg(feature = "bindgen")] +#[wasm_bindgen] pub fn get_sample_count(ctx: *const WaveformRendererCtx) -> usize { unsafe { (*ctx).waveform_buf.len() } } diff --git a/engine/wavegen/Cargo.toml b/engine/wavegen/Cargo.toml index 006365f0..fb1a5568 100644 --- a/engine/wavegen/Cargo.toml +++ b/engine/wavegen/Cargo.toml @@ -9,14 +9,10 @@ crate-type = ["cdylib", "rlib"] [dependencies] rustfft = "6.1" -wasm-bindgen = { version = "=0.2.82", optional = true } waveform_renderer = { path = "../waveform_renderer", default-features = false, features = [] } -console_error_panic_hook = "0.1.6" fastapprox = "0.3" +common = { path = "../common", default-features = false, features = [] } [dev-dependencies] textplots = "0.8" -[features] -bindgen = ["wasm-bindgen"] -default = ["bindgen"] diff --git a/engine/wavegen/src/bindings.rs b/engine/wavegen/src/bindings.rs index 45c8b5b6..723f9166 100644 --- a/engine/wavegen/src/bindings.rs +++ b/engine/wavegen/src/bindings.rs @@ -1,29 +1,45 @@ use std::f32::consts::PI; -use wasm_bindgen::prelude::*; use waveform_renderer::WaveformRendererCtx; +extern "C" { + fn log_err(s: *const u8, len: usize); +} + const WAVEFORM_LENGTH_SAMPLES: usize = 1024 * 4; const HARMONIC_COUNT: usize = 64; const WAVEFORM_HEIGHT_PX: u32 = 256; const WAVEFORM_WIDTH_PX: u32 = 1024; const SAMPLE_RATE: u32 = 44_100; -fn build_waveform(buf: &mut Vec, magnitudes: &[f32], phases: &[f32]) { +fn build_waveform(buf: &mut Vec, magnitudes: &[f32], phases: &[f32], fast: bool) { buf.fill(0.); buf.resize(WAVEFORM_LENGTH_SAMPLES, 0.); for (harmonic_ix, (magnitude, phase)) in magnitudes.iter().zip(phases).enumerate() { + if *magnitude == 0. { + continue; + } + for (sample_ix, sample) in buf.iter_mut().enumerate() { let phase = (sample_ix as f32 / WAVEFORM_LENGTH_SAMPLES as f32) * PI * 2. * harmonic_ix as f32 + -*phase * PI * 2.; - // *sample += magnitude * phase.sin(); - *sample += magnitude * fastapprox::fast::sinfull(phase); + if fast { + *sample += magnitude * fastapprox::fast::sinfull(phase); + } else { + *sample += magnitude * phase.sin(); + } } } } static mut WAVEFORM_RENDERER_CTX: *mut WaveformRendererCtx = std::ptr::null_mut(); +static mut ENCODED_STATE_BUF: [f32; HARMONIC_COUNT * 2] = [0.; HARMONIC_COUNT * 2]; + +#[no_mangle] +pub extern "C" fn get_encoded_state_buf_ptr() -> *mut f32 { + unsafe { ENCODED_STATE_BUF.as_mut_ptr() } +} fn get_waveform_renderer_ctx() -> &'static mut WaveformRendererCtx { if !unsafe { WAVEFORM_RENDERER_CTX.is_null() } { @@ -43,15 +59,30 @@ fn get_waveform_renderer_ctx() -> &'static mut WaveformRendererCtx { /// State format: /// [WAVEFORM_SIZE * HARMONIC_COUNT] f32s for the magnitudes /// [WAVEFORM_SIZE * HARMONIC_COUNT] f32s for the phases -#[wasm_bindgen] -pub fn render_waveform(state: &[f32]) -> *const u8 { - console_error_panic_hook::set_once(); +#[no_mangle] +pub extern "C" fn wavegen_render_waveform() -> *const u8 { + common::set_raw_panic_hook(log_err); - assert_eq!(state.len(), HARMONIC_COUNT * 2, "Invalid state length"); + let state = unsafe { &mut ENCODED_STATE_BUF }; + let magnitudes = &mut state[..HARMONIC_COUNT]; + // normalize magnitudes + let max_magnitude = magnitudes.iter().fold(0.0f32, |acc, x| acc.max(*x)); + if max_magnitude > 0. { + for magnitude in magnitudes.iter_mut() { + *magnitude /= max_magnitude; + } + } + drop(magnitudes); let magnitudes = &state[..HARMONIC_COUNT]; let phases = &state[HARMONIC_COUNT..]; let ctx = get_waveform_renderer_ctx(); - build_waveform(&mut ctx.waveform_buf, magnitudes, phases); + build_waveform(&mut ctx.waveform_buf, magnitudes, phases, true); waveform_renderer::render_waveform(ctx, 0, 100_000_000) } + +#[no_mangle] +pub extern "C" fn wavegen_get_waveform_buf_ptr() -> *const f32 { + let ctx = get_waveform_renderer_ctx(); + ctx.waveform_buf.as_ptr() +} diff --git a/engine/wavegen/src/lib.rs b/engine/wavegen/src/lib.rs index 76f90a53..1c4752b0 100644 --- a/engine/wavegen/src/lib.rs +++ b/engine/wavegen/src/lib.rs @@ -2,8 +2,6 @@ //! Utilities for generating waveforms and wavetables using inverse FFT. +mod bindings; #[cfg(test)] mod tests; - -#[cfg(feature = "bindgen")] -mod bindings; diff --git a/engine/wavetable/src/fm/mod.rs b/engine/wavetable/src/fm/mod.rs index cfdca8f5..78a07511 100644 --- a/engine/wavetable/src/fm/mod.rs +++ b/engine/wavetable/src/fm/mod.rs @@ -24,7 +24,7 @@ use self::{ }; extern "C" { - fn log_err(ptr: *const u8, len: usize); + pub(crate) fn log_err(ptr: *const u8, len: usize); fn on_gate_cb(midi_number: usize, voice_ix: usize); diff --git a/engine/wavetable/src/lib.rs b/engine/wavetable/src/lib.rs index 87a3d6ff..c8d2f3f1 100644 --- a/engine/wavetable/src/lib.rs +++ b/engine/wavetable/src/lib.rs @@ -1,9 +1,4 @@ -#![feature( - box_syntax, - stdsimd, - const_maybe_uninit_assume_init, - get_mut_unchecked -)] +#![feature(box_syntax, stdsimd, const_maybe_uninit_assume_init, get_mut_unchecked)] pub mod fm; pub mod lookup_tables; @@ -175,6 +170,8 @@ pub fn init_wavetable( waveform_length: usize, base_frequency: f32, ) -> *mut WaveTable { + common::set_raw_panic_hook(crate::fm::log_err); + let settings = WaveTableSettings { waveforms_per_dimension, dimension_count, @@ -190,6 +187,11 @@ pub fn get_data_table_ptr(handle_ptr: *mut WaveTable) -> *mut f32 { unsafe { (*handle_ptr).samples.as_mut_ptr() } } +#[no_mangle] +pub extern "C" fn set_base_frequency(handle_ptr: *mut WaveTable, base_frequency: f32) { + unsafe { (*handle_ptr).settings.base_frequency = base_frequency } +} + #[no_mangle] pub fn drop_wavetable(table: *mut WaveTable) { drop(unsafe { Box::from_raw(table) }) } @@ -237,6 +239,12 @@ pub fn get_samples(handle_ptr: *mut WaveTableHandle, sample_count: usize) -> *co } for sample_ix in 0..sample_count { + let frequency = handle.frequencies_buffer[sample_ix]; + if frequency == 0.0 { + handle.sample_buffer[sample_ix] = 0.0; + continue; + } + for dimension_ix in 0..handle.table.settings.dimension_count { handle.mixes_for_sample[dimension_ix * 2] = handle.mixes[(dimension_ix * 2 * sample_count) + sample_ix]; @@ -244,7 +252,6 @@ pub fn get_samples(handle_ptr: *mut WaveTableHandle, sample_count: usize) -> *co handle.mixes[(dimension_ix * 2 * sample_count) + sample_count + sample_ix]; } - let frequency = handle.frequencies_buffer[sample_ix]; handle.sample_buffer[sample_ix] = handle.get_sample(frequency); } diff --git a/package.json b/package.json index f7fb4c3d..f17bd148 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "ace-builds": "^1.15", "ameo-utils": "^0.8.0", "chartist": "^0.11.4", + "comlink": "^4.4.1", "d3": "^7.8.2", "dexie": "^3.2.3", "downloadjs": "^1.4.7", diff --git a/public/WaveTableNodeProcessor.js b/public/WaveTableNodeProcessor.js index 35f4ba36..74928c21 100644 --- a/public/WaveTableNodeProcessor.js +++ b/public/WaveTableNodeProcessor.js @@ -37,7 +37,7 @@ class WaveTableNodeProcessor extends AudioWorkletProcessor { } handleWasmPanic = (ptr, len) => { - const mem = new Uint8Array(this.getWasmMemoryBuffer().buffer); + const mem = new Uint8Array(this.wasmInstance.exports.memory.buffer); const slice = mem.subarray(ptr, ptr + len); const str = String.fromCharCode(...slice); throw new Error(str); @@ -66,6 +66,7 @@ class WaveTableNodeProcessor extends AudioWorkletProcessor { data.waveformLength, data.baseFrequency ); + console.log(data); // Wasm memory doesn't become available until after some function in the Wasm module has been called, apparently, // so we wait to set this reference until after calling one of the Wasm functions. @@ -107,17 +108,62 @@ class WaveTableNodeProcessor extends AudioWorkletProcessor { this.frequencyBufArrayOffset = frequencyBufPtr / BYTES_PER_F32; } + updateWavetable(data) { + if (!this.lastWavetableParams) { + throw new Error('Tried to update wavetable before initializing it'); + } + if ( + data.waveformsPerDimension !== this.lastWavetableParams.waveformsPerDimension || + data.dimensionCount !== this.lastWavetableParams.dimensionCount || + data.waveformLength !== this.lastWavetableParams.waveformLength + ) { + console.error( + 'Tried to update wavetable with different params than it was initialized with:', + { old: this.lastWavetableParams, new: data } + ); + throw new Error( + 'Tried to update wavetable with different params than it was initialized with' + ); + } + + this.float32WasmMemory = new Float32Array(this.wasmInstance.exports.memory.buffer); + + const wavetableDataPtr = this.wasmInstance.exports.get_data_table_ptr(this.waveTablePtr); + const wavetableDataArrayOffset = wavetableDataPtr / BYTES_PER_F32; + + // Write the table's data into the Wasm heap + // TODO: Interpolate between the old and new tables + if (data.tableSamples.some(isNaN)) { + console.error('NaN in table samples', data.tableSamples); + throw new Error('NaN in table samples'); + } + this.float32WasmMemory.set(data.tableSamples, wavetableDataArrayOffset); + + this.wasmInstance.exports.set_base_frequency(this.waveTablePtr, data.baseFrequency); + } + constructor() { super(); this.isShutdown = false; + this.lastWavetableParams = null; + this.port.onmessage = event => { if (event.data === 'shutdown') { this.isShutdown = true; return; + } else if (event.data.type === 'init') { + this.lastWavetableParams = { + waveformsPerDimension: event.data.waveformsPerDimension, + dimensionCount: event.data.dimensionCount, + waveformLength: event.data.waveformLength, + }; + this.initWasmInstance(event.data); + } else if (event.data.type === 'update') { + this.updateWavetable(event.data); + } else { + console.error('Unknown message type', event.data); } - - this.initWasmInstance(event.data); }; } diff --git a/src/fmSynth/Wavetable/BuildWavetable.svelte b/src/fmSynth/Wavetable/BuildWavetable.svelte index fdfad941..364de757 100644 --- a/src/fmSynth/Wavetable/BuildWavetable.svelte +++ b/src/fmSynth/Wavetable/BuildWavetable.svelte @@ -1,17 +1,57 @@ + + @@ -24,8 +64,27 @@
{ + sliderMode = + sliderMode === BuildWavetableSliderMode.Magnitude + ? BuildWavetableSliderMode.Phase + : BuildWavetableSliderMode.Magnitude; + inst?.setSliderMode(sliderMode); + }, + }, + { type: 'button', label: 'cancel', action: onCancel }, + ]} + onChange={handleChange} />
diff --git a/src/fmSynth/Wavetable/BuildWavetableInstance.ts b/src/fmSynth/Wavetable/BuildWavetableInstance.ts index 272f28ff..690880fd 100644 --- a/src/fmSynth/Wavetable/BuildWavetableInstance.ts +++ b/src/fmSynth/Wavetable/BuildWavetableInstance.ts @@ -1,18 +1,23 @@ +import * as Comlink from 'comlink'; import * as R from 'ramda'; import * as PIXI from 'src/controls/pixi'; import { makeDraggable } from 'src/controls/pixiUtils'; +import { WavetableConfiguratorWorker } from 'src/fmSynth/Wavetable/WavetableConfiguratorWorker.worker'; +import WaveTable, { + type WavetableDef, +} from 'src/graphEditor/nodes/CustomAudio/WaveTable/WaveTable'; import { getSentry } from 'src/sentry'; - -type BuildWavetableWasmEngine = typeof import('src/wavegen'); +import { AsyncOnce, SAMPLE_RATE } from 'src/util'; const dpr = window.devicePixelRatio ?? 1; const BACKGROUND_COLOR = 0x020202; -export const BUILD_WAVETABLE_INST_HEIGHT_PX = 450; +export const BUILD_WAVETABLE_INST_HEIGHT_PX = 800; export const BUILD_WAVETABLE_INST_WIDTH_PX = Math.round(BUILD_WAVETABLE_INST_HEIGHT_PX * 1.618); const HARMONICS_COUNT = 64; +const WAVEFORM_LENGTH_SAMPLES = 1024 * 4; const SLIDER_BG_COLOR = 0x1f1f1f; const SLIDER_BORDER_COLOR = 0x2f2f2f; @@ -22,28 +27,41 @@ const SLIDER_LABEL_COLOR = 0x727272; const SLIDER_LABEL_FONT_FAMILY = 'Hack'; const SLIDER_LABEL_FONT_SIZE = 9.5; const SLIDER_HANDLE_COLOR = 0x4a4a4a; -const SLIDER_HANDLE_HEIGHT = 10; +const SLIDER_HANDLE_HEIGHT = 8; +const SLIDER_GHOST_TICK_HEIGHT = 4; +const SLIDER_GHOST_TICK_PHASE_COLOR = 0x798dbd; +const SLIDER_GHOST_TICK_MAGNITUDE_COLOR = 0x83c996; const WAVEFORM_IMAGE_HEIGHT_PX = 256; const WAVEFORM_IMAGE_WIDTH_PX = 1024; -export enum SliderMode { +export enum BuildWavetableSliderMode { Magnitude, Phase, } export interface BuildWavetableInstanceState { harmonics: { magnitude: number; phase: number }[]; - sliderMode: SliderMode; + sliderMode: BuildWavetableSliderMode; } const buildDefaultBuildWavetableInstanceState = (): BuildWavetableInstanceState => ({ harmonics: new Array(HARMONICS_COUNT) .fill(null) .map((_, i) => ({ magnitude: i === 1 ? 1 : 0, phase: 0 })), - sliderMode: SliderMode.Magnitude, + sliderMode: BuildWavetableSliderMode.Magnitude, }); +const WavegenWasm = new AsyncOnce( + () => + fetch( + process.env.ASSET_PATH + + 'wavegen.wasm' + + (window.location.host.includes('localhost') ? '' : `?${crypto.randomUUID()}`) + ).then(res => res.arrayBuffer()), + true +); + /** * Expects values to be in the range [0, 1]. */ @@ -52,22 +70,50 @@ class VerticalSlider { private handle: PIXI.Graphics; private onChange: (value: number) => void; private value: number; + private ghostValue: number; + private ghostTickMode: BuildWavetableSliderMode; public dragData: PIXI.InteractionData | null = null; public get graphics() { return this.bg; } - constructor(onChange: (value: number) => void, initialValue: number, ix: number) { - this.onChange = onChange; - this.value = initialValue; - - this.bg = new PIXI.Graphics() - .beginFill(SLIDER_BG_COLOR) + private renderBackgroundGraphics(bg: PIXI.Graphics) { + const ghostTickY = + SLIDER_HEIGHT - + SLIDER_GHOST_TICK_HEIGHT - + this.ghostValue * (SLIDER_HEIGHT - SLIDER_GHOST_TICK_HEIGHT) + + 1; + bg.beginFill(SLIDER_BG_COLOR) .drawRect(0, 0, SLIDER_WIDTH, SLIDER_HEIGHT) .endFill() .lineStyle(1, SLIDER_BORDER_COLOR) - .drawRect(0, 0, SLIDER_WIDTH, SLIDER_HEIGHT); + .drawRect(0, 0, SLIDER_WIDTH, SLIDER_HEIGHT) + // ghost tick + .lineStyle( + 1, + this.ghostTickMode === BuildWavetableSliderMode.Magnitude + ? SLIDER_GHOST_TICK_MAGNITUDE_COLOR + : SLIDER_GHOST_TICK_PHASE_COLOR + ) + .moveTo(0, ghostTickY) + .lineTo(SLIDER_WIDTH, ghostTickY); + } + + constructor( + onChange: (value: number) => void, + initialValue: number, + initialGhostValue: number, + initialGhostTickMode: BuildWavetableSliderMode, + ix: number + ) { + this.onChange = onChange; + this.value = initialValue; + this.ghostValue = initialGhostValue; + this.ghostTickMode = initialGhostTickMode; + + this.bg = new PIXI.Graphics(); + this.renderBackgroundGraphics(this.bg); this.bg.interactive = true; // this.bg.cacheAsBitmap = true; this.bg.on('pointerdown', this.onPointerDown); @@ -110,29 +156,57 @@ class VerticalSlider { this.handleChange(newPos.y - SLIDER_HANDLE_HEIGHT); }; - public setValue(newValue: number) { + public setValue(newValue: number, ghostValue: number, ghostTickMode: BuildWavetableSliderMode) { this.value = newValue; - this.handle.y = this.value * (SLIDER_HEIGHT - SLIDER_HANDLE_HEIGHT); + this.ghostValue = ghostValue; + this.ghostTickMode = ghostTickMode; + this.bg.clear(); + this.renderBackgroundGraphics(this.bg); + this.handle.y = (1 - this.value) * (SLIDER_HEIGHT - SLIDER_HANDLE_HEIGHT); } } +const buildPlaceholderWavetableDef = (): WavetableDef => [ + [new Float32Array(WAVEFORM_LENGTH_SAMPLES)], +]; + export class BuildWavetableInstance { + private ctx = new AudioContext(); private app: PIXI.Application; - private engine: BuildWavetableWasmEngine | null = null; - private getWasmMemory: () => Uint8Array = () => new Uint8Array(0); - private slidersContainer: PIXI.Container; - private state: BuildWavetableInstanceState = buildDefaultBuildWavetableInstanceState(); + private worker: Comlink.Remote; private waveformImage: PIXI.Sprite; private waveformContainer: PIXI.Container; + private slidersContainer: PIXI.Container; + private sliders: VerticalSlider[] = []; + private commitDispatchSeq = 0; + private commitRenderSeq = 0; + private wavetable: WaveTable; + private gainNode: GainNode; + private frequencyCSN: ConstantSourceNode; + + private state: BuildWavetableInstanceState = buildDefaultBuildWavetableInstanceState(); constructor(canvas: HTMLCanvasElement) { - Promise.all([import('src/wavegen'), import('src/wavegen_bg.wasm')] as const).then( - ([wavegenMod, wasm]) => { - this.getWasmMemory = () => new Uint8Array(wasm.memory.buffer); - this.engine = wavegenMod; - this.commit(); - } + this.worker = Comlink.wrap( + new Worker(new URL('./WavetableConfiguratorWorker.worker.ts', import.meta.url)) ); + WavegenWasm.get().then(async wasm => { + await this.worker.setWasmBytes(Comlink.transfer(wasm, [wasm])); + this.commit(); + }); + + this.gainNode = new GainNode(this.ctx); + this.frequencyCSN = new ConstantSourceNode(this.ctx); + this.wavetable = new WaveTable(this.ctx, '', { + wavetableDef: buildPlaceholderWavetableDef(), + onInitialized: (wavetable: WaveTable) => { + wavetable.workletHandle!.connect(this.gainNode); + this.frequencyCSN.connect( + (wavetable.workletHandle!.parameters as Map).get('frequency')! + ); + this.frequencyCSN.start(); + }, + }); try { this.app = new PIXI.Application({ @@ -158,11 +232,14 @@ export class BuildWavetableInstance { const slider = new VerticalSlider( newValue => this.handleSliderChange(harmonicIx, newValue), this.state.harmonics[harmonicIx].magnitude, + this.state.harmonics[harmonicIx].phase, + BuildWavetableSliderMode.Magnitude, harmonicIx ); slider.graphics.x = harmonicIx * SLIDER_WIDTH; slider.graphics.y = 10; this.slidersContainer.addChild(slider.graphics); + this.sliders.push(slider); } this.slidersContainer.y = BUILD_WAVETABLE_INST_HEIGHT_PX - SLIDER_HEIGHT - 10; this.app.stage.addChild(this.slidersContainer); @@ -176,24 +253,54 @@ export class BuildWavetableInstance { this.waveformContainer.addChild(this.waveformImage); } - public setSliderMode = (sliderMode: SliderMode) => { + public setSliderMode = (sliderMode: BuildWavetableSliderMode) => { this.state.sliderMode = sliderMode; for (let harmonicIx = 0; harmonicIx < HARMONICS_COUNT; harmonicIx++) { if (harmonicIx === 0) { continue; } - const slider = this.slidersContainer.getChildAt(harmonicIx - 1) as unknown as VerticalSlider; - if (sliderMode === SliderMode.Magnitude) { - slider.setValue(this.state.harmonics[harmonicIx].magnitude); + const slider = this.sliders[harmonicIx - 1]; + if (sliderMode === BuildWavetableSliderMode.Magnitude) { + slider.setValue( + this.state.harmonics[harmonicIx].magnitude, + this.state.harmonics[harmonicIx].phase, + BuildWavetableSliderMode.Phase + ); } else { - slider.setValue(this.state.harmonics[harmonicIx].phase); + slider.setValue( + this.state.harmonics[harmonicIx].phase, + this.state.harmonics[harmonicIx].magnitude, + BuildWavetableSliderMode.Magnitude + ); + } + } + }; + + public setIsPlaying = (isPlaying: boolean) => { + if (isPlaying) { + this.gainNode.connect(this.ctx.destination); + } else { + try { + this.gainNode.disconnect(); + } catch (err) { + // pass } } }; + public setVolumeDb = (volumeDb: number) => { + const gain = Math.pow(10, volumeDb / 20); + console.log('gain', gain); + this.gainNode.gain.value = gain; + }; + + public setFrequency = (frequency: number) => { + this.frequencyCSN.offset.value = frequency; + }; + private handleSliderChange = (sliderIx: number, value: number) => { - if (this.state.sliderMode === SliderMode.Magnitude) { + if (this.state.sliderMode === BuildWavetableSliderMode.Magnitude) { this.setHarmonicMagnitude(sliderIx, value); } else { this.setHarmonicPhase(sliderIx, value); @@ -220,29 +327,18 @@ export class BuildWavetableInstance { this.commit(); }; - private encodeState = (): Float32Array => { - const encodedState = new Float32Array(HARMONICS_COUNT * 2); - // magnitude, phase - encodedState.set(this.state.harmonics.map(h => h.magnitude)); - encodedState.set( - this.state.harmonics.map(h => h.phase), - HARMONICS_COUNT - ); - console.log('encodedState', encodedState); - return encodedState; - }; - private commit = async () => { - if (!this.engine) { + const dispatchID = this.commitDispatchSeq++; + const res = await this.worker.renderWaveform(this.state.harmonics).catch(err => { + console.error('Failed to render waveform image', err); + getSentry()?.captureException(err); + return null; + }); + if (!res || this.commitRenderSeq > dispatchID) { return; } + const { waveformImage, waveformSamples } = res; - const encodedState = this.encodeState(); - const waveformImagePtr = this.engine.render_waveform(encodedState); - const waveformImage = this.getWasmMemory().slice( - waveformImagePtr, - waveformImagePtr + WAVEFORM_IMAGE_HEIGHT_PX * WAVEFORM_IMAGE_WIDTH_PX * 4 - ); const imageBitmap = await createImageBitmap( new ImageData( new Uint8ClampedArray(waveformImage), @@ -257,13 +353,30 @@ export class BuildWavetableInstance { format: PIXI.FORMATS.RGBA, type: PIXI.TYPES.UNSIGNED_BYTE, }); - if (this.waveformImage) { - this.waveformImage.texture.destroy(true); - this.waveformImage.texture = texture; + + this.waveformImage.texture.destroy(true); + this.waveformImage.texture = texture; + + this.commitRenderSeq = dispatchID; + + // We need to compute the frequency of the base harmonic, which has a wavelength of `WAVEFORM_LENGTH_SAMPLES`. + const baseFrequency = SAMPLE_RATE / WAVEFORM_LENGTH_SAMPLES; + // normalize the samples to [-1, 1] + const maxSample = Math.max(...waveformSamples); + const minSample = Math.min(...waveformSamples); + const maxAbsSample = Math.max(Math.abs(maxSample), Math.abs(minSample)); + if (maxAbsSample > 0) { + for (let i = 0; i < waveformSamples.length; i++) { + waveformSamples[i] /= maxAbsSample; + } } + this.wavetable.setWavetableDef([[waveformSamples]], baseFrequency); }; public destroy() { this.app.destroy(false, { children: true, texture: true, baseTexture: true }); + this.wavetable.shutdown(); + this.gainNode.disconnect(); + this.frequencyCSN.disconnect(); } } diff --git a/src/fmSynth/Wavetable/WavetableConfiguratorWorker.worker.ts b/src/fmSynth/Wavetable/WavetableConfiguratorWorker.worker.ts new file mode 100644 index 00000000..adc5f680 --- /dev/null +++ b/src/fmSynth/Wavetable/WavetableConfiguratorWorker.worker.ts @@ -0,0 +1,80 @@ +import * as Comlink from 'comlink'; + +const WAVEFORM_IMAGE_HEIGHT_PX = 256; +const WAVEFORM_IMAGE_WIDTH_PX = 1024; +const WAVEFORM_LENGTH_SAMPLES = 1024 * 4; + +export class WavetableConfiguratorWorker { + private wasmInstance: Promise; + private setWasmInstance!: (instance: WebAssembly.Instance) => void; + + constructor() { + this.wasmInstance = new Promise(resolve => { + this.setWasmInstance = resolve; + }); + } + + public setWasmBytes = async (wasmBytes: ArrayBuffer) => { + const wasmModule = await WebAssembly.compile(wasmBytes); + const wasmInstance = await WebAssembly.instantiate(wasmModule, { + env: { + log_err: (ptr: number, len: number) => { + console.error( + 'WASM error', + new TextDecoder().decode(new Uint8Array(wasmBytes.slice(ptr, ptr + len))) + ); + }, + }, + }); + this.setWasmInstance(wasmInstance); + }; + + private encodeState = (harmonics: { magnitude: number; phase: number }[]): Float32Array => { + const encodedState = new Float32Array(harmonics.length * 2); + // magnitude, phase + encodedState.set(harmonics.map(h => h.magnitude)); + encodedState.set( + harmonics.map(h => h.phase), + harmonics.length + ); + return encodedState; + }; + + public renderWaveform = async (harmonics: { magnitude: number; phase: number }[]) => { + const inst = await this.wasmInstance; + const memory = inst.exports.memory as WebAssembly.Memory; + + const encodedState = this.encodeState(harmonics); + const encodedStateBufPtr: number = (inst.exports.get_encoded_state_buf_ptr as any)(); + const encodedStateBuf = new Float32Array(memory.buffer).subarray( + encodedStateBufPtr / 4, + encodedStateBufPtr / 4 + encodedState.length + ); + encodedStateBuf.set(encodedState); + + const waveformImagePtr = (inst.exports.wavegen_render_waveform as any)(); + const waveformImage = new Uint8Array( + memory.buffer.slice( + waveformImagePtr, + waveformImagePtr + WAVEFORM_IMAGE_HEIGHT_PX * WAVEFORM_IMAGE_WIDTH_PX * 4 + ) + ); + + const waveformBufPtr: number = (inst.exports.wavegen_get_waveform_buf_ptr as any)(); + const waveformSamples = new Float32Array(memory.buffer).slice( + waveformBufPtr / 4, + waveformBufPtr / 4 + WAVEFORM_LENGTH_SAMPLES + ); + if (waveformSamples.some(isNaN)) { + console.error('NaN in waveform samples', waveformSamples); + throw new Error('NaN in waveform samples'); + } + + return Comlink.transfer({ waveformImage, waveformSamples }, [ + waveformImage.buffer, + waveformSamples.buffer, + ]); + }; +} + +Comlink.expose(new WavetableConfiguratorWorker()); diff --git a/src/graphEditor/nodes/CustomAudio/Sidechain.tsx b/src/graphEditor/nodes/CustomAudio/Sidechain.tsx index 1e7f4e9e..a052c160 100644 --- a/src/graphEditor/nodes/CustomAudio/Sidechain.tsx +++ b/src/graphEditor/nodes/CustomAudio/Sidechain.tsx @@ -17,7 +17,7 @@ const SidechainAWPRegistered = new AsyncOnce( new AudioContext().audioWorklet.addModule( process.env.ASSET_PATH + 'SidechainWorkletProcessor.js?cacheBust=' + - (window.location.host.includes('localhost') ? '' : btoa(Math.random().toString())) + (window.location.host.includes('localhost') ? '' : crypto.randomUUID()) ), true ); @@ -26,7 +26,7 @@ const SidechainWasm = new AsyncOnce( fetch( process.env.ASSET_PATH + 'sidechain.wasm' + - (window.location.host.includes('localhost') ? '' : `?${btoa(Math.random().toString())}`) + (window.location.host.includes('localhost') ? '' : `?${crypto.randomUUID()}`) ).then(res => res.arrayBuffer()), true ); diff --git a/src/graphEditor/nodes/CustomAudio/WaveTable/WaveTable.ts b/src/graphEditor/nodes/CustomAudio/WaveTable/WaveTable.ts index 38337581..cb8a534b 100644 --- a/src/graphEditor/nodes/CustomAudio/WaveTable/WaveTable.ts +++ b/src/graphEditor/nodes/CustomAudio/WaveTable/WaveTable.ts @@ -14,7 +14,7 @@ import WaveTableSmallView from './WaveTableSmallView.svelte'; // Manually generate some waveforms... for science -const SAMPLE_RATE = 44100; +const SAMPLE_RATE = 44_100; const baseFrequency = 30; // 30hz // Number of samples per waveform @@ -56,7 +56,9 @@ for (let i = 0; i < waveformLength; i++) { bufs[3][i] = periodIxFract * 2 - 1; } -export const getDefaultWavetableDef = () => [ +export type WavetableDef = Float32Array[][]; + +export const getDefaultWavetableDef = (): WavetableDef => [ [bufs[0], bufs[1]], [bufs[2], bufs[3]], ]; @@ -86,7 +88,7 @@ export default class WaveTable implements ForeignNode { private ctx: AudioContext; private vcId: string; public workletHandle: AudioWorkletNode | undefined; - private wavetableDef: Float32Array[][] = getDefaultWavetableDef(); + private wavetableDef: WavetableDef = getDefaultWavetableDef(); private onInitialized?: (inst: WaveTable) => void; static typeName = 'Wave Table Synthesizer'; @@ -124,12 +126,14 @@ export default class WaveTable implements ForeignNode { } }); - this.renderSmallView = mkSvelteContainerRenderHelper({ - Comp: WaveTableSmallView, - getProps: () => ({}), - }); + if (this.vcId) { + this.renderSmallView = mkSvelteContainerRenderHelper({ + Comp: WaveTableSmallView, + getProps: () => ({}), + }); - this.cleanupSmallView = mkSvelteContainerCleanupHelper({ preserveRoot: true }); + this.cleanupSmallView = mkSvelteContainerCleanupHelper({ preserveRoot: true }); + } } private buildParamOverrides(workletHandle: AudioWorkletNode): ForeignNode['paramOverrides'] { @@ -209,9 +213,10 @@ export default class WaveTable implements ForeignNode { ); } - private async initWaveTable() { + private encodeTableDef() { const dimensionCount = this.wavetableDef.length; const waveformsPerDimension = this.wavetableDef[0].length; + const waveformLength = this.wavetableDef[0][0].length; const samplesPerDimension = waveformLength * waveformsPerDimension; const tableSamples = new Float32Array(dimensionCount * waveformsPerDimension * waveformLength); @@ -224,7 +229,15 @@ export default class WaveTable implements ForeignNode { } } + return { dimensionCount, waveformsPerDimension, tableSamples, waveformLength }; + } + + private async initWaveTable() { + const { dimensionCount, waveformsPerDimension, tableSamples, waveformLength } = + this.encodeTableDef(); + this.workletHandle!.port.postMessage({ + type: 'init', arrayBuffer: await getWavetableWasmBytes(), waveformsPerDimension, dimensionCount, @@ -238,7 +251,7 @@ export default class WaveTable implements ForeignNode { await this.ctx.audioWorklet.addModule( process.env.ASSET_PATH + 'WaveTableNodeProcessor.js?cacheBust=' + - btoa(Math.random().toString()) + (window.location.href.includes('localhost') ? '' : crypto.randomUUID()) ); this.workletHandle = new AudioWorkletNode(this.ctx, 'wavetable-node-processor'); @@ -247,6 +260,24 @@ export default class WaveTable implements ForeignNode { return this.workletHandle; } + public setWavetableDef(wavetableDef: WavetableDef, baseFrequency: number) { + this.wavetableDef = wavetableDef; + if (!this.workletHandle) { + return; + } + + const { dimensionCount, waveformsPerDimension, tableSamples, waveformLength } = + this.encodeTableDef(); + this.workletHandle.port.postMessage({ + type: 'update', + waveformsPerDimension, + dimensionCount, + waveformLength, + baseFrequency, + tableSamples, + }); + } + public buildConnectables() { return { // TODO: get dimension count dynamically diff --git a/webpack.base.js b/webpack.base.js index 77593418..ac2729ab 100644 --- a/webpack.base.js +++ b/webpack.base.js @@ -120,7 +120,7 @@ const config = { experiments: { syncWebAssembly: true, backCompat: false, - outputModule: true, + // outputModule: true, }, }; diff --git a/webpack.headless.js b/webpack.headless.js index 72070b85..0f02f647 100644 --- a/webpack.headless.js +++ b/webpack.headless.js @@ -34,6 +34,10 @@ const config = { plugins: [...baseConfig.plugins, new webpack.EnvironmentPlugin({ ASSET_PATH })], mode: 'production', devtool: 'source-map', + experiments: { + ...(baseConfig.experiments || {}), + outputModule: true, + }, }; module.exports = config; diff --git a/yarn.lock b/yarn.lock index 11420eb2..75b58bb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2201,6 +2201,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +comlink@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.4.1.tgz#e568b8e86410b809e8600eb2cf40c189371ef981" + integrity sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q== + commander@7, commander@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"