From d0c0c28435e3778212e8ff0e59e7249337e1fd03 Mon Sep 17 00:00:00 2001 From: git-chad Date: Thu, 30 Jul 2026 13:15:15 -0300 Subject: [PATCH 01/21] feat(audio): pure analysis and modulation core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the dependency-free half of audio-reactive parameters. Nothing imports it yet, so there is no runtime impact. Two-stage analysis, which is the load-bearing design choice: Stage A (expensive, once per source) decodes to mono PCM, runs an STFT and reduces each frame to 64 log-spaced band magnitudes plus a windowed RMS scalar, then discards the PCM. Stage B (~5ms for a 5 minute track) turns that into one normalized, smoothed envelope per band. Because it is cheap it can re-run on every band-config edit, which is what makes editing frequency ranges feel instant instead of triggering a multi-second re-analysis. Normalization is self-calibrating: each band maps its own 5th..99th percentile onto [0,1]. A fixed dB window does not survive real material — measured against a loudness-compressed master, a 48dB window left every band clustered between 0.70 and 0.95, which reads as visually static. A 9dB span floor stops a near-constant band being stretched into full-swing noise, and a 60dB cross-band gate stops FFT spectral leakage from faking a band with no content. Smoothing runs after normalization, which gives a provable invariant: every envelope sample lies in [0,1]. The modulation math depends on it. Everything here is free of browser globals so it runs under `bun test`; the 83 tests cover band isolation on synthetic PCM, temporal discrimination, quiet-track usability, silence, attack/release curves, and the [0,1] invariant. Modulation deliberately avoids a new AnimatedPropertyBinding kind: the binding says what is driven, the band says what drives it. That keeps the existing non-exhaustive `binding.kind` branches correct and means the published runtime package needs no mirrored type changes. --- .../audio/__tests__/envelope-lookup.test.ts | 121 +++++ .../editor/audio/__tests__/envelope.test.ts | 375 +++++++++++++ src/lib/editor/audio/__tests__/fft.test.ts | 203 +++++++ src/lib/editor/audio/__tests__/links.test.ts | 508 ++++++++++++++++++ .../editor/audio/__tests__/modulate.test.ts | 256 +++++++++ .../audio/__tests__/spectrogram.test.ts | 281 ++++++++++ src/lib/editor/audio/analysis-protocol.ts | 32 ++ src/lib/editor/audio/analysis.worker.ts | 61 +++ src/lib/editor/audio/analyze-client.ts | 171 ++++++ src/lib/editor/audio/bands.ts | 213 ++++++++ src/lib/editor/audio/decode.ts | 91 ++++ src/lib/editor/audio/envelope-lookup.ts | 107 ++++ src/lib/editor/audio/envelope.ts | 397 ++++++++++++++ src/lib/editor/audio/fft.ts | 259 +++++++++ src/lib/editor/audio/links.ts | 254 +++++++++ src/lib/editor/audio/live-band-driver.ts | 86 +++ src/lib/editor/audio/modulate.ts | 235 ++++++++ src/lib/editor/audio/spectrogram.ts | 231 ++++++++ src/lib/editor/binding-key.ts | 32 ++ src/lib/editor/parameter-schema.ts | 20 + src/types/editor.ts | 60 ++- 21 files changed, 3992 insertions(+), 1 deletion(-) create mode 100644 src/lib/editor/audio/__tests__/envelope-lookup.test.ts create mode 100644 src/lib/editor/audio/__tests__/envelope.test.ts create mode 100644 src/lib/editor/audio/__tests__/fft.test.ts create mode 100644 src/lib/editor/audio/__tests__/links.test.ts create mode 100644 src/lib/editor/audio/__tests__/modulate.test.ts create mode 100644 src/lib/editor/audio/__tests__/spectrogram.test.ts create mode 100644 src/lib/editor/audio/analysis-protocol.ts create mode 100644 src/lib/editor/audio/analysis.worker.ts create mode 100644 src/lib/editor/audio/analyze-client.ts create mode 100644 src/lib/editor/audio/bands.ts create mode 100644 src/lib/editor/audio/decode.ts create mode 100644 src/lib/editor/audio/envelope-lookup.ts create mode 100644 src/lib/editor/audio/envelope.ts create mode 100644 src/lib/editor/audio/fft.ts create mode 100644 src/lib/editor/audio/links.ts create mode 100644 src/lib/editor/audio/live-band-driver.ts create mode 100644 src/lib/editor/audio/modulate.ts create mode 100644 src/lib/editor/audio/spectrogram.ts create mode 100644 src/lib/editor/binding-key.ts diff --git a/src/lib/editor/audio/__tests__/envelope-lookup.test.ts b/src/lib/editor/audio/__tests__/envelope-lookup.test.ts new file mode 100644 index 0000000..c890d84 --- /dev/null +++ b/src/lib/editor/audio/__tests__/envelope-lookup.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, test } from "bun:test" +import type { AudioEnvelopeSet } from "@/lib/editor/audio/envelope" +import { + sampleAllBands, + sampleBand, + sampleEnvelopeToPeaks, +} from "@/lib/editor/audio/envelope-lookup" +import { AUDIO_BAND_IDS } from "@/types/editor" + +const ENVELOPE_RATE = 60 + +function makeEnvelopes(bass: number[]): AudioEnvelopeSet { + return { + bands: { + bass: Float32Array.from(bass), + high: new Float32Array(bass.length), + level: new Float32Array(bass.length), + mid: new Float32Array(bass.length), + }, + durationSeconds: bass.length / ENVELOPE_RATE, + envelopeRate: ENVELOPE_RATE, + sampleCount: bass.length, + silentBands: [], + } +} + +describe("sampleBand", () => { + const envelopes = makeEnvelopes([0, 0.5, 1]) + + test("returns the first sample at and before t=0", () => { + expect(sampleBand(envelopes, "bass", 0, 0)).toBe(0) + expect(sampleBand(envelopes, "bass", 0, -10)).toBe(0) + }) + + test("returns the last sample past the end, clamping rather than wrapping", () => { + // A short timeline over a long track must hold, not loop the intro. + expect(sampleBand(envelopes, "bass", 0, 999)).toBe(1) + }) + + test("hits sample boundaries exactly", () => { + expect(sampleBand(envelopes, "bass", 0, 1 / ENVELOPE_RATE)).toBeCloseTo( + 0.5, + 6 + ) + }) + + test("interpolates linearly between samples", () => { + const midpoint = 0.5 / ENVELOPE_RATE + + expect(sampleBand(envelopes, "bass", 0, midpoint)).toBeCloseTo(0.25, 6) + }) + + test("shifts by offsetSeconds", () => { + // Offsetting by one sample period should read the next sample. + expect(sampleBand(envelopes, "bass", 1 / ENVELOPE_RATE, 0)).toBeCloseTo( + 0.5, + 6 + ) + }) + + test("returns 0 for non-finite time", () => { + expect(sampleBand(envelopes, "bass", 0, Number.NaN)).toBe(0) + expect(sampleBand(envelopes, "bass", 0, Number.POSITIVE_INFINITY)).toBe(0) + }) + + test("returns 0 for an empty envelope", () => { + expect(sampleBand(makeEnvelopes([]), "bass", 0, 0.5)).toBe(0) + }) + + test("returns the single value for a one-sample envelope", () => { + const single = makeEnvelopes([0.42]) + + expect(sampleBand(single, "bass", 0, 0)).toBeCloseTo(0.42, 6) + expect(sampleBand(single, "bass", 0, 5)).toBeCloseTo(0.42, 6) + }) +}) + +describe("sampleAllBands", () => { + test("returns a value for every band id", () => { + const values = sampleAllBands(makeEnvelopes([0, 1]), 0, 0) + + for (const bandId of AUDIO_BAND_IDS) { + expect(typeof values[bandId]).toBe("number") + } + }) +}) + +describe("sampleEnvelopeToPeaks", () => { + test("keeps transients by taking the bucket maximum", () => { + // A mean-based reduction would flatten the spike to 0.25. + const envelope = Float32Array.from([0, 0, 0, 1, 0, 0, 0, 0]) + const peaks = sampleEnvelopeToPeaks(envelope, 2) + + expect(peaks).toHaveLength(2) + expect(peaks[0]).toBe(1) + expect(peaks[1]).toBe(0) + }) + + test("returns a copy when the envelope is already short enough", () => { + // Exactly representable in Float32 so the equality check is meaningful. + const envelope = Float32Array.from([0.25, 0.5]) + const peaks = sampleEnvelopeToPeaks(envelope, 8) + + expect(Array.from(peaks)).toEqual([0.25, 0.5]) + expect(peaks).not.toBe(envelope) + }) + + test("returns empty for a non-positive target or empty input", () => { + expect(sampleEnvelopeToPeaks(Float32Array.from([1]), 0)).toHaveLength(0) + expect(sampleEnvelopeToPeaks(new Float32Array(0), 10)).toHaveLength(0) + }) + + test("covers the whole envelope with no dropped tail", () => { + const envelope = new Float32Array(1000) + envelope[999] = 1 + + const peaks = sampleEnvelopeToPeaks(envelope, 10) + + expect(peaks[9]).toBe(1) + }) +}) diff --git a/src/lib/editor/audio/__tests__/envelope.test.ts b/src/lib/editor/audio/__tests__/envelope.test.ts new file mode 100644 index 0000000..6574580 --- /dev/null +++ b/src/lib/editor/audio/__tests__/envelope.test.ts @@ -0,0 +1,375 @@ +import { describe, expect, test } from "bun:test" +import { createDefaultAudioBands, ENVELOPE_RATE } from "@/lib/editor/audio/bands" +import { + computeBandNormalization, + computeEnvelopeSet, + computeReferenceLevel, + extractBandSeries, + smoothEnvelopeInPlace, +} from "@/lib/editor/audio/envelope" +import { sampleBand } from "@/lib/editor/audio/envelope-lookup" +import { analyzeSpectrogram } from "@/lib/editor/audio/spectrogram" +import { AUDIO_BAND_IDS, type AudioBandId } from "@/types/editor" + +const SAMPLE_RATE = 48000 + +function makeSine( + frequencyHz: number, + amplitude: number, + seconds: number +): Float32Array { + const sampleCount = Math.round(SAMPLE_RATE * seconds) + const samples = new Float32Array(sampleCount) + + for (let index = 0; index < sampleCount; index += 1) { + samples[index] = + amplitude * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE) + } + + return samples +} + +function analyzeSine(frequencyHz: number, amplitude: number, seconds = 2) { + const samples = makeSine(frequencyHz, amplitude, seconds) + const spectrogram = analyzeSpectrogram(samples, SAMPLE_RATE) + + return computeEnvelopeSet(spectrogram, createDefaultAudioBands()) +} + +/** Value once the attack has settled, away from the start and end ramps. */ +function steadyState( + envelopes: ReturnType, + bandId: AudioBandId +): number { + return sampleBand(envelopes, bandId, 0, 1.5) +} + +describe("computeReferenceLevel", () => { + test("returns 0 for an empty series", () => { + expect(computeReferenceLevel(new Float32Array(0))).toBe(0) + }) + + test("approximates the 99th percentile, not the peak", () => { + // 99 samples at 0.1 and one at 100. A peak-based reference would return + // ~100 and crush everything else; the percentile must stay near 0.1. + const series = new Float32Array(100).fill(0.1) + series[99] = 100 + + const reference = computeReferenceLevel(series) + + expect(reference).toBeGreaterThan(0.05) + expect(reference).toBeLessThan(0.5) + }) +}) + +describe("raw band bucketing", () => { + // Isolation is asserted on the *raw* magnitudes, before normalization. + // Normalized envelopes cannot express it: normalization is per-band and + // relative, so a constant tone drives every band that receives any energy to + // full scale (its p99 equals its own steady value). That is correct for real, + // dynamic music and is why the user-facing guarantee is temporal + // discrimination, tested below. + function rawSteady(frequencyHz: number, bandId: AudioBandId): number { + const spectrogram = analyzeSpectrogram( + makeSine(frequencyHz, 0.8, 1), + SAMPLE_RATE + ) + const series = extractBandSeries( + spectrogram, + bandId, + createDefaultAudioBands()[bandId] + ) + + return series[Math.round(series.length / 2)] ?? 0 + } + + test("a 60Hz sine deposits far more energy in bass than in mid or high", () => { + const bass = rawSteady(60, "bass") + + expect(bass).toBeGreaterThan(rawSteady(60, "mid") * 20) + expect(bass).toBeGreaterThan(rawSteady(60, "high") * 1000) + }) + + test("an 800Hz sine deposits far more energy in mid than in bass or high", () => { + const mid = rawSteady(800, "mid") + + expect(mid).toBeGreaterThan(rawSteady(800, "bass") * 20) + expect(mid).toBeGreaterThan(rawSteady(800, "high") * 20) + }) + + test("an 8kHz sine deposits far more energy in high than in bass or mid", () => { + const high = rawSteady(8000, "high") + + expect(high).toBeGreaterThan(rawSteady(8000, "bass") * 100) + expect(high).toBeGreaterThan(rawSteady(8000, "mid") * 100) + }) + + test("a band with genuinely no content is gated to silence", () => { + const envelopes = analyzeSine(60, 0.8) + + expect(envelopes.silentBands).toContain("high") + expect(steadyState(envelopes, "high")).toBe(0) + }) +}) + +describe("temporal band discrimination", () => { + // The user-facing guarantee: when the music moves between bands, the + // envelopes follow. + function analyzeTwoTone(): ReturnType { + const seconds = 2 + const sampleCount = SAMPLE_RATE * seconds + const samples = new Float32Array(sampleCount) + const half = sampleCount / 2 + + for (let index = 0; index < sampleCount; index += 1) { + const frequencyHz = index < half ? 60 : 800 + samples[index] = + 0.8 * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE) + } + + return computeEnvelopeSet( + analyzeSpectrogram(samples, SAMPLE_RATE), + createDefaultAudioBands() + ) + } + + test("bass leads while the low tone plays and falls back when it stops", () => { + const envelopes = analyzeTwoTone() + + const bassDuringLow = sampleBand(envelopes, "bass", 0, 0.8) + const bassDuringHigh = sampleBand(envelopes, "bass", 0, 1.8) + + expect(bassDuringLow).toBeGreaterThan(0.9) + expect(bassDuringHigh).toBeLessThan(bassDuringLow * 0.6) + }) + + test("mid rises only once the higher tone starts", () => { + const envelopes = analyzeTwoTone() + + const midDuringLow = sampleBand(envelopes, "mid", 0, 0.8) + const midDuringHigh = sampleBand(envelopes, "mid", 0, 1.8) + + expect(midDuringHigh).toBeGreaterThan(0.9) + expect(midDuringLow).toBeLessThan(midDuringHigh * 0.6) + }) + + test("the level band responds to any content", () => { + expect(steadyState(analyzeSine(60, 0.8), "level")).toBeGreaterThan(0.9) + expect(steadyState(analyzeSine(8000, 0.8), "level")).toBeGreaterThan(0.9) + }) +}) + +describe("quiet tracks", () => { + test("a very quiet 60Hz sine still drives bass to full scale", () => { + // The whole point of per-band percentile normalization: a track mastered + // 40dB down must still be usable without the user touching a gain control. + const envelopes = analyzeSine(60, 0.008) + + expect(steadyState(envelopes, "bass")).toBeGreaterThan(0.9) + }) + + test("loud and quiet versions of the same signal agree closely", () => { + const loud = steadyState(analyzeSine(60, 0.9), "bass") + const quiet = steadyState(analyzeSine(60, 0.004), "bass") + + expect(Math.abs(loud - quiet)).toBeLessThan(0.05) + }) +}) + +describe("self-calibrating normalization", () => { + test("derives a window from the series' own distribution", () => { + // Two levels 20dB apart: 1.0 and 0.1. + const series = new Float32Array(200) + for (let index = 0; index < series.length; index += 1) { + series[index] = index % 2 === 0 ? 1 : 0.1 + } + + const { lowDb, spanDb } = computeBandNormalization(series) + + expect(spanDb).toBeGreaterThan(15) + expect(spanDb).toBeLessThanOrEqual(48) + expect(lowDb).toBeLessThan(0) + }) + + test("floors the span so a near-constant band is not stretched to full swing", () => { + const series = new Float32Array(200).fill(0.5) + + expect(computeBandNormalization(series).spanDb).toBeCloseTo(9, 6) + }) + + test("returns a usable window for an empty series", () => { + expect(computeBandNormalization(new Float32Array(0)).spanDb).toBeGreaterThan( + 0 + ) + }) + + test("a loudness-compressed source still uses most of the [0,1] range", () => { + // Regression guard for the real-music failure this replaced: a fixed 48dB + // window left a mastered track's bands clustered between 0.7 and 0.95, + // which reads as visually static. Here bass energy varies only ~12dB, as on + // a compressed master. + const seconds = 4 + const sampleCount = SAMPLE_RATE * seconds + const samples = new Float32Array(sampleCount) + + for (let index = 0; index < sampleCount; index += 1) { + const t = index / SAMPLE_RATE + const amplitude = Math.floor(t * 2) % 2 === 0 ? 0.9 : 0.225 + samples[index] = amplitude * Math.sin(2 * Math.PI * 60 * t) + } + + const envelopes = computeEnvelopeSet( + analyzeSpectrogram(samples, SAMPLE_RATE), + createDefaultAudioBands() + ) + + const values = Array.from(envelopes.bands.bass).sort((a, b) => a - b) + const low = values[Math.floor(values.length * 0.05)] ?? 0 + const high = values[Math.floor(values.length * 0.95)] ?? 0 + + expect(high - low).toBeGreaterThan(0.5) + }) +}) + +describe("silence", () => { + test("a silent track yields all-zero envelopes and reports every band silent", () => { + const spectrogram = analyzeSpectrogram( + new Float32Array(SAMPLE_RATE), + SAMPLE_RATE + ) + const envelopes = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + + expect(envelopes.silentBands).toHaveLength(AUDIO_BAND_IDS.length) + + for (const bandId of AUDIO_BAND_IDS) { + for (const value of envelopes.bands[bandId]) { + expect(value).toBe(0) + } + } + }) +}) + +describe("attack and release", () => { + test("rises during attack and decays after the signal stops", () => { + // 0.5s of 60Hz then 0.5s of silence. + const samples = new Float32Array(SAMPLE_RATE) + for (let index = 0; index < SAMPLE_RATE / 2; index += 1) { + samples[index] = 0.8 * Math.sin((2 * Math.PI * 60 * index) / SAMPLE_RATE) + } + + const spectrogram = analyzeSpectrogram(samples, SAMPLE_RATE) + const envelopes = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + + const atOnset = sampleBand(envelopes, "bass", 0, 0.002) + const settled = sampleBand(envelopes, "bass", 0, 0.4) + const released = sampleBand(envelopes, "bass", 0, 0.75) + + expect(atOnset).toBeLessThan(settled) + expect(settled).toBeGreaterThan(0.9) + expect(released).toBeLessThan(0.2) + }) + + test("zero attack reaches the target within a single step", () => { + const envelope = new Float32Array(8).fill(1) + smoothEnvelopeInPlace(envelope, 0, 100, ENVELOPE_RATE) + + expect(envelope[0]).toBeCloseTo(1, 6) + }) + + test("release follows a one-pole decay", () => { + const envelope = new Float32Array(ENVELOPE_RATE) + envelope[0] = 1 + + const releaseMs = 100 + smoothEnvelopeInPlace(envelope, 0, releaseMs, ENVELOPE_RATE) + + // After `releaseMs`, a one-pole decay from 1 sits at exp(-1) ~= 0.368. + const afterOneTimeConstant = envelope[Math.round(ENVELOPE_RATE / 10)] ?? 0 + + expect(afterOneTimeConstant).toBeGreaterThan(Math.exp(-1) * 0.85) + expect(afterOneTimeConstant).toBeLessThan(Math.exp(-1) * 1.15) + }) + + test("smoothing never leaves the [0,1] range of its input", () => { + const envelope = new Float32Array([0, 1, 0, 1, 1, 0, 0.5, 1]) + smoothEnvelopeInPlace(envelope, 1, 1, ENVELOPE_RATE) + + for (const value of envelope) { + expect(value).toBeGreaterThanOrEqual(0) + expect(value).toBeLessThanOrEqual(1) + } + }) +}) + +describe("the [0,1] invariant", () => { + const fixtures: [string, Float32Array][] = [ + ["60Hz loud", makeSine(60, 0.95, 1)], + ["60Hz very quiet", makeSine(60, 0.002, 1)], + ["800Hz", makeSine(800, 0.5, 1)], + ["8kHz", makeSine(8000, 0.5, 1)], + ["silence", new Float32Array(SAMPLE_RATE)], + ["clipped square-ish", Float32Array.from(makeSine(120, 4, 1), Math.sign)], + ] + + for (const [label, samples] of fixtures) { + test(`holds for ${label}`, () => { + const spectrogram = analyzeSpectrogram(samples, SAMPLE_RATE) + const envelopes = computeEnvelopeSet( + spectrogram, + createDefaultAudioBands() + ) + + for (const bandId of AUDIO_BAND_IDS) { + for (const value of envelopes.bands[bandId]) { + expect(value).toBeGreaterThanOrEqual(0) + expect(value).toBeLessThanOrEqual(1) + } + } + }) + } +}) + +describe("stage B determinism and locality", () => { + test("is idempotent for the same spectrogram and bands", () => { + const spectrogram = analyzeSpectrogram(makeSine(220, 0.6, 1), SAMPLE_RATE) + + const first = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + const second = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + + for (const bandId of AUDIO_BAND_IDS) { + expect(Array.from(first.bands[bandId])).toEqual( + Array.from(second.bands[bandId]) + ) + } + }) + + test("editing the high band leaves bass byte-identical", () => { + // This is the property the instant band editor depends on. + const spectrogram = analyzeSpectrogram( + makeSine(60, 0.6, 1), + SAMPLE_RATE + ) + + const before = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + + const edited = createDefaultAudioBands() + edited.high = { ...edited.high, lowHz: 4000 } + const after = computeEnvelopeSet(spectrogram, edited) + + expect(Array.from(after.bands.bass)).toEqual(Array.from(before.bands.bass)) + }) + + test("raising a band's gain raises its envelope", () => { + const spectrogram = analyzeSpectrogram(makeSine(60, 0.02, 2), SAMPLE_RATE) + + const quietBands = createDefaultAudioBands() + quietBands.bass = { ...quietBands.bass, gainDb: -30 } + + const attenuated = computeEnvelopeSet(spectrogram, quietBands) + const neutral = computeEnvelopeSet(spectrogram, createDefaultAudioBands()) + + expect(steadyState(attenuated, "bass")).toBeLessThan( + steadyState(neutral, "bass") + ) + }) +}) diff --git a/src/lib/editor/audio/__tests__/fft.test.ts b/src/lib/editor/audio/__tests__/fft.test.ts new file mode 100644 index 0000000..42b5d0a --- /dev/null +++ b/src/lib/editor/audio/__tests__/fft.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, test } from "bun:test" +import { + binToFrequency, + computeFrameMagnitudes, + createFftWorkspace, + fftInPlace, + frequencyToBin, + getHannWindow, + getWindowSum, + isPowerOfTwo, +} from "@/lib/editor/audio/fft" + +const SAMPLE_RATE = 48000 +const FFT_SIZE = 2048 + +function makeSine( + frequencyHz: number, + amplitude: number, + sampleCount: number +): Float32Array { + const samples = new Float32Array(sampleCount) + for (let index = 0; index < sampleCount; index += 1) { + samples[index] = + amplitude * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE) + } + + return samples +} + +function peakBin(magnitudes: Float32Array): number { + let best = 0 + for (let bin = 1; bin < magnitudes.length; bin += 1) { + if ((magnitudes[bin] ?? 0) > (magnitudes[best] ?? 0)) { + best = bin + } + } + + return best +} + +describe("isPowerOfTwo", () => { + test("accepts powers of two", () => { + expect(isPowerOfTwo(2)).toBe(true) + expect(isPowerOfTwo(1024)).toBe(true) + expect(isPowerOfTwo(2048)).toBe(true) + }) + + test("rejects non-powers of two, zero, negatives and fractions", () => { + expect(isPowerOfTwo(0)).toBe(false) + expect(isPowerOfTwo(3)).toBe(false) + expect(isPowerOfTwo(1000)).toBe(false) + expect(isPowerOfTwo(-1024)).toBe(false) + expect(isPowerOfTwo(2.5)).toBe(false) + }) +}) + +describe("getHannWindow", () => { + test("is periodic: starts at zero and peaks at the centre", () => { + const window = getHannWindow(FFT_SIZE) + + expect(window[0]).toBeCloseTo(0, 10) + expect(window[FFT_SIZE / 2]).toBeCloseTo(1, 10) + }) + + test("sums to half the window size", () => { + // The periodic Hann window has mean 0.5 exactly. + expect(getWindowSum(getHannWindow(FFT_SIZE))).toBeCloseTo(FFT_SIZE / 2, 6) + }) + + test("returns a cached instance for repeated sizes", () => { + expect(getHannWindow(512)).toBe(getHannWindow(512)) + }) +}) + +describe("fftInPlace", () => { + test("throws on a non-power-of-two length", () => { + expect(() => { + fftInPlace(new Float64Array(3), new Float64Array(3)) + }).toThrow(/power of two/) + }) + + test("throws when real and imag lengths disagree", () => { + expect(() => { + fftInPlace(new Float64Array(8), new Float64Array(4)) + }).toThrow(/match in length/) + }) + + test("puts a constant signal entirely in bin 0", () => { + const size = 16 + const real = new Float64Array(size).fill(1) + const imag = new Float64Array(size) + + fftInPlace(real, imag) + + expect(real[0]).toBeCloseTo(size, 10) + for (let bin = 1; bin < size; bin += 1) { + expect(Math.hypot(real[bin] ?? 0, imag[bin] ?? 0)).toBeCloseTo(0, 10) + } + }) +}) + +describe("computeFrameMagnitudes", () => { + test("locates a 1kHz sine at the expected bin and rejects distant bins", () => { + // 1000 Hz / (48000 / 2048) = 42.67 -> nearest bin is 43. + const workspace = createFftWorkspace(FFT_SIZE) + const samples = makeSine(1000, 1, FFT_SIZE * 2) + + const magnitudes = computeFrameMagnitudes(workspace, samples, 0) + + expect(peakBin(magnitudes)).toBe(43) + expect(magnitudes[43] ?? 0).toBeGreaterThan((magnitudes[60] ?? 0) * 20) + }) + + test("a bin-centred full-scale sine reads a magnitude near 1.0", () => { + // Bin 43 centre is exactly 43 * 48000 / 2048 Hz, so there is no scalloping + // loss and the windowScale normalization is directly observable. + const workspace = createFftWorkspace(FFT_SIZE) + const centreHz = binToFrequency(43, FFT_SIZE, SAMPLE_RATE) + const samples = makeSine(centreHz, 1, FFT_SIZE * 2) + + const magnitudes = computeFrameMagnitudes(workspace, samples, 0) + + expect(magnitudes[43] ?? 0).toBeCloseTo(1, 2) + }) + + test("amplitude scales the peak magnitude linearly", () => { + const workspace = createFftWorkspace(FFT_SIZE) + const centreHz = binToFrequency(43, FFT_SIZE, SAMPLE_RATE) + + const loud = computeFrameMagnitudes( + workspace, + makeSine(centreHz, 1, FFT_SIZE * 2), + 0 + ) + const loudPeak = loud[43] ?? 0 + + const quiet = computeFrameMagnitudes( + workspace, + makeSine(centreHz, 0.01, FFT_SIZE * 2), + 0 + ) + const quietPeak = quiet[43] ?? 0 + + expect(quietPeak).toBeCloseTo(loudPeak * 0.01, 4) + }) + + test("a unit impulse produces a flat magnitude spectrum", () => { + const workspace = createFftWorkspace(FFT_SIZE) + const samples = new Float32Array(FFT_SIZE) + // Window peak is exactly 1.0 at size/2, so every bin should read windowScale. + samples[FFT_SIZE / 2] = 1 + + const magnitudes = computeFrameMagnitudes(workspace, samples, 0) + const expected = workspace.windowScale + + for (const magnitude of magnitudes) { + expect(magnitude).toBeCloseTo(expected, 8) + } + }) + + test("zero-pads out-of-bounds offsets instead of wrapping", () => { + const workspace = createFftWorkspace(64) + const samples = new Float32Array(8).fill(1) + + // Entirely before the buffer: nothing to window, so every bin is silent. + const magnitudes = computeFrameMagnitudes(workspace, samples, -1000) + + for (const magnitude of magnitudes) { + expect(magnitude).toBe(0) + } + }) + + test("is deterministic across repeated runs", () => { + const samples = makeSine(440, 0.7, FFT_SIZE * 2) + + const first = Float32Array.from( + computeFrameMagnitudes(createFftWorkspace(FFT_SIZE), samples, 128) + ) + const second = Float32Array.from( + computeFrameMagnitudes(createFftWorkspace(FFT_SIZE), samples, 128) + ) + + expect(Array.from(first)).toEqual(Array.from(second)) + }) +}) + +describe("frequencyToBin / binToFrequency", () => { + test("round-trips a bin centre", () => { + const hz = binToFrequency(43, FFT_SIZE, SAMPLE_RATE) + + expect(frequencyToBin(hz, FFT_SIZE, SAMPLE_RATE)).toBe(43) + }) + + test("clamps above nyquist and below zero", () => { + expect(frequencyToBin(999_999, FFT_SIZE, SAMPLE_RATE)).toBe(FFT_SIZE / 2) + expect(frequencyToBin(-500, FFT_SIZE, SAMPLE_RATE)).toBe(0) + }) + + test("returns 0 for non-finite input", () => { + expect(frequencyToBin(Number.NaN, FFT_SIZE, SAMPLE_RATE)).toBe(0) + expect(frequencyToBin(Number.POSITIVE_INFINITY, FFT_SIZE, 0)).toBe(0) + }) +}) diff --git a/src/lib/editor/audio/__tests__/links.test.ts b/src/lib/editor/audio/__tests__/links.test.ts new file mode 100644 index 0000000..7c97d45 --- /dev/null +++ b/src/lib/editor/audio/__tests__/links.test.ts @@ -0,0 +1,508 @@ +import { describe, expect, test } from "bun:test" +import type { AudioEnvelopeSet } from "@/lib/editor/audio/envelope" +import { + applyAudioModulation, + createAudioLink, + findAudioLink, + findConflictingAudioLinks, + hasAudioLink, + patchAudioLink, +} from "@/lib/editor/audio/links" +import { getLayerDefinition } from "@/lib/editor/config/layer-registry" +import { createLayer } from "@/lib/editor/layers" +import type { EvaluatedLayerState } from "@/lib/editor/timeline/evaluate" +import { LINEAR_EASING } from "@/lib/easing-curve" +import { AUDIO_BAND_IDS, type AudioBandId } from "@/types/editor" +import type { + AnimatedPropertyBinding, + EditorLayer, + NumberParameterDefinition, + TimelineTrack, + Vec2ParameterDefinition, +} from "@/types/editor" + +const gradientParams = getLayerDefinition("gradient").params + +const numberParam = gradientParams.find( + (definition): definition is NumberParameterDefinition => + definition.type === "number" && + definition.max !== undefined && + definition.min !== undefined +) +const vec2Param = gradientParams.find( + (definition): definition is Vec2ParameterDefinition => + definition.type === "vec2" +) + +if (numberParam === undefined) { + throw new Error("gradient registry is missing a bounded number param") +} + +if (vec2Param === undefined) { + throw new Error("gradient registry is missing a vec2 param") +} + +function paramBinding(key: string, label = key): AnimatedPropertyBinding { + return { key, kind: "param", label, valueType: "number" } +} + +/** Envelopes where every band holds a constant value. */ +function constantEnvelopes(value: number): AudioEnvelopeSet { + const bands = {} as Record + for (const bandId of AUDIO_BAND_IDS) { + bands[bandId] = Float32Array.from([value, value]) + } + + return { + bands, + durationSeconds: 1, + envelopeRate: 60, + sampleCount: 2, + silentBands: [], + } +} + +function makeLayer(): EditorLayer { + return createLayer("gradient", 0) +} + +describe("createAudioLink", () => { + test("omits optional fields rather than setting them to undefined", () => { + // exactOptionalPropertyTypes makes `undefined` a type error, and a stray + // undefined key survives a .lab round-trip as an explicit null. + const link = createAudioLink({ + band: "bass", + binding: paramBinding("strength"), + id: "l1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + }) + + expect("component" in link).toBe(false) + expect("threshold" in link).toBe(false) + expect("quantize" in link).toBe(false) + expect(link.enabled).toBe(true) + }) + + test("keeps optional fields that were provided", () => { + const link = createAudioLink({ + band: "mid", + binding: paramBinding("strength"), + component: "x", + enabled: false, + id: "l1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + threshold: 0.8, + }) + + expect(link.component).toBe("x") + expect(link.threshold).toBe(0.8) + expect(link.enabled).toBe(false) + }) +}) + +describe("patchAudioLink", () => { + const base = createAudioLink({ + band: "bass", + binding: paramBinding("strength"), + component: "x", + id: "l1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + }) + + test("updates provided fields", () => { + expect(patchAudioLink(base, { band: "high" }).band).toBe("high") + }) + + test("deletes a field set to undefined rather than storing undefined", () => { + const patched = patchAudioLink(base, { component: undefined }) + + expect("component" in patched).toBe(false) + }) + + test("does not mutate the original", () => { + patchAudioLink(base, { band: "high" }) + + expect(base.band).toBe("bass") + }) +}) + +describe("findAudioLink / hasAudioLink", () => { + const link = createAudioLink({ + band: "bass", + binding: paramBinding("strength"), + id: "l1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + }) + + test("matches on layer and binding", () => { + expect(findAudioLink([link], "layer-1", paramBinding("strength"))).toBe(link) + expect(hasAudioLink([link], "layer-1", paramBinding("strength"))).toBe(true) + }) + + test("does not match another layer or another binding", () => { + expect(hasAudioLink([link], "layer-2", paramBinding("strength"))).toBe(false) + expect(hasAudioLink([link], "layer-1", paramBinding("other"))).toBe(false) + }) +}) + +describe("findConflictingAudioLinks", () => { + const link = createAudioLink({ + band: "bass", + binding: paramBinding("strength"), + id: "l1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + }) + + function makeTrack(layerId: string, key: string): TimelineTrack { + return { + binding: paramBinding(key), + enabled: true, + id: `track-${key}`, + keyframes: [{ easing: LINEAR_EASING, id: "k1", time: 0, value: 0 }], + layerId, + } + } + + test("reports a link and track on the same layer and binding", () => { + const conflicts = findConflictingAudioLinks( + [link], + [makeTrack("layer-1", "strength")] + ) + + expect(conflicts).toHaveLength(1) + expect(conflicts[0]?.link).toBe(link) + }) + + test("ignores tracks on other bindings or layers", () => { + expect( + findConflictingAudioLinks([link], [makeTrack("layer-1", "other")]) + ).toHaveLength(0) + expect( + findConflictingAudioLinks([link], [makeTrack("layer-2", "strength")]) + ).toHaveLength(0) + }) + + test("short-circuits on empty input", () => { + expect(findConflictingAudioLinks([], [makeTrack("l", "k")])).toHaveLength(0) + expect(findConflictingAudioLinks([link], [])).toHaveLength(0) + }) +}) + +describe("applyAudioModulation", () => { + function apply( + layers: EditorLayer[], + links: ReturnType[], + keyframeStates: EvaluatedLayerState[] = [], + bandValue = 1 + ): EvaluatedLayerState[] { + return applyAudioModulation( + layers, + keyframeStates, + { + envelopes: constantEnvelopes(bandValue), + links, + offsetSeconds: 0, + }, + 0 + ) + } + + test("drives a bounded number parameter to its output maximum", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: layer.id, + outMax: numberParam.max ?? 1, + outMin: numberParam.min ?? 0, + }) + + const states = apply([layer], [link], [], 1) + + expect(states[0]?.params[numberParam.key]).toBe(numberParam.max) + }) + + test("returns the keyframe states untouched when no links are active", () => { + const layer = makeLayer() + const keyframeStates: EvaluatedLayerState[] = [ + { layerId: layer.id, params: { a: 1 }, properties: {} }, + ] + + const disabled = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + enabled: false, + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + expect(apply([layer], [disabled], keyframeStates)).toBe(keyframeStates) + }) + + test("overrides a keyframed value, because audio wins", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: layer.id, + outMax: numberParam.max ?? 1, + outMin: numberParam.min ?? 0, + }) + + const keyframeStates: EvaluatedLayerState[] = [ + { + layerId: layer.id, + params: { [numberParam.key]: 0 }, + properties: {}, + }, + ] + + const states = apply([layer], [link], keyframeStates, 1) + + expect(states[0]?.params[numberParam.key]).toBe(numberParam.max) + }) + + test("preserves other keyframed params on the same layer", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + const keyframeStates: EvaluatedLayerState[] = [ + { + layerId: layer.id, + params: { untouched: 7 }, + properties: { opacity: 0.3 }, + }, + ] + + const states = apply([layer], [link], keyframeStates, 1) + + expect(states[0]?.params.untouched).toBe(7) + expect(states[0]?.properties.opacity).toBe(0.3) + }) + + test("does not mutate the incoming keyframe states", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + const original: EvaluatedLayerState = { + layerId: layer.id, + params: { [numberParam.key]: 0 }, + properties: {}, + } + + apply([layer], [link], [original], 1) + + expect(original.params[numberParam.key]).toBe(0) + }) + + test("ignores links pointing at a deleted layer", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: "does-not-exist", + outMax: 1, + outMin: 0, + }) + + expect(apply([layer], [link], [], 1)).toHaveLength(0) + }) + + test("ignores links on parameters that are not audio-modulatable", () => { + const layer = makeLayer() + const colorParam = gradientParams.find((d) => d.type === "color") + + if (!colorParam) { + throw new Error("expected a colour param in the gradient registry") + } + + const link = createAudioLink({ + band: "bass", + binding: paramBinding(colorParam.key), + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + expect(apply([layer], [link], [], 1)).toHaveLength(0) + }) + + test("ignores links on parameters the layer does not define", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding("definitelyNotAParam"), + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + expect(apply([layer], [link], [], 1)).toHaveLength(0) + }) + + test("routes layer-property bindings into properties", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: { + kind: "layer", + label: "Opacity", + property: "opacity", + valueType: "number", + }, + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 0, + }) + + const states = apply([layer], [link], [], 1) + + expect(states[0]?.properties.opacity).toBe(1) + expect(states[0]?.params).toEqual({}) + }) + + test("two component links on one vector compose", () => { + const layer = makeLayer() + const links = (["x", "y"] as const).map((component, index) => + createAudioLink({ + band: "bass", + binding: paramBinding(vec2Param.key), + component, + id: `l${index}`, + layerId: layer.id, + outMax: index === 0 ? 1 : -1, + outMin: index === 0 ? 1 : -1, + }) + ) + + const states = apply([layer], links, [], 1) + + expect(states[0]?.params[vec2Param.key]).toEqual([1, -1]) + }) + + test("a component link preserves the keyframed value of the other axis", () => { + const layer = makeLayer() + const link = createAudioLink({ + band: "bass", + binding: paramBinding(vec2Param.key), + component: "x", + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 1, + }) + + const keyframeStates: EvaluatedLayerState[] = [ + { + layerId: layer.id, + params: { [vec2Param.key]: [0.1, 0.9] }, + properties: {}, + }, + ] + + const states = apply([layer], [link], keyframeStates, 1) + + expect(states[0]?.params[vec2Param.key]).toEqual([1, 0.9]) + }) + + test("ALIASING GUARD: never returns or mutates a tuple owned by the layer store", () => { + // A missed clone here corrupts both layer.params and the paramsCloneCache + // WeakMap in renderer/contracts.ts, producing drift that persists across + // frames and survives re-renders. + const layer = makeLayer() + const storedTuple = layer.params[vec2Param.key] + const snapshot = Array.isArray(storedTuple) ? [...storedTuple] : null + + const link = createAudioLink({ + band: "bass", + binding: paramBinding(vec2Param.key), + component: "x", + id: "l1", + layerId: layer.id, + outMax: 1, + outMin: 1, + }) + + const states = apply([layer], [link], [], 1) + const result = states[0]?.params[vec2Param.key] + + expect(result).not.toBe(storedTuple) + expect(layer.params[vec2Param.key]).toEqual(snapshot) + }) + + test("samples the band at the requested time", () => { + const layer = makeLayer() + // Use the definition's own bounds so the clamp cannot mask the result. + const low = numberParam.min ?? 0 + const high = numberParam.max ?? 1 + const link = createAudioLink({ + band: "bass", + binding: paramBinding(numberParam.key), + id: "l1", + layerId: layer.id, + outMax: high, + outMin: low, + }) + + const envelopes: AudioEnvelopeSet = { + bands: { + bass: Float32Array.from([0, 1]), + high: Float32Array.from([0, 0]), + level: Float32Array.from([0, 0]), + mid: Float32Array.from([0, 0]), + }, + durationSeconds: 2 / 60, + envelopeRate: 60, + sampleCount: 2, + silentBands: [], + } + + const atStart = applyAudioModulation( + [layer], + [], + { envelopes, links: [link], offsetSeconds: 0 }, + 0 + ) + const atEnd = applyAudioModulation( + [layer], + [], + { envelopes, links: [link], offsetSeconds: 0 }, + 1 / 60 + ) + + expect(atStart[0]?.params[numberParam.key]).toBe(low) + expect(atEnd[0]?.params[numberParam.key]).toBe(high) + }) +}) diff --git a/src/lib/editor/audio/__tests__/modulate.test.ts b/src/lib/editor/audio/__tests__/modulate.test.ts new file mode 100644 index 0000000..dcb2cc0 --- /dev/null +++ b/src/lib/editor/audio/__tests__/modulate.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, test } from "bun:test" +import { createAudioLink } from "@/lib/editor/audio/links" +import { + remapBand, + resolveAudioLinkValue, + resolveBooleanValue, + resolveLayerPropertyValue, + resolveNumberValue, + resolveVectorValue, +} from "@/lib/editor/audio/modulate" +import type { AudioLink, ParameterDefinition } from "@/types/editor" + +function makeLink(overrides: Partial = {}): AudioLink { + return { + ...createAudioLink({ + band: "bass", + binding: { key: "strength", kind: "param", label: "Strength", valueType: "number" }, + id: "link-1", + layerId: "layer-1", + outMax: 1, + outMin: 0, + }), + ...overrides, + } +} + +const boundedNumber: ParameterDefinition = { + defaultValue: 0.5, + key: "strength", + label: "Strength", + max: 2, + min: -1, + step: 0.25, + type: "number", +} + +const unboundedNumber: ParameterDefinition = { + defaultValue: 1, + key: "scale", + label: "Scale", + type: "number", +} + +const intNumber: ParameterDefinition = { + defaultValue: 3, + input: "int", + key: "count", + label: "Count", + max: 8, + min: 1, + step: 1, + type: "number", +} + +const vec2Definition: ParameterDefinition = { + defaultValue: [0, 0], + key: "offset", + label: "Offset", + max: 1, + min: -1, + step: 0.01, + type: "vec2", +} + +describe("remapBand", () => { + test("maps the band range onto the output range", () => { + expect(remapBand(0, 10, 20)).toBe(10) + expect(remapBand(1, 10, 20)).toBe(20) + expect(remapBand(0.5, 10, 20)).toBe(15) + }) + + test("supports an inverted range", () => { + expect(remapBand(0, 20, 10)).toBe(20) + expect(remapBand(1, 20, 10)).toBe(10) + }) + + test("clamps band values outside [0,1]", () => { + expect(remapBand(-5, 0, 1)).toBe(0) + expect(remapBand(5, 0, 1)).toBe(1) + }) + + test("falls back to outMin for non-finite input", () => { + expect(remapBand(Number.NaN, 3, 9)).toBe(3) + }) +}) + +describe("resolveNumberValue", () => { + test("clamps to declared bounds", () => { + const link = makeLink({ outMax: 99, outMin: -99 }) + + expect(resolveNumberValue(link, boundedNumber, 1)).toBe(2) + expect(resolveNumberValue(link, boundedNumber, 0)).toBe(-1) + }) + + test("does not invent bounds the definition omits", () => { + // The sidebar's min 0 / max 100 slider fallback must not leak in here. + const link = makeLink({ outMax: 5000, outMin: 0 }) + + expect(resolveNumberValue(link, unboundedNumber, 1)).toBe(5000) + }) + + test("does not quantize to step by default", () => { + const link = makeLink({ outMax: 1, outMin: 0 }) + + expect(resolveNumberValue(link, boundedNumber, 0.5)).toBeCloseTo(0.5, 6) + }) + + test("quantizes only when the link opts in", () => { + const link = makeLink({ outMax: 1, outMin: 0, quantize: true }) + + // step 0.25 -> 0.6 snaps to 0.5 + expect(resolveNumberValue(link, boundedNumber, 0.6)).toBeCloseTo(0.5, 6) + }) + + test("rounds integer inputs", () => { + const link = makeLink({ outMax: 8, outMin: 1 }) + const value = resolveNumberValue(link, intNumber, 0.5) + + expect(Number.isInteger(value)).toBe(true) + }) + + test("re-clamps after rounding pushes past a bound", () => { + const link = makeLink({ outMax: 8.4, outMin: 8.4 }) + + expect(resolveNumberValue(link, intNumber, 0.5)).toBeLessThanOrEqual(8) + }) +}) + +describe("resolveVectorValue", () => { + test("drives every component when component is 'all'", () => { + const link = makeLink({ component: "all", outMax: 1, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, [0, 0], 1)).toEqual([1, 1]) + }) + + test("drives every component when component is omitted", () => { + const link = makeLink({ outMax: 1, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, [0, 0], 1)).toEqual([1, 1]) + }) + + test("merges a single component into the base, preserving the other axis", () => { + const link = makeLink({ component: "x", outMax: 1, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, [-0.5, 0.25], 1)).toEqual([ + 1, 0.25, + ]) + }) + + test("never mutates or aliases the base tuple", () => { + const base: [number, number] = [-0.5, 0.25] + const link = makeLink({ component: "x", outMax: 1, outMin: 0 }) + + const result = resolveVectorValue(link, vec2Definition, base, 1) + + expect(result).not.toBe(base) + expect(base).toEqual([-0.5, 0.25]) + }) + + test("returns null for a component the vector does not have", () => { + const link = makeLink({ component: "z", outMax: 1, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, [0, 0], 1)).toBeNull() + }) + + test("drives all components when the base is unusable", () => { + const link = makeLink({ component: "x", outMax: 1, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, 5, 1)).toEqual([1, 1]) + }) + + test("clamps components to the definition bounds", () => { + const link = makeLink({ component: "all", outMax: 99, outMin: 0 }) + + expect(resolveVectorValue(link, vec2Definition, [0, 0], 1)).toEqual([1, 1]) + }) +}) + +describe("resolveBooleanValue", () => { + test("gates at 0.5 by default", () => { + const link = makeLink() + + expect(resolveBooleanValue(link, 0.49)).toBe(false) + expect(resolveBooleanValue(link, 0.5)).toBe(true) + }) + + test("honours a custom threshold", () => { + const link = makeLink({ threshold: 0.8 }) + + expect(resolveBooleanValue(link, 0.79)).toBe(false) + expect(resolveBooleanValue(link, 0.81)).toBe(true) + }) +}) + +describe("resolveLayerPropertyValue", () => { + test("clamps opacity to [0,1]", () => { + const link = makeLink({ outMax: 50, outMin: -50 }) + + expect(resolveLayerPropertyValue(link, "opacity", 1)).toBe(1) + expect(resolveLayerPropertyValue(link, "opacity", 0)).toBe(0) + }) + + test("clamps hue to [-180,180]", () => { + const link = makeLink({ outMax: 900, outMin: -900 }) + + expect(resolveLayerPropertyValue(link, "hue", 1)).toBe(180) + expect(resolveLayerPropertyValue(link, "hue", 0)).toBe(-180) + }) + + test("clamps saturation to [0,2]", () => { + const link = makeLink({ outMax: 50, outMin: -50 }) + + expect(resolveLayerPropertyValue(link, "saturation", 1)).toBe(2) + }) + + test("gates visible as a boolean", () => { + const link = makeLink() + + expect(resolveLayerPropertyValue(link, "visible", 0.9)).toBe(true) + expect(resolveLayerPropertyValue(link, "visible", 0.1)).toBe(false) + }) +}) + +describe("resolveAudioLinkValue", () => { + test("routes layer bindings without needing a definition", () => { + const link = makeLink({ + binding: { + kind: "layer", + label: "Opacity", + property: "opacity", + valueType: "number", + }, + outMax: 1, + outMin: 0, + }) + + expect(resolveAudioLinkValue(link, null, undefined, 1)).toBe(1) + }) + + test("returns null for a param binding with no definition", () => { + expect(resolveAudioLinkValue(makeLink(), null, undefined, 1)).toBeNull() + }) + + test("returns null for colour and select parameters", () => { + const colorDefinition: ParameterDefinition = { + defaultValue: "#ffffff", + key: "tint", + label: "Tint", + type: "color", + } + + expect( + resolveAudioLinkValue(makeLink(), colorDefinition, undefined, 1) + ).toBeNull() + }) +}) diff --git a/src/lib/editor/audio/__tests__/spectrogram.test.ts b/src/lib/editor/audio/__tests__/spectrogram.test.ts new file mode 100644 index 0000000..0491b02 --- /dev/null +++ b/src/lib/editor/audio/__tests__/spectrogram.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, test } from "bun:test" +import { + clampBandConfig, + createDefaultAudioBands, + createSpectroBandLayout, + DEFAULT_FFT_SIZE, + ENVELOPE_RATE, + frequencyToBinRange, + MIN_RELEASE_MS, + resolveSpectroBandRange, + SPECTRO_BAND_COUNT, +} from "@/lib/editor/audio/bands" +import { + analyzeSpectrogram, + analyzeSpectrogramStepwise, + downmixToMono, +} from "@/lib/editor/audio/spectrogram" + +const SAMPLE_RATE = 48000 +const HOP = SAMPLE_RATE / ENVELOPE_RATE + +describe("frequencyToBinRange", () => { + test("maps the default bass range and always excludes DC", () => { + // 20Hz -> bin 0.85 (floored to 0, raised to 1); 140Hz -> bin 5.97 (ceil 6). + expect(frequencyToBinRange(20, 140, DEFAULT_FFT_SIZE, SAMPLE_RATE)).toEqual({ + endBin: 6, + startBin: 1, + }) + }) + + test("clamps above nyquist", () => { + const range = frequencyToBinRange( + 1000, + 999_999, + DEFAULT_FFT_SIZE, + SAMPLE_RATE + ) + + expect(range.endBin).toBe(DEFAULT_FFT_SIZE / 2) + }) + + test("normalizes an inverted range instead of throwing", () => { + expect(frequencyToBinRange(140, 20, DEFAULT_FFT_SIZE, SAMPLE_RATE)).toEqual( + frequencyToBinRange(20, 140, DEFAULT_FFT_SIZE, SAMPLE_RATE) + ) + }) + + test("degrades to a single bin for a sub-bin-width range", () => { + const range = frequencyToBinRange( + 1000, + 1000.1, + DEFAULT_FFT_SIZE, + SAMPLE_RATE + ) + + expect(range.startBin).toBeLessThanOrEqual(range.endBin) + }) + + test("returns a safe range for non-finite input", () => { + expect( + frequencyToBinRange( + Number.NaN, + Number.NaN, + DEFAULT_FFT_SIZE, + SAMPLE_RATE + ) + ).toEqual({ endBin: 1, startBin: 1 }) + }) +}) + +describe("createSpectroBandLayout", () => { + test("produces ascending edges and centres between them", () => { + const layout = createSpectroBandLayout(SAMPLE_RATE) + + expect(layout.edgeHz).toHaveLength(SPECTRO_BAND_COUNT + 1) + expect(layout.centerHz).toHaveLength(SPECTRO_BAND_COUNT) + + for (let index = 0; index < SPECTRO_BAND_COUNT; index += 1) { + const lower = layout.edgeHz[index] ?? 0 + const upper = layout.edgeHz[index + 1] ?? 0 + const center = layout.centerHz[index] ?? 0 + + expect(upper).toBeGreaterThan(lower) + expect(center).toBeGreaterThan(lower) + expect(center).toBeLessThan(upper) + } + }) + + test("never exceeds nyquist for a low sample rate", () => { + const layout = createSpectroBandLayout(8000) + const top = layout.edgeHz[SPECTRO_BAND_COUNT] ?? 0 + + expect(top).toBeLessThanOrEqual(4000 + 1e-3) + }) +}) + +describe("resolveSpectroBandRange", () => { + const centerHz = createSpectroBandLayout(SAMPLE_RATE).centerHz + + test("selects the bands whose centres fall inside the range", () => { + const range = resolveSpectroBandRange(centerHz, 20, 140) + + expect(range.startIndex).toBe(0) + expect(centerHz[range.endIndex] ?? 0).toBeLessThan(140) + expect(centerHz[range.endIndex + 1] ?? 0).toBeGreaterThanOrEqual(140) + }) + + test("falls back to the nearest single band for a very narrow range", () => { + const range = resolveSpectroBandRange(centerHz, 1000, 1000.5) + + expect(range.startIndex).toBe(range.endIndex) + expect(centerHz[range.startIndex] ?? 0).toBeGreaterThan(800) + expect(centerHz[range.startIndex] ?? 0).toBeLessThan(1250) + }) + + test("handles an empty layout", () => { + expect(resolveSpectroBandRange(new Float32Array(0), 20, 140)).toEqual({ + endIndex: 0, + startIndex: 0, + }) + }) +}) + +describe("clampBandConfig", () => { + test("floors release at one envelope step to avoid aliasing on export", () => { + const clamped = clampBandConfig({ + attackMs: -5, + gainDb: 0, + highHz: 2000, + lowHz: 140, + releaseMs: 0, + }) + + expect(clamped.releaseMs).toBeCloseTo(MIN_RELEASE_MS, 6) + expect(clamped.attackMs).toBe(0) + }) + + test("keeps highHz above lowHz", () => { + const clamped = clampBandConfig({ + attackMs: 10, + gainDb: 0, + highHz: 50, + lowHz: 500, + releaseMs: 100, + }) + + expect(clamped.highHz).toBeGreaterThan(clamped.lowHz) + }) + + test("clamps frequencies to nyquist", () => { + const clamped = clampBandConfig( + { attackMs: 10, gainDb: 0, highHz: 99_999, lowHz: 20, releaseMs: 100 }, + 48000 + ) + + expect(clamped.highHz).toBeLessThanOrEqual(24000) + }) +}) + +describe("createDefaultAudioBands", () => { + test("returns independent copies so edits cannot leak between projects", () => { + const first = createDefaultAudioBands() + const second = createDefaultAudioBands() + + first.bass.lowHz = 999 + + expect(second.bass.lowHz).toBe(20) + }) + + test("bands tile the spectrum without gaps", () => { + const bands = createDefaultAudioBands() + + expect(bands.bass.highHz).toBe(bands.mid.lowHz) + expect(bands.mid.highHz).toBe(bands.high.lowHz) + }) +}) + +describe("analyzeSpectrogram", () => { + test("derives frame count from the hop size", () => { + const samples = new Float32Array(SAMPLE_RATE) + const spectrogram = analyzeSpectrogram(samples, SAMPLE_RATE) + + expect(spectrogram.frameCount).toBe(Math.ceil(SAMPLE_RATE / HOP)) + expect(spectrogram.envelopeRate).toBe(ENVELOPE_RATE) + expect(spectrogram.durationSeconds).toBeCloseTo(1, 6) + }) + + test("centres each frame on its timestamp", () => { + // An impulse at t=0.5s must peak at frame 30 (0.5 * 60), not ~half a window + // later. Left-aligned framing would report a kick ~21ms late. + const samples = new Float32Array(SAMPLE_RATE) + samples[Math.round(0.5 * SAMPLE_RATE)] = 1 + + const spectrogram = analyzeSpectrogram(samples, SAMPLE_RATE) + + let peakFrame = 0 + for (let frame = 0; frame < spectrogram.frameCount; frame += 1) { + if ( + (spectrogram.rms[frame] ?? 0) > (spectrogram.rms[peakFrame] ?? 0) + ) { + peakFrame = frame + } + } + + expect(peakFrame).toBe(Math.round(0.5 * ENVELOPE_RATE)) + }) + + test("the generator yields ascending progress ending at 1", () => { + const iterator = analyzeSpectrogramStepwise( + new Float32Array(SAMPLE_RATE), + SAMPLE_RATE + ) + + const progress: number[] = [] + let step = iterator.next() + while (!step.done) { + progress.push(step.value) + step = iterator.next() + } + + expect(progress[0]).toBe(0) + expect(progress.at(-1)).toBe(1) + for (let index = 1; index < progress.length; index += 1) { + expect(progress[index] ?? 0).toBeGreaterThanOrEqual( + progress[index - 1] ?? 0 + ) + } + }) + + test("the sync path matches a drained generator exactly", () => { + const samples = new Float32Array(SAMPLE_RATE / 2) + for (let index = 0; index < samples.length; index += 1) { + samples[index] = Math.sin((2 * Math.PI * 220 * index) / SAMPLE_RATE) + } + + const direct = analyzeSpectrogram(samples, SAMPLE_RATE) + + const iterator = analyzeSpectrogramStepwise(samples, SAMPLE_RATE) + let step = iterator.next() + while (!step.done) { + step = iterator.next() + } + + expect(Array.from(step.value.bands)).toEqual(Array.from(direct.bands)) + expect(Array.from(step.value.rms)).toEqual(Array.from(direct.rms)) + }) + + test("throws on a non-positive sample rate", () => { + expect(() => { + analyzeSpectrogram(new Float32Array(128), 0) + }).toThrow(/sampleRate/) + }) + + test("handles a shorter-than-one-window buffer", () => { + const spectrogram = analyzeSpectrogram(new Float32Array(16), SAMPLE_RATE) + + expect(spectrogram.frameCount).toBeGreaterThanOrEqual(1) + expect(spectrogram.bands).toHaveLength( + spectrogram.frameCount * SPECTRO_BAND_COUNT + ) + }) +}) + +describe("downmixToMono", () => { + test("averages channels rather than summing", () => { + const left = new Float32Array([1, 0, -1]) + const right = new Float32Array([1, 1, -1]) + + expect(Array.from(downmixToMono([left, right]))).toEqual([1, 0.5, -1]) + }) + + test("returns the single channel unchanged", () => { + const mono = new Float32Array([0.5]) + + expect(downmixToMono([mono])).toBe(mono) + }) + + test("returns an empty buffer for no channels", () => { + expect(downmixToMono([])).toHaveLength(0) + }) +}) diff --git a/src/lib/editor/audio/analysis-protocol.ts b/src/lib/editor/audio/analysis-protocol.ts new file mode 100644 index 0000000..8712613 --- /dev/null +++ b/src/lib/editor/audio/analysis-protocol.ts @@ -0,0 +1,32 @@ +import type { AudioSpectrogram } from "@/lib/editor/audio/spectrogram" + +/** + * Message shapes shared between the main thread and the analysis Worker. + * + * Kept in its own module so the Worker entry point imports no React, no stores + * and no DOM helpers — importing any of those would break the Worker bundle. + */ + +export type AnalysisRequest = { + bandCount?: number + envelopeRate?: number + fftSize?: number + sampleRate: number + samples: Float32Array +} + +export type AnalysisResponse = + | { message: string; type: "error" } + | { progress: number; type: "progress" } + | { spectrogram: AudioSpectrogram; type: "done" } + +/** Buffers to hand over rather than copy when posting a finished spectrogram. */ +export function collectSpectrogramTransfers( + spectrogram: AudioSpectrogram +): ArrayBuffer[] { + return [ + spectrogram.bands.buffer as ArrayBuffer, + spectrogram.centerHz.buffer as ArrayBuffer, + spectrogram.rms.buffer as ArrayBuffer, + ] +} diff --git a/src/lib/editor/audio/analysis.worker.ts b/src/lib/editor/audio/analysis.worker.ts new file mode 100644 index 0000000..d6870d0 --- /dev/null +++ b/src/lib/editor/audio/analysis.worker.ts @@ -0,0 +1,61 @@ +/// + +import { + type AnalysisRequest, + type AnalysisResponse, + collectSpectrogramTransfers, +} from "@/lib/editor/audio/analysis-protocol" +import { analyzeSpectrogramStepwise } from "@/lib/editor/audio/spectrogram" + +/** + * Stage A off the main thread. Decoding stays on the main thread because Web + * Audio is unavailable here; only the FFT pass runs in the Worker, receiving + * already-decoded PCM. + */ + +const scope = self as unknown as DedicatedWorkerGlobalScope + +function post(message: AnalysisResponse, transfer?: ArrayBuffer[]): void { + if (transfer) { + scope.postMessage(message, transfer) + return + } + + scope.postMessage(message) +} + +scope.onmessage = (event: MessageEvent) => { + const request = event.data + + try { + const iterator = analyzeSpectrogramStepwise( + request.samples, + request.sampleRate, + { + ...(request.bandCount === undefined + ? {} + : { bandCount: request.bandCount }), + ...(request.envelopeRate === undefined + ? {} + : { envelopeRate: request.envelopeRate }), + ...(request.fftSize === undefined ? {} : { fftSize: request.fftSize }), + } + ) + + let step = iterator.next() + while (!step.done) { + post({ progress: step.value, type: "progress" }) + step = iterator.next() + } + + post( + { spectrogram: step.value, type: "done" }, + collectSpectrogramTransfers(step.value) + ) + } catch (error) { + post({ + message: error instanceof Error ? error.message : "Audio analysis failed", + type: "error", + }) + } +} diff --git a/src/lib/editor/audio/analyze-client.ts b/src/lib/editor/audio/analyze-client.ts new file mode 100644 index 0000000..a62d57e --- /dev/null +++ b/src/lib/editor/audio/analyze-client.ts @@ -0,0 +1,171 @@ +import type { + AnalysisRequest, + AnalysisResponse, +} from "@/lib/editor/audio/analysis-protocol" +import { decodeAudioToMono } from "@/lib/editor/audio/decode" +import { + analyzeSpectrogramStepwise, + type AudioSpectrogram, + type SpectrogramOptions, +} from "@/lib/editor/audio/spectrogram" + +export type AnalyzeAudioOptions = SpectrogramOptions & { + onProgress?: (progress: number) => void + signal?: AbortSignal +} + +/** How often the main-thread fallback yields to the event loop. */ +const FALLBACK_YIELD_INTERVAL_MS = 8 + +function createAnalysisWorker(): Worker | null { + if (typeof Worker === "undefined") { + return null + } + + try { + return new Worker(new URL("./analysis.worker.ts", import.meta.url), { + type: "module", + }) + } catch { + return null + } +} + +function runInWorker( + worker: Worker, + request: AnalysisRequest, + options: AnalyzeAudioOptions +): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + worker.onmessage = null + worker.onerror = null + options.signal?.removeEventListener("abort", onAbort) + worker.terminate() + } + + const onAbort = () => { + cleanup() + reject(new DOMException("Audio analysis aborted", "AbortError")) + } + + worker.onmessage = (event: MessageEvent) => { + const message = event.data + + if (message.type === "progress") { + options.onProgress?.(message.progress) + return + } + + if (message.type === "error") { + cleanup() + reject(new Error(message.message)) + return + } + + cleanup() + resolve(message.spectrogram) + } + + worker.onerror = (event) => { + cleanup() + reject(new Error(event.message || "Audio analysis worker failed")) + } + + options.signal?.addEventListener("abort", onAbort, { once: true }) + + // Hand the PCM over rather than copying it — it can be ~57MB for a 5 minute + // track, and the main thread has no further use for it. + worker.postMessage(request, [request.samples.buffer as ArrayBuffer]) + }) +} + +/** + * Main-thread stage A, yielding to the event loop periodically. Used when + * Workers are unavailable (SSR, restrictive environments). + */ +async function runOnMainThread( + request: AnalysisRequest, + options: AnalyzeAudioOptions +): Promise { + const iterator = analyzeSpectrogramStepwise( + request.samples, + request.sampleRate, + { + ...(request.bandCount === undefined + ? {} + : { bandCount: request.bandCount }), + ...(request.envelopeRate === undefined + ? {} + : { envelopeRate: request.envelopeRate }), + ...(request.fftSize === undefined ? {} : { fftSize: request.fftSize }), + } + ) + + let lastYield = performance.now() + let step = iterator.next() + + while (!step.done) { + options.onProgress?.(step.value) + + if (performance.now() - lastYield > FALLBACK_YIELD_INTERVAL_MS) { + await new Promise((resolve) => setTimeout(resolve, 0)) + + if (options.signal?.aborted) { + throw new DOMException("Audio analysis aborted", "AbortError") + } + + lastYield = performance.now() + } + + step = iterator.next() + } + + return step.value +} + +/** + * Decode an audio URL and produce its spectrogram (stage A). + * + * Decoding must happen on the main thread — Web Audio is not available inside a + * Worker — so only the FFT pass is offloaded. + */ +export async function analyzeAudioSource( + url: string, + options: AnalyzeAudioOptions = {} +): Promise { + const decoded = await decodeAudioToMono(url, options.signal) + + const request: AnalysisRequest = { + sampleRate: decoded.sampleRate, + samples: decoded.samples, + ...(options.bandCount === undefined ? {} : { bandCount: options.bandCount }), + ...(options.envelopeRate === undefined + ? {} + : { envelopeRate: options.envelopeRate }), + ...(options.fftSize === undefined ? {} : { fftSize: options.fftSize }), + } + + const worker = createAnalysisWorker() + + if (!worker) { + return runOnMainThread(request, options) + } + + try { + return await runInWorker(worker, request, options) + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") { + throw error + } + + // The buffer was transferred away, so the fallback cannot reuse it. Re-decode + // rather than failing outright. + const retry = await decodeAudioToMono(url, options.signal) + + return runOnMainThread( + { ...request, samples: retry.samples, sampleRate: retry.sampleRate }, + options + ) + } +} diff --git a/src/lib/editor/audio/bands.ts b/src/lib/editor/audio/bands.ts new file mode 100644 index 0000000..8049614 --- /dev/null +++ b/src/lib/editor/audio/bands.ts @@ -0,0 +1,213 @@ +import { + AUDIO_BAND_IDS, + type AudioBandConfig, + type AudioBandId, +} from "@/types/editor" + +/** Envelope samples per second. Matches the rAF loop; lookup interpolates. */ +export const ENVELOPE_RATE = 60 + +export const DEFAULT_FFT_SIZE = 2048 + +/** + * Log-spaced magnitude bands retained per STFT frame. Far finer than needed to + * place three boundaries (~1/7 octave), which is what lets the band editor + * recompute envelopes without re-running the FFT. Doubles as spectrum-analyser + * data for the advanced UI. + */ +export const SPECTRO_BAND_COUNT = 64 + +export const SPECTRO_MIN_HZ = 20 +export const SPECTRO_MAX_HZ = 20000 + +/** dB window mapped onto `[0,1]` after normalization. */ +export const DYNAMIC_RANGE_DB = 48 + +/** Below this, a band is treated as silent rather than amplified to noise. */ +export const SILENCE_REFERENCE_FLOOR = 1e-6 + +/** + * An envelope smoothed faster than the sample rate can represent will alias + * when sampled at low export frame rates, so release is floored at one + * envelope step. + */ +export const MIN_RELEASE_MS = 1000 / ENVELOPE_RATE + +export const DEFAULT_AUDIO_BANDS: Record = { + bass: { attackMs: 8, gainDb: 0, highHz: 140, lowHz: 20, releaseMs: 140 }, + high: { attackMs: 6, gainDb: 0, highHz: 16000, lowHz: 2000, releaseMs: 90 }, + // `level` is full-band RMS; its lowHz/highHz are inert but kept for shape + // uniformity so the UI can render one control set for every band. + level: { attackMs: 20, gainDb: 0, highHz: 20000, lowHz: 20, releaseMs: 220 }, + mid: { attackMs: 8, gainDb: 0, highHz: 2000, lowHz: 140, releaseMs: 110 }, +} + +export function createDefaultAudioBands(): Record< + AudioBandId, + AudioBandConfig +> { + const bands = {} as Record + + for (const bandId of AUDIO_BAND_IDS) { + bands[bandId] = { ...DEFAULT_AUDIO_BANDS[bandId] } + } + + return bands +} + +/** `level` ignores its frequency range because it is derived from RMS. */ +export function isFullBandBand(bandId: AudioBandId): boolean { + return bandId === "level" +} + +export type SpectroBandLayout = { + /** Geometric centre of each band, ascending. Length `bandCount`. */ + centerHz: Float32Array + /** Band edges, ascending. Length `bandCount + 1`. */ + edgeHz: Float32Array +} + +/** + * Geometrically spaced band edges from {@link SPECTRO_MIN_HZ} up to the lesser + * of {@link SPECTRO_MAX_HZ} and nyquist. + */ +export function createSpectroBandLayout( + sampleRate: number, + bandCount: number = SPECTRO_BAND_COUNT +): SpectroBandLayout { + const nyquist = sampleRate / 2 + const maxHz = Math.min(SPECTRO_MAX_HZ, nyquist) + const minHz = Math.min(SPECTRO_MIN_HZ, maxHz / 2) + const ratio = maxHz / minHz + + const edgeHz = new Float32Array(bandCount + 1) + for (let index = 0; index <= bandCount; index += 1) { + edgeHz[index] = minHz * ratio ** (index / bandCount) + } + + const centerHz = new Float32Array(bandCount) + for (let index = 0; index < bandCount; index += 1) { + const lower = edgeHz[index] ?? minHz + const upper = edgeHz[index + 1] ?? maxHz + centerHz[index] = Math.sqrt(lower * upper) + } + + return { centerHz, edgeHz } +} + +export type BinRange = { + /** Inclusive. */ + endBin: number + /** Inclusive, never 0 — the DC bin carries no musical information. */ + startBin: number +} + +/** + * FFT bins overlapping `[lowHz, highHz]`. Always excludes DC and always returns + * at least one bin, so a narrow or inverted range degrades to a single bin + * rather than producing an empty mean. + */ +export function frequencyToBinRange( + lowHz: number, + highHz: number, + fftSize: number, + sampleRate: number +): BinRange { + const nyquistBin = Math.floor(fftSize / 2) + + if (!(Number.isFinite(lowHz) && Number.isFinite(highHz)) || sampleRate <= 0) { + return { endBin: 1, startBin: 1 } + } + + const lower = Math.min(lowHz, highHz) + const upper = Math.max(lowHz, highHz) + const binsPerHz = fftSize / sampleRate + + const startBin = Math.min( + Math.max(Math.floor(lower * binsPerHz), 1), + nyquistBin + ) + const endBin = Math.min( + Math.max(Math.ceil(upper * binsPerHz), startBin), + nyquistBin + ) + + return { endBin, startBin } +} + +export type SpectroBandRange = { + /** Inclusive. */ + endIndex: number + /** Inclusive. */ + startIndex: number +} + +/** + * Spectro bands whose centre falls inside `[lowHz, highHz)`. Falls back to the + * single nearest band when the range is narrower than the spectro resolution, + * so every band config yields a usable envelope. + */ +export function resolveSpectroBandRange( + centerHz: Float32Array, + lowHz: number, + highHz: number +): SpectroBandRange { + const bandCount = centerHz.length + + if (bandCount === 0) { + return { endIndex: 0, startIndex: 0 } + } + + const lower = Math.min(lowHz, highHz) + const upper = Math.max(lowHz, highHz) + + let startIndex = -1 + let endIndex = -1 + + for (let index = 0; index < bandCount; index += 1) { + const center = centerHz[index] ?? 0 + if (center >= lower && center < upper) { + if (startIndex === -1) { + startIndex = index + } + endIndex = index + } + } + + if (startIndex !== -1) { + return { endIndex, startIndex } + } + + // No centre landed inside the range — snap to the nearest band. + const target = (lower + upper) / 2 + let nearest = 0 + let nearestDistance = Number.POSITIVE_INFINITY + + for (let index = 0; index < bandCount; index += 1) { + const distance = Math.abs((centerHz[index] ?? 0) - target) + if (distance < nearestDistance) { + nearestDistance = distance + nearest = index + } + } + + return { endIndex: nearest, startIndex: nearest } +} + +/** Keeps a user-edited band config inside physically sensible bounds. */ +export function clampBandConfig( + config: AudioBandConfig, + sampleRate = 48000 +): AudioBandConfig { + const nyquist = sampleRate / 2 + const lowHz = Math.min(Math.max(config.lowHz, 1), nyquist) + const highHz = Math.min(Math.max(config.highHz, lowHz + 1), nyquist) + + return { + attackMs: Math.min(Math.max(config.attackMs, 0), 2000), + gainDb: Math.min(Math.max(config.gainDb, -24), 24), + highHz, + lowHz, + releaseMs: Math.min(Math.max(config.releaseMs, MIN_RELEASE_MS), 4000), + } +} diff --git a/src/lib/editor/audio/decode.ts b/src/lib/editor/audio/decode.ts new file mode 100644 index 0000000..c417f5c --- /dev/null +++ b/src/lib/editor/audio/decode.ts @@ -0,0 +1,91 @@ +import { downmixToMono } from "@/lib/editor/audio/spectrogram" + +/** + * The only module in `lib/editor/audio` permitted to touch Web Audio. + * + * Everything else is pure so the analysis pipeline stays unit-testable under + * `bun test` and runnable inside a Worker. Keep it that way. + */ + +export type DecodedAudio = { + sampleRate: number + samples: Float32Array +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) { + throw new DOMException("Audio analysis aborted", "AbortError") + } +} + +/** + * Decode any browser-supported audio container to mono PCM. + * + * Uses `OfflineAudioContext` rather than a live `AudioContext`: it has no + * autoplay-policy user-gesture requirement and nothing to leak. This also + * transparently handles the audio track of an mp4/webm, which is what makes + * "use a video layer's audio" the same code path with no extra work. + */ +export async function decodeAudioToMono( + url: string, + signal?: AbortSignal +): Promise { + throwIfAborted(signal) + + const response = await fetch(url, signal ? { signal } : {}) + + if (!response.ok) { + throw new Error(`Failed to fetch audio (${response.status})`) + } + + const encoded = await response.arrayBuffer() + throwIfAborted(signal) + + const buffer = await decodeArrayBuffer(encoded) + throwIfAborted(signal) + + const channels: Float32Array[] = [] + for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) { + channels.push(buffer.getChannelData(channel)) + } + + return { + sampleRate: buffer.sampleRate, + samples: downmixToMono(channels), + } +} + +async function decodeArrayBuffer(encoded: ArrayBuffer): Promise { + // Sample rate and length here are placeholders — decodeAudioData returns the + // file's own rate regardless. + const OfflineCtor = + globalThis.OfflineAudioContext ?? + (globalThis as { webkitOfflineAudioContext?: typeof OfflineAudioContext }) + .webkitOfflineAudioContext + + if (OfflineCtor) { + const context = new OfflineCtor(1, 1, 44100) + + try { + return await context.decodeAudioData(encoded) + } catch (error) { + // Some Safari versions reject OfflineAudioContext.decodeAudioData for + // certain containers; fall through to a real context below. + if (!globalThis.AudioContext) { + throw error + } + } + } + + if (!globalThis.AudioContext) { + throw new Error("This browser cannot decode audio.") + } + + const context = new AudioContext() + + try { + return await context.decodeAudioData(encoded) + } finally { + void context.close() + } +} diff --git a/src/lib/editor/audio/envelope-lookup.ts b/src/lib/editor/audio/envelope-lookup.ts new file mode 100644 index 0000000..4fbd35f --- /dev/null +++ b/src/lib/editor/audio/envelope-lookup.ts @@ -0,0 +1,107 @@ +import type { AudioEnvelopeSet } from "@/lib/editor/audio/envelope" +import { AUDIO_BAND_IDS, type AudioBandId } from "@/types/editor" + +/** + * Band value at `time`, linearly interpolated between envelope samples. + * + * This is the single seam through which every consumer — the live render loop, + * the offline exporter, agent screenshots, the sidebar readout — reads audio. + * Keeping it a pure function of `time` is what makes exported video match the + * preview frame for frame. + * + * Clamps rather than wraps at both ends: a 6 second timeline over a 4 minute + * track should hold the last value, not loop the intro. `offsetSeconds` is how + * the user chooses which part of a long track a short timeline sees. + */ +export function sampleBand( + envelopes: AudioEnvelopeSet, + bandId: AudioBandId, + offsetSeconds: number, + time: number +): number { + const envelope = envelopes.bands[bandId] + + if (!envelope || envelope.length === 0) { + return 0 + } + + const shifted = time + offsetSeconds + + if (!Number.isFinite(shifted)) { + return 0 + } + + const position = shifted * envelopes.envelopeRate + + if (position <= 0) { + return envelope[0] ?? 0 + } + + const lastIndex = envelope.length - 1 + + if (position >= lastIndex) { + return envelope[lastIndex] ?? 0 + } + + const lowerIndex = Math.floor(position) + const lower = envelope[lowerIndex] ?? 0 + const upper = envelope[lowerIndex + 1] ?? lower + + return lower + (upper - lower) * (position - lowerIndex) +} + +/** Every band at one instant. Four array lookups — safe to call per frame. */ +export function sampleAllBands( + envelopes: AudioEnvelopeSet, + offsetSeconds: number, + time: number +): Record { + const values = {} as Record + + for (const bandId of AUDIO_BAND_IDS) { + values[bandId] = sampleBand(envelopes, bandId, offsetSeconds, time) + } + + return values +} + +/** + * Downsample an envelope to `targetCount` peaks for drawing. Takes the maximum + * of each bucket rather than the mean so transients stay visible at any zoom + * level — a mean-reduced waveform looks limp. + */ +export function sampleEnvelopeToPeaks( + envelope: Float32Array, + targetCount: number +): Float32Array { + if (targetCount <= 0 || envelope.length === 0) { + return new Float32Array(0) + } + + if (envelope.length <= targetCount) { + return Float32Array.from(envelope) + } + + const peaks = new Float32Array(targetCount) + const bucketSize = envelope.length / targetCount + + for (let index = 0; index < targetCount; index += 1) { + const start = Math.floor(index * bucketSize) + const end = Math.min( + Math.max(Math.floor((index + 1) * bucketSize), start + 1), + envelope.length + ) + + let peak = 0 + for (let cursor = start; cursor < end; cursor += 1) { + const value = envelope[cursor] ?? 0 + if (value > peak) { + peak = value + } + } + + peaks[index] = peak + } + + return peaks +} diff --git a/src/lib/editor/audio/envelope.ts b/src/lib/editor/audio/envelope.ts new file mode 100644 index 0000000..722b3b5 --- /dev/null +++ b/src/lib/editor/audio/envelope.ts @@ -0,0 +1,397 @@ +import { + DYNAMIC_RANGE_DB, + isFullBandBand, + resolveSpectroBandRange, + SILENCE_REFERENCE_FLOOR, +} from "@/lib/editor/audio/bands" +import type { AudioSpectrogram } from "@/lib/editor/audio/spectrogram" +import { + AUDIO_BAND_IDS, + type AudioBandConfig, + type AudioBandId, +} from "@/types/editor" + +/** + * Stage B output: one normalized, smoothed envelope per band. Cheap to rebuild + * from a cached {@link AudioSpectrogram}, which is what makes editing band + * frequency ranges feel instant. + * + * Every sample is guaranteed to lie in `[0,1]` — normalization clamps and a + * one-pole filter cannot exceed its input range. Downstream modulation relies + * on this. + */ +export type AudioEnvelopeSet = { + bands: Record + durationSeconds: number + envelopeRate: number + sampleCount: number + /** Bands with no usable energy, so the UI can explain a dead control. */ + silentBands: AudioBandId[] +} + +const HISTOGRAM_BUCKETS = 1024 +const HISTOGRAM_MIN_DB = -120 +const HISTOGRAM_MAX_DB = 20 +const HISTOGRAM_SPAN_DB = HISTOGRAM_MAX_DB - HISTOGRAM_MIN_DB + +/** Floor for the log of a magnitude, so silence maps to the bottom of the range. */ +const MAGNITUDE_EPSILON = 1e-9 + +/** Fraction of frames allowed to exceed the reference level and clip at 1.0. */ +const REFERENCE_PERCENTILE = 0.99 + +/** + * A frequency band this far below the loudest frequency band is treated as + * silent. + * + * Necessary because normalization is per-band: without a gate, a band + * containing nothing but FFT spectral leakage would have its own tiny reference + * level and therefore normalize that leakage up to full scale — a bass-only + * track would show a fake, full-swing "high" band that merely tracks the bass. + * 60 dB is far below the spread between bands in real music, so this only ever + * catches bands with genuinely no content. + */ +const RELATIVE_SILENCE_DB = 60 + +/** + * Floor on the self-calibrating normalization span. Without it, a band whose + * level barely changes would have its tiny variation stretched to a full 0-to-1 + * swing, turning noise into apparent signal. + */ +const MIN_NORMALIZATION_SPAN_DB = 9 + +function clamp01(value: number): number { + if (!Number.isFinite(value)) { + return 0 + } + + return Math.min(Math.max(value, 0), 1) +} + +function toDecibels(magnitude: number): number { + return 20 * Math.log10(Math.max(magnitude, MAGNITUDE_EPSILON)) +} + +function bucketToDecibels(bucket: number): number { + return ( + HISTOGRAM_MIN_DB + + ((bucket + 0.5) / HISTOGRAM_BUCKETS) * HISTOGRAM_SPAN_DB + ) +} + +/** + * dB-domain histogram of a series. O(n), no per-sample allocation and no sort, + * so percentiles over an 18k-frame track cost one pass. + */ +function computeDbHistogram(series: Float32Array): Uint32Array { + const histogram = new Uint32Array(HISTOGRAM_BUCKETS) + + for (const value of series) { + const decibels = toDecibels(value) + const normalized = (decibels - HISTOGRAM_MIN_DB) / HISTOGRAM_SPAN_DB + const bucket = Math.min( + Math.max(Math.floor(normalized * HISTOGRAM_BUCKETS), 0), + HISTOGRAM_BUCKETS - 1 + ) + histogram[bucket] = (histogram[bucket] ?? 0) + 1 + } + + return histogram +} + +function percentileDecibels( + histogram: Uint32Array, + sampleCount: number, + percentile: number +): number { + const threshold = Math.max(1, Math.ceil(sampleCount * percentile)) + let cumulative = 0 + + for (let bucket = 0; bucket < HISTOGRAM_BUCKETS; bucket += 1) { + cumulative += histogram[bucket] ?? 0 + if (cumulative >= threshold) { + return bucketToDecibels(bucket) + } + } + + return HISTOGRAM_MAX_DB +} + +/** + * Reference level for the cross-band silence gate: the + * {@link REFERENCE_PERCENTILE} of the series. + * + * Deliberately not the peak — one clap or one clipped sample would otherwise + * dominate, misreporting how much content a band really has. + */ +export function computeReferenceLevel(series: Float32Array): number { + if (series.length === 0) { + return 0 + } + + const histogram = computeDbHistogram(series) + + return ( + 10 ** + (percentileDecibels(histogram, series.length, REFERENCE_PERCENTILE) / 20) + ) +} + +export type BandNormalization = { + /** Input dB that maps to 0. */ + lowDb: number + /** dB span mapped onto `[0,1]`. */ + spanDb: number +} + +/** + * Self-calibrating normalization window, derived from the band's own dB + * distribution: the 5th percentile maps to 0 and the 99th to 1. + * + * A *fixed* dB window cannot work across real material. Measured against a + * loudness-compressed master, a 48dB window left every band clustered between + * 0.7 and 0.95 — visually static, because a mastered track's band energy only + * varies 10-20dB. Anchoring to the track's own distribution means every band + * uses the full range whether the source is a compressed club master or a + * dynamic acoustic recording. + * + * The span is clamped: {@link MIN_NORMALIZATION_SPAN_DB} stops a nearly + * constant band from being stretched into a full-swing signal built from noise, + * and {@link DYNAMIC_RANGE_DB} caps how much of a very dynamic track's range is + * compressed into `[0,1]`. + */ +export function computeBandNormalization( + series: Float32Array +): BandNormalization { + if (series.length === 0) { + return { lowDb: -DYNAMIC_RANGE_DB, spanDb: DYNAMIC_RANGE_DB } + } + + const histogram = computeDbHistogram(series) + const highDb = percentileDecibels(histogram, series.length, 0.99) + const lowDb = percentileDecibels(histogram, series.length, 0.05) + + const spanDb = Math.min( + Math.max(highDb - lowDb, MIN_NORMALIZATION_SPAN_DB), + DYNAMIC_RANGE_DB + ) + + return { lowDb: highDb - spanDb, spanDb } +} + +/** + * Raw per-frame magnitude series for one band: the mean of the spectro bands + * overlapping its frequency range, or the precomputed RMS track for `level`. + */ +export function extractBandSeries( + spectrogram: AudioSpectrogram, + bandId: AudioBandId, + config: AudioBandConfig +): Float32Array { + if (isFullBandBand(bandId)) { + return spectrogram.rms + } + + const { bandCount, bands, centerHz, frameCount } = spectrogram + const range = resolveSpectroBandRange(centerHz, config.lowHz, config.highHz) + const width = range.endIndex - range.startIndex + 1 + const series = new Float32Array(frameCount) + + for (let frame = 0; frame < frameCount; frame += 1) { + const rowStart = frame * bandCount + let sum = 0 + + for (let index = range.startIndex; index <= range.endIndex; index += 1) { + sum += bands[rowStart + index] ?? 0 + } + + series[frame] = width > 0 ? sum / width : 0 + } + + return series +} + +/** + * One-pole asymmetric smoothing. Applied *after* normalization so the `[0,1]` + * invariant is preserved: a one-pole filter is a convex blend of its previous + * output and its input, so it can never leave their shared range. + */ +export function smoothEnvelopeInPlace( + envelope: Float32Array, + attackMs: number, + releaseMs: number, + envelopeRate: number +): void { + const hopSeconds = 1 / envelopeRate + const attackCoefficient = + 1 - Math.exp(-hopSeconds / Math.max(attackMs / 1000, 1e-6)) + const releaseCoefficient = + 1 - Math.exp(-hopSeconds / Math.max(releaseMs / 1000, 1e-6)) + + let current = 0 + + for (let index = 0; index < envelope.length; index += 1) { + const target = envelope[index] ?? 0 + const coefficient = + target > current ? attackCoefficient : releaseCoefficient + current += (target - current) * coefficient + envelope[index] = current + } +} + +export type BandEnvelopeResult = { + envelope: Float32Array + silent: boolean +} + +type BandMeasurement = { + normalization: BandNormalization + peak: number + reference: number + series: Float32Array +} + +/** + * Extract a band's raw series along with the two statistics the gate and + * normalization need. Measuring once and reusing the result is what keeps stage + * B cheap enough to run on every band-config change. + */ +function measureBand( + spectrogram: AudioSpectrogram, + bandId: AudioBandId, + config: AudioBandConfig +): BandMeasurement { + const series = extractBandSeries(spectrogram, bandId, config) + + // Absolute silence is detected from the peak, not the reference level: the + // histogram's lowest bucket maps to a small-but-nonzero magnitude, so an + // all-zero series would otherwise slip past a reference-based check. + let peak = 0 + for (const value of series) { + peak = Math.max(peak, value) + } + + return { + normalization: computeBandNormalization(series), + peak, + reference: computeReferenceLevel(series), + series, + } +} + +function normalizeAndSmooth( + measurement: BandMeasurement, + config: AudioBandConfig, + envelopeRate: number +): Float32Array { + const { normalization, series } = measurement + const { lowDb, spanDb } = normalization + const envelope = new Float32Array(series.length) + + for (let index = 0; index < series.length; index += 1) { + // dB rather than linear magnitude: perceptually proportional, so a value + // driven by it tracks how loud the music *sounds*. + const decibels = toDecibels(series[index] ?? 0) + config.gainDb + envelope[index] = clamp01((decibels - lowDb) / spanDb) + } + + smoothEnvelopeInPlace( + envelope, + config.attackMs, + config.releaseMs, + envelopeRate + ) + + return envelope +} + +function isSilent(measurement: BandMeasurement, referenceFloor: number): boolean { + return ( + measurement.peak < SILENCE_REFERENCE_FLOOR || + measurement.reference < referenceFloor + ) +} + +/** + * Stage B for a single band. + * + * `referenceFloor` gates bands with no real content; pass 0 to disable. See + * {@link RELATIVE_SILENCE_DB}. + */ +export function computeBandEnvelope( + spectrogram: AudioSpectrogram, + bandId: AudioBandId, + config: AudioBandConfig, + referenceFloor = 0 +): BandEnvelopeResult { + const measurement = measureBand(spectrogram, bandId, config) + + if (isSilent(measurement, referenceFloor)) { + return { + envelope: new Float32Array(measurement.series.length), + silent: true, + } + } + + return { + envelope: normalizeAndSmooth( + measurement, + config, + spectrogram.envelopeRate + ), + silent: false, + } +} + +/** + * Stage B for every band. Pure and fast (sub-millisecond for a 5 minute track), + * so it can run synchronously on every band-config change. + */ +export function computeEnvelopeSet( + spectrogram: AudioSpectrogram, + bands: Record +): AudioEnvelopeSet { + const envelopes = {} as Record + const silentBands: AudioBandId[] = [] + + // Measure every band once, then gate relative to the loudest *frequency* + // band. `level` is excluded from the comparison because it is derived from + // RMS and so is not on a scale comparable to the magnitude means. + const measurements = {} as Record + let loudestReference = 0 + + for (const bandId of AUDIO_BAND_IDS) { + const measurement = measureBand(spectrogram, bandId, bands[bandId]) + measurements[bandId] = measurement + + if (!isFullBandBand(bandId)) { + loudestReference = Math.max(loudestReference, measurement.reference) + } + } + + const referenceFloor = loudestReference * 10 ** (-RELATIVE_SILENCE_DB / 20) + + for (const bandId of AUDIO_BAND_IDS) { + const measurement = measurements[bandId] + + if (isSilent(measurement, isFullBandBand(bandId) ? 0 : referenceFloor)) { + envelopes[bandId] = new Float32Array(measurement.series.length) + silentBands.push(bandId) + continue + } + + envelopes[bandId] = normalizeAndSmooth( + measurement, + bands[bandId], + spectrogram.envelopeRate + ) + } + + return { + bands: envelopes, + durationSeconds: spectrogram.durationSeconds, + envelopeRate: spectrogram.envelopeRate, + sampleCount: spectrogram.frameCount, + silentBands, + } +} diff --git a/src/lib/editor/audio/fft.ts b/src/lib/editor/audio/fft.ts new file mode 100644 index 0000000..3888ba2 --- /dev/null +++ b/src/lib/editor/audio/fft.ts @@ -0,0 +1,259 @@ +/** + * Radix-2 Cooley-Tukey FFT with cached twiddle factors, bit-reversal tables and + * Hann windows. + * + * Deliberately dependency-free and free of browser globals so the whole audio + * analysis pipeline stays unit-testable under `bun test` and portable into a + * Web Worker. Only `decode.ts` may touch Web Audio. + */ + +type TwiddleTable = { + cos: Float64Array + sin: Float64Array +} + +const bitReversalCache = new Map() +const hannWindowCache = new Map() +const twiddleCache = new Map() + +export function isPowerOfTwo(value: number): boolean { + return Number.isInteger(value) && value > 0 && (value & (value - 1)) === 0 +} + +function assertFftSize(size: number): void { + if (!isPowerOfTwo(size) || size < 2) { + throw new Error( + `fft size must be a power of two and at least 2, received ${size}` + ) + } +} + +/** + * Periodic Hann window. Periodic (dividing by `size`) rather than symmetric + * (`size - 1`) because consecutive STFT frames overlap — the periodic form is + * what sums to a constant under overlap-add. + */ +export function getHannWindow(size: number): Float64Array { + assertFftSize(size) + + const cached = hannWindowCache.get(size) + if (cached) { + return cached + } + + const window = new Float64Array(size) + for (let index = 0; index < size; index += 1) { + window[index] = 0.5 * (1 - Math.cos((2 * Math.PI * index) / size)) + } + + hannWindowCache.set(size, window) + + return window +} + +export function getWindowSum(window: Float64Array): number { + let sum = 0 + for (const coefficient of window) { + sum += coefficient + } + + return sum +} + +function getBitReversalTable(size: number): Uint32Array { + const cached = bitReversalCache.get(size) + if (cached) { + return cached + } + + const bitCount = Math.log2(size) + const table = new Uint32Array(size) + + for (let index = 0; index < size; index += 1) { + let reversed = 0 + for (let bit = 0; bit < bitCount; bit += 1) { + if ((index & (1 << bit)) !== 0) { + reversed |= 1 << (bitCount - 1 - bit) + } + } + table[index] = reversed + } + + bitReversalCache.set(size, table) + + return table +} + +/** + * Twiddle factors `exp(-2i*pi*j/size)` for `j < size / 2`. A butterfly at stage + * `len` and offset `k` reads index `k * (size / len)`, so one table serves every + * stage. + */ +function getTwiddleTable(size: number): TwiddleTable { + const cached = twiddleCache.get(size) + if (cached) { + return cached + } + + const half = size / 2 + const cos = new Float64Array(half) + const sin = new Float64Array(half) + + for (let index = 0; index < half; index += 1) { + const angle = (-2 * Math.PI * index) / size + cos[index] = Math.cos(angle) + sin[index] = Math.sin(angle) + } + + const table: TwiddleTable = { cos, sin } + twiddleCache.set(size, table) + + return table +} + +/** In-place forward FFT. `real` and `imag` must be the same power-of-two length. */ +export function fftInPlace(real: Float64Array, imag: Float64Array): void { + const size = real.length + assertFftSize(size) + + if (imag.length !== size) { + throw new Error( + `fft real and imag buffers must match in length, received ${size} and ${imag.length}` + ) + } + + const reversal = getBitReversalTable(size) + for (let index = 0; index < size; index += 1) { + const target = reversal[index] ?? 0 + if (target > index) { + const realTemp = real[index] ?? 0 + const imagTemp = imag[index] ?? 0 + real[index] = real[target] ?? 0 + imag[index] = imag[target] ?? 0 + real[target] = realTemp + imag[target] = imagTemp + } + } + + const twiddle = getTwiddleTable(size) + + for (let length = 2; length <= size; length *= 2) { + const half = length / 2 + const stride = size / length + + for (let start = 0; start < size; start += length) { + for (let offset = 0; offset < half; offset += 1) { + const twiddleIndex = offset * stride + const cos = twiddle.cos[twiddleIndex] ?? 0 + const sin = twiddle.sin[twiddleIndex] ?? 0 + + const lowIndex = start + offset + const highIndex = lowIndex + half + + const highReal = real[highIndex] ?? 0 + const highImag = imag[highIndex] ?? 0 + const rotatedReal = highReal * cos - highImag * sin + const rotatedImag = highReal * sin + highImag * cos + + const lowReal = real[lowIndex] ?? 0 + const lowImag = imag[lowIndex] ?? 0 + + real[highIndex] = lowReal - rotatedReal + imag[highIndex] = lowImag - rotatedImag + real[lowIndex] = lowReal + rotatedReal + imag[lowIndex] = lowImag + rotatedImag + } + } + } +} + +export type FftWorkspace = { + imag: Float64Array + magnitudes: Float32Array + real: Float64Array + size: number + window: Float64Array + windowScale: number +} + +/** + * Allocate reusable buffers for repeated STFT frames. Analysing a 5 minute + * track runs ~18k frames, so per-frame allocation is not an option. + * + * `windowScale` normalizes magnitudes so a full-scale sine reads ~1.0 at its + * peak bin regardless of window or fft size, which keeps thresholds meaningful + * and tests readable. + */ +export function createFftWorkspace(size: number): FftWorkspace { + assertFftSize(size) + + const window = getHannWindow(size) + + return { + imag: new Float64Array(size), + magnitudes: new Float32Array(size / 2 + 1), + real: new Float64Array(size), + size, + window, + windowScale: 2 / getWindowSum(window), + } +} + +/** + * Window `samples` (starting at `offset`, zero-padded outside its bounds), run + * the FFT, and write the normalized magnitude spectrum into + * `workspace.magnitudes`. Returns that same buffer — it is overwritten on every + * call, so callers must consume it before the next frame. + */ +export function computeFrameMagnitudes( + workspace: FftWorkspace, + samples: Float32Array, + offset: number +): Float32Array { + const { imag, magnitudes, real, size, window, windowScale } = workspace + + for (let index = 0; index < size; index += 1) { + const sampleIndex = offset + index + const sample = + sampleIndex >= 0 && sampleIndex < samples.length + ? (samples[sampleIndex] ?? 0) + : 0 + real[index] = sample * (window[index] ?? 0) + imag[index] = 0 + } + + fftInPlace(real, imag) + + const binCount = size / 2 + 1 + for (let bin = 0; bin < binCount; bin += 1) { + const binReal = real[bin] ?? 0 + const binImag = imag[bin] ?? 0 + magnitudes[bin] = Math.hypot(binReal, binImag) * windowScale + } + + return magnitudes +} + +/** Centre frequency of an fft bin, in Hz. */ +export function binToFrequency( + bin: number, + size: number, + sampleRate: number +): number { + return (bin * sampleRate) / size +} + +/** Nearest fft bin for a frequency, clamped to the usable range. */ +export function frequencyToBin( + frequencyHz: number, + size: number, + sampleRate: number +): number { + if (!Number.isFinite(frequencyHz) || sampleRate <= 0) { + return 0 + } + + const bin = Math.round((frequencyHz * size) / sampleRate) + + return Math.min(Math.max(bin, 0), size / 2) +} diff --git a/src/lib/editor/audio/links.ts b/src/lib/editor/audio/links.ts new file mode 100644 index 0000000..9dafbdc --- /dev/null +++ b/src/lib/editor/audio/links.ts @@ -0,0 +1,254 @@ +import { getLayerBindingKey } from "@/lib/editor/binding-key" +import { getLayerDefinition } from "@/lib/editor/config/layer-registry" +import { resolveAudioLinkValue } from "@/lib/editor/audio/modulate" +import type { AudioEnvelopeSet } from "@/lib/editor/audio/envelope" +import { sampleAllBands } from "@/lib/editor/audio/envelope-lookup" +import { + getParameterDefinition, + isParameterAudioModulatable, +} from "@/lib/editor/parameter-schema" +import type { EvaluatedLayerState } from "@/lib/editor/timeline/evaluate" +import type { + AnimatedPropertyBinding, + AudioBandId, + AudioLink, + AudioLinkComponent, + EditorLayer, + ParameterDefinition, + ParameterValue, + TimelineTrack, +} from "@/types/editor" + +export type AudioModulationInput = { + envelopes: AudioEnvelopeSet + links: AudioLink[] + offsetSeconds: number +} + +export type CreateAudioLinkInput = { + band: AudioBandId + binding: AnimatedPropertyBinding + component?: AudioLinkComponent + enabled?: boolean + id: string + layerId: string + outMax: number + outMin: number + quantize?: boolean + threshold?: number +} + +/** + * Only constructor for {@link AudioLink}. + * + * `exactOptionalPropertyTypes` makes `{ component: undefined }` a type error and + * a latent `.lab` round-trip hazard, so the optional fields are *omitted* rather + * than set to undefined. Building these literals inline invites that bug. + */ +export function createAudioLink(input: CreateAudioLinkInput): AudioLink { + const link: AudioLink = { + band: input.band, + binding: input.binding, + enabled: input.enabled ?? true, + id: input.id, + layerId: input.layerId, + outMax: input.outMax, + outMin: input.outMin, + } + + return { + ...link, + ...(input.component === undefined ? {} : { component: input.component }), + ...(input.quantize === undefined ? {} : { quantize: input.quantize }), + ...(input.threshold === undefined ? {} : { threshold: input.threshold }), + } +} + +export type AudioLinkPatch = Partial> + +/** Apply a patch while preserving the omit-not-undefined invariant. */ +export function patchAudioLink(link: AudioLink, patch: AudioLinkPatch): AudioLink { + const next: AudioLink = { ...link } + + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) { + delete next[key as keyof AudioLink] + continue + } + + Object.assign(next, { [key]: value }) + } + + return next +} + +export function getAudioLinkKey(link: AudioLink): string { + return getLayerBindingKey(link.layerId, link.binding) +} + +export function findAudioLink( + links: AudioLink[], + layerId: string, + binding: AnimatedPropertyBinding +): AudioLink | null { + const key = getLayerBindingKey(layerId, binding) + + return links.find((link) => getAudioLinkKey(link) === key) ?? null +} + +export function hasAudioLink( + links: AudioLink[], + layerId: string, + binding: AnimatedPropertyBinding +): boolean { + return findAudioLink(links, layerId, binding) !== null +} + +export type AudioLinkConflict = { + link: AudioLink + track: TimelineTrack +} + +/** + * Links that target the same layer+binding as an existing keyframe track. + * + * Audio takes precedence in the engine, so a conflict is never an error — but + * the user should be told their keyframes are being overridden, and offered + * either removal or a bake-to-keyframes conversion. + */ +export function findConflictingAudioLinks( + links: AudioLink[], + tracks: TimelineTrack[] +): AudioLinkConflict[] { + if (links.length === 0 || tracks.length === 0) { + return [] + } + + const trackByKey = new Map() + for (const track of tracks) { + trackByKey.set(getLayerBindingKey(track.layerId, track.binding), track) + } + + const conflicts: AudioLinkConflict[] = [] + for (const link of links) { + const track = trackByKey.get(getAudioLinkKey(link)) + if (track) { + conflicts.push({ link, track }) + } + } + + return conflicts +} + +function resolveDefinition( + layer: EditorLayer, + binding: AnimatedPropertyBinding +): ParameterDefinition | null { + if (binding.kind === "layer") { + return null + } + + return getParameterDefinition(getLayerDefinition(layer.type).params, binding.key) +} + +/** + * Layer audio-driven values on top of the keyframe evaluation result. + * + * Runs *after* `evaluateTimelineForLayers` and receives its output, so audio + * wins on conflict and so a per-component vec link merges into a keyframed + * value rather than discarding it. + * + * Returns a fresh array of fresh states — never aliases anything from + * `layers`, which is what keeps `renderer/contracts.ts`'s `paramsCloneCache` + * safe. + */ +export function applyAudioModulation( + layers: EditorLayer[], + keyframeStates: EvaluatedLayerState[], + audio: AudioModulationInput, + time: number +): EvaluatedLayerState[] { + const activeLinks = audio.links.filter((link) => link.enabled) + + if (activeLinks.length === 0) { + return keyframeStates + } + + const layerById = new Map(layers.map((layer) => [layer.id, layer])) + + // Copy the incoming states so callers keep their originals intact. + const stateByLayerId = new Map() + for (const state of keyframeStates) { + stateByLayerId.set(state.layerId, { + layerId: state.layerId, + params: { ...state.params }, + properties: { ...state.properties }, + }) + } + + const bandValues = sampleAllBands(audio.envelopes, audio.offsetSeconds, time) + + for (const link of activeLinks) { + const layer = layerById.get(link.layerId) + + // Links pointing at deleted layers are inert, matching how + // `evaluateTimelineForLayers` filters orphaned tracks rather than pruning. + if (!layer) { + continue + } + + const definition = resolveDefinition(layer, link.binding) + + if (link.binding.kind === "param") { + if (definition === null) { + continue + } + + if (!isParameterAudioModulatable(definition)) { + continue + } + } + + let state = stateByLayerId.get(link.layerId) + if (!state) { + state = { layerId: link.layerId, params: {}, properties: {} } + stateByLayerId.set(link.layerId, state) + } + + // Base is what this parameter would otherwise be this frame: the keyframe + // result if one exists, else the stored value. Only vec merges use it. + const base: ParameterValue | undefined = + link.binding.kind === "param" + ? (state.params[link.binding.key] ?? layer.params[link.binding.key]) + : undefined + + const value = resolveAudioLinkValue( + link, + definition, + base, + bandValues[link.band] + ) + + if (value === null) { + continue + } + + if (link.binding.kind === "param") { + state.params[link.binding.key] = value + continue + } + + if (link.binding.property === "visible") { + if (typeof value === "boolean") { + state.properties.visible = value + } + continue + } + + if (typeof value === "number") { + state.properties[link.binding.property] = value + } + } + + return [...stateByLayerId.values()] +} diff --git a/src/lib/editor/audio/live-band-driver.ts b/src/lib/editor/audio/live-band-driver.ts new file mode 100644 index 0000000..cf6dfd7 --- /dev/null +++ b/src/lib/editor/audio/live-band-driver.ts @@ -0,0 +1,86 @@ +"use client" + +import { sampleBand } from "@/lib/editor/audio/envelope-lookup" +import { useAudioStore } from "@/store/audio-store" +import { useTimelineStore } from "@/store/timeline-store" +import { AUDIO_BAND_IDS, type AudioBandId } from "@/types/editor" + +/** + * Publishes live band values as CSS custom properties on the document root. + * + * Deliberately not React state. These change 60 times a second; routing them + * through `useState` would re-render the properties sidebar every frame and make + * any audio-driven slider fight the user's pointer. Meters and fills read + * `var(--audio-band-bass)` directly, so the browser animates them with no React + * involvement at all. + * + * Reference-counted singleton: many components can ask for it, one loop runs. + */ + +const bandVariableNames: Record = { + bass: "--audio-band-bass", + high: "--audio-band-high", + level: "--audio-band-level", + mid: "--audio-band-mid", +} + +export function getBandVariableName(bandId: AudioBandId): string { + return bandVariableNames[bandId] +} + +let subscriberCount = 0 +let frameId: number | null = null + +function writeBandValues(): void { + const root = document.documentElement + const { envelopes, offsetSeconds } = useAudioStore.getState() + const time = useTimelineStore.getState().currentTime + + for (const bandId of AUDIO_BAND_IDS) { + const value = envelopes + ? sampleBand(envelopes, bandId, offsetSeconds, time) + : 0 + + root.style.setProperty(bandVariableNames[bandId], value.toFixed(4)) + } +} + +function tick(): void { + writeBandValues() + frameId = window.requestAnimationFrame(tick) +} + +/** Start the driver (if needed) and return a release function. */ +export function acquireLiveBandDriver(): () => void { + if (typeof window === "undefined") { + return () => undefined + } + + subscriberCount += 1 + + if (frameId === null) { + frameId = window.requestAnimationFrame(tick) + } + + let released = false + + return () => { + if (released) { + return + } + + released = true + subscriberCount -= 1 + + if (subscriberCount <= 0 && frameId !== null) { + window.cancelAnimationFrame(frameId) + frameId = null + subscriberCount = 0 + + const root = document.documentElement + for (const bandId of AUDIO_BAND_IDS) { + root.style.setProperty(bandVariableNames[bandId], "0") + } + } + } +} diff --git a/src/lib/editor/audio/modulate.ts b/src/lib/editor/audio/modulate.ts new file mode 100644 index 0000000..c2a9aeb --- /dev/null +++ b/src/lib/editor/audio/modulate.ts @@ -0,0 +1,235 @@ +import { cloneParameterValue } from "@/lib/editor/parameter-schema" +import type { + AudioLink, + LayerAnimatableProperty, + ParameterDefinition, + ParameterValue, +} from "@/types/editor" + +/** Matches the bounds enforced by `clampLayerAdjustments` in `lib/editor/layers.ts`. */ +const LAYER_PROPERTY_BOUNDS: Record< + "hue" | "opacity" | "saturation", + { max: number; min: number } +> = { + hue: { max: 180, min: -180 }, + opacity: { max: 1, min: 0 }, + saturation: { max: 2, min: 0 }, +} + +const DEFAULT_BOOLEAN_THRESHOLD = 0.5 + +function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max) +} + +/** + * Range remap: the band's `[0,1]` envelope onto the link's output range. + * + * `outMin > outMax` is legal and inverts the response. Because envelope samples + * are guaranteed to be in `[0,1]`, the result always lies between the two + * bounds — the clamps applied afterwards guard against a bad `outMin`/`outMax`, + * not against the envelope. + */ +export function remapBand( + bandValue: number, + outMin: number, + outMax: number +): number { + if (!Number.isFinite(bandValue)) { + return outMin + } + + return outMin + (outMax - outMin) * clamp(bandValue, 0, 1) +} + +function quantizeToStep(value: number, step: number | undefined): number { + if (!step || step <= 0) { + return value + } + + return Math.round(value / step) * step +} + +/** + * Resolve a numeric parameter value. + * + * Clamps only against bounds the definition actually declares. The properties + * sidebar falls back to `min 0 / max 100` for sliders, but those are widget + * affordances — applying them here would invent a ceiling the shader does not + * have (e.g. on an unbounded `scale`). + * + * `step` is not honoured by default: `updateLayerParam` does not enforce it and + * keyframe interpolation produces continuous values, so quantizing would make + * audio the only modulation source that snaps. Opt in per link via `quantize`. + */ +export function resolveNumberValue( + link: AudioLink, + definition: ParameterDefinition, + bandValue: number +): number { + const raw = remapBand(bandValue, link.outMin, link.outMax) + + const min = "min" in definition ? definition.min : undefined + const max = "max" in definition ? definition.max : undefined + const step = "step" in definition ? definition.step : undefined + + let value = clamp( + raw, + min ?? Number.NEGATIVE_INFINITY, + max ?? Number.POSITIVE_INFINITY + ) + + if (link.quantize) { + value = quantizeToStep(value, step) + } + + if (definition.type === "number" && definition.input === "int") { + value = Math.round(value) + } + + // Rounding or quantizing can push back past a bound. + return clamp( + value, + min ?? Number.NEGATIVE_INFINITY, + max ?? Number.POSITIVE_INFINITY + ) +} + +function componentIndex(link: AudioLink): number | "all" { + switch (link.component) { + case "x": + return 0 + case "y": + return 1 + case "z": + return 2 + default: + return "all" + } +} + +/** + * Resolve a vec2/vec3 value. + * + * `base` is the value the parameter would otherwise have this frame (keyframe + * result if present, else the stored value). It is cloned before any + * per-component write — mutating it would corrupt both the layer store and the + * `paramsCloneCache` WeakMap in `renderer/contracts.ts`, producing drift that + * persists across frames. + */ +export function resolveVectorValue( + link: AudioLink, + definition: ParameterDefinition, + base: ParameterValue | undefined, + bandValue: number +): ParameterValue | null { + const expectedLength = definition.type === "vec2" ? 2 : 3 + + const min = "min" in definition ? definition.min : undefined + const max = "max" in definition ? definition.max : undefined + const scalar = clamp( + remapBand(bandValue, link.outMin, link.outMax), + min ?? Number.NEGATIVE_INFINITY, + max ?? Number.POSITIVE_INFINITY + ) + + const index = componentIndex(link) + + if (index === "all") { + return expectedLength === 2 + ? [scalar, scalar] + : [scalar, scalar, scalar] + } + + if (index >= expectedLength) { + // e.g. component "z" on a vec2 — skip rather than throw. + return null + } + + if (!Array.isArray(base) || base.length !== expectedLength) { + // No usable base to merge into; fall back to driving every component. + return expectedLength === 2 ? [scalar, scalar] : [scalar, scalar, scalar] + } + + const next = cloneParameterValue(base) + + if (!Array.isArray(next)) { + return null + } + + next[index] = scalar + + return next as ParameterValue +} + +/** + * Threshold gate for boolean bindings. + * + * Deliberately has no hysteresis: a Schmitt trigger depends on the previous + * output, which would make the result differ when scrubbing backwards, and + * differ between the live loop and `prewarmExportFrame`'s repeated renders of + * one timestamp. Chatter is controlled by the band's release time instead. + */ +export function resolveBooleanValue( + link: AudioLink, + bandValue: number +): boolean { + return bandValue >= (link.threshold ?? DEFAULT_BOOLEAN_THRESHOLD) +} + +/** Resolve a value for an animatable *layer* property rather than a parameter. */ +export function resolveLayerPropertyValue( + link: AudioLink, + property: LayerAnimatableProperty, + bandValue: number +): boolean | number | null { + if (property === "visible") { + return resolveBooleanValue(link, bandValue) + } + + const bounds = LAYER_PROPERTY_BOUNDS[property] + + if (!bounds) { + return null + } + + return clamp( + remapBand(bandValue, link.outMin, link.outMax), + bounds.min, + bounds.max + ) +} + +/** + * Resolve the value a single link produces, or `null` when it cannot apply. + * + * Exported so the UI can render a live readout for a modulated parameter + * without running the whole frame pipeline. + */ +export function resolveAudioLinkValue( + link: AudioLink, + definition: ParameterDefinition | null, + base: ParameterValue | undefined, + bandValue: number +): ParameterValue | null { + if (link.binding.kind === "layer") { + return resolveLayerPropertyValue(link, link.binding.property, bandValue) + } + + if (!definition) { + return null + } + + switch (definition.type) { + case "number": + return resolveNumberValue(link, definition, bandValue) + case "boolean": + return resolveBooleanValue(link, bandValue) + case "vec2": + case "vec3": + return resolveVectorValue(link, definition, base, bandValue) + default: + // color/select/text carry no numeric range to remap onto. + return null + } +} diff --git a/src/lib/editor/audio/spectrogram.ts b/src/lib/editor/audio/spectrogram.ts new file mode 100644 index 0000000..26e4289 --- /dev/null +++ b/src/lib/editor/audio/spectrogram.ts @@ -0,0 +1,231 @@ +import { + type BinRange, + createSpectroBandLayout, + DEFAULT_FFT_SIZE, + ENVELOPE_RATE, + frequencyToBinRange, + SPECTRO_BAND_COUNT, +} from "@/lib/editor/audio/bands" +import { + computeFrameMagnitudes, + createFftWorkspace, + type FftWorkspace, +} from "@/lib/editor/audio/fft" + +/** + * Stage A output: the expensive, source-dependent reduction of PCM. Computed + * once per audio source, then retained so band-config edits (stage B) never + * re-decode or re-FFT. + * + * Not serializable and never persisted — a 5 minute track is ~4.6 MB. + */ +export type AudioSpectrogram = { + /** `frameCount * bandCount` log-spaced magnitudes, row-major by frame. */ + bands: Float32Array + bandCount: number + /** Geometric centre frequency of each band. Length `bandCount`. */ + centerHz: Float32Array + durationSeconds: number + envelopeRate: number + fftSize: number + frameCount: number + /** Window-weighted RMS per frame. Drives the full-band `level`. */ + rms: Float32Array + sampleRate: number +} + +export type SpectrogramOptions = { + bandCount?: number + envelopeRate?: number + fftSize?: number +} + +const PROGRESS_FRAME_INTERVAL = 256 + +type ReductionPlan = { + bandCount: number + binRanges: BinRange[] + centerHz: Float32Array + /** Sum of squared window coefficients, for the weighted RMS denominator. */ + windowEnergy: number +} + +function createReductionPlan( + workspace: FftWorkspace, + sampleRate: number, + bandCount: number +): ReductionPlan { + const layout = createSpectroBandLayout(sampleRate, bandCount) + const binRanges: BinRange[] = [] + + for (let index = 0; index < bandCount; index += 1) { + binRanges.push( + frequencyToBinRange( + layout.edgeHz[index] ?? 0, + layout.edgeHz[index + 1] ?? 0, + workspace.size, + sampleRate + ) + ) + } + + let windowEnergy = 0 + for (let index = 0; index < workspace.size; index += 1) { + const coefficient = workspace.window[index] ?? 0 + windowEnergy += coefficient * coefficient + } + + return { + bandCount, + binRanges, + centerHz: layout.centerHz, + windowEnergy: windowEnergy > 0 ? windowEnergy : 1, + } +} + +function computeWindowedRms( + workspace: FftWorkspace, + samples: Float32Array, + offset: number, + windowEnergy: number +): number { + let weighted = 0 + + for (let index = 0; index < workspace.size; index += 1) { + const sampleIndex = offset + index + if (sampleIndex < 0 || sampleIndex >= samples.length) { + continue + } + + const coefficient = workspace.window[index] ?? 0 + const sample = samples[sampleIndex] ?? 0 + weighted += coefficient * coefficient * sample * sample + } + + return Math.sqrt(weighted / windowEnergy) +} + +/** + * Stage A as a generator, yielding progress in `[0,1]`. The driver decides + * whether to run it synchronously, chunked on the main thread, or inside a + * Worker — this module stays free of scheduling concerns. + */ +export function* analyzeSpectrogramStepwise( + samples: Float32Array, + sampleRate: number, + options: SpectrogramOptions = {} +): Generator { + const fftSize = options.fftSize ?? DEFAULT_FFT_SIZE + const envelopeRate = options.envelopeRate ?? ENVELOPE_RATE + const bandCount = options.bandCount ?? SPECTRO_BAND_COUNT + + if (sampleRate <= 0) { + throw new Error(`sampleRate must be positive, received ${sampleRate}`) + } + + const workspace = createFftWorkspace(fftSize) + const plan = createReductionPlan(workspace, sampleRate, bandCount) + + const hopSamples = Math.max(1, Math.round(sampleRate / envelopeRate)) + const frameCount = Math.max(1, Math.ceil(samples.length / hopSamples)) + const halfWindow = Math.floor(fftSize / 2) + + const bands = new Float32Array(frameCount * bandCount) + const rms = new Float32Array(frameCount) + + for (let frame = 0; frame < frameCount; frame += 1) { + // Centred framing: frame `i` describes the audio *at* t = i / envelopeRate. + // Left-aligned windows would report a kick ~fftSize/2 samples late (~21ms + // at 2048/48k), which is perceptible when projected on stage. + const offset = frame * hopSamples - halfWindow + + const magnitudes = computeFrameMagnitudes(workspace, samples, offset) + const rowStart = frame * bandCount + + for (let band = 0; band < bandCount; band += 1) { + const range = plan.binRanges[band] + if (!range) { + continue + } + + let sum = 0 + let count = 0 + for (let bin = range.startBin; bin <= range.endBin; bin += 1) { + sum += magnitudes[bin] ?? 0 + count += 1 + } + + // Mean, not sum: keeps a band's reading scale-invariant when the user + // narrows its frequency range. + bands[rowStart + band] = count > 0 ? sum / count : 0 + } + + rms[frame] = computeWindowedRms( + workspace, + samples, + offset, + plan.windowEnergy + ) + + if (frame % PROGRESS_FRAME_INTERVAL === 0) { + yield frame / frameCount + } + } + + yield 1 + + return { + bandCount, + bands, + centerHz: plan.centerHz, + durationSeconds: samples.length / sampleRate, + envelopeRate, + fftSize, + frameCount, + rms, + sampleRate, + } +} + +/** Synchronous stage A. Drains {@link analyzeSpectrogramStepwise}. */ +export function analyzeSpectrogram( + samples: Float32Array, + sampleRate: number, + options: SpectrogramOptions = {} +): AudioSpectrogram { + const iterator = analyzeSpectrogramStepwise(samples, sampleRate, options) + + let step = iterator.next() + while (!step.done) { + step = iterator.next() + } + + return step.value +} + +/** Average all channels. Summing would clip; taking channel 0 loses hard pans. */ +export function downmixToMono(channels: Float32Array[]): Float32Array { + const first = channels[0] + + if (!first) { + return new Float32Array(0) + } + + if (channels.length === 1) { + return first + } + + const mono = new Float32Array(first.length) + + for (const channel of channels) { + for (let index = 0; index < mono.length; index += 1) { + mono[index] = (mono[index] ?? 0) + (channel[index] ?? 0) + } + } + + for (let index = 0; index < mono.length; index += 1) { + mono[index] = (mono[index] ?? 0) / channels.length + } + + return mono +} diff --git a/src/lib/editor/binding-key.ts b/src/lib/editor/binding-key.ts new file mode 100644 index 0000000..5ef2ff7 --- /dev/null +++ b/src/lib/editor/binding-key.ts @@ -0,0 +1,32 @@ +import type { AnimatedPropertyBinding } from "@/types/editor" + +/** + * Stable identity for an animated/modulated property binding, for use as a map + * key or dedupe key. + * + * Lives here rather than alongside the sidebar so pure lib modules (audio + * modulation, timeline evaluation) can share it without importing component + * code. + */ +export function getBindingKey(binding: AnimatedPropertyBinding): string { + if (binding.kind === "layer") { + return `layer:${binding.property}` + } + + return `param:${binding.key}` +} + +export function bindingEquals( + left: AnimatedPropertyBinding, + right: AnimatedPropertyBinding +): boolean { + return getBindingKey(left) === getBindingKey(right) +} + +/** Identity of a binding *on a specific layer*. */ +export function getLayerBindingKey( + layerId: string, + binding: AnimatedPropertyBinding +): string { + return `${layerId}:${getBindingKey(binding)}` +} diff --git a/src/lib/editor/parameter-schema.ts b/src/lib/editor/parameter-schema.ts index 474a3e0..4784207 100644 --- a/src/lib/editor/parameter-schema.ts +++ b/src/lib/editor/parameter-schema.ts @@ -48,6 +48,26 @@ export function isParameterAnimatable(definition: ParameterDefinition): boolean return definition.animatable ?? true } +/** + * Whether a parameter can be driven by an audio band. + * + * Stricter than {@link isParameterAnimatable}: `color` has no numeric range to + * remap a band onto, and `select` values are not ordered. `vec3` is excluded + * because `ParameterField` renders no control for it, so a link would be + * invisible and un-editable. + */ +export function isParameterAudioModulatable(definition: ParameterDefinition): boolean { + if ( + definition.type === "color" || + definition.type === "select" || + definition.type === "vec3" + ) { + return false + } + + return isParameterAnimatable(definition) +} + export function isParameterValueEqual(left: ParameterValue, right: ParameterValue): boolean { if (Array.isArray(left) && Array.isArray(right)) { return ( diff --git a/src/types/editor.ts b/src/types/editor.ts index 447ce9e..ca3dd23 100644 --- a/src/types/editor.ts +++ b/src/types/editor.ts @@ -120,7 +120,7 @@ export interface MaskConfig { source: MaskSource } -export const ASSET_KINDS = ["image", "video", "model"] as const +export const ASSET_KINDS = ["image", "video", "model", "audio"] as const export type AssetKind = (typeof ASSET_KINDS)[number] export type Vector2 = { x: number; y: number } @@ -360,6 +360,63 @@ export interface TimelineStateSnapshot { tracks: TimelineTrack[] } +/** + * Ordered for display, not alphabetically — the UI iterates this directly and + * bass/mid/high/level is the order a musician expects. + */ +export const AUDIO_BAND_IDS = ["bass", "mid", "high", "level"] as const +export type AudioBandId = (typeof AUDIO_BAND_IDS)[number] + +export interface AudioBandConfig { + attackMs: number + gainDb: number + /** Ignored for the `level` band, which is full-band RMS. */ + highHz: number + /** Ignored for the `level` band, which is full-band RMS. */ + lowHz: number + releaseMs: number +} + +export type AudioSourceRef = + | { assetId: string; kind: "asset" } + | { kind: "video-layer"; layerId: string } + +/** Which component of a vec parameter a link drives. */ +export type AudioLinkComponent = "all" | "x" | "y" | "z" + +/** + * A link remaps one band's `[0,1]` envelope onto `[outMin, outMax]` for one + * bound parameter. `binding` reuses the keyframe binding type verbatim: binding + * describes *what is driven*, the band describes *what drives it*, so no new + * `AnimatedPropertyBinding` kind is needed. + * + * `component` and `threshold` must be omitted rather than set to `undefined` + * (`exactOptionalPropertyTypes`) — construct via `createAudioLink`. + */ +export interface AudioLink { + band: AudioBandId + binding: AnimatedPropertyBinding + /** vec2/vec3 bindings only. */ + component?: AudioLinkComponent + enabled: boolean + id: string + layerId: string + outMax: number + outMin: number + /** Boolean bindings only; the band value at which the output flips true. */ + threshold?: number + /** Opt-in snapping to the parameter definition's `step`. */ + quantize?: boolean +} + +/** The serializable slice of audio state — what lands in a `.lab` file. */ +export interface EditorAudioSnapshot { + bands: Record + links: AudioLink[] + offsetSeconds: number + source: AudioSourceRef | null +} + export type SidebarView = "properties" | "scene" export type MobileEditorPanel = | "none" @@ -477,6 +534,7 @@ export interface EditorStateSnapshot { } export interface EditorHistorySnapshot { + audio: EditorAudioSnapshot hoveredLayerId: string | null layers: EditorLayer[] selectedLayerId: string | null From 977d861aeaf90d3668c1f0c6993d02d6f9db4faf Mon Sep 17 00:00:00 2001 From: git-chad Date: Thu, 30 Jul 2026 13:15:28 -0300 Subject: [PATCH 02/21] feat(audio): accept audio assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds "audio" to ASSET_KINDS with mime + extension sniffing (browsers report audio types inconsistently, and often not at all for .m4a), and an `