Merge pull request #30 from diffusionstudio/kostantin/feature/fast-sampler

Kostantin/feature/fast sampler

Author: k9p5

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@diffusionstudio/core",
   "private": false,
-  "version": "1.0.1",
+  "version": "1.1.0",
   "type": "module",
   "description": "Build bleeding edge video processing applications",
   "files": [

src/composition/composition.ts

@@ -438,8 +438,6 @@ export class Composition extends EventEmitterMixin<CompositionEvents, typeof Ser
 					output.getChannelData(i).set(outputData);
 				}
 			} catch (_) { }
-
-			clip.source.audioBuffer = undefined;
 		}
 
 		return output;

src/sources/audio.spec.ts

@@ -16,7 +16,8 @@ class MockOfflineAudioContext {
 		const audioBuffer = {
 			duration: 5, // Mock duration
 			sampleRate: this.sampleRate,
-			getChannelData: () => new Float32Array(5000), // Return a dummy Float32Array
+			length: 5000,
+			getChannelData: () => new Float32Array(5000).fill(0.5), // Return a dummy Float32Array
 		} as any as AudioBuffer;
 		return Promise.resolve(audioBuffer);
 	}
@@ -33,11 +34,59 @@ describe('AudioSource', () => {
 	});
 
 	it('should decode an audio buffer correctly', async () => {
-		const buffer = await audioSource.decode(2, 44100);
+		const buffer = await audioSource.decode(2, 44100, true);
 		expect(buffer.duration).toBe(5); // Mock duration
 		expect(buffer.sampleRate).toBe(44100);
 		expect(audioSource.audioBuffer).toBe(buffer);
 		expect(audioSource.duration.seconds).toBe(5); // Ensure duration is set
+
+		audioSource.audioBuffer = undefined;
+		await audioSource.decode(2, 44100, false);
+		expect(audioSource.audioBuffer).toBe(undefined);
+	});
+
+	it('should (fast) sample an audio buffer correctly', async () => {
+		const samples = await audioSource.fastsampler({ length: 20 });
+		expect(samples.length).toBe(20);
+
+		for (const sample of samples) {
+			expect(sample).toBe(0.5);
+		}
+	});
+
+	it('should (fast) sample an audio buffer correctly with start', async () => {
+		const samples = await audioSource.fastsampler({
+			length: 20,
+			start: 10,
+		});
+		expect(samples.length).toBe(20);
+
+		for (const sample of samples) {
+			expect(sample).toBe(0.5);
+		}
+	});
+
+	it('should (fast) sample an audio buffer correctly with stop', async () => {
+		const samples = await audioSource.fastsampler({
+			length: 20,
+			stop: 1000,
+			start: 20,
+		});
+		expect(samples.length).toBe(20);
+
+		for (const sample of samples) {
+			expect(sample).toBe(0.5);
+		}
+	});
+
+	it('should (fast) sample an audio buffer correctly with logarithmic scale', async () => {
+		const samples = await audioSource.fastsampler({ logarithmic: true });
+		expect(samples.length).toBe(60);
+
+		for (const sample of samples) {
+			expect(sample).toBeGreaterThanOrEqual(0.5);
+			expect(sample).toBeLessThanOrEqual(1);
+		}
 	});
 
 	it('should create a thumbnail with correct DOM elements', async () => {

src/sources/audio.ts

@@ -9,29 +9,48 @@ import { Source } from './source';
 
 import type { ClipType } from '../clips';
 import type { ArgumentTypes } from '../types';
+import type { FastSamplerOptions } from './audio.types';
 
 export class AudioSource extends Source {
+	private decoding = false;
+
 	public readonly type: ClipType = 'audio';
 	public audioBuffer?: AudioBuffer;
 
 	public async decode(
 		numberOfChannels: number = 2,
 		sampleRate: number = 48000,
+		cache = false,
 	): Promise<AudioBuffer> {
+		// make sure audio is not decoded multiple times
+		if (this.decoding && cache) {
+			await new Promise(this.resolve('update'));
+
+			if (this.audioBuffer) {
+				return this.audioBuffer;
+			}
+		}
+
+		this.decoding = true;
 		const buffer = await this.arrayBuffer();
 
 		const ctx = new OfflineAudioContext(numberOfChannels, 1, sampleRate);
 
-		this.audioBuffer = await ctx.decodeAudioData(buffer);
-		this.duration.seconds = this.audioBuffer.duration;
+		const audioBuffer = await ctx.decodeAudioData(buffer);
+		this.duration.seconds = audioBuffer.duration;
+		if (cache) this.audioBuffer = audioBuffer;
 
+		this.decoding = false;
 		this.trigger('update', undefined);
 
-		return this.audioBuffer;
+		return audioBuffer;
 	}
 
+	/**
+	 * @deprecated Use fastsampler instead.
+	 */
 	public async samples(numberOfSampes = 60, windowSize = 50, min = 0): Promise<number[]> {
-		const buffer = this.audioBuffer ?? (await this.decode(1, 16e3));
+		const buffer = this.audioBuffer ?? (await this.decode(1, 3000, true));
 
 		const window = Math.round(buffer.sampleRate / windowSize);
 		const length = buffer.sampleRate * buffer.duration - window;
@@ -50,6 +69,43 @@ export class AudioSource extends Source {
 		return res.map((v) => Math.round((v / Math.max(...res)) * (100 - min)) + min);
 	}
 
+	/**
+	 * Fast sampler that uses a window size to calculate the max value of the samples in the window.
+	 * @param options - Sampling options.
+	 * @returns An array of the max values of the samples in the window.
+	 */
+	public async fastsampler({ length = 60, start = 0, stop, logarithmic = false }: FastSamplerOptions): Promise<Float32Array> {
+		if (typeof start === 'object') start = start.millis;
+		if (typeof stop === 'object') stop = stop.millis;
+
+		const sampleRate = 3000;
+		const audioBuffer = this.audioBuffer ?? (await this.decode(1, sampleRate, true));
+		const channelData = audioBuffer.getChannelData(0);
+
+		const firstSample = Math.floor(Math.max(start * sampleRate / 1000, 0));
+		const lastSample = stop
+			? Math.floor(Math.min(stop * sampleRate / 1000, audioBuffer.length))
+			: audioBuffer.length;
+
+		const windowSize = Math.floor((lastSample - firstSample) / length);
+		const result = new Float32Array(length);
+
+		for (let i = 0; i < length; i++) {
+			const start = firstSample + i * windowSize;
+			const end = start + windowSize;
+			let min = Infinity;
+			let max = -Infinity;
+
+			for (let j = start; j < end; j++) {
+				const sample = channelData[j];
+				if (sample < min) min = sample;
+				if (sample > max) max = sample;
+			}
+			result[i] = logarithmic ? Math.log2(1 + max) : max;
+		}
+		return result;
+	}
+
 	public async thumbnail(...args: ArgumentTypes<this['samples']>): Promise<HTMLElement> {
 		const samples = await this.samples(...args);
 

src/sources/audio.types.ts

@@ -0,0 +1,23 @@
+import type { Timestamp } from '../models';
+
+/**
+ * Fast sampler options.
+ */
+export type FastSamplerOptions = {
+  /**
+   * The number of samples to return.
+   */
+  length?: number;
+  /**
+   * The start time in **milliseconds** relative to the beginning of the clip.
+   */
+  start?: Timestamp | number;
+  /**
+   * The stop time in **milliseconds** relative to the beginning of the clip.
+   */
+  stop?: Timestamp | number;
+  /**
+   * Whether to use a logarithmic scale.
+   */
+  logarithmic?: boolean;
+};

src/sources/index.ts

@@ -8,5 +8,6 @@
 export * from './source';
 export * from './html';
 export * from './audio';
+export * from './audio.types';
 export * from './image';
 export * from './video';