From 86734825965d23a306c92df89becb3aaf75930b6 Mon Sep 17 00:00:00 2001 From: AHMED Date: Thu, 17 Sep 2026 15:34:20 +0500 Subject: [PATCH 1/4] fix(export): ensure WAV companion audio is rendered and exported with sound --- src/lib/exporter/audioEncoder.test.ts | 142 ++++++++++++++++++-- src/lib/exporter/audioEncoder.ts | 65 +++++++-- src/lib/exporter/audioMediaProcessor.ts | 26 +++- src/lib/exporter/audioProcessorShared.ts | 26 ++++ src/lib/exporter/audioTranscodeProcessor.ts | 38 ++++-- src/lib/exporter/offlineAudioProcessor.ts | 44 ++++-- 6 files changed, 300 insertions(+), 41 deletions(-) diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index f3e8b8f61..89cce529e 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it, vi } from "vitest"; -import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder"; +import { AudioProcessor, isWavAudioPath, softLimitOfflineMixPeaksInPlace } from "./audioEncoder"; type OfflineRenderTestHarness = AudioProcessor & { + bulkDecodeFromUrl(url: string, sampleRate: number): Promise; decodeAudioFromUrl(url: string): Promise; getMediaDurationSec(url: string): Promise; loadAudioFileDemuxer(audioPath: string): Promise; + processTrimOnlyAudio( + demuxer: unknown, + muxer: unknown, + sortedTrims: unknown[], + readEndSec?: number, + ): Promise; + streamDecodeFromUrl(url: string): Promise; prepareOfflineRender( videoUrl: string, trimRegions: never[], @@ -55,14 +63,37 @@ function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer { describe("AudioProcessor offline render preparation", () => { it("routes a muted full-track clip through offline audio rendering", async () => { const processor = new AudioProcessor(); - const render = vi.spyOn(processor as unknown as OfflineRenderTestHarness, - "renderAndMuxOfflineAudio").mockResolvedValue(); - const clips = [{ id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }]; + const render = vi + .spyOn(processor as unknown as OfflineRenderTestHarness, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + const clips = [ + { id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }, + ]; const muxer = {} as never; - await processor.process(null, muxer, "recording.mp4", [], [], undefined, - [], [], undefined, undefined, clips); - expect(render).toHaveBeenCalledWith("recording.mp4", [], [], [], [], - undefined, undefined, clips, muxer); + await processor.process( + null, + muxer, + "recording.mp4", + [], + [], + undefined, + [], + [], + undefined, + undefined, + clips, + ); + expect(render).toHaveBeenCalledWith( + "recording.mp4", + [], + [], + [], + [], + undefined, + undefined, + clips, + muxer, + ); }); it("rejects a cancelled chunked render instead of returning a partial WAV", async () => { @@ -208,6 +239,101 @@ describe("AudioProcessor offline render preparation", () => { expect(renderAndMuxOfflineAudio).toHaveBeenCalled(); }); + it("detects WAV audio paths and URLs correctly", () => { + expect(isWavAudioPath("C:\\Recordly\\recording.system.wav")).toBe(true); + expect(isWavAudioPath("/tmp/recording.mic.WAV")).toBe(true); + expect(isWavAudioPath("http://localhost:4321/video?path=audio.wav&sig=123")).toBe(true); + expect(isWavAudioPath("file:///tmp/recording.mp4")).toBe(false); + expect(isWavAudioPath("/tmp/recording.mic.m4a")).toBe(false); + expect(isWavAudioPath("/tmp/recording.mic.webm")).toBe(false); + expect(isWavAudioPath(null)).toBe(false); + expect(isWavAudioPath(undefined)).toBe(false); + }); + + it("avoids the single-sidecar fast path for WAV companion audio (such as system audio) and routes to offline rendering", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["C:\\Recordly\\recording.system.wav"], + ); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith( + "file:///tmp/recording.mp4", + [], + [], + [], + ["C:\\Recordly\\recording.system.wav"], + undefined, + undefined, + undefined, + expect.anything(), + ); + }); + + it("falls back to offline rendering when fast sidecar demux fails or outputs no audio", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const mockDemuxer = { destroy: vi.fn() }; + vi.spyOn(processor, "loadAudioFileDemuxer").mockResolvedValue(mockDemuxer); + vi.spyOn(processor, "processTrimOnlyAudio").mockResolvedValue(false); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mic.webm"], + ); + + expect(mockDemuxer.destroy).toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mic.webm"], + undefined, + undefined, + undefined, + expect.anything(), + ); + }); + + it("bypasses streaming decode and uses bulk decode directly for WAV files", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const fakeBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const streamDecodeFromUrl = vi.spyOn(processor, "streamDecodeFromUrl"); + const bulkDecodeFromUrl = vi + .spyOn(processor, "bulkDecodeFromUrl") + .mockResolvedValue(fakeBuffer); + + const result = await processor.decodeAudioFromUrl("C:\\Recordly\\recording.system.wav"); + + expect(result).toBe(fakeBuffer); + expect(streamDecodeFromUrl).not.toHaveBeenCalled(); + expect(bulkDecodeFromUrl).toHaveBeenCalledWith( + "C:\\Recordly\\recording.system.wav", + expect.any(Number), + ); + }); + it("soft-limits mixed peaks before encoding or WAV conversion", () => { const samples = new Float32Array([ -1.6, diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 26050c0a2..1e78f3892 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -10,6 +10,7 @@ import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; import { AudioTranscodeProcessor } from "./audioTranscodeProcessor"; import { hasNonDefaultSourceTrackSettings, + isWavAudioPath, MIN_SPEED_REGION_DELTA_MS, MP4_AUDIO_CODEC, type TrimLikeRegion, @@ -20,6 +21,7 @@ export { getSourceTrackIdFromPath, hasNonDefaultSourceTrackSettings, isAacAudioEncodingSupported, + isWavAudioPath, softLimitOfflineMixPeaksInPlace, } from "./audioProcessorShared"; @@ -142,11 +144,15 @@ export class AudioProcessor extends AudioTranscodeProcessor { const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); + const hasWavCompanionAudio = routingPolicy.playbackPaths.some((audioPath) => + isWavAudioPath(audioPath), + ); const needsSourceAudioMixing = routingPolicy.playbackPaths.length > 1 || (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || requiresLegacyMacMicSidecarMix || - hasTimedCompanionAudio; + hasTimedCompanionAudio || + hasWavCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( @@ -173,22 +179,34 @@ export class AudioProcessor extends AudioTranscodeProcessor { // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); - if (sidecarDemuxer) { - try { - await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); - } finally { + const sidecarPath = routingPolicy.playbackPaths[0]; + if (!isWavAudioPath(sidecarPath)) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(sidecarPath); + if (sidecarDemuxer) { + let wroteAudio = false; try { - sidecarDemuxer.destroy(); - } catch { - /* cleanup */ + wroteAudio = await this.processTrimOnlyAudio( + sidecarDemuxer, + muxer, + sortedTrims, + ); + } catch (error) { + console.warn("[AudioProcessor] Fast sidecar demux failed:", error); + } finally { + try { + sidecarDemuxer.destroy(); + } catch { + /* cleanup */ + } + } + if (wroteAudio) { + return; } } - return; } - // Fallback to offline rendering if demuxer creation failed + // Fallback to offline rendering if demuxer creation failed, unsupported codec, or no audio was written console.warn( - "[AudioProcessor] Fast sidecar demux failed, falling back to offline rendering", + "[AudioProcessor] Fast sidecar demux unavailable or failed, falling back to offline rendering", ); await this.renderAndMuxOfflineAudio( videoUrl, @@ -235,7 +253,28 @@ export class AudioProcessor extends AudioTranscodeProcessor { } } - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + const wroteTrimOnlyAudio = await this.processTrimOnlyAudio( + demuxer, + muxer, + sortedTrims, + readEndSec, + ); + if (!wroteTrimOnlyAudio && routingPolicy.playbackPaths.length > 0) { + console.warn( + "[AudioProcessor] Main demuxer audio trim failed, falling back to offline rendering for playback paths", + ); + await this.renderAndMuxOfflineAudio( + videoUrl, + sortedTrims, + [], + [], + routingPolicy.playbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, + muxer, + ); + } } async renderEditedAudioTrack( diff --git a/src/lib/exporter/audioMediaProcessor.ts b/src/lib/exporter/audioMediaProcessor.ts index 3359873a2..13da10124 100644 --- a/src/lib/exporter/audioMediaProcessor.ts +++ b/src/lib/exporter/audioMediaProcessor.ts @@ -1,10 +1,17 @@ import { WebDemuxer } from "web-demuxer"; +import { + DECODE_BACKPRESSURE_LIMIT, + isWavAudioPath, + OFFLINE_AUDIO_SAMPLE_RATE, +} from "./audioProcessorShared"; import { AudioTimelineProcessor } from "./audioTimelineProcessor"; -import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared"; import { resolveMediaElementSource } from "./localMediaSource"; export class AudioMediaProcessor extends AudioTimelineProcessor { protected async decodeAudioFromUrl(url: string): Promise { + if (isWavAudioPath(url)) { + return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE); + } try { const buffer = await this.streamDecodeFromUrl(url); if (buffer) return buffer; @@ -36,6 +43,11 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { return null; // No audio track } + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + return null; + } + const sampleRate = audioConfig.sampleRate || 48_000; const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); @@ -137,7 +149,14 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { } if (decoder.state === "configured") { - await decoder.flush(); + try { + await decoder.flush(); + } catch (flushError) { + console.warn( + "[AudioMediaProcessor] Non-fatal audio decoder flush warning:", + flushError, + ); + } } if (decodeError) throw decodeError; } finally { @@ -236,6 +255,9 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { // Get the duration of a media file by loading only its metadata. protected async getMediaDurationSec(url: string): Promise { + if (typeof document === "undefined") { + return 0; + } const source = await resolveMediaElementSource(url); try { const media = document.createElement("video"); diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts index a48062a78..3af9c244c 100644 --- a/src/lib/exporter/audioProcessorShared.ts +++ b/src/lib/exporter/audioProcessorShared.ts @@ -96,6 +96,32 @@ export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { return "mixed"; } +export function isWavAudioPath(audioPath: string | null | undefined): boolean { + if (!audioPath) { + return false; + } + const normalized = audioPath.toLowerCase().trim(); + if (normalized.endsWith(".wav")) { + return true; + } + try { + const parsedUrl = new URL(audioPath, "http://localhost"); + const pathParam = parsedUrl.searchParams.get("path"); + if (pathParam && pathParam.toLowerCase().trim().endsWith(".wav")) { + return true; + } + if (parsedUrl.pathname.toLowerCase().endsWith(".wav")) { + return true; + } + } catch { + const beforeQuery = normalized.split("?")[0]; + if (beforeQuery.endsWith(".wav")) { + return true; + } + } + return false; +} + export function hasNonDefaultSourceTrackSettings( sourceAudioTrackSettings?: SourceAudioTrackSettings, ) { diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts index 209a280ac..fb95f541f 100644 --- a/src/lib/exporter/audioTranscodeProcessor.ts +++ b/src/lib/exporter/audioTranscodeProcessor.ts @@ -1,5 +1,4 @@ import type { WebDemuxer } from "web-demuxer"; -import { OfflineAudioProcessor } from "./offlineAudioProcessor"; import { AUDIO_BITRATE, DECODE_BACKPRESSURE_LIMIT, @@ -8,6 +7,7 @@ import { type TrimLikeRegion, } from "./audioProcessorShared"; import type { VideoMuxer } from "./muxer"; +import { OfflineAudioProcessor } from "./offlineAudioProcessor"; export class AudioTranscodeProcessor extends OfflineAudioProcessor { protected async processTrimOnlyAudio( @@ -15,19 +15,19 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { muxer: VideoMuxer, sortedTrims: TrimLikeRegion[], readEndSec?: number, - ): Promise { + ): Promise { let audioConfig: AudioDecoderConfig; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { console.warn("[AudioProcessor] No audio track found, skipping"); - return; + return false; } const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); if (!codecCheck.supported) { console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); - return; + return false; } const audioStream = @@ -37,7 +37,7 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { let sourceTimestampOffsetUs: number | null = null; - await this.transcodeAudioStream( + return await this.transcodeAudioStream( audioStream as ReadableStream, audioConfig, muxer, @@ -68,7 +68,7 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { shouldSkipChunk?: (timestampMs: number) => boolean; transformAudioData?: (data: AudioData) => AudioData | null; } = {}, - ): Promise { + ): Promise { const pendingFrames: AudioData[] = []; let decodeError: Error | null = null; let encodeError: Error | null = null; @@ -138,11 +138,13 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); if (!encodeSupport.supported) { console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); - return; + return false; } + let wroteAudio = false; const encoder = new AudioEncoder({ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + wroteAudio = true; pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) { @@ -223,7 +225,14 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { } if (decoder.state === "configured") { - await decoder.flush(); + try { + await decoder.flush(); + } catch (flushError) { + console.warn( + "[AudioTranscodeProcessor] Non-fatal audio decoder flush warning:", + flushError, + ); + } } while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { @@ -237,7 +246,14 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { failIfNeeded(); if (encoder.state === "configured") { - await encoder.flush(); + try { + await encoder.flush(); + } catch (flushError) { + console.warn( + "[AudioTranscodeProcessor] Non-fatal audio encoder flush warning:", + flushError, + ); + } } await pendingMuxing; @@ -264,8 +280,10 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { } if (this.cancelled) { - return; + return false; } + + return wroteAudio; } // ---------- Offline audio rendering pipeline ---------- diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index e1080db3b..8d19fbc5f 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -104,9 +104,12 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { startDelaySec: number; gain: number; }> = []; - const refDuration = + const mediaDuration = mainBuffer?.duration ?? - (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + (resolvedPlan.playbackPaths.length > 0 || audioRegions.length > 0 + ? await this.getMediaDurationSec(videoUrl) + : 0); + const refDuration = mediaDuration; for (const audioPath of resolvedPlan.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); @@ -147,7 +150,10 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { if (mainBufferEntry?.buffer) { sourceDurationSec = mainBufferEntry.buffer.duration; } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { - sourceDurationSec = await this.getMediaDurationSec(videoUrl); + sourceDurationSec = Math.max( + mediaDuration > 0 ? mediaDuration : 0, + primaryBuffer?.duration ?? 0, + ); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; } @@ -215,6 +221,7 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { let encodeError: Error | null = null; let muxError: Error | null = null; + const getEncodeError = (): Error | null => encodeError; let pendingMuxing = Promise.resolve(); let wroteFirstChunk = false; @@ -249,6 +256,8 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { }); encoder.configure(encodeConfig); + let totalFramesEncoded = 0; + try { await this.renderChunked( prepared, @@ -256,20 +265,35 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { async (rendered, outputOffsetSec) => { if (encodeError) throw encodeError; if (muxError) throw muxError; - await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec); + const frames = await this.feedBufferToEncoder( + encoder, + rendered, + outputOffsetSec, + ); + totalFramesEncoded += frames; }, ); if (encodeError) throw encodeError; if (muxError) throw muxError; - if (encoder.state === "configured") { - await encoder.flush(); + if (totalFramesEncoded > 0 && encoder.state === "configured") { + try { + await encoder.flush(); + } catch (flushError) { + console.warn( + "[OfflineAudioProcessor] Non-fatal audio encoder flush warning:", + flushError, + ); + } } await pendingMuxing; - if (encodeError) throw encodeError; + const finalEncodeError = getEncodeError(); + const isIgnorableFlushError = + wroteFirstChunk && Boolean(finalEncodeError?.message.includes("Flushing error")); + if (finalEncodeError && !isIgnorableFlushError) throw finalEncodeError; if (muxError) throw muxError; } finally { if (encoder.state === "configured") { @@ -425,10 +449,11 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { encoder: AudioEncoder, buffer: AudioBuffer, timestampOffsetSec: number, - ): Promise { + ): Promise { const sampleRate = buffer.sampleRate; const numChannels = buffer.numberOfChannels; const totalFrames = buffer.length; + let encodedFrames = 0; for ( let offset = 0; @@ -454,11 +479,14 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { encoder.encode(audioData); audioData.close(); + encodedFrames += frameCount; while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { await new Promise((r) => setTimeout(r, 1)); } } + + return encodedFrames; } // Decode audio from a URL using streaming WebCodecs decode with bulk fallback. From 0f8aab9174da2527cc14c24c81eccaa46e0d10d2 Mon Sep 17 00:00:00 2001 From: AHMED Date: Sat, 10 Oct 2026 13:19:44 +0500 Subject: [PATCH 2/4] fix(export): prevent offline audio fallback after audio chunks are committed to muxer --- src/lib/exporter/audioEncoder.test.ts | 53 +++++++++++++++++++++ src/lib/exporter/audioEncoder.ts | 20 ++++++-- src/lib/exporter/audioTranscodeProcessor.ts | 2 +- src/lib/exporter/muxer.ts | 7 +++ src/lib/exporter/offlineAudioProcessor.ts | 5 ++ 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 89cce529e..8740b1c29 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -316,6 +316,59 @@ describe("AudioProcessor offline render preparation", () => { ); }); + it("does not fall back to offline rendering if audio chunks were already committed to muxer", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const mockDemuxer = { destroy: vi.fn() }; + vi.spyOn(processor, "loadAudioFileDemuxer").mockResolvedValue(mockDemuxer); + vi.spyOn(processor, "processTrimOnlyAudio").mockRejectedValue( + new Error("Muxer write failure"), + ); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + const muxer = { hasCommittedAudio: true } as never; + + await expect( + processor.process( + null, + muxer, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mic.webm"], + ), + ).rejects.toThrow("Muxer write failure"); + + expect(mockDemuxer.destroy).toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled(); + }); + + it("rejects offline audio rendering if muxer already has committed audio", async () => { + const processor = new AudioProcessor() as unknown as { + renderAndMuxOfflineAudio: (...args: unknown[]) => Promise; + }; + const muxer = { hasCommittedAudio: true } as never; + + await expect( + processor.renderAndMuxOfflineAudio( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mic.webm"], + undefined, + undefined, + undefined, + muxer, + ), + ).rejects.toThrow( + "Cannot render offline audio after audio chunks have already been committed to muxer", + ); + }); + it("bypasses streaming decode and uses bulk decode directly for WAV files", async () => { const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; const fakeBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 1e78f3892..f04b0389c 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,13 +1,10 @@ import type { WebDemuxer } from "web-demuxer"; -import { requiresClipTimelineRendering } from "./clipTimeline"; import type { AudioRegion, ClipRegion, SourceAudioTrackSettings, SpeedRegion, } from "@/components/video-editor/types"; -import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; -import { AudioTranscodeProcessor } from "./audioTranscodeProcessor"; import { hasNonDefaultSourceTrackSettings, isWavAudioPath, @@ -15,7 +12,10 @@ import { MP4_AUDIO_CODEC, type TrimLikeRegion, } from "./audioProcessorShared"; +import { AudioTranscodeProcessor } from "./audioTranscodeProcessor"; +import { requiresClipTimelineRendering } from "./clipTimeline"; import type { VideoMuxer } from "./muxer"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; export { getSourceTrackIdFromPath, @@ -192,6 +192,9 @@ export class AudioProcessor extends AudioTranscodeProcessor { ); } catch (error) { console.warn("[AudioProcessor] Fast sidecar demux failed:", error); + if (muxer?.hasCommittedAudio) { + throw error; + } } finally { try { sidecarDemuxer.destroy(); @@ -199,11 +202,14 @@ export class AudioProcessor extends AudioTranscodeProcessor { /* cleanup */ } } - if (wroteAudio) { + if (wroteAudio || muxer?.hasCommittedAudio) { return; } } } + if (muxer?.hasCommittedAudio) { + return; + } // Fallback to offline rendering if demuxer creation failed, unsupported codec, or no audio was written console.warn( "[AudioProcessor] Fast sidecar demux unavailable or failed, falling back to offline rendering", @@ -259,7 +265,11 @@ export class AudioProcessor extends AudioTranscodeProcessor { sortedTrims, readEndSec, ); - if (!wroteTrimOnlyAudio && routingPolicy.playbackPaths.length > 0) { + if ( + !wroteTrimOnlyAudio && + routingPolicy.playbackPaths.length > 0 && + !muxer?.hasCommittedAudio + ) { console.warn( "[AudioProcessor] Main demuxer audio trim failed, falling back to offline rendering for playback paths", ); diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts index fb95f541f..8a716e2cc 100644 --- a/src/lib/exporter/audioTranscodeProcessor.ts +++ b/src/lib/exporter/audioTranscodeProcessor.ts @@ -144,13 +144,13 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { let wroteAudio = false; const encoder = new AudioEncoder({ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { - wroteAudio = true; pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) { return; } await muxer.addAudioChunk(chunk, meta); + wroteAudio = true; }) .catch((error) => { muxError = error instanceof Error ? error : new Error(String(error)); diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index 54e551c9b..6510f1156 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -81,6 +81,7 @@ export class VideoMuxer { private output: Output | null = null; private videoSource: EncodedVideoPacketSource | null = null; private audioSource: EncodedAudioPacketSource | null = null; + private committedAudioChunks = 0; private hasAudio: boolean; private target: MediabunnyTarget | null = null; private config: ExportConfig; @@ -93,6 +94,10 @@ export class VideoMuxer { this.mode = mode ?? (shouldUseStreamTarget() ? "stream" : "buffer"); } + get hasCommittedAudio(): boolean { + return this.committedAudioChunks > 0; + } + getTargetMode(): MuxerTargetMode { return this.mode; } @@ -158,6 +163,7 @@ export class VideoMuxer { const packet = EncodedPacket.fromEncodedChunk(chunk); await this.audioSource.add(packet, meta); + this.committedAudioChunks += 1; } async finalize(): Promise { @@ -210,6 +216,7 @@ export class VideoMuxer { this.videoSource = null; this.audioSource = null; this.target = null; + this.committedAudioChunks = 0; if (this.streamSink) { void this.abortStream(); } diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index 8d19fbc5f..2fdd517b0 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -41,6 +41,11 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, ): Promise { + if (muxer?.hasCommittedAudio) { + throw new Error( + "[AudioProcessor] Cannot render offline audio after audio chunks have already been committed to muxer", + ); + } const prepared = await this.prepareOfflineRender( videoUrl, trimRegions, From d6d141fe45f1233b7540f23a2a2ac82fd23a2762 Mon Sep 17 00:00:00 2001 From: AHMED Date: Sat, 10 Oct 2026 14:16:59 +0500 Subject: [PATCH 3/4] fix(export): preserve video duration and propagate encoder flush errors in offline audio processor --- src/lib/exporter/audioEncoder.test.ts | 17 +++++++++++++++++ src/lib/exporter/offlineAudioProcessor.ts | 18 ++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 8740b1c29..8c011eee8 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -369,6 +369,23 @@ describe("AudioProcessor offline render preparation", () => { ); }); + it("uses video duration instead of companion audio duration when video duration is available", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const companionBuffer = { duration: 15, numberOfChannels: 2 } as AudioBuffer; + vi.spyOn(processor, "decodeAudioFromUrl").mockResolvedValue(companionBuffer); + vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10); + + const prepared = await processor.prepareOfflineRender( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.system.wav"], + ); + + expect(prepared.outputDurationMs).toBe(10_000); + }); + it("bypasses streaming decode and uses bulk decode directly for WAV files", async () => { const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; const fakeBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index 2fdd517b0..311ba066d 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -151,17 +151,9 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { throw new Error("No decodable audio sources found"); } - let sourceDurationSec: number; - if (mainBufferEntry?.buffer) { - sourceDurationSec = mainBufferEntry.buffer.duration; - } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { - sourceDurationSec = Math.max( - mediaDuration > 0 ? mediaDuration : 0, - primaryBuffer?.duration ?? 0, - ); - } else { - sourceDurationSec = primaryBuffer?.duration ?? 0; - } + const sourceDurationSec = + mainBufferEntry?.buffer?.duration ?? + (mediaDuration > 0 ? mediaDuration : (primaryBuffer?.duration ?? 0)); const sourceDurationMs = sourceDurationSec * 1000; // Build timeline slices (non-trimmed segments with speed info) @@ -296,9 +288,7 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { await pendingMuxing; const finalEncodeError = getEncodeError(); - const isIgnorableFlushError = - wroteFirstChunk && Boolean(finalEncodeError?.message.includes("Flushing error")); - if (finalEncodeError && !isIgnorableFlushError) throw finalEncodeError; + if (finalEncodeError) throw finalEncodeError; if (muxError) throw muxError; } finally { if (encoder.state === "configured") { From 2d57da76c036f8a4b0444da5db6bc7e71a57c207 Mon Sep 17 00:00:00 2001 From: AHMED Date: Sat, 10 Oct 2026 15:10:14 +0500 Subject: [PATCH 4/4] docs(export): add docstrings for touched exporter methods --- src/lib/exporter/audioProcessorShared.ts | 3 +++ src/lib/exporter/audioTranscodeProcessor.ts | 8 ++++++++ src/lib/exporter/muxer.ts | 3 +++ src/lib/exporter/offlineAudioProcessor.ts | 8 +++++++- 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts index 3af9c244c..e0bf38fa3 100644 --- a/src/lib/exporter/audioProcessorShared.ts +++ b/src/lib/exporter/audioProcessorShared.ts @@ -96,6 +96,9 @@ export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { return "mixed"; } +/** + * Checks whether an audio path or URL points to a WAV audio file. + */ export function isWavAudioPath(audioPath: string | null | undefined): boolean { if (!audioPath) { return false; diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts index 8a716e2cc..b8768f206 100644 --- a/src/lib/exporter/audioTranscodeProcessor.ts +++ b/src/lib/exporter/audioTranscodeProcessor.ts @@ -10,6 +10,10 @@ import type { VideoMuxer } from "./muxer"; import { OfflineAudioProcessor } from "./offlineAudioProcessor"; export class AudioTranscodeProcessor extends OfflineAudioProcessor { + /** + * Demuxes and transcodes audio for trim-only operations without decoding to PCM. + * Returns true if audio chunks were successfully encoded and written to the muxer. + */ protected async processTrimOnlyAudio( demuxer: WebDemuxer, muxer: VideoMuxer, @@ -59,6 +63,10 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { ); } + /** + * Transcodes a stream of encoded audio chunks through WebCodecs AudioDecoder and AudioEncoder. + * Returns true if at least one audio packet was committed to the muxer. + */ protected async transcodeAudioStream( audioStream: ReadableStream, audioConfig: AudioDecoderConfig, diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index 6510f1156..8d39db8e3 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -94,6 +94,9 @@ export class VideoMuxer { this.mode = mode ?? (shouldUseStreamTarget() ? "stream" : "buffer"); } + /** + * Indicates whether any encoded audio packets have been committed to the muxer. + */ get hasCommittedAudio(): boolean { return this.committedAudioChunks > 0; } diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index 311ba066d..766517e79 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -30,6 +30,9 @@ import { import type { VideoMuxer } from "./muxer"; export class OfflineAudioProcessor extends AudioMediaProcessor { + /** + * Renders multi-source or processed audio offline and muxes it into the final output. + */ protected async renderAndMuxOfflineAudio( videoUrl: string, trimRegions: TrimLikeRegion[], @@ -439,7 +442,10 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { source.start(localStartSec, bufferOffsetSec, duration); } - // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset. + /** + * Feeds a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset. + * Returns the count of frames successfully submitted to the encoder. + */ protected async feedBufferToEncoder( encoder: AudioEncoder, buffer: AudioBuffer,