FFmpeg.wasm: Bringing Multimedia Processing to the Browser

┌─────────────────────────────────────────┐
│           Main Thread (UI)              │
│  - User Interface                       │
│  - File Input/Output                    │
│  - Async Communication                  │
└─────────────┬───────────────────────────┘
              │ PostMessage API
              ▼
┌─────────────────────────────────────────┐
│         Web Worker Thread               │
│  ┌───────────────────────────────────┐  │
│  │     FFmpeg WASM Module            │  │
│  │  ┌─────────────────────────────┐  │  │
│  │  │   Emscripten Runtime        │  │  │
│  │  │  - Module loader            │  │  │
│  │  │  - Memory management        │  │  │
│  │  │  - System call emulation    │  │  │
│  │  └─────────────────────────────┘  │  │
│  │                                   │  │
│  │  ┌─────────────────────────────┐  │  │
│  │  │   Virtual File System       │  │  │
│  │  │  - MEMFS (in-memory)        │  │  │
│  │  │  - IDBFS (persistent)       │  │  │
│  │  │  - WORKERFS (shared)        │  │  │
│  │  └─────────────────────────────┘  │  │
│  │                                   │  │
│  │  ┌─────────────────────────────┐  │  │
│  │  │   FFmpeg Libraries          │  │  │
│  │  │  - libavcodec               │  │  │
│  │  │  - libavformat              │  │  │
│  │  │  - libavfilter              │  │  │
│  │  │  - libswscale               │  │  │
│  │  └─────────────────────────────┘  │  │
│  └───────────────────────────────────┘  │
└─────────────────────────────────────────┘

// Main thread
import { createFFmpeg, fetchFile } from '@ffmpeg/ffmpeg';

const ffmpeg = createFFmpeg({ 
  log: true,
  corePath: 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd/ffmpeg-core.js'
});

// Load FFmpeg (async)
await ffmpeg.load();

// Write input file to virtual FS
ffmpeg.FS('writeFile', 'input.mp4', await fetchFile('video.mp4'));

// Execute FFmpeg command (runs in worker)
await ffmpeg.run(
  '-i', 'input.mp4',
  '-vf', 'scale=640:480',
  '-c:a', 'copy',
  'output.mp4'
);

// Read output from virtual FS
const data = ffmpeg.FS('readFile', 'output.mp4');

// Create download blob
const blob = new Blob([data.buffer], { type: 'video/mp4' });
const url = URL.createObjectURL(blob);

// File System API Methods
ffmpeg.FS(method, ...args)

// Write file from various sources
ffmpeg.FS('writeFile', 'input.mp4', uint8Array);
ffmpeg.FS('writeFile', 'image.jpg', await fetchFile(url));
ffmpeg.FS('writeFile', 'audio.wav', await fetchFile(file));

// Create directories
ffmpeg.FS('mkdir', '/output');

// Read file
const data = ffmpeg.FS('readFile', '/output/result.mp4');

// List directory
const files = ffmpeg.FS('readdir', '/');

// Delete file
ffmpeg.FS('unlink', 'temp.mp4');

// Get file stats
const stats = ffmpeg.FS('stat', 'input.mp4');
console.log('File size:', stats.size);

// Mount persistent storage (IndexedDB)
ffmpeg.FS('mount', ffmpeg.IDBFS, {}, '/persistent');
ffmpeg.FS('syncfs', true, () => {
  console.log('File system synced to IndexedDB');
});

// Direct C function calls from JavaScript
const Module = {
  onRuntimeInitialized: function() {
    // Call exported C functions
    const result = Module.ccall(
      'process_video',    // Function name
      'number',           // Return type
      ['string', 'number'], // Argument types
      ['input.mp4', 30]  // Arguments
    );
    
    // Create wrapped function for repeated calls
    const processFrame = Module.cwrap(
      'process_frame',
      'number',
      ['number', 'number']
    );
    
    // Memory management
    const ptr = Module._malloc(1024 * 1024); // Allocate 1MB
    const heap = Module.HEAPU8.subarray(ptr, ptr + 1024 * 1024);
    
    // Write data to WASM memory
    heap.set(uint8ArrayData);
    
    // Process in WASM
    processFrame(ptr, 1024 * 1024);
    
    // Read result
    const result = heap.slice();
    
    // Free memory
    Module._free(ptr);
  }
};

// String operations
const strPtr = Module.allocateUTF8("Hello WASM");
Module._process_string(strPtr);
Module._free(strPtr);

Test: Transcode 1080p WebM → MP4 (60 seconds)
Hardware: Intel Core i5 11th Gen

┌─────────────────────┬────────────┬────────────┬──────────────┐
│ Platform            │ Avg Time   │ Max Time   │ Performance  │
├─────────────────────┼────────────┼────────────┼──────────────┤
│ Native FFmpeg       │ 5.2 sec    │ 5.3 sec    │ 1.0× (base)  │
│ FFmpeg.wasm (ST)    │ 128.8 sec  │ 130.7 sec  │ 0.04× (~25×) │
│ FFmpeg.wasm (MT)    │ 60.4 sec   │ 63.9 sec   │ 0.08× (~12×) │
│ Native + NVENC      │ 0.8 sec    │ 0.9 sec    │ 6.5× faster  │
└─────────────────────┴────────────┴────────────┴──────────────┘

Performance Factors:
• No hardware acceleration (GPU/SIMD)
• Memory copy overhead (JS ↔ WASM)
• WebAssembly runtime overhead
• Limited threading efficiency
• Browser sandbox restrictions

class MemoryOptimizedProcessor {
  constructor() {
    this.ffmpeg = createFFmpeg({
      log: false, // Reduce memory for logs
      progress: ({ ratio }) => {
        console.log(`Progress: ${(ratio * 100).toFixed(2)}%`);
      }
    });
  }

  async processLargeFile(file, maxMemoryMB = 512) {
    // Check file size
    const fileSizeMB = file.size / (1024 * 1024);
    
    if (fileSizeMB > maxMemoryMB) {
      // Use streaming approach
      return this.streamProcess(file);
    }
    
    // Direct processing for smaller files
    return this.directProcess(file);
  }

  async streamProcess(file) {
    // Create blob URL for direct access
    const blobUrl = URL.createObjectURL(file);
    
    // Use FFmpeg's network protocols
    await this.ffmpeg.run(
      '-i', blobUrl,
      '-c:v', 'copy',  // Avoid re-encoding if possible
      '-c:a', 'copy',
      '-movflags', 'frag_keyframe+empty_moov', // Streaming-friendly
      'output.mp4'
    );
    
    URL.revokeObjectURL(blobUrl);
  }

  // Monitor memory usage
  getMemoryUsage() {
    if (performance.memory) {
      return {
        used: Math.round(performance.memory.usedJSHeapSize / 1048576),
        total: Math.round(performance.memory.totalJSHeapSize / 1048576),
        limit: Math.round(performance.memory.jsHeapSizeLimit / 1048576)
      };
    }
    return null;
  }

  // Cleanup
  async cleanup() {
    // Clear virtual file system
    const files = this.ffmpeg.FS('readdir', '/');
    for (const file of files) {
      if (file !== '.' && file !== '..') {
        try {
          this.ffmpeg.FS('unlink', file);
        } catch (e) {
          // Directory, skip
        }
      }
    }
    
    // Force garbage collection if available
    if (global.gc) {
      global.gc();
    }
  }
}