Skip to main content


A package to connect @pexip/media-control and @pexip/media-processor to create a streamlined media process


npm install @pexip/media



It create's an object to interact with the media stream, which is usually used for our main stream.

Its major features:

  • Create a media pipeline to get and process the MediaStream, see createMediaPipeline. It also provides some media features including:

    - Video Processing - background blur
    - Audio Processing - Noise Suppression
    - Audio Signal Detection - To detect if the active audio input device is capturing any audio
    - Voice Activity Detection - To detect if there is any voice activity
  • Provide media input related information

  • Verify if we need to request a new MediaStream, see updateMedia

  • Subscribe events from @pexip/media-control and update the state accordingly


A processor to process audio from the stream to be used together with createMedia as one of the media processor of the pipeline.


A processor to process video from the stream to be used together with createMedia as one of the media processor of the pipeline.


It create's an object to control the provided media stream, e.g. change the audio/video input device and apply changes to the main stream when necessary.


The package tries to follow the MediaStream API's Constraints pattern. Beside the MediaTrackConstraintSet that's specified from the Media Capture and Streams spec, we have extended some additional media features as the following:

  • Able to pass MediaDeviceInfo directly to audio/video
  • Add vad to use our own implementation of Voice Activity Detection feature
  • Add asd to use our own implementation of Audio Signal Detection feature
  • Add denoise to use our own implementation of noise suppression
  • Add backgroundBlurAmount to adjust the blur amount when using the background blur feature
  • Add edgeBlurAmount to adjust the blur amount for edge of the segmented person
  • Add foregroundThreshold to adjust the threshold to be considered as foreground aka the segmented person
  • Add bgImageUrl for the image to be used for background replacement
  • Add videoSegmentation for specified effects of video segmentation
  • Add videoSegmentationModel for specifying segmentation model to be used
  • Add flipHorizontal to flip horizontally
export interface InputConstraintSet extends MediaTrackConstraints {
* Same purpose as `deviceId` but it gives more information about the device
* so that we can have extra tolerance on device selection
device?: DeviceConstraint | ConstraintDeviceParameters;
* Whether or not using video segmentation, e.g. background
* blur/replacement, to specify the effects, intended to be applied to the
* segment. Available effects are `none`, `blur`, `overlay` or `remove`
videoSegmentation?: ConstrainDOMString;
* Segmentation model to be used for video segmentation, currently only
* supports `mediapipeSelfie` and `personify`
videoSegmentationModel?: ConstrainDOMString;
* Whether or not using our own noise suppression
denoise?: ConstrainBoolean;
* Voice Activity Detection
vad?: ConstrainBoolean;
* Audio Signal Detection for the purpose of checking if the audio input is
* hardware muted or unusable
asd?: ConstrainBoolean;
* Flip the video horizontally
flipHorizontal?: ConstrainBoolean;
* Blur size/level parameter when using video segmentation with `blur`
* effects
backgroundBlurAmount?: ConstrainULong;
* Blur amount applied to the segmented person's edge
edgeBlurAmount?: ConstrainULong;
* Erode level for edge smoothing when using video segmentation
foregroundThreshold?: ConstrainDouble;
* Image Url that is being used for video overlay effects
bgImageUrl?: ConstrainDOMString;


import {createMedia, createMediaSignals, UserMediaStatus} from '@pexip/media';
import type {VideoRenderParams} from '@pexip/media';
import {
urls as mpUrls,
} from '@pexip/media-processor';

// Need some state for the app
interface State extends VideoRenderParams {
mute: {
audio: boolean;
video: boolean;
noiseSuppression: boolean;
vad: boolean;
asd: boolean;
updateFrequency: number;
silentDetectionDuration: number;
vadThrottleMS: number;
monitor: boolean;
channelCount: number;
export const mediaState: State = {
mute: {
audio: false,
video: false,
videoSegmentation: 'none',
noiseSuppression: false,
vad: false,
asd: false,
updateFrequency: 0.5,
silentDetectionDuration: 4.0,
vadThrottleMS: 3000,
frameRate: 30,
bgImageUrl: '',
monitor: false,
channelCount: 1,
foregroundThreshold: FOREGROUND_THRESHOLD,
backgroundBlurAmount: BACKGROUND_BLUR_AMOUNT,
edgeBlurAmount: EDGE_BLUR_AMOUNT,
flipHorizontal: FLIP_HORIZONTAL,

// Create required signals with additional `onDevicesChanged`,
// `onStatusChanged` and `onStreamTrackEnabled` signals
export const signals = createMediaSignals('demo', [

const audioProcessor = createAudioStreamProcess({
shouldEnable: () => true,
denoiseParams: {
wasmURL: denoiseWasm.href,
workletModule: mpUrls.denoise.href,
analyzerUpdateFrequency: mediaState.updateFrequency,
audioSignalDetectionDuration: mediaState.silentDetectionDuration,
throttleMs: mediaState.vadThrottleMS,
onAudioSignalDetected: signals.onSilentDetected.emit,
onVoiceActivityDetected: signals.onVAD.emit,

// Create symbolic link to link bg-blur/selfie_segmentation to a public folder
// Thus we can create a URL object to get the link to be used for the `gluePath`
const selfieJs = new URL(
const mediapipeSegmenter = createMediapipeSegmenter('selfie_segmentation', {
modelType: 'general',
processingWidth: PROCESSING_WIDTH,
processingHeight: PROCESSING_HEIGHT,
gluePath: selfieJs.href,

const segmenters: Segmenters = {
mediapipeSelfie: mediapipeSegmenter,

const transformer = createCanvasTransform(mediapipeSegmenter, {
effects: mediaState.videoSegmentation,
width: mediaState.width,
height: mediaState.height,
foregroundThreshold: mediaState.foregroundThreshold,
backgroundBlurAmount: mediaState.backgroundBlurAmount,
edgeBlurAmount: mediaState.edgeBlurAmount,
flipHorizontal: mediaState.flipHorizontal,
const videoProcessor = createVideoProcessor([transformer]);

const videoStreamProcessor = createVideoStreamProcess({
trackProcessorAPI: 'stream', // Use Stream API when available
shouldEnable: () => true, // Always Process Video Track
videoSegmentationModel: 'mediapipeSelfie', // Segmentation Modal
processingWidth: mediaState.width,
processingHeight: mediaState.height,
onError: error => {

// Instantiate the media object
export const mediaController = createMedia({
getMuteState: () => mediaState.mute,
mediaProcessors: [videoStreamProcessor, audioProcessor],

// Hook-up the signals to get the update
// Subscribe the onMediaChanged signal to get the latest Media object change event
signals.onMediaChanged.add(media => setLocalMedia(media)); // Assume there is a local function `setLocalMedia`
// Subscribe the onStatusChanged signal to get the latest Media status change event
signals.onStatusChanged.add(media => setMediaStatus(media)); // Assume there is a local function `setMediaStatus`
// Subscribe onStreamTrackEnabled signal to get the MediaStreamTrack['enabled'] change event
signals.onStreamTrackEnabled.add(track => {

// Later we can initialize a gUM to get a MediaStream
audio: {
sampleRate: 48000, // Ideal sample rate for our own noise suppression implementation
denoise: true, // Use our own implementation of noise suppression
noiseSuppression: false, // Disable the built-in noise suppression
vad: true, // Use Voice Activity Detection
asd: true, // Use Audio Signal Detection to check if the microphone is malfunctioned
video: {
videoSegmentation: 'blur', // Use background blur
blurLevel: 21,

// Access the current `MediaStream`;
// Get audio mute state;
// Mute audio;
// Get video mute state;
// Mute video;
// Get the status;
// Update the status = UserMediaStatus.Initial;
// Stop the `MediaStream`

// Later we can make changes to the media on-demand without initiating a new gUM call

// Turn off background blur
video: {videoSegmentation: 'none'},
// Turn on background blur
video: {videoSegmentation: 'blur'},
// Use our own noise suppression
audio: {denoise: true, noiseSuppression: false},
// Use built-in noise suppression
audio: {denoise: false, noiseSuppression: true},


const {createPreviewController} = require('@pexip/media');

// Some states from the App
interface AppState {
media: Media;
devices: MediaDeviceInfo[];
// Some functions from the App
const updateStream = (constraints: MediaDeviceRequest) => Promise<void>;

const onMediaChangedSignal = createSignal<Media>();

// Instantiate the controller
const controller = createPreviewController({
getDesiredResolution: () => ({}),
getCurrentDevices: () => state.devices,
getCurrentMedia: () =>,
updateMainStream: updateStream,
mediaSignal: onMediaChangedSignal,
getSegmentationOptions: () => ({
effects: mediaState.videoSegmentation,
erode: mediaState.erode,
blurLevel: mediaState.blurLevel,
videoProcessingEnabled: true,
updateMainStream: mc.getUserMediaAsync,
onEnded: () => {
psySegScriptLoaded: true,