@pexip/media

A package to connect @pexip/media-control and @pexip/media-processor to create a streamlined media process

Install

npm install @pexip/media

Overview

createMedia

It create's an object to interact with the media stream, which is usually used for our main stream.

Its major features:

• Create a media pipeline to get and process the MediaStream, see createMediaPipeline. It also provides some media features including:

  - Video Processing - background blur
  - Audio Processing - Noise Suppression
  - Audio Signal Detection - To detect if the active audio input device is capturing any audio
  - Voice Activity Detection - To detect if there is any voice activity

• Provide media input related information

• Verify if we need to request a new MediaStream, see updateMedia

• Subscribe events from @pexip/media-control and update the state accordingly

createAudioStreamProcess

A processor to process audio from the stream to be used together with createMedia as one of the media processor of the pipeline.

createVideoStreamProcess

A processor to process video from the stream to be used together with createMedia as one of the media processor of the pipeline.

createPreviewController

It create's an object to control the provided media stream, e.g. change the audio/video input device and apply changes to the main stream when necessary.

Usage

The package tries to follow the MediaStream API's Constraints pattern. Beside the MediaTrackConstraintSet that's specified from the Media Capture and Streams spec, we have extended some additional media features as the following:

• Able to pass MediaDeviceInfo directly to audio/video
• Add vad to use our own implementation of Voice Activity Detection feature
• Add asd to use our own implementation of Audio Signal Detection feature
• Add denoise to use our own implementation of noise suppression
• Add backgroundBlurAmount to adjust the blur amount when using the background blur feature
• Add edgeBlurAmount to adjust the blur amount for edge of the segmented person
• Add foregroundThreshold to adjust the threshold to be considered as foreground aka the segmented person
• Add bgImageUrl for the image to be used for background replacement
• Add videoSegmentation for specified effects of video segmentation
• Add videoSegmentationModel for specifying segmentation model to be used
• Add flipHorizontal to flip horizontally

export interface InputConstraintSet extends MediaTrackConstraints {
  /**
   * Same purpose as `deviceId` but it gives more information about the device
   * so that we can have extra tolerance on device selection
   */
  device?: DeviceConstraint | ConstraintDeviceParameters;
  /**
   * Whether or not using video segmentation, e.g. background
   * blur/replacement, to specify the effects, intended to be applied to the
   * segment. Available effects are `none`, `blur`, `overlay` or `remove`
   */
  videoSegmentation?: ConstrainDOMString;
  /**
   * Segmentation model to be used for video segmentation, currently only
   * supports `mediapipeSelfie` and `personify`
   */
  videoSegmentationModel?: ConstrainDOMString;
  /**
   * Whether or not using our own noise suppression
   */
  denoise?: ConstrainBoolean;
  /**
   * Voice Activity Detection
   */
  vad?: ConstrainBoolean;
  /**
   * Audio Signal Detection for the purpose of checking if the audio input is
   * hardware muted or unusable
   */
  asd?: ConstrainBoolean;
  /**
   * Flip the video horizontally
   */
  flipHorizontal?: ConstrainBoolean;
  /**
   * Blur size/level parameter when using video segmentation with `blur`
   * effects
   */
  backgroundBlurAmount?: ConstrainULong;
  /**
   * Blur amount applied to the segmented person's edge
   */
  edgeBlurAmount?: ConstrainULong;
  /**
   * Erode level for edge smoothing when using video segmentation
   */
  foregroundThreshold?: ConstrainDouble;
  /**
   * Image Url that is being used for video overlay effects
   */
  bgImageUrl?: ConstrainDOMString;
}

createMedia

import {createMedia, createMediaSignals, UserMediaStatus} from '@pexip/media';
import type {VideoRenderParams} from '@pexip/media';
import {
  urls as mpUrls,
  createMediapipeSegmenter,
  createPsySegSegmenter,
  createCanvasTransform,
  createVideoTrackProcessor,
  createVideoTrackProcessorWithFallback,
  createVideoProcessor,
  PROCESSING_WIDTH,
  PROCESSING_HEIGHT,
  FOREGROUND_THRESHOLD,
  BACKGROUND_BLUR_AMOUNT,
  EDGE_BLUR_AMOUNT,
  FLIP_HORIZONTAL,
} from '@pexip/media-processor';

// Need some state for the app
interface State extends VideoRenderParams {
  mute: {
    audio: boolean;
    video: boolean;
  };
  noiseSuppression: boolean;
  vad: boolean;
  asd: boolean;
  updateFrequency: number;
  silentDetectionDuration: number;
  vadThrottleMS: number;
  monitor: boolean;
  channelCount: number;
}
export const mediaState: State = {
  mute: {
    audio: false,
    video: false,
  },
  videoSegmentation: 'none',
  noiseSuppression: false,
  vad: false,
  asd: false,
  updateFrequency: 0.5,
  silentDetectionDuration: 4.0,
  vadThrottleMS: 3000,
  width: PROCESSING_WIDTH,
  height: PROCESSING_HEIGHT,
  frameRate: 30,
  bgImageUrl: '',
  monitor: false,
  channelCount: 1,
  foregroundThreshold: FOREGROUND_THRESHOLD,
  backgroundBlurAmount: BACKGROUND_BLUR_AMOUNT,
  edgeBlurAmount: EDGE_BLUR_AMOUNT,
  flipHorizontal: FLIP_HORIZONTAL,
};

// Create required signals with additional `onDevicesChanged`,
// `onStatusChanged` and `onStreamTrackEnabled` signals
export const signals = createMediaSignals('demo', [
  'onDevicesChanged',
  'onStatusChanged',
  'onStreamTrackEnabled',
]);

const audioProcessor = createAudioStreamProcess({
  shouldEnable: () => true,
  denoiseParams: {
    wasmURL: denoiseWasm.href,
    workletModule: mpUrls.denoise.href,
  },
  analyzerUpdateFrequency: mediaState.updateFrequency,
  audioSignalDetectionDuration: mediaState.silentDetectionDuration,
  throttleMs: mediaState.vadThrottleMS,
  onAudioSignalDetected: signals.onSilentDetected.emit,
  onVoiceActivityDetected: signals.onVAD.emit,
});

// Create symbolic link to link bg-blur/selfie_segmentation to a public folder
// Thus we can create a URL object to get the link to be used for the `gluePath`
const selfieJs = new URL(
  `./selfie_segmentation/selfie_segmentation.js`,
  document.baseURI,
);
const mediapipeSegmenter = createMediapipeSegmenter('selfie_segmentation', {
  modelType: 'general',
  processingWidth: PROCESSING_WIDTH,
  processingHeight: PROCESSING_HEIGHT,
  gluePath: selfieJs.href,
});

const segmenters: Segmenters = {
  mediapipeSelfie: mediapipeSegmenter,
};

const transformer = createCanvasTransform(mediapipeSegmenter, {
  effects: mediaState.videoSegmentation,
  width: mediaState.width,
  height: mediaState.height,
  foregroundThreshold: mediaState.foregroundThreshold,
  backgroundBlurAmount: mediaState.backgroundBlurAmount,
  edgeBlurAmount: mediaState.edgeBlurAmount,
  flipHorizontal: mediaState.flipHorizontal,
});
const videoProcessor = createVideoProcessor([transformer]);

const videoStreamProcessor = createVideoStreamProcess({
  trackProcessorAPI: 'stream', // Use Stream API when available
  shouldEnable: () => true, // Always Process Video Track
  videoSegmentationModel: 'mediapipeSelfie', // Segmentation Modal
  segmenters,
  transformer,
  videoProcessor,
  processingWidth: mediaState.width,
  processingHeight: mediaState.height,
  onError: error => {
    console.error(error);
  },
});

// Instantiate the media object
export const mediaController = createMedia({
  getMuteState: () => mediaState.mute,
  signals,
  mediaProcessors: [videoStreamProcessor, audioProcessor],
});

// Hook-up the signals to get the update
// Subscribe the onMediaChanged signal to get the latest Media object change event
signals.onMediaChanged.add(media => setLocalMedia(media)); // Assume there is a local function `setLocalMedia`
// Subscribe the onStatusChanged signal to get the latest Media status change event
signals.onStatusChanged.add(media => setMediaStatus(media)); // Assume there is a local function `setMediaStatus`
// Subscribe onStreamTrackEnabled signal to get the MediaStreamTrack['enabled'] change event
signals.onStreamTrackEnabled.add(track => {
  console.log(track);
});

// Later we can initialize a gUM to get a MediaStream
mediaController.getUserMedia({
  audio: {
    sampleRate: 48000, // Ideal sample rate for our own noise suppression implementation
    denoise: true, // Use our own implementation of noise suppression
    noiseSuppression: false, // Disable the built-in noise suppression
    vad: true, // Use Voice Activity Detection
    asd: true, // Use Audio Signal Detection to check if the microphone is malfunctioned
  },
  video: {
    videoSegmentation: 'blur', // Use background blur
    blurLevel: 21,
  },
});

// Access the current `MediaStream`
mediaController.media.stream;
// Get audio mute state
mediaController.media.audioMuted;
// Mute audio
mediaController.media.muteAudio(true);
// Get video mute state
mediaController.media.videoMuted;
// Mute video
mediaController.media.muteVideo(true);
// Get the status
mediaController.media.status;
// Update the status
mediaController.media.status = UserMediaStatus.Initial;
// Stop the `MediaStream`
await mediaController.media.release();

// Later we can make changes to the media on-demand without initiating a new gUM call

// Turn off background blur
await mediaController.media.applyConstraints({
  video: {videoSegmentation: 'none'},
});
// Turn on background blur
await mediaController.media.applyConstraints({
  video: {videoSegmentation: 'blur'},
});
// Use our own noise suppression
await mediaController.media.applyConstraints({
  audio: {denoise: true, noiseSuppression: false},
});
// Use built-in noise suppression
await mediaController.media.applyConstraints({
  audio: {denoise: false, noiseSuppression: true},
});

createPreviewController

const {createPreviewController} = require('@pexip/media');

// Some states from the App
interface AppState {
  media: Media;
  devices: MediaDeviceInfo[];
}
// Some functions from the App
const updateStream = (constraints: MediaDeviceRequest) => Promise<void>;

const onMediaChangedSignal = createSignal<Media>();

// Instantiate the controller
const controller = createPreviewController({
  getDesiredResolution: () => ({}),
  getCurrentDevices: () => state.devices,
  getCurrentMedia: () => state.media,
  updateMainStream: updateStream,
  mediaSignal: onMediaChangedSignal,
  getSegmentationOptions: () => ({
    effects: mediaState.videoSegmentation,
    erode: mediaState.erode,
    blurLevel: mediaState.blurLevel,
  }),
  videoProcessingEnabled: true,
  updateMainStream: mc.getUserMediaAsync,
  onEnded: () => {
    close();
  },
  psySegScriptLoaded: true,
});