feat: av1 #4500
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| import {RemoteVideoResolution} from '../types'; | ||
| import {AV1CodecInfo, H264CodecInfo, SupportedResolution} from './types'; | ||
|
|
||
| export const H264_CODEC_PARAMETERS = { | ||
| '90p': { | ||
| maxFs: 60, | ||
| }, | ||
| '180p': { | ||
| maxFs: 240, | ||
| }, | ||
| '360p': { | ||
| maxFs: 920, | ||
| }, | ||
| '540p': { | ||
| maxFs: 2040, | ||
| }, | ||
| '720p': { | ||
| maxFs: 3600, | ||
| }, | ||
| '1080p': { | ||
| maxFs: 8192, | ||
| }, | ||
| } satisfies Record<SupportedResolution, Omit<H264CodecInfo, 'codec'>>; | ||
|
|
||
| export const AV1_CODEC_PARAMETERS = { | ||
| '90p': { | ||
| maxPicSize: 147_456, | ||
| levelIdx: 0, | ||
| maxWidth: 1152, | ||
| maxHeight: 2048, | ||
| maxDecodeRate: 5_529_600, | ||
| }, | ||
| '180p': { | ||
| maxPicSize: 147_456, | ||
| levelIdx: 0, | ||
| maxWidth: 1152, | ||
| maxHeight: 2048, | ||
| maxDecodeRate: 5_529_600, | ||
| }, | ||
| '360p': { | ||
| maxPicSize: 278_784, | ||
| levelIdx: 1, | ||
| maxWidth: 2816, | ||
| maxHeight: 1584, | ||
| maxDecodeRate: 10_454_400, | ||
| }, | ||
| '540p': { | ||
| maxPicSize: 665_856, | ||
| levelIdx: 4, | ||
| maxWidth: 4352, | ||
| maxHeight: 2448, | ||
| maxDecodeRate: 24_969_600, | ||
| }, | ||
| '720p': { | ||
| maxPicSize: 1_065_024, | ||
| levelIdx: 5, | ||
| maxWidth: 5504, | ||
| maxHeight: 3096, | ||
| maxDecodeRate: 39_938_400, | ||
| }, | ||
| '1080p': { | ||
| maxPicSize: 2_359_296, | ||
| levelIdx: 9, | ||
| maxWidth: 6144, | ||
| maxHeight: 3456, | ||
| maxDecodeRate: 155_713_536, | ||
| }, | ||
| } satisfies Record<SupportedResolution, Omit<AV1CodecInfo, 'codec'>>; | ||
|
|
||
| export const CODEC_DEFAULTS = { | ||
| h264: { | ||
| ...H264_CODEC_PARAMETERS['1080p'], | ||
| maxFps: 3000, | ||
| maxMbps: 245760, | ||
| }, | ||
| av1: { | ||
| ...AV1_CODEC_PARAMETERS['1080p'], | ||
| tier: 0, | ||
| }, | ||
| }; | ||
|
|
||
| export const PANE_SIZE_TO_RESOLUTION = { | ||
| thumbnail: '90p', | ||
| 'very small': '180p', | ||
| small: '360p', | ||
| medium: '720p', | ||
| large: '1080p', | ||
| best: '1080p', | ||
| } satisfies Record<RemoteVideoResolution, SupportedResolution>; | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,175 @@ | ||
| /* eslint-disable class-methods-use-this */ | ||
| import { | ||
| AV1Codec, | ||
| getRecommendedMaxBitrateForPicSize, | ||
| getFrameSizeForPicSize, | ||
| CodecInfo as WcmeCodecInfo, | ||
| } from '@webex/internal-media-core'; | ||
| import {AV1_CODEC_PARAMETERS, CODEC_DEFAULTS, PANE_SIZE_TO_RESOLUTION} from './constants'; | ||
| import {MediaCodecHelper, AV1CodecInfo, SupportedResolution} from './types'; | ||
| import {MediaRequest, RemoteVideoResolution} from '../types'; | ||
| import LoggerProxy from '../../common/logs/logger-proxy'; | ||
|
|
||
| /** | ||
| * Class for AV1 media codec info | ||
| */ | ||
| export default class MediaCodecHelperAV1 implements MediaCodecHelper { | ||
| /** | ||
| * Gets the AV1 codec info | ||
| * | ||
| * @param {Object} options - The options for the AV1 codec info | ||
| * @returns {AV1CodecInfo} The AV1 codec info | ||
| */ | ||
| getCodecInfo(options: {getMaxPicSize?: () => number}): AV1CodecInfo { | ||
| if (!options.getMaxPicSize) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return this.getParameters(options.getMaxPicSize()); | ||
| } | ||
|
|
||
| /** | ||
| * Degrades the media request | ||
| * | ||
| * @param {MediaRequest} mr - The media request to degrade | ||
| * @param {Resolution} resolution - The resolution to degrade to | ||
| * @returns {number} The total macroblocks requested | ||
| */ | ||
| degradeMediaRequest(mr: MediaRequest, resolution: SupportedResolution): number { | ||
| if (mr.codecInfo?.codec !== 'av1') { | ||
| return 0; | ||
| } | ||
|
|
||
| const preferredMaxPicSize = mr.preferredMaxPicSize | ||
| ? mr.preferredMaxPicSize | ||
| : CODEC_DEFAULTS.av1.maxPicSize; | ||
|
|
||
| mr.codecInfo.maxPicSize = Math.min( | ||
| preferredMaxPicSize, | ||
| mr.codecInfo.maxPicSize || CODEC_DEFAULTS.av1.maxPicSize, | ||
| AV1_CODEC_PARAMETERS[resolution].maxPicSize | ||
| ); | ||
|
|
||
| // we only consider sources with "live" state | ||
| const slotsWithLiveSource = mr.receiveSlots.filter((rs) => rs.sourceState === 'live'); | ||
|
|
||
| return getFrameSizeForPicSize(mr.codecInfo.maxPicSize) * slotsWithLiveSource.length; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the max payload bits per second | ||
| * | ||
| * @param {MediaRequest} mediaRequest - The media request to get the max payload bits per second from | ||
| * @returns {number} The max payload bits per second | ||
| */ | ||
| getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number { | ||
| if (mediaRequest.codecInfo?.codec !== 'av1') { | ||
| return 0; | ||
| } | ||
|
|
||
| return getRecommendedMaxBitrateForPicSize(mediaRequest.codecInfo.maxPicSize); | ||
| } | ||
|
|
||
| /** | ||
| * Gets the WCME codec infos | ||
| * | ||
| * @param {MediaRequest} mr - The media request to get the WCME codec infos from | ||
| * @returns {WcmeCodecInfo[]} The WCME codec infos | ||
| */ | ||
| getWCMECodecInfos(mr: MediaRequest): WcmeCodecInfo[] { | ||
| if (mr.codecInfo?.codec !== 'av1') { | ||
| return []; | ||
| } | ||
|
|
||
| return [ | ||
| WcmeCodecInfo.fromAv1( | ||
| 45, | ||
|
Contributor
There was a problem hiding this comment. could we have any constant for this param here? |
||
| new AV1Codec( | ||
| mr.codecInfo.levelIdx || CODEC_DEFAULTS.av1.levelIdx, | ||
| mr.codecInfo.tier || CODEC_DEFAULTS.av1.tier, | ||
| mr.codecInfo.maxWidth || CODEC_DEFAULTS.av1.maxWidth, | ||
| mr.codecInfo.maxHeight || CODEC_DEFAULTS.av1.maxHeight, | ||
| mr.codecInfo.maxPicSize || CODEC_DEFAULTS.av1.maxPicSize, | ||
| mr.codecInfo.maxDecodeRate || CODEC_DEFAULTS.av1.maxDecodeRate | ||
| ) | ||
| ), | ||
| ]; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the highest compatible AV1 codec parameters for the given maximum picture size | ||
| * | ||
| * @param {number} maxPicSize - The maximum picture size | ||
| * @returns {AV1CodecInfo} The AV1 codec parameters | ||
| */ | ||
| private getParameters(maxPicSize: number): AV1CodecInfo { | ||
| const parameters = Object.values(AV1_CODEC_PARAMETERS) | ||
| // filter out parameters with a max picture size greater than the given max picture size | ||
| .filter((entry) => maxPicSize <= entry.maxPicSize) | ||
| // sort by max picture size descending | ||
| .sort((a, b) => b.maxPicSize - a.maxPicSize); | ||
|
|
||
| // return the highest compatible AV1 codec parameters | ||
| return { | ||
| codec: 'av1', | ||
| ...parameters[0], | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Converts pane size into av1 maxPicSize | ||
| * | ||
| * @param {RemoteVideoResolution} paneSize - The pane size to get the max pic size for | ||
| * @returns {number} The max pic size | ||
| */ | ||
| getMaxPicSize(paneSize: RemoteVideoResolution): number { | ||
| let resolution: SupportedResolution; | ||
|
|
||
| if (paneSize in PANE_SIZE_TO_RESOLUTION) { | ||
| resolution = PANE_SIZE_TO_RESOLUTION[paneSize]; | ||
| } else { | ||
| LoggerProxy.logger.warn( | ||
| `MediaCodecHelperAV1#getMaxPicSize --> unsupported paneSize: ${paneSize}, using "medium" instead` | ||
|
Contributor
There was a problem hiding this comment. nit: you can use in log |
||
| ); | ||
| resolution = PANE_SIZE_TO_RESOLUTION.medium; | ||
| } | ||
|
|
||
| return AV1_CODEC_PARAMETERS[resolution].maxPicSize; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the max pic size for the given width and height | ||
| * | ||
| * @param {number} width - The width of the video element | ||
| * @param {number} height - The height of the video element | ||
| * @returns {number | undefined} The max pic size for the given width and height, or undefined if the width or height is 0 | ||
| */ | ||
| getSizeHintMaxPicSize(width: number, height: number): number | undefined { | ||
| if (width === 0 || height === 0) { | ||
| return undefined; | ||
| } | ||
|
|
||
| // we switch to the next resolution level when the height is 10% more than the current resolution height | ||
| // except for 1080p - we switch to it immediately when the height is more than 720p | ||
| const threshold = 1.1; | ||
| const getThresholdHeight = (h: number) => Math.round(h * threshold); | ||
|
|
||
| if (height < getThresholdHeight(90)) { | ||
| return AV1_CODEC_PARAMETERS['90p'].maxPicSize; | ||
| } | ||
| if (height < getThresholdHeight(180)) { | ||
| return AV1_CODEC_PARAMETERS['180p'].maxPicSize; | ||
| } | ||
| if (height < getThresholdHeight(360)) { | ||
| return AV1_CODEC_PARAMETERS['360p'].maxPicSize; | ||
| } | ||
| if (height < getThresholdHeight(540)) { | ||
| return AV1_CODEC_PARAMETERS['540p'].maxPicSize; | ||
| } | ||
| if (height <= 720) { | ||
| return AV1_CODEC_PARAMETERS['720p'].maxPicSize; | ||
| } | ||
|
|
||
| return AV1_CODEC_PARAMETERS['1080p'].maxPicSize; | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So far, I believe @k-wasniowski said that we won't support the different
levelIdx, ot I'm missing something?