From e510e733cd44d2dee6d0f417f5a93c5e83e3eae7 Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Tue, 29 Aug 2023 05:01:42 -0400 Subject: [PATCH] fix(server): extract motion photo android single frame (#3903) --- server/src/domain/media/media.repository.ts | 5 +- server/src/domain/media/media.service.spec.ts | 140 +++++++++--------- server/src/domain/media/media.service.ts | 23 ++- server/src/domain/media/media.util.ts | 114 +++++++------- .../infra/repositories/media.repository.ts | 3 + server/test/e2e/setup.ts | 2 +- server/test/fixtures/media.stub.ts | 23 ++- 7 files changed, 163 insertions(+), 147 deletions(-) diff --git a/server/src/domain/media/media.repository.ts b/server/src/domain/media/media.repository.ts index 57ddafe0e1..25486cc4ed 100644 --- a/server/src/domain/media/media.repository.ts +++ b/server/src/domain/media/media.repository.ts @@ -8,6 +8,7 @@ export interface ResizeOptions { } export interface VideoStreamInfo { + index: number; height: number; width: number; rotation: number; @@ -18,8 +19,10 @@ export interface VideoStreamInfo { } export interface AudioStreamInfo { + index: number; codecName?: string; codecType?: string; + frameCount: number; } export interface VideoFormat { @@ -55,7 +58,7 @@ export interface BitrateDistribution { } export interface VideoCodecSWConfig { - getOptions(stream: VideoStreamInfo): TranscodeOptions; + getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions; } export interface VideoCodecHWConfig extends VideoCodecSWConfig { diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 6c2d10c799..d59da447aa 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -311,8 +311,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -350,8 +350,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -374,8 +374,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -401,8 +401,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -426,8 +426,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -451,8 +451,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -476,8 +476,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -525,8 +525,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -555,8 +555,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -582,8 +582,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -611,8 +611,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec vp9', - '-acodec aac', + '-c:v:0 vp9', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -642,8 +642,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec vp9', - '-acodec aac', + '-c:v:0 vp9', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -672,8 +672,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec vp9', - '-acodec aac', + '-c:v:0 vp9', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -701,8 +701,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec vp9', - '-acodec aac', + '-c:v:0 vp9', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -729,8 +729,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -757,8 +757,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -785,8 +785,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec hevc', - '-acodec aac', + '-c:v:0 hevc', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -816,8 +816,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec hevc', - '-acodec aac', + '-c:v:0 hevc', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -876,7 +876,6 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ - `-vcodec h264_nvenc`, '-tune hq', '-qmin 0', '-g 250', @@ -886,7 +885,8 @@ describe(MediaService.name, () => { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - '-acodec aac', + `-c:v:0 h264_nvenc`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -916,7 +916,6 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ - `-vcodec h264_nvenc`, '-tune hq', '-qmin 0', '-g 250', @@ -926,7 +925,8 @@ describe(MediaService.name, () => { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - '-acodec aac', + `-c:v:0 h264_nvenc`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -952,7 +952,6 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ - `-vcodec h264_nvenc`, '-tune hq', '-qmin 0', '-g 250', @@ -962,7 +961,8 @@ describe(MediaService.name, () => { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - '-acodec aac', + `-c:v:0 h264_nvenc`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -989,7 +989,6 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ - `-vcodec h264_nvenc`, '-tune hq', '-qmin 0', '-g 250', @@ -999,7 +998,8 @@ describe(MediaService.name, () => { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - '-acodec aac', + `-c:v:0 h264_nvenc`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1022,7 +1022,6 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ - `-vcodec h264_nvenc`, '-tune hq', '-qmin 0', '-g 250', @@ -1032,7 +1031,8 @@ describe(MediaService.name, () => { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - '-acodec aac', + `-c:v:0 h264_nvenc`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1060,12 +1060,12 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ - `-vcodec h264_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7', - '-acodec aac', + `-c:v:0 h264_qsv`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1095,12 +1095,12 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ - `-vcodec h264_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7', - '-acodec aac', + `-c:v:0 h264_qsv`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1127,12 +1127,12 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ - `-vcodec vp9_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7', - '-acodec aac', + `-c:v:0 vp9_qsv`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-low_power 1', @@ -1170,8 +1170,8 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ - `-vcodec h264_vaapi`, - '-acodec aac', + `-c:v:0 h264_vaapi`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1199,8 +1199,8 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ - `-vcodec h264_vaapi`, - '-acodec aac', + `-c:v:0 h264_vaapi`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1230,8 +1230,8 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ - `-vcodec h264_vaapi`, - '-acodec aac', + `-c:v:0 h264_vaapi`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1257,8 +1257,8 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'], outputOptions: [ - `-vcodec h264_vaapi`, - '-acodec aac', + `-c:v:0 h264_vaapi`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1280,8 +1280,8 @@ describe(MediaService.name, () => { { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'], outputOptions: [ - `-vcodec h264_vaapi`, - '-acodec aac', + `-c:v:0 h264_vaapi`, + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1310,8 +1310,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1345,8 +1345,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1370,8 +1370,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', @@ -1395,8 +1395,8 @@ describe(MediaService.name, () => { { inputOptions: [], outputOptions: [ - '-vcodec h264', - '-acodec aac', + '-c:v:0 h264', + '-c:a:0 aac', '-movflags faststart', '-fps_mode passthrough', '-v verbose', diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index c4ba66cdd0..732aadde7c 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -73,15 +73,16 @@ export class MediaService { this.logger.log(`Successfully generated image thumbnail ${asset.id}`); break; case AssetType.VIDEO: - const { videoStreams } = await this.mediaRepository.probe(asset.originalPath); - const mainVideoStream = this.getMainVideoStream(videoStreams); + const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const mainVideoStream = this.getMainStream(videoStreams); if (!mainVideoStream) { this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`); return false; } + const mainAudioStream = this.getMainStream(audioStreams); const { ffmpeg } = await this.configCore.getConfig(); const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false }; - const options = new ThumbnailConfig(config).getOptions(mainVideoStream); + const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options); this.logger.log(`Successfully generated video thumbnail ${asset.id}`); break; @@ -149,8 +150,8 @@ export class MediaService { this.storageRepository.mkdirSync(outputFolder); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input); - const mainVideoStream = this.getMainVideoStream(videoStreams); - const mainAudioStream = this.getMainAudioStream(audioStreams); + const mainVideoStream = this.getMainStream(videoStreams); + const mainAudioStream = this.getMainStream(audioStreams); const containerExtension = format.formatName; if (!mainVideoStream || !containerExtension) { return false; @@ -165,7 +166,7 @@ export class MediaService { let transcodeOptions; try { - transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); + transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream)); } catch (err) { this.logger.error(`An error occurred while configuring transcoding options: ${err}`); return false; @@ -176,13 +177,13 @@ export class MediaService { await this.mediaRepository.transcode(input, output, transcodeOptions); } catch (err) { this.logger.error(err); - if (config.accel && config.accel !== TranscodeHWAccel.DISABLED) { + if (config.accel !== TranscodeHWAccel.DISABLED) { this.logger.error( `Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`, ); } config.accel = TranscodeHWAccel.DISABLED; - transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); + transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream)); await this.mediaRepository.transcode(input, output, transcodeOptions); } @@ -193,14 +194,10 @@ export class MediaService { return true; } - private getMainVideoStream(streams: VideoStreamInfo[]): VideoStreamInfo | null { + private getMainStream(streams: T[]): T { return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0]; } - private getMainAudioStream(streams: AudioStreamInfo[]): AudioStreamInfo | null { - return streams[0]; - } - private isTranscodeRequired( asset: AssetEntity, videoStream: VideoStreamInfo, diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index 4e2d18dc2c..546555d26f 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -1,6 +1,7 @@ import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities'; import { SystemConfigFFmpegDto } from '../system-config/dto'; import { + AudioStreamInfo, BitrateDistribution, TranscodeOptions, VideoCodecHWConfig, @@ -10,13 +11,13 @@ import { class BaseConfig implements VideoCodecSWConfig { constructor(protected config: SystemConfigFFmpegDto) {} - getOptions(stream: VideoStreamInfo) { + getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) { const options = { inputOptions: this.getBaseInputOptions(), - outputOptions: this.getBaseOutputOptions().concat('-v verbose'), + outputOptions: this.getBaseOutputOptions(videoStream, audioStream).concat('-v verbose'), twoPass: this.eligibleForTwoPass(), } as TranscodeOptions; - const filters = this.getFilterOptions(stream); + const filters = this.getFilterOptions(videoStream); if (filters.length > 0) { options.outputOptions.push(`-vf ${filters.join(',')}`); } @@ -31,9 +32,10 @@ class BaseConfig implements VideoCodecSWConfig { return []; } - getBaseOutputOptions() { + getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) { return [ - `-acodec ${this.config.targetAudioCodec}`, + `-c:v:${videoStream.index} ${this.getVideoCodec()}`, + `-c:a:${audioStream.index} ${this.getAudioCodec()}`, // Makes a second pass moving the moov atom to the // beginning of the file for improved playback speed. '-movflags faststart', @@ -41,13 +43,13 @@ class BaseConfig implements VideoCodecSWConfig { ]; } - getFilterOptions(stream: VideoStreamInfo) { + getFilterOptions(videoStream: VideoStreamInfo) { const options = []; - if (this.shouldScale(stream)) { - options.push(`scale=${this.getScaling(stream)}`); + if (this.shouldScale(videoStream)) { + options.push(`scale=${this.getScaling(videoStream)}`); } - if (this.shouldToneMap(stream)) { + if (this.shouldToneMap(videoStream)) { options.push(...this.getToneMapping()); } options.push('format=yuv420p'); @@ -103,34 +105,34 @@ class BaseConfig implements VideoCodecSWConfig { return { max, target, min, unit } as BitrateDistribution; } - getTargetResolution(stream: VideoStreamInfo) { + getTargetResolution(videoStream: VideoStreamInfo) { if (this.config.targetResolution === 'original') { - return Math.min(stream.height, stream.width); + return Math.min(videoStream.height, videoStream.width); } return Number.parseInt(this.config.targetResolution); } - shouldScale(stream: VideoStreamInfo) { - return Math.min(stream.height, stream.width) > this.getTargetResolution(stream); + shouldScale(videoStream: VideoStreamInfo) { + return Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream); } - shouldToneMap(stream: VideoStreamInfo) { - return stream.isHDR && this.config.tonemap !== ToneMapping.DISABLED; + shouldToneMap(videoStream: VideoStreamInfo) { + return videoStream.isHDR && this.config.tonemap !== ToneMapping.DISABLED; } - getScaling(stream: VideoStreamInfo) { - const targetResolution = this.getTargetResolution(stream); + getScaling(videoStream: VideoStreamInfo) { + const targetResolution = this.getTargetResolution(videoStream); const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1 - return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; + return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; } - isVideoRotated(stream: VideoStreamInfo) { - return Math.abs(stream.rotation) === 90; + isVideoRotated(videoStream: VideoStreamInfo) { + return Math.abs(videoStream.rotation) === 90; } - isVideoVertical(stream: VideoStreamInfo) { - return stream.height > stream.width || this.isVideoRotated(stream); + isVideoVertical(videoStream: VideoStreamInfo) { + return videoStream.height > videoStream.width || this.isVideoRotated(videoStream); } isBitrateConstrained() { @@ -171,6 +173,14 @@ class BaseConfig implements VideoCodecSWConfig { `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`, ]; } + + getAudioCodec(): string { + return this.config.targetAudioCodec; + } + + getVideoCodec(): string { + return this.config.targetVideoCodec; + } } export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { @@ -202,6 +212,10 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { return -a.localeCompare(b); }); } + + getVideoCodec(): string { + return `${this.config.targetVideoCodec}_${this.config.accel}`; + } } export class ThumbnailConfig extends BaseConfig { @@ -217,9 +231,9 @@ export class ThumbnailConfig extends BaseConfig { return []; } - getScaling(stream: VideoStreamInfo) { - let options = super.getScaling(stream); - if (!this.shouldToneMap(stream)) { + getScaling(videoStream: VideoStreamInfo) { + let options = super.getScaling(videoStream); + if (!this.shouldToneMap(videoStream)) { options += ':out_color_matrix=bt601:out_range=pc'; } return options; @@ -236,10 +250,6 @@ export class ThumbnailConfig extends BaseConfig { } export class H264Config extends BaseConfig { - getBaseOutputOptions() { - return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; - } - getThreadOptions() { if (this.config.threads <= 0) { return []; @@ -253,10 +263,6 @@ export class H264Config extends BaseConfig { } export class HEVCConfig extends BaseConfig { - getBaseOutputOptions() { - return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; - } - getThreadOptions() { if (this.config.threads <= 0) { return []; @@ -270,10 +276,6 @@ export class HEVCConfig extends BaseConfig { } export class VP9Config extends BaseConfig { - getBaseOutputOptions() { - return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; - } - getPresetOptions() { const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads if (speed >= 0) { @@ -309,9 +311,8 @@ export class NVENCConfig extends BaseHWConfig { return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda']; } - getBaseOutputOptions() { + getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) { return [ - `-vcodec ${this.config.targetVideoCodec}_nvenc`, // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding '-tune hq', '-qmin 0', @@ -322,15 +323,15 @@ export class NVENCConfig extends BaseHWConfig { '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', - ...super.getBaseOutputOptions(), + ...super.getBaseOutputOptions(videoStream, audioStream), ]; } - getFilterOptions(stream: VideoStreamInfo) { - const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; + getFilterOptions(videoStream: VideoStreamInfo) { + const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload_cuda'); - if (this.shouldScale(stream)) { - options.push(`scale_cuda=${this.getScaling(stream)}`); + if (this.shouldScale(videoStream)) { + options.push(`scale_cuda=${this.getScaling(videoStream)}`); } return options; @@ -378,15 +379,14 @@ export class QSVConfig extends BaseHWConfig { return ['-init_hw_device qsv=hw', '-filter_hw_device hw']; } - getBaseOutputOptions() { + getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) { // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md const options = [ - `-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7', - ...super.getBaseOutputOptions(), + ...super.getBaseOutputOptions(videoStream, audioStream), ]; // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a if (this.config.targetVideoCodec === VideoCodec.VP9) { @@ -395,11 +395,11 @@ export class QSVConfig extends BaseHWConfig { return options; } - getFilterOptions(stream: VideoStreamInfo) { - const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; + getFilterOptions(videoStream: VideoStreamInfo) { + const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload=extra_hw_frames=64'); - if (this.shouldScale(stream)) { - options.push(`scale_qsv=${this.getScaling(stream)}`); + if (this.shouldScale(videoStream)) { + options.push(`scale_qsv=${this.getScaling(videoStream)}`); } return options; } @@ -437,15 +437,11 @@ export class VAAPIConfig extends BaseHWConfig { return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel']; } - getBaseOutputOptions() { - return [`-vcodec ${this.config.targetVideoCodec}_vaapi`, ...super.getBaseOutputOptions()]; - } - - getFilterOptions(stream: VideoStreamInfo) { - const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; + getFilterOptions(videoStream: VideoStreamInfo) { + const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload'); - if (this.shouldScale(stream)) { - options.push(`scale_vaapi=${this.getScaling(stream)}`); + if (this.shouldScale(videoStream)) { + options.push(`scale_vaapi=${this.getScaling(videoStream)}`); } return options; diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index b4c3f8abdf..a261708654 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -42,6 +42,7 @@ export class MediaRepository implements IMediaRepository { videoStreams: results.streams .filter((stream) => stream.codec_type === 'video') .map((stream) => ({ + index: stream.index, height: stream.height || 0, width: stream.width || 0, codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name, @@ -53,8 +54,10 @@ export class MediaRepository implements IMediaRepository { audioStreams: results.streams .filter((stream) => stream.codec_type === 'audio') .map((stream) => ({ + index: stream.index, codecType: stream.codec_type, codecName: stream.codec_name, + frameCount: Number.parseInt(stream.nb_frames ?? '0'), })), }; } diff --git a/server/test/e2e/setup.ts b/server/test/e2e/setup.ts index 8f0654d86f..6c2395b688 100644 --- a/server/test/e2e/setup.ts +++ b/server/test/e2e/setup.ts @@ -1,5 +1,5 @@ -import { GenericContainer } from 'testcontainers'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { GenericContainer } from 'testcontainers'; export default async () => { process.env.NODE_ENV = 'development'; process.env.TYPESENSE_API_KEY = 'abc123'; diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index a08e02a44d..f5988b9c60 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -7,10 +7,21 @@ const probeStubDefaultFormat: VideoFormat = { }; const probeStubDefaultVideoStream: VideoStreamInfo[] = [ - { height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0, isHDR: false }, + { + index: 0, + height: 1080, + width: 1920, + codecName: 'hevc', + codecType: 'video', + frameCount: 100, + rotation: 0, + isHDR: false, + }, ]; -const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }]; +const probeStubDefaultAudioStream: AudioStreamInfo[] = [ + { index: 0, codecName: 'aac', codecType: 'audio', frameCount: 100 }, +]; const probeStubDefault: VideoInfo = { format: probeStubDefaultFormat, @@ -25,6 +36,7 @@ export const probeStub = { ...probeStubDefault, videoStreams: [ { + index: 0, height: 1080, width: 400, codecName: 'hevc', @@ -34,6 +46,7 @@ export const probeStub = { isHDR: false, }, { + index: 1, height: 1080, width: 400, codecName: 'h7000', @@ -48,6 +61,7 @@ export const probeStub = { ...probeStubDefault, videoStreams: [ { + index: 0, height: 0, width: 400, codecName: 'hevc', @@ -62,6 +76,7 @@ export const probeStub = { ...probeStubDefault, videoStreams: [ { + index: 0, height: 2160, width: 3840, codecName: 'h264', @@ -76,6 +91,7 @@ export const probeStub = { ...probeStubDefault, videoStreams: [ { + index: 0, height: 480, width: 480, codecName: 'h264', @@ -90,6 +106,7 @@ export const probeStub = { ...probeStubDefault, videoStreams: [ { + index: 0, height: 2160, width: 3840, codecName: 'h264', @@ -102,7 +119,7 @@ export const probeStub = { }), audioStreamMp3: Object.freeze({ ...probeStubDefault, - audioStreams: [{ codecType: 'audio', codecName: 'aac' }], + audioStreams: [{ index: 0, codecType: 'audio', codecName: 'aac', frameCount: 100 }], }), matroskaContainer: Object.freeze({ ...probeStubDefault,