cotugno/jellyfin
2
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2024-11-15 01:48:52 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11250 from nyanmisaka/fix-hwa-video-rotation

Browse Source 
Fix the broken video orientation (+-90/180)
This commit is contained in:
Bond-009 2024-08-27 22:24:34 +02:00 committed by GitHub
commit 6281cd707d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 323 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Emby.Server.Implementations/Data/SqliteItemRepository.cs
View File

@ -183,7 +183,8 @@ namespace Emby.Server.Implementations.Data
"ElPresentFlag",
"BlPresentFlag",
"DvBlSignalCompatibilityId",
"IsHearingImpaired"
"IsHearingImpaired",
"Rotation"
};
private static readonly string _mediaStreamSaveColumnsInsertQuery =
@ -343,7 +344,7 @@ namespace Emby.Server.Implementations.Data
base.Initialize();
const string CreateMediaStreamsTableCommand
= "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, PRIMARY KEY (ItemId, StreamIndex))";
= "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, Rotation INT NULL, PRIMARY KEY (ItemId, StreamIndex))";
const string CreateMediaAttachmentsTableCommand
= "create table if not exists mediaattachments (ItemId GUID, AttachmentIndex INT, Codec TEXT, CodecTag TEXT NULL, Comment TEXT NULL, Filename TEXT NULL, MIMEType TEXT NULL, PRIMARY KEY (ItemId, AttachmentIndex))";
@ -538,6 +539,8 @@ namespace Emby.Server.Implementations.Data
AddColumn(connection, "MediaStreams", "IsHearingImpaired", "BIT", existingColumnNames);
AddColumn(connection, "MediaStreams", "Rotation", "INT", existingColumnNames);
connection.Execute(string.Join(';', postQueries));
transaction.Commit();
@ -5483,6 +5486,8 @@ AND Type = @InternalPersonType)");
statement.TryBind("@DvBlSignalCompatibilityId" + index, stream.DvBlSignalCompatibilityId);
statement.TryBind("@IsHearingImpaired" + index, stream.IsHearingImpaired);
statement.TryBind("@Rotation" + index, stream.Rotation);
}
statement.ExecuteNonQuery();
@ -5694,6 +5699,11 @@ AND Type = @InternalPersonType)");
item.IsHearingImpaired = reader.TryGetBoolean(43, out var result) && result;
if (reader.TryGetInt32(44, out var rotation))
{
item.Rotation = rotation;
}
if (item.Type is MediaStreamType.Audio or MediaStreamType.Subtitle)
{
item.LocalizedDefault = _localization.GetLocalizedString("Default");

11
Jellyfin.Api/Helpers/DynamicHlsHelper.cs
View File

@ -198,6 +198,17 @@ public class DynamicHlsHelper
AddSubtitles(state, subtitleStreams, builder, _httpContextAccessor.HttpContext.User);
}
// Video rotation metadata is only supported in fMP4 remuxing
if (state.VideoStream is not null
&& state.VideoRequest is not null
&& (state.VideoStream?.Rotation ?? 0) != 0
&& EncodingHelper.IsCopyCodec(state.OutputVideoCodec)
&& !string.IsNullOrWhiteSpace(state.Request.SegmentContainer)
&& !string.Equals(state.Request.SegmentContainer, "mp4", StringComparison.OrdinalIgnoreCase))
{
playlistUrl += "&AllowVideoStreamCopy=false";
}
var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup);
if (state.VideoStream is not null && state.VideoRequest is not null)

362
MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
View File

@ -65,6 +65,7 @@ namespace MediaBrowser.Controller.MediaEncoding
private readonly Version _minFFmpegVaapiH26xEncA53CcSei = new Version(6, 0);
private readonly Version _minFFmpegReadrateOption = new Version(5, 0);
private readonly Version _minFFmpegWorkingVtHwSurface = new Version(7, 0, 1);
private readonly Version _minFFmpegDisplayRotationOption = new Version(6, 0);
private static readonly Regex _validationRegex = new(ValidationRegex, RegexOptions.Compiled);
@ -231,6 +232,7 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilter("tonemap_vaapi")
&& _mediaEncoder.SupportsFilter("procamp_vaapi")
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVaapiFrameSync)
&& _mediaEncoder.SupportsFilter("transpose_vaapi")
&& _mediaEncoder.SupportsFilter("hwupload_vaapi");
}
@ -248,6 +250,8 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilter("scale_opencl")
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapOpenclBt2390)
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclFrameSync);
// Let transpose_opencl optional for the time being.
}
private bool IsCudaFullSupported()
@ -258,6 +262,8 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapCudaName)
&& _mediaEncoder.SupportsFilter("overlay_cuda")
&& _mediaEncoder.SupportsFilter("hwupload_cuda");
// Let transpose_cuda optional for the time being.
}
private bool IsVulkanFullSupported()
@ -265,7 +271,9 @@ namespace MediaBrowser.Controller.MediaEncoding
return _mediaEncoder.SupportsHwaccel("vulkan")
&& _mediaEncoder.SupportsFilter("libplacebo")
&& _mediaEncoder.SupportsFilter("scale_vulkan")
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync)
&& _mediaEncoder.SupportsFilter("transpose_vulkan")
&& _mediaEncoder.SupportsFilter("flip_vulkan");
}
private bool IsVideoToolboxFullSupported()
@ -275,6 +283,8 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilter("overlay_videotoolbox")
&& _mediaEncoder.SupportsFilter("tonemap_videotoolbox")
&& _mediaEncoder.SupportsFilter("scale_vt");
// Let transpose_vt optional for the time being.
}
private bool IsSwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
@ -1147,9 +1157,6 @@ namespace MediaBrowser.Controller.MediaEncoding
args.Append(vidDecoder);
}
// hw transpose filters should be added manually.
args.Append(" -noautorotate");
return args.ToString().Trim();
}
@ -2947,8 +2954,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
public static string GetHwScaleFilter(
string hwScalePrefix,
string hwScaleSuffix,
string videoFormat,
bool swapOutputWandH,
int? videoWidth,
int? videoHeight,
int? requestedWidth,
@ -2970,8 +2979,11 @@ namespace MediaBrowser.Controller.MediaEncoding
|| !videoHeight.HasValue
|| outHeight.Value != videoHeight.Value;
var arg1 = isSizeFixed ? ("=w=" + outWidth.Value + ":h=" + outHeight.Value) : string.Empty;
var arg2 = isFormatFixed ? ("format=" + videoFormat) : string.Empty;
var swpOutW = swapOutputWandH ? outHeight.Value : outWidth.Value;
var swpOutH = swapOutputWandH ? outWidth.Value : outHeight.Value;
var arg1 = isSizeFixed ? $"=w={swpOutW}:h={swpOutH}" : string.Empty;
var arg2 = isFormatFixed ? $"format={videoFormat}" : string.Empty;
if (isFormatFixed)
{
arg2 = (isSizeFixed ? ':' : '=') + arg2;
@ -2981,7 +2993,8 @@ namespace MediaBrowser.Controller.MediaEncoding
{
return string.Format(
CultureInfo.InvariantCulture,
"scale_{0}{1}{2}",
"{0}_{1}{2}{3}",
hwScalePrefix ?? "scale",
hwScaleSuffix,
arg1,
arg2);
@ -3384,6 +3397,18 @@ namespace MediaBrowser.Controller.MediaEncoding
tonemapArg);
}
public string GetVideoTransposeDirection(EncodingJobInfo state)
{
return (state.VideoStream?.Rotation ?? 0) switch
{
90 => "cclock",
180 => "reversal",
-90 => "clock",
-180 => "reversal",
_ => string.Empty
};
}
/// <summary>
/// Gets the parameter of software filter chain.
/// </summary>
@ -3418,6 +3443,11 @@ namespace MediaBrowser.Controller.MediaEncoding
var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
var rotation = state.VideoStream?.Rotation ?? 0;
var swapWAndH = Math.Abs(rotation) == 90;
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -3432,7 +3462,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = isSwDecoder ? "yuv420p" : "nv12";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
if (isVaapiEncoder)
{
outFormat = "nv12";
@ -3481,7 +3511,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
else if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -3555,6 +3585,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doCuTranspose = !string.IsNullOrEmpty(tranposeDir) && _mediaEncoder.SupportsFilter("transpose_cuda");
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isNvDecoder && doCuTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -3571,10 +3608,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doCuTonemap ? "yuv420p10le" : "yuv420p";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// sw => hw
if (doCuTonemap)
@ -3593,8 +3630,14 @@ namespace MediaBrowser.Controller.MediaEncoding
mainFilters.Add(deintFilter);
}
// hw transpose
if (doCuTranspose)
{
mainFilters.Add($"transpose_cuda=dir={tranposeDir}");
}
var outFormat = doCuTonemap ? string.Empty : "yuv420p";
var hwScaleFilter = GetHwScaleFilter("cuda", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "cuda", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
// hw scale
mainFilters.Add(hwScaleFilter);
}
@ -3644,7 +3687,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=yuva420p");
}
@ -3654,7 +3697,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
// alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=yuva420p");
@ -3669,7 +3712,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -3745,6 +3788,14 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doOclTranspose = !string.IsNullOrEmpty(tranposeDir)
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TransposeOpenclReversal);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isD3d11vaDecoder && doOclTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -3761,10 +3812,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// keep video at memory except ocl tonemap,
// since the overhead caused by hwupload >>> using sw filter.
@ -3773,7 +3824,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
mainFilters.Add("hwupload=derive_device=d3d11va:extra_hw_frames=24");
mainFilters.Add("format=d3d11");
mainFilters.Add("hwmap=derive_device=opencl");
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
}
}
@ -3781,12 +3832,18 @@ namespace MediaBrowser.Controller.MediaEncoding
{
// INPUT d3d11 surface(vram)
// map from d3d11va to opencl via d3d11-opencl interop.
mainFilters.Add("hwmap=derive_device=opencl");
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
// hw deint <= TODO: finsh the 'yadif_opencl' filter
// hw transpose
if (doOclTranspose)
{
mainFilters.Add($"transpose_opencl=dir={tranposeDir}");
}
var outFormat = doOclTonemap ? string.Empty : "nv12";
var hwScaleFilter = GetHwScaleFilter("opencl", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "opencl", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
// hw scale
mainFilters.Add(hwScaleFilter);
}
@ -3831,7 +3888,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
// OUTPUT d3d11(nv12) surface(vram)
// reverse-mapping via d3d11-opencl interop.
mainFilters.Add("hwmap=derive_device=d3d11va:reverse=1");
mainFilters.Add("hwmap=derive_device=d3d11va:mode=write:reverse=1");
mainFilters.Add("format=d3d11");
}
@ -3844,7 +3901,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=yuva420p");
}
@ -3854,7 +3911,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
// alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=yuva420p");
@ -3863,7 +3920,7 @@ namespace MediaBrowser.Controller.MediaEncoding
subFilters.Add("hwupload=derive_device=opencl");
overlayFilters.Add("overlay_opencl=eof_action=pass:repeatlast=0");
overlayFilters.Add("hwmap=derive_device=d3d11va:reverse=1");
overlayFilters.Add("hwmap=derive_device=d3d11va:mode=write:reverse=1");
overlayFilters.Add("format=d3d11");
}
}
@ -3871,7 +3928,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -3967,6 +4024,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doVppTranspose = !string.IsNullOrEmpty(tranposeDir);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || ((isD3d11vaDecoder || isQsvDecoder) && doVppTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -3983,10 +4047,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12");
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// keep video at memory except ocl tonemap,
// since the overhead caused by hwupload >>> using sw filter.
@ -3998,8 +4062,15 @@ namespace MediaBrowser.Controller.MediaEncoding
}
else if (isD3d11vaDecoder || isQsvDecoder)
{
var outFormat = doOclTonemap ? string.Empty : "nv12";
var hwScaleFilter = GetHwScaleFilter("qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var outFormat = doOclTonemap ? (doVppTranspose ? "p010" : string.Empty) : "nv12";
var swapOutputWandH = doVppTranspose && swapWAndH;
var hwScalePrefix = doVppTranspose ? "vpp" : "scale";
var hwScaleFilter = GetHwScaleFilter(hwScalePrefix, "qsv", outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
if (!string.IsNullOrEmpty(hwScaleFilter) && doVppTranspose)
{
hwScaleFilter += $":transpose={tranposeDir}";
}
if (isD3d11vaDecoder)
{
@ -4018,14 +4089,14 @@ namespace MediaBrowser.Controller.MediaEncoding
mainFilters.Add(deintFilter);
}
// hw scale
// hw transpose & scale
mainFilters.Add(hwScaleFilter);
}
if (doOclTonemap && isHwDecoder)
{
// map from qsv to opencl via qsv(d3d11)-opencl interop.
mainFilters.Add("hwmap=derive_device=opencl");
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
}
// hw tonemap
@ -4069,7 +4140,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
// OUTPUT qsv(nv12) surface(vram)
// reverse-mapping via qsv(d3d11)-opencl interop.
mainFilters.Add("hwmap=derive_device=qsv:reverse=1");
mainFilters.Add("hwmap=derive_device=qsv:mode=write:reverse=1");
mainFilters.Add("format=qsv");
}
@ -4083,7 +4154,7 @@ namespace MediaBrowser.Controller.MediaEncoding
if (hasGraphicalSubs)
{
// overlay_qsv can handle overlay scaling, setup a smaller height to reduce transfer overhead
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, 1080);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -4093,7 +4164,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
// alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -4104,9 +4175,9 @@ namespace MediaBrowser.Controller.MediaEncoding
// default to 64 otherwise it will fail on certain iGPU.
subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64");
var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
var overlaySize = (overlayW.HasValue && overlayH.HasValue)
? (":w=" + overlayW.Value + ":h=" + overlayH.Value)
? $":w={overlayW.Value}:h={overlayH.Value}"
: string.Empty;
var overlayQsvFilter = string.Format(
CultureInfo.InvariantCulture,
@ -4119,7 +4190,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -4164,6 +4235,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doVppTranspose = !string.IsNullOrEmpty(tranposeDir);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || ((isVaapiDecoder || isQsvDecoder) && doVppTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -4180,10 +4258,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12");
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// keep video at memory except ocl tonemap,
// since the overhead caused by hwupload >>> using sw filter.
@ -4195,24 +4273,39 @@ namespace MediaBrowser.Controller.MediaEncoding
}
else if (isVaapiDecoder || isQsvDecoder)
{
var hwFilterSuffix = isVaapiDecoder ? "vaapi" : "qsv";
// INPUT vaapi/qsv surface(vram)
// hw deint
if (doDeintH2645)
{
var deintFilter = GetHwDeinterlaceFilter(state, options, isVaapiDecoder ? "vaapi" : "qsv");
var deintFilter = GetHwDeinterlaceFilter(state, options, hwFilterSuffix);
mainFilters.Add(deintFilter);
}
var outFormat = doTonemap ? string.Empty : "nv12";
var hwScaleFilter = GetHwScaleFilter(isVaapiDecoder ? "vaapi" : "qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
// hw transpose(vaapi vpp)
if (isVaapiDecoder && doVppTranspose)
{
mainFilters.Add($"transpose_vaapi=dir={tranposeDir}");
}
// allocate extra pool sizes for vaapi vpp
var outFormat = doOclTonemap ? ((isQsvDecoder && doVppTranspose) ? "p010" : string.Empty) : "nv12";
var swapOutputWandH = isQsvDecoder && doVppTranspose && swapWAndH;
var hwScalePrefix = (isQsvDecoder && doVppTranspose) ? "vpp" : "scale";
var hwScaleFilter = GetHwScaleFilter(hwScalePrefix, hwFilterSuffix, outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
if (!string.IsNullOrEmpty(hwScaleFilter) && isQsvDecoder && doVppTranspose)
{
hwScaleFilter += $":transpose={tranposeDir}";
}
// allocate extra pool sizes for vaapi vpp scale
if (!string.IsNullOrEmpty(hwScaleFilter) && isVaapiDecoder)
{
hwScaleFilter += ":extra_hw_frames=24";
}
// hw scale
// hw transpose(qsv vpp) & scale
mainFilters.Add(hwScaleFilter);
}
@ -4240,7 +4333,7 @@ namespace MediaBrowser.Controller.MediaEncoding
if (doOclTonemap && isHwDecoder)
{
// map from qsv to opencl via qsv(vaapi)-opencl interop.
mainFilters.Add("hwmap=derive_device=opencl");
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
}
// ocl tonemap
@ -4287,7 +4380,7 @@ namespace MediaBrowser.Controller.MediaEncoding
// OUTPUT qsv(nv12) surface(vram)
// reverse-mapping via qsv(vaapi)-opencl interop.
// add extra pool size to avoid the 'cannot allocate memory' error on hevc_qsv.
mainFilters.Add("hwmap=derive_device=qsv:reverse=1:extra_hw_frames=16");
mainFilters.Add("hwmap=derive_device=qsv:mode=write:reverse=1:extra_hw_frames=16");
mainFilters.Add("format=qsv");
}
else if (isVaapiDecoder)
@ -4307,7 +4400,7 @@ namespace MediaBrowser.Controller.MediaEncoding
if (hasGraphicalSubs)
{
// overlay_qsv can handle overlay scaling, setup a smaller height to reduce transfer overhead
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, 1080);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -4316,7 +4409,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var framerate = state.VideoStream?.RealFrameRate;
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -4327,9 +4420,9 @@ namespace MediaBrowser.Controller.MediaEncoding
// default to 64 otherwise it will fail on certain iGPU.
subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64");
var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
var overlaySize = (overlayW.HasValue && overlayH.HasValue)
? (":w=" + overlayW.Value + ":h=" + overlayH.Value)
? $":w={overlayW.Value}:h={overlayH.Value}"
: string.Empty;
var overlayQsvFilter = string.Format(
CultureInfo.InvariantCulture,
@ -4342,7 +4435,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -4453,6 +4546,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doVaVppTranspose = !string.IsNullOrEmpty(tranposeDir);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isVaapiDecoder && doVaVppTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -4469,10 +4569,10 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doOclTonemap ? "yuv420p10le" : "nv12";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// keep video at memory except ocl tonemap,
// since the overhead caused by hwupload >>> using sw filter.
@ -4492,8 +4592,14 @@ namespace MediaBrowser.Controller.MediaEncoding
mainFilters.Add(deintFilter);
}
// hw transpose
if (doVaVppTranspose)
{
mainFilters.Add($"transpose_vaapi=dir={tranposeDir}");
}
var outFormat = doTonemap ? string.Empty : "nv12";
var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
// allocate extra pool sizes for vaapi vpp
if (!string.IsNullOrEmpty(hwScaleFilter))
@ -4515,7 +4621,7 @@ namespace MediaBrowser.Controller.MediaEncoding
if (doOclTonemap && isVaapiDecoder)
{
// map from vaapi to opencl via vaapi-opencl interop(Intel only).
mainFilters.Add("hwmap=derive_device=opencl");
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
}
// ocl tonemap
@ -4529,7 +4635,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
// OUTPUT vaapi(nv12) surface(vram)
// reverse-mapping via vaapi-opencl interop.
mainFilters.Add("hwmap=derive_device=vaapi:reverse=1");
mainFilters.Add("hwmap=derive_device=vaapi:mode=write:reverse=1");
mainFilters.Add("format=vaapi");
}
@ -4580,7 +4686,7 @@ namespace MediaBrowser.Controller.MediaEncoding
if (hasGraphicalSubs)
{
// overlay_vaapi can handle overlay scaling, setup a smaller height to reduce transfer overhead
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, 1080);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -4589,7 +4695,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var framerate = state.VideoStream?.RealFrameRate;
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -4598,9 +4704,9 @@ namespace MediaBrowser.Controller.MediaEncoding
subFilters.Add("hwupload=derive_device=vaapi");
var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
var overlaySize = (overlayW.HasValue && overlayH.HasValue)
? (":w=" + overlayW.Value + ":h=" + overlayH.Value)
? $":w={overlayW.Value}:h={overlayH.Value}"
: string.Empty;
var overlayVaapiFilter = string.Format(
CultureInfo.InvariantCulture,
@ -4613,7 +4719,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
@ -4658,6 +4764,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doVkTranspose = isVaapiDecoder && !string.IsNullOrEmpty(tranposeDir);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isVaapiDecoder && doVkTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -4682,7 +4795,7 @@ namespace MediaBrowser.Controller.MediaEncoding
else
{
// sw scale
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=nv12");
}
@ -4690,7 +4803,7 @@ namespace MediaBrowser.Controller.MediaEncoding
else if (isVaapiDecoder)
{
// INPUT vaapi surface(vram)
if (doVkTonemap || hasSubs)
if (doVkTranspose || doVkTonemap || hasSubs)
{
// map from vaapi to vulkan/drm via interop (Polaris/gfx8+).
mainFilters.Add("hwmap=derive_device=vulkan");
@ -4706,15 +4819,28 @@ namespace MediaBrowser.Controller.MediaEncoding
}
// hw scale
var hwScaleFilter = GetHwScaleFilter("vaapi", "nv12", inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", "nv12", false, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
mainFilters.Add(hwScaleFilter);
}
}
// vk transpose
if (doVkTranspose)
{
if (string.Equals(tranposeDir, "reversal", StringComparison.OrdinalIgnoreCase))
{
mainFilters.Add("flip_vulkan");
}
else
{
mainFilters.Add($"transpose_vulkan=dir={tranposeDir}");
}
}
// vk libplacebo
if (doVkTonemap || hasSubs)
{
var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
mainFilters.Add(libplaceboFilter);
}
@ -4758,7 +4884,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -4767,7 +4893,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var framerate = state.VideoStream?.RealFrameRate;
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -4839,6 +4965,11 @@ namespace MediaBrowser.Controller.MediaEncoding
var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
var rotation = state.VideoStream?.Rotation ?? 0;
var swapWAndH = Math.Abs(rotation) == 90 && isSwDecoder;
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -4856,7 +4987,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
outFormat = doOclTonemap ? "yuv420p10le" : "nv12";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
@ -4880,7 +5011,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
outFormat = doOclTonemap ? string.Empty : "nv12";
var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", outFormat, false, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
// allocate extra pool sizes for vaapi vpp
if (!string.IsNullOrEmpty(hwScaleFilter))
@ -4976,7 +5107,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
@ -5030,6 +5161,15 @@ namespace MediaBrowser.Controller.MediaEncoding
string vidDecoder,
string vidEncoder)
{
var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
var isVtDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
if (!isVtEncoder)
{
// should not happen.
return (null, null, null);
}
var inW = state.VideoStream?.Width;
var inH = state.VideoStream?.Height;
var reqW = state.BaseRequest.Width;
@ -5038,9 +5178,6 @@ namespace MediaBrowser.Controller.MediaEncoding
var reqMaxH = state.BaseRequest.MaxHeight;
var threeDFormat = state.MediaSource.Video3DFormat;
var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
var isVtDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
var doDeintH2645 = doDeintH264 || doDeintHevc;
@ -5048,6 +5185,13 @@ namespace MediaBrowser.Controller.MediaEncoding
var doMetalTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
var usingHwSurface = isVtDecoder && (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface);
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doVtTranspose = !string.IsNullOrEmpty(tranposeDir) && _mediaEncoder.SupportsFilter("transpose_vt");
var swapWAndH = Math.Abs(rotation) == 90 && doVtTranspose;
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
var scaleFormat = string.Empty;
// Use P010 for Metal tone mapping, otherwise force an 8bit output.
if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase))
@ -5065,7 +5209,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
}
var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter = GetHwScaleFilter("scale", "vt", scaleFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
@ -5074,12 +5218,6 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
if (!isVtEncoder)
{
// should not happen.
return (null, null, null);
}
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -5090,6 +5228,12 @@ namespace MediaBrowser.Controller.MediaEncoding
mainFilters.Add(deintFilter);
}
// hw transpose
if (doVtTranspose)
{
mainFilters.Add($"transpose_vt=dir={tranposeDir}");
}
if (doVtTonemap)
{
const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709";
@ -5118,7 +5262,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -5127,7 +5271,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var framerate = state.VideoStream?.RealFrameRate;
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -5229,6 +5373,13 @@ namespace MediaBrowser.Controller.MediaEncoding
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
var rotation = state.VideoStream?.Rotation ?? 0;
var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
var doRkVppTranspose = !string.IsNullOrEmpty(tranposeDir);
var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isRkmppDecoder && doRkVppTranspose));
var swpInW = swapWAndH ? inH : inW;
var swpInH = swapWAndH ? inW : inH;
/* Make main filters for video stream */
var mainFilters = new List<string>();
@ -5245,7 +5396,7 @@ namespace MediaBrowser.Controller.MediaEncoding
}
var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12");
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
if (!string.IsNullOrEmpty(swScaleFilter))
{
swScaleFilter += ":flags=fast_bilinear";
@ -5253,7 +5404,7 @@ namespace MediaBrowser.Controller.MediaEncoding
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
mainFilters.Add($"format={outFormat}");
// keep video at memory except ocl tonemap,
// since the overhead caused by hwupload >>> using sw filter.
@ -5268,21 +5419,29 @@ namespace MediaBrowser.Controller.MediaEncoding
// INPUT rkmpp/drm surface(gem/dma-heap)
var isFullAfbcPipeline = isDrmInDrmOut && !doOclTonemap;
var swapOutputWandH = doRkVppTranspose && swapWAndH;
var outFormat = doOclTonemap ? "p010" : "nv12";
var hwScaleFilter = GetHwScaleFilter("rkrga", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter2 = GetHwScaleFilter("rkrga", string.Empty, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var hwScalePrefix = doRkVppTranspose ? "vpp" : "scale";
var hwScaleFilter = GetHwScaleFilter(hwScalePrefix, "rkrga", outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
var hwScaleFilter2 = GetHwScaleFilter(hwScalePrefix, "rkrga", string.Empty, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
if (!hasSubs
|| doRkVppTranspose
|| !isFullAfbcPipeline
|| !string.IsNullOrEmpty(hwScaleFilter2))
{
if (!string.IsNullOrEmpty(hwScaleFilter) && doRkVppTranspose)
{
hwScaleFilter += $":transpose={tranposeDir}";
}
// try enabling AFBC to save DDR bandwidth
if (!string.IsNullOrEmpty(hwScaleFilter) && isFullAfbcPipeline)
{
hwScaleFilter += ":afbc=1";
}
// hw scale
// hw transpose & scale
mainFilters.Add(hwScaleFilter);
}
}
@ -5353,7 +5512,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
subFilters.Add("format=bgra");
}
@ -5363,7 +5522,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
// alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
@ -5380,7 +5539,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (hasGraphicalSubs)
{
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subPreProcFilters);
overlayFilters.Add("overlay=eof_action=pass:repeatlast=0");
}
@ -5795,6 +5954,11 @@ namespace MediaBrowser.Controller.MediaEncoding
// Disable the extra internal copy in nvdec. We already handle it in filter chain.
var nvdecNoInternalCopy = ffmpegVersion >= _minFFmpegHwaUnsafeOutput;
// Strip the display rotation side data from the transposed fmp4 output stream.
var stripRotationData = (state.VideoStream?.Rotation ?? 0) != 0
&& ffmpegVersion >= _minFFmpegDisplayRotationOption;
var stripRotationDataArgs = stripRotationData ? " -display_rotation 0" : string.Empty;
if (bitDepth == 10 && isCodecAvailable)
{
if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase)
@ -5819,13 +5983,13 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (isVaapiSupported && isCodecAvailable)
{
return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty)
return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi -noautorotate" + stripRotationDataArgs : string.Empty)
+ (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty);
}
if (isD3d11Supported && isCodecAvailable)
{
return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty)
return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11 -noautorotate" + stripRotationDataArgs : string.Empty)
+ (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + " -threads 2" + (isAv1 ? " -c:v av1" : string.Empty);
}
}
@ -5833,7 +5997,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (isQsvSupported && isCodecAvailable)
{
return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv" : string.Empty);
return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv -noautorotate" + stripRotationDataArgs : string.Empty);
}
}
}
@ -5846,12 +6010,12 @@ namespace MediaBrowser.Controller.MediaEncoding
if (options.EnableEnhancedNvdecDecoder)
{
// set -threads 1 to nvdec decoder explicitly since it doesn't implement threading support.
return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty)
return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda -noautorotate" + stripRotationDataArgs : string.Empty)
+ (nvdecNoInternalCopy ? " -hwaccel_flags +unsafe_output" : string.Empty) + " -threads 1" + (isAv1 ? " -c:v av1" : string.Empty);
}
// cuvid decoder doesn't have threading issue.
return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty);
return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda -noautorotate" + stripRotationDataArgs : string.Empty);
}
}
@ -5860,7 +6024,7 @@ namespace MediaBrowser.Controller.MediaEncoding
{
if (isD3d11Supported && isCodecAvailable)
{
return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty)
return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11 -noautorotate" + stripRotationDataArgs : string.Empty)
+ (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty);
}
}
@ -5870,7 +6034,7 @@ namespace MediaBrowser.Controller.MediaEncoding
&& isVaapiSupported
&& isCodecAvailable)
{
return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty)
return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi -noautorotate" + stripRotationDataArgs : string.Empty)
+ (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty);
}
@ -5879,7 +6043,7 @@ namespace MediaBrowser.Controller.MediaEncoding
&& isVideotoolboxSupported
&& isCodecAvailable)
{
return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty);
return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty) + "-noautorotate" + stripRotationDataArgs;
}
// Rockchip rkmpp
@ -5887,7 +6051,7 @@ namespace MediaBrowser.Controller.MediaEncoding
&& isRkmppSupported
&& isCodecAvailable)
{
return " -hwaccel rkmpp" + (outputHwSurface ? " -hwaccel_output_format drm_prime" : string.Empty);
return " -hwaccel rkmpp" + (outputHwSurface ? " -hwaccel_output_format drm_prime -noautorotate" + stripRotationDataArgs : string.Empty);
}
return null;

7
MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs
View File

@ -33,6 +33,11 @@ namespace MediaBrowser.Controller.MediaEncoding
/// <summary>
/// The overlay_vulkan_framesync.
/// </summary>
OverlayVulkanFrameSync = 5
OverlayVulkanFrameSync = 5,
/// <summary>
/// The transpose_opencl_reversal.
/// </summary>
TransposeOpenclReversal = 6
}
}

9
MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
View File

@ -112,25 +112,31 @@ namespace MediaBrowser.MediaEncoding.Encoder
"yadif_cuda",
"tonemap_cuda",
"overlay_cuda",
"transpose_cuda",
"hwupload_cuda",
// opencl
"scale_opencl",
"tonemap_opencl",
"overlay_opencl",
"transpose_opencl",
// vaapi
"scale_vaapi",
"deinterlace_vaapi",
"tonemap_vaapi",
"procamp_vaapi",
"overlay_vaapi",
"transpose_vaapi",
"hwupload_vaapi",
// vulkan
"libplacebo",
"scale_vulkan",
"overlay_vulkan",
"transpose_vulkan",
"flip_vulkan",
// videotoolbox
"yadif_videotoolbox",
"scale_vt",
"transpose_vt",
"overlay_videotoolbox",
"tonemap_videotoolbox",
// rkrga
@ -146,7 +152,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
{ 2, new string[] { "tonemap_opencl", "bt2390" } },
{ 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
{ 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } },
{ 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } }
{ 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } },
{ 6, new string[] { "transpose_opencl", "rotate by half-turn" } }
};
// These are the library versions that corresponds to our minimum ffmpeg version 4.4 according to the version table below

7
MediaBrowser.MediaEncoding/Probing/MediaStreamInfoSideData.cs
View File

@ -69,5 +69,12 @@ namespace MediaBrowser.MediaEncoding.Probing
/// <value>The DvBlSignalCompatibilityId.</value>
[JsonPropertyName("dv_bl_signal_compatibility_id")]
public int? DvBlSignalCompatibilityId { get; set; }
/// <summary>
/// Gets or sets the Rotation in degrees.
/// </summary>
/// <value>The Rotation.</value>
[JsonPropertyName("rotation")]
public int? Rotation { get; set; }
}
}

6
MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
View File

@ -892,8 +892,12 @@ namespace MediaBrowser.MediaEncoding.Probing
stream.ElPresentFlag = data.ElPresentFlag;
stream.BlPresentFlag = data.BlPresentFlag;
stream.DvBlSignalCompatibilityId = data.DvBlSignalCompatibilityId;
}
break;
// Parse video rotation metadata from side_data
else if (string.Equals(data.SideDataType, "Display Matrix", StringComparison.OrdinalIgnoreCase))
{
stream.Rotation = data.Rotation;
}
}
}

6
MediaBrowser.Model/Entities/MediaStream.cs
View File

@ -123,6 +123,12 @@ namespace MediaBrowser.Model.Entities
/// <value>The Dolby Vision bl signal compatibility id.</value>
public int? DvBlSignalCompatibilityId { get; set; }
/// <summary>
/// Gets or sets the Rotation in degrees.
/// </summary>
/// <value>The video rotation.</value>
public int? Rotation { get; set; }
/// <summary>
/// Gets or sets the comment.
/// </summary>

1
tests/Jellyfin.MediaEncoding.Tests/Probing/ProbeResultNormalizerTests.cs
View File

@ -84,6 +84,7 @@ namespace Jellyfin.MediaEncoding.Tests.Probing
Assert.Equal(0, res.VideoStream.ElPresentFlag);
Assert.Equal(1, res.VideoStream.BlPresentFlag);
Assert.Equal(0, res.VideoStream.DvBlSignalCompatibilityId);
Assert.Equal(-180, res.VideoStream.Rotation);
var audio1 = res.MediaStreams[1];
Assert.Equal("eac3", audio1.Codec);

4
tests/Jellyfin.MediaEncoding.Tests/Test Data/Probing/video_metadata.json
View File

@ -59,6 +59,10 @@
"el_present_flag": 0,
"bl_present_flag": 1,
"dv_bl_signal_compatibility_id": 0
},
{
"side_data_type": "Display Matrix",
"rotation": -180
}
]
},