using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Jellyfin.Data.Entities; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Net; using Microsoft.Extensions.Logging; using Photo = MediaBrowser.Controller.Entities.Photo; namespace Emby.Drawing { /// /// Class ImageProcessor. /// public sealed class ImageProcessor : IImageProcessor, IDisposable { // Increment this when there's a change requiring caches to be invalidated private const string Version = "3"; private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".webp", ".gif" }; private readonly ILogger _logger; private readonly IFileSystem _fileSystem; private readonly IServerApplicationPaths _appPaths; private readonly IImageEncoder _imageEncoder; private readonly IMediaEncoder _mediaEncoder; private bool _disposed; /// /// Initializes a new instance of the class. /// /// The logger. /// The server application paths. /// The filesystem. /// The image encoder. /// The media encoder. public ImageProcessor( ILogger logger, IServerApplicationPaths appPaths, IFileSystem fileSystem, IImageEncoder imageEncoder, IMediaEncoder mediaEncoder) { _logger = logger; _fileSystem = fileSystem; _imageEncoder = imageEncoder; _mediaEncoder = mediaEncoder; _appPaths = appPaths; } private string ResizedImageCachePath => Path.Combine(_appPaths.ImageCachePath, "resized-images"); /// public IReadOnlyCollection SupportedInputFormats => new HashSet(StringComparer.OrdinalIgnoreCase) { "tiff", "tif", "jpeg", "jpg", "png", "aiff", "cr2", "crw", "nef", "orf", "pef", "arw", "webp", "gif", "bmp", "erf", "raf", "rw2", "nrw", "dng", "ico", "astc", "ktx", "pkm", "wbmp" }; /// public bool SupportsImageCollageCreation => _imageEncoder.SupportsImageCollageCreation; /// public async Task ProcessImage(ImageProcessingOptions options, Stream toStream) { var file = await ProcessImage(options).ConfigureAwait(false); using (var fileStream = new FileStream(file.Item1, FileMode.Open, FileAccess.Read, FileShare.Read, IODefaults.FileStreamBufferSize, AsyncFile.UseAsyncIO)) { await fileStream.CopyToAsync(toStream).ConfigureAwait(false); } } /// public IReadOnlyCollection GetSupportedImageOutputFormats() => _imageEncoder.SupportedOutputFormats; /// public bool SupportsTransparency(string path) => _transparentImageTypes.Contains(Path.GetExtension(path)); /// public async Task<(string path, string? mimeType, DateTime dateModified)> ProcessImage(ImageProcessingOptions options) { ItemImageInfo originalImage = options.Image; BaseItem item = options.Item; string originalImagePath = originalImage.Path; DateTime dateModified = originalImage.DateModified; ImageDimensions? originalImageSize = null; if (originalImage.Width > 0 && originalImage.Height > 0) { originalImageSize = new ImageDimensions(originalImage.Width, originalImage.Height); } if (!_imageEncoder.SupportsImageEncoding) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } var supportedImageInfo = await GetSupportedImage(originalImagePath, dateModified).ConfigureAwait(false); originalImagePath = supportedImageInfo.path; if (!File.Exists(originalImagePath)) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } dateModified = supportedImageInfo.dateModified; bool requiresTransparency = _transparentImageTypes.Contains(Path.GetExtension(originalImagePath)); bool autoOrient = false; ImageOrientation? orientation = null; if (item is Photo photo) { if (photo.Orientation.HasValue) { if (photo.Orientation.Value != ImageOrientation.TopLeft) { autoOrient = true; orientation = photo.Orientation; } } else { // Orientation unknown, so do it autoOrient = true; orientation = photo.Orientation; } } if (options.HasDefaultOptions(originalImagePath, originalImageSize) && (!autoOrient || !options.RequiresAutoOrientation)) { // Just spit out the original file if all the options are default return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } int quality = options.Quality; ImageFormat outputFormat = GetOutputFormat(options.SupportedOutputFormats, requiresTransparency); string cacheFilePath = GetCacheFilePath( originalImagePath, options.Width, options.Height, options.MaxWidth, options.MaxHeight, options.FillWidth, options.FillHeight, quality, dateModified, outputFormat, options.AddPlayedIndicator, options.PercentPlayed, options.UnplayedCount, options.Blur, options.BackgroundColor, options.ForegroundLayer); try { if (!File.Exists(cacheFilePath)) { string resultPath = _imageEncoder.EncodeImage(originalImagePath, dateModified, cacheFilePath, autoOrient, orientation, quality, options, outputFormat); if (string.Equals(resultPath, originalImagePath, StringComparison.OrdinalIgnoreCase)) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } } return (cacheFilePath, GetMimeType(outputFormat, cacheFilePath), _fileSystem.GetLastWriteTimeUtc(cacheFilePath)); } catch (Exception ex) { // If it fails for whatever reason, return the original image _logger.LogError(ex, "Error encoding image"); return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } } private ImageFormat GetOutputFormat(IReadOnlyCollection clientSupportedFormats, bool requiresTransparency) { var serverFormats = GetSupportedImageOutputFormats(); // Client doesn't care about format, so start with webp if supported if (serverFormats.Contains(ImageFormat.Webp) && clientSupportedFormats.Contains(ImageFormat.Webp)) { return ImageFormat.Webp; } // If transparency is needed and webp isn't supported, than png is the only option if (requiresTransparency && clientSupportedFormats.Contains(ImageFormat.Png)) { return ImageFormat.Png; } foreach (var format in clientSupportedFormats) { if (serverFormats.Contains(format)) { return format; } } // We should never actually get here return ImageFormat.Jpg; } private string? GetMimeType(ImageFormat format, string path) => format switch { ImageFormat.Bmp => MimeTypes.GetMimeType("i.bmp"), ImageFormat.Gif => MimeTypes.GetMimeType("i.gif"), ImageFormat.Jpg => MimeTypes.GetMimeType("i.jpg"), ImageFormat.Png => MimeTypes.GetMimeType("i.png"), ImageFormat.Webp => MimeTypes.GetMimeType("i.webp"), _ => MimeTypes.GetMimeType(path) }; /// /// Gets the cache file path based on a set of parameters. /// private string GetCacheFilePath( string originalPath, int? width, int? height, int? maxWidth, int? maxHeight, int? fillWidth, int? fillHeight, int quality, DateTime dateModified, ImageFormat format, bool addPlayedIndicator, double percentPlayed, int? unwatchedCount, int? blur, string backgroundColor, string foregroundLayer) { var filename = new StringBuilder(256); filename.Append(originalPath); filename.Append(",quality="); filename.Append(quality); filename.Append(",datemodified="); filename.Append(dateModified.Ticks); filename.Append(",f="); filename.Append(format); if (width.HasValue) { filename.Append(",width="); filename.Append(width.Value); } if (height.HasValue) { filename.Append(",height="); filename.Append(height.Value); } if (maxWidth.HasValue) { filename.Append(",maxwidth="); filename.Append(maxWidth.Value); } if (maxHeight.HasValue) { filename.Append(",maxheight="); filename.Append(maxHeight.Value); } if (fillWidth.HasValue) { filename.Append(",fillwidth="); filename.Append(fillWidth.Value); } if (fillHeight.HasValue) { filename.Append(",fillheight="); filename.Append(fillHeight.Value); } if (addPlayedIndicator) { filename.Append(",pl=true"); } if (percentPlayed > 0) { filename.Append(",p="); filename.Append(percentPlayed); } if (unwatchedCount.HasValue) { filename.Append(",p="); filename.Append(unwatchedCount.Value); } if (blur.HasValue) { filename.Append(",blur="); filename.Append(blur.Value); } if (!string.IsNullOrEmpty(backgroundColor)) { filename.Append(",b="); filename.Append(backgroundColor); } if (!string.IsNullOrEmpty(foregroundLayer)) { filename.Append(",fl="); filename.Append(foregroundLayer); } filename.Append(",v="); filename.Append(Version); return GetCachePath(ResizedImageCachePath, filename.ToString(), "." + format.ToString().ToLowerInvariant()); } /// public ImageDimensions GetImageDimensions(BaseItem item, ItemImageInfo info) { int width = info.Width; int height = info.Height; if (height > 0 && width > 0) { return new ImageDimensions(width, height); } string path = info.Path; _logger.LogDebug("Getting image size for item {ItemType} {Path}", item.GetType().Name, path); ImageDimensions size = GetImageDimensions(path); info.Width = size.Width; info.Height = size.Height; return size; } /// public ImageDimensions GetImageDimensions(string path) => _imageEncoder.GetImageSize(path); /// public string GetImageBlurHash(string path) { var size = GetImageDimensions(path); if (size.Width <= 0 || size.Height <= 0) { return string.Empty; } // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components float xCompF = MathF.Sqrt(16.0f * size.Width / size.Height); float yCompF = xCompF * size.Height / size.Width; int xComp = Math.Min((int)xCompF + 1, 9); int yComp = Math.Min((int)yCompF + 1, 9); return _imageEncoder.GetImageBlurHash(xComp, yComp, path); } /// public string GetImageCacheTag(BaseItem item, ItemImageInfo image) => (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture); /// public string GetImageCacheTag(BaseItem item, ChapterInfo chapter) { return GetImageCacheTag(item, new ItemImageInfo { Path = chapter.ImagePath, Type = ImageType.Chapter, DateModified = chapter.ImageDateModified }); } /// public string? GetImageCacheTag(User user) { if (user.ProfileImage == null) { return null; } return (user.ProfileImage.Path + user.ProfileImage.LastModified.Ticks).GetMD5() .ToString("N", CultureInfo.InvariantCulture); } private async Task<(string path, DateTime dateModified)> GetSupportedImage(string originalImagePath, DateTime dateModified) { var inputFormat = Path.GetExtension(originalImagePath) .TrimStart('.') .Replace("jpeg", "jpg", StringComparison.OrdinalIgnoreCase); // These are just jpg files renamed as tbn if (string.Equals(inputFormat, "tbn", StringComparison.OrdinalIgnoreCase)) { return (originalImagePath, dateModified); } if (!_imageEncoder.SupportedInputFormats.Contains(inputFormat)) { try { string filename = (originalImagePath + dateModified.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("N", CultureInfo.InvariantCulture); string cacheExtension = _mediaEncoder.SupportsEncoder("libwebp") ? ".webp" : ".png"; var outputPath = Path.Combine(_appPaths.ImageCachePath, "converted-images", filename + cacheExtension); var file = _fileSystem.GetFileInfo(outputPath); if (!file.Exists) { await _mediaEncoder.ConvertImage(originalImagePath, outputPath).ConfigureAwait(false); dateModified = _fileSystem.GetLastWriteTimeUtc(outputPath); } else { dateModified = file.LastWriteTimeUtc; } originalImagePath = outputPath; } catch (Exception ex) { _logger.LogError(ex, "Image conversion failed for {Path}", originalImagePath); } } return (originalImagePath, dateModified); } /// /// Gets the cache path. /// /// The path. /// Name of the unique. /// The file extension. /// System.String. /// /// path /// or /// uniqueName /// or /// fileExtension. /// public string GetCachePath(string path, string uniqueName, string fileExtension) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(nameof(path)); } if (string.IsNullOrEmpty(uniqueName)) { throw new ArgumentNullException(nameof(uniqueName)); } if (string.IsNullOrEmpty(fileExtension)) { throw new ArgumentNullException(nameof(fileExtension)); } var filename = uniqueName.GetMD5() + fileExtension; return GetCachePath(path, filename); } /// /// Gets the cache path. /// /// The path. /// The filename. /// System.String. /// /// path /// or /// filename. /// public string GetCachePath(ReadOnlySpan path, ReadOnlySpan filename) { if (path.IsEmpty) { throw new ArgumentException("Path can't be empty.", nameof(path)); } if (filename.IsEmpty) { throw new ArgumentException("Filename can't be empty.", nameof(filename)); } var prefix = filename.Slice(0, 1); return Path.Join(path, prefix, filename); } /// public void CreateImageCollage(ImageCollageOptions options, string? libraryName) { _logger.LogInformation("Creating image collage and saving to {Path}", options.OutputPath); _imageEncoder.CreateImageCollage(options, libraryName); _logger.LogInformation("Completed creation of image collage and saved to {Path}", options.OutputPath); } /// public void Dispose() { if (_disposed) { return; } if (_imageEncoder is IDisposable disposable) { disposable.Dispose(); } _disposed = true; } } }