using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; using Jellyfin.Data.Entities; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Net; using Microsoft.Extensions.Logging; using Photo = MediaBrowser.Controller.Entities.Photo; namespace Emby.Drawing { /// /// Class ImageProcessor. /// public sealed class ImageProcessor : IImageProcessor, IDisposable { // Increment this when there's a change requiring caches to be invalidated private const string Version = "3"; private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".webp", ".gif" }; private readonly ILogger _logger; private readonly IFileSystem _fileSystem; private readonly IServerApplicationPaths _appPaths; private readonly IImageEncoder _imageEncoder; private readonly IMediaEncoder _mediaEncoder; private bool _disposed = false; /// /// Initializes a new instance of the class. /// /// The logger. /// The server application paths. /// The filesystem. /// The image encoder. /// The media encoder. public ImageProcessor( ILogger logger, IServerApplicationPaths appPaths, IFileSystem fileSystem, IImageEncoder imageEncoder, IMediaEncoder mediaEncoder) { _logger = logger; _fileSystem = fileSystem; _imageEncoder = imageEncoder; _mediaEncoder = mediaEncoder; _appPaths = appPaths; } private string ResizedImageCachePath => Path.Combine(_appPaths.ImageCachePath, "resized-images"); /// public IReadOnlyCollection SupportedInputFormats => new HashSet(StringComparer.OrdinalIgnoreCase) { "tiff", "tif", "jpeg", "jpg", "png", "aiff", "cr2", "crw", "nef", "orf", "pef", "arw", "webp", "gif", "bmp", "erf", "raf", "rw2", "nrw", "dng", "ico", "astc", "ktx", "pkm", "wbmp" }; /// public bool SupportsImageCollageCreation => _imageEncoder.SupportsImageCollageCreation; /// public async Task ProcessImage(ImageProcessingOptions options, Stream toStream) { var file = await ProcessImage(options).ConfigureAwait(false); using (var fileStream = new FileStream(file.Item1, FileMode.Open, FileAccess.Read, FileShare.Read, IODefaults.FileStreamBufferSize, true)) { await fileStream.CopyToAsync(toStream).ConfigureAwait(false); } } /// public IReadOnlyCollection GetSupportedImageOutputFormats() => _imageEncoder.SupportedOutputFormats; /// public bool SupportsTransparency(string path) => _transparentImageTypes.Contains(Path.GetExtension(path)); /// public async Task<(string path, string? mimeType, DateTime dateModified)> ProcessImage(ImageProcessingOptions options) { ItemImageInfo originalImage = options.Image; BaseItem item = options.Item; string originalImagePath = originalImage.Path; DateTime dateModified = originalImage.DateModified; ImageDimensions? originalImageSize = null; if (originalImage.Width > 0 && originalImage.Height > 0) { originalImageSize = new ImageDimensions(originalImage.Width, originalImage.Height); } if (!_imageEncoder.SupportsImageEncoding) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } var supportedImageInfo = await GetSupportedImage(originalImagePath, dateModified).ConfigureAwait(false); originalImagePath = supportedImageInfo.path; if (!File.Exists(originalImagePath)) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } dateModified = supportedImageInfo.dateModified; bool requiresTransparency = _transparentImageTypes.Contains(Path.GetExtension(originalImagePath)); bool autoOrient = false; ImageOrientation? orientation = null; if (item is Photo photo) { if (photo.Orientation.HasValue) { if (photo.Orientation.Value != ImageOrientation.TopLeft) { autoOrient = true; orientation = photo.Orientation; } } else { // Orientation unknown, so do it autoOrient = true; orientation = photo.Orientation; } } if (options.HasDefaultOptions(originalImagePath, originalImageSize) && (!autoOrient || !options.RequiresAutoOrientation)) { // Just spit out the original file if all the options are default return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } ImageDimensions newSize = ImageHelper.GetNewImageSize(options, null); int quality = options.Quality; ImageFormat outputFormat = GetOutputFormat(options.SupportedOutputFormats, requiresTransparency); string cacheFilePath = GetCacheFilePath(originalImagePath, newSize, quality, dateModified, outputFormat, options.AddPlayedIndicator, options.PercentPlayed, options.UnplayedCount, options.Blur, options.BackgroundColor, options.ForegroundLayer); try { if (!File.Exists(cacheFilePath)) { if (options.CropWhiteSpace && !SupportsTransparency(originalImagePath)) { options.CropWhiteSpace = false; } string resultPath = _imageEncoder.EncodeImage(originalImagePath, dateModified, cacheFilePath, autoOrient, orientation, quality, options, outputFormat); if (string.Equals(resultPath, originalImagePath, StringComparison.OrdinalIgnoreCase)) { return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } } return (cacheFilePath, GetMimeType(outputFormat, cacheFilePath), _fileSystem.GetLastWriteTimeUtc(cacheFilePath)); } catch (Exception ex) { // If it fails for whatever reason, return the original image _logger.LogError(ex, "Error encoding image"); return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); } } private ImageFormat GetOutputFormat(IReadOnlyCollection clientSupportedFormats, bool requiresTransparency) { var serverFormats = GetSupportedImageOutputFormats(); // Client doesn't care about format, so start with webp if supported if (serverFormats.Contains(ImageFormat.Webp) && clientSupportedFormats.Contains(ImageFormat.Webp)) { return ImageFormat.Webp; } // If transparency is needed and webp isn't supported, than png is the only option if (requiresTransparency && clientSupportedFormats.Contains(ImageFormat.Png)) { return ImageFormat.Png; } foreach (var format in clientSupportedFormats) { if (serverFormats.Contains(format)) { return format; } } // We should never actually get here return ImageFormat.Jpg; } private string? GetMimeType(ImageFormat format, string path) => format switch { ImageFormat.Bmp => MimeTypes.GetMimeType("i.bmp"), ImageFormat.Gif => MimeTypes.GetMimeType("i.gif"), ImageFormat.Jpg => MimeTypes.GetMimeType("i.jpg"), ImageFormat.Png => MimeTypes.GetMimeType("i.png"), ImageFormat.Webp => MimeTypes.GetMimeType("i.webp"), _ => MimeTypes.GetMimeType(path) }; /// /// Gets the cache file path based on a set of parameters. /// private string GetCacheFilePath(string originalPath, ImageDimensions outputSize, int quality, DateTime dateModified, ImageFormat format, bool addPlayedIndicator, double percentPlayed, int? unwatchedCount, int? blur, string backgroundColor, string foregroundLayer) { var filename = originalPath + "width=" + outputSize.Width + "height=" + outputSize.Height + "quality=" + quality + "datemodified=" + dateModified.Ticks + "f=" + format; if (addPlayedIndicator) { filename += "pl=true"; } if (percentPlayed > 0) { filename += "p=" + percentPlayed; } if (unwatchedCount.HasValue) { filename += "p=" + unwatchedCount.Value; } if (blur.HasValue) { filename += "blur=" + blur.Value; } if (!string.IsNullOrEmpty(backgroundColor)) { filename += "b=" + backgroundColor; } if (!string.IsNullOrEmpty(foregroundLayer)) { filename += "fl=" + foregroundLayer; } filename += "v=" + Version; return GetCachePath(ResizedImageCachePath, filename, "." + format.ToString().ToLowerInvariant()); } /// public ImageDimensions GetImageDimensions(BaseItem item, ItemImageInfo info) { int width = info.Width; int height = info.Height; if (height > 0 && width > 0) { return new ImageDimensions(width, height); } string path = info.Path; _logger.LogDebug("Getting image size for item {ItemType} {Path}", item.GetType().Name, path); ImageDimensions size = GetImageDimensions(path); info.Width = size.Width; info.Height = size.Height; return size; } /// public ImageDimensions GetImageDimensions(string path) => _imageEncoder.GetImageSize(path); /// public string GetImageBlurHash(string path) { var size = GetImageDimensions(path); if (size.Width <= 0 || size.Height <= 0) { return string.Empty; } // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components float xCompF = MathF.Sqrt(16.0f * size.Width / size.Height); float yCompF = xCompF * size.Height / size.Width; int xComp = Math.Min((int)xCompF + 1, 9); int yComp = Math.Min((int)yCompF + 1, 9); return _imageEncoder.GetImageBlurHash(xComp, yComp, path); } /// public string GetImageCacheTag(BaseItem item, ItemImageInfo image) => (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture); /// public string GetImageCacheTag(BaseItem item, ChapterInfo chapter) { return GetImageCacheTag(item, new ItemImageInfo { Path = chapter.ImagePath, Type = ImageType.Chapter, DateModified = chapter.ImageDateModified }); } /// public string GetImageCacheTag(User user) { return (user.ProfileImage.Path + user.ProfileImage.LastModified.Ticks).GetMD5() .ToString("N", CultureInfo.InvariantCulture); } private async Task<(string path, DateTime dateModified)> GetSupportedImage(string originalImagePath, DateTime dateModified) { var inputFormat = Path.GetExtension(originalImagePath) .TrimStart('.') .Replace("jpeg", "jpg", StringComparison.OrdinalIgnoreCase); // These are just jpg files renamed as tbn if (string.Equals(inputFormat, "tbn", StringComparison.OrdinalIgnoreCase)) { return (originalImagePath, dateModified); } if (!_imageEncoder.SupportedInputFormats.Contains(inputFormat)) { try { string filename = (originalImagePath + dateModified.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("N", CultureInfo.InvariantCulture); string cacheExtension = _mediaEncoder.SupportsEncoder("libwebp") ? ".webp" : ".png"; var outputPath = Path.Combine(_appPaths.ImageCachePath, "converted-images", filename + cacheExtension); var file = _fileSystem.GetFileInfo(outputPath); if (!file.Exists) { await _mediaEncoder.ConvertImage(originalImagePath, outputPath).ConfigureAwait(false); dateModified = _fileSystem.GetLastWriteTimeUtc(outputPath); } else { dateModified = file.LastWriteTimeUtc; } originalImagePath = outputPath; } catch (Exception ex) { _logger.LogError(ex, "Image conversion failed for {Path}", originalImagePath); } } return (originalImagePath, dateModified); } /// /// Gets the cache path. /// /// The path. /// Name of the unique. /// The file extension. /// System.String. /// /// path /// or /// uniqueName /// or /// fileExtension. /// public string GetCachePath(string path, string uniqueName, string fileExtension) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(nameof(path)); } if (string.IsNullOrEmpty(uniqueName)) { throw new ArgumentNullException(nameof(uniqueName)); } if (string.IsNullOrEmpty(fileExtension)) { throw new ArgumentNullException(nameof(fileExtension)); } var filename = uniqueName.GetMD5() + fileExtension; return GetCachePath(path, filename); } /// /// Gets the cache path. /// /// The path. /// The filename. /// System.String. /// /// path /// or /// filename. /// public string GetCachePath(string path, string filename) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(nameof(path)); } if (string.IsNullOrEmpty(filename)) { throw new ArgumentNullException(nameof(filename)); } var prefix = filename.Substring(0, 1); return Path.Combine(path, prefix, filename); } /// public void CreateImageCollage(ImageCollageOptions options) { _logger.LogInformation("Creating image collage and saving to {Path}", options.OutputPath); _imageEncoder.CreateImageCollage(options); _logger.LogInformation("Completed creation of image collage and saved to {Path}", options.OutputPath); } /// public void Dispose() { if (_disposed) { return; } if (_imageEncoder is IDisposable disposable) { disposable.Dispose(); } _disposed = true; } } }