cotugno/jellyfin
2
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2024-11-15 18:08:53 -07:00
Code Issues Packages Projects Releases Wiki Activity

reduce fanart tv requests by using their updates api

Browse Source
This commit is contained in:
Luke Pulverenti 2013-05-27 14:33:43 -04:00
parent 9a820efde3
commit d4e3c6aa52
4 changed files with 298 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MediaBrowser.Controller/MediaBrowser.Controller.csproj
View File

@ -79,6 +79,7 @@
<Compile Include="Providers\Music\ArtistsPostScanTask.cs" />
<Compile Include="Providers\Music\FanArtUpdatesPrescanTask.cs" />
<Compile Include="Providers\TV\FanArtSeasonProvider.cs" />
<Compile Include="Providers\TV\FanArtTvUpdatesPrescanTask.cs" />
<Compile Include="Providers\TV\TvdbPrescanTask.cs" />
<Compile Include="Providers\TV\TvdbSeriesImageProvider.cs" />
<Compile Include="Session\ISessionManager.cs" />

8
MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs
View File

@ -68,7 +68,7 @@ namespace MediaBrowser.Controller.Providers.Music
return;
}
// Find out the last time we queried tvdb for updates
// Find out the last time we queried for updates
var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
@ -93,11 +93,11 @@ namespace MediaBrowser.Controller.Providers.Music
/// <summary>
/// Gets the artist ids to update.
/// </summary>
/// <param name="existingSeriesIds">The existing series ids.</param>
/// <param name="existingArtistIds">The existing series ids.</param>
/// <param name="lastUpdateTime">The last update time.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task{IEnumerable{System.String}}.</returns>
private async Task<IEnumerable<string>> GetArtistIdsToUpdate(IEnumerable<string> existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken)
private async Task<IEnumerable<string>> GetArtistIdsToUpdate(IEnumerable<string> existingArtistIds, string lastUpdateTime, CancellationToken cancellationToken)
{
// First get last time
using (var stream = await _httpClient.Get(new HttpRequestOptions
@ -121,7 +121,7 @@ namespace MediaBrowser.Controller.Providers.Music
var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdate>>(json);
return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase));
return updates.Select(i => i.id).Where(i => existingArtistIds.Contains(i, StringComparer.OrdinalIgnoreCase));
}
}
}

134
MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs
View File

@ -10,6 +10,7 @@ using MediaBrowser.Model.Logging;
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
@ -20,6 +21,8 @@ namespace MediaBrowser.Controller.Providers.TV
{
protected string FanArtBaseUrl = "http://api.fanart.tv/webservice/series/{0}/{1}/xml/all/1/1";
internal static FanArtTvProvider Current { get; private set; }
/// <summary>
/// Gets the HTTP client.
/// </summary>
@ -37,6 +40,7 @@ namespace MediaBrowser.Controller.Providers.TV
}
HttpClient = httpClient;
_providerManager = providerManager;
Current = this;
}
public override bool Supports(BaseItem item)
@ -80,7 +84,23 @@ namespace MediaBrowser.Controller.Providers.TV
/// <returns>Guid.</returns>
private Guid GetComparisonData(string id)
{
return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5();
if (!string.IsNullOrEmpty(id))
{
// Process images
var path = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, id);
var files = new DirectoryInfo(path)
.EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly)
.Select(i => i.FullName + i.LastWriteTimeUtc.Ticks)
.ToArray();
if (files.Length > 0)
{
return string.Join(string.Empty, files).GetMD5();
}
}
return Guid.Empty;
}
/// <summary>
@ -148,7 +168,6 @@ namespace MediaBrowser.Controller.Providers.TV
{
cancellationToken.ThrowIfCancellationRequested();
var status = ProviderRefreshStatus.Success;
BaseProviderInfo data;
if (!item.ProviderData.TryGetValue(Id, out data))
@ -157,86 +176,95 @@ namespace MediaBrowser.Controller.Providers.TV
item.ProviderData[Id] = data;
}
var series = (Series)item;
var seriesId = item.GetProviderId(MetadataProviders.Tvdb);
string language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
var seriesDataPath = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId);
var xmlPath = Path.Combine(seriesDataPath, "fanart.xml");
var seriesId = series.GetProviderId(MetadataProviders.Tvdb);
string url = string.Format(FanArtBaseUrl, ApiKey, seriesId);
var xmlPath = Path.Combine(GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId), "fanart.xml");
using (var response = await HttpClient.Get(new HttpRequestOptions
// Only download the xml if it doesn't already exist. The prescan task will take care of getting updates
if (!File.Exists(xmlPath))
{
Url = url,
ResourcePool = FanArtResourcePool,
CancellationToken = cancellationToken
}).ConfigureAwait(false))
{
using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
{
await response.CopyToAsync(xmlFileStream).ConfigureAwait(false);
}
await DownloadSeriesXml(seriesDataPath, seriesId, cancellationToken).ConfigureAwait(false);
}
if (File.Exists(xmlPath))
{
await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false);
}
data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb));
SetLastRefreshed(item, DateTime.UtcNow);
return true;
}
/// <summary>
/// Fetches from XML.
/// </summary>
/// <param name="item">The item.</param>
/// <param name="xmlFilePath">The XML file path.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken)
{
var doc = new XmlDocument();
doc.Load(xmlPath);
doc.Load(xmlFilePath);
cancellationToken.ThrowIfCancellationRequested();
string path;
var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
var hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hdtv" : "clear";
if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !series.HasImage(ImageType.Logo))
if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !item.HasImage(ImageType.Logo))
{
var node = doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo/@url") ??
doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo/@url");
path = node != null ? node.Value : null;
var path = node != null ? node.Value : null;
if (!string.IsNullOrEmpty(path))
{
series.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(series, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
}
}
cancellationToken.ThrowIfCancellationRequested();
hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hd" : "";
if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !series.HasImage(ImageType.Art))
if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !item.HasImage(ImageType.Art))
{
var node = doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/cleararts/clearart[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart/@url") ??
doc.SelectSingleNode("//fanart/series/cleararts/clearart/@url");
path = node != null ? node.Value : null;
var path = node != null ? node.Value : null;
if (!string.IsNullOrEmpty(path))
{
series.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(series, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
}
}
cancellationToken.ThrowIfCancellationRequested();
if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !series.HasImage(ImageType.Thumb))
if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !item.HasImage(ImageType.Thumb))
{
var node = doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb/@url");
path = node != null ? node.Value : null;
var path = node != null ? node.Value : null;
if (!string.IsNullOrEmpty(path))
{
series.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(series, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
}
}
if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !series.HasImage(ImageType.Banner))
if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !item.HasImage(ImageType.Banner))
{
var node = doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner[@lang = \"" + language + "\"]/@url") ??
doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner/@url");
path = node != null ? node.Value : null;
var path = node != null ? node.Value : null;
if (!string.IsNullOrEmpty(path))
{
series.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(series, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
}
}
@ -250,7 +278,7 @@ namespace MediaBrowser.Controller.Providers.TV
foreach (XmlNode node in nodes)
{
path = node.Value;
var path = node.Value;
if (!string.IsNullOrEmpty(path))
{
@ -265,11 +293,37 @@ namespace MediaBrowser.Controller.Providers.TV
}
}
data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb));
SetLastRefreshed(series, DateTime.UtcNow, status);
return true;
}
/// <summary>
/// Downloads the series XML.
/// </summary>
/// <param name="seriesDataPath">The series data path.</param>
/// <param name="tvdbId">The TVDB id.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
internal async Task DownloadSeriesXml(string seriesDataPath, string tvdbId, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
string url = string.Format(FanArtBaseUrl, ApiKey, tvdbId);
var xmlPath = Path.Combine(seriesDataPath, "fanart.xml");
using (var response = await HttpClient.Get(new HttpRequestOptions
{
Url = url,
ResourcePool = FanArtResourcePool,
CancellationToken = cancellationToken
}).ConfigureAwait(false))
{
using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
{
await response.CopyToAsync(xmlFileStream).ConfigureAwait(false);
}
}
}
}
}

199
MediaBrowser.Controller/Providers/TV/FanArtTvUpdatesPrescanTask.cs Normal file
View File

@ -0,0 +1,199 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Common.Net;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Providers.Music;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Net;
using MediaBrowser.Model.Serialization;
namespace MediaBrowser.Controller.Providers.TV
{
class FanArtTvUpdatesPrescanTask : ILibraryPrescanTask
{
private const string UpdatesUrl = "http://api.fanart.tv/webservice/newtv/{0}/{1}/";
/// <summary>
/// The _HTTP client
/// </summary>
private readonly IHttpClient _httpClient;
/// <summary>
/// The _logger
/// </summary>
private readonly ILogger _logger;
/// <summary>
/// The _config
/// </summary>
private readonly IServerConfigurationManager _config;
private readonly IJsonSerializer _jsonSerializer;
private static readonly CultureInfo UsCulture = new CultureInfo("en-US");
public FanArtTvUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient)
{
_jsonSerializer = jsonSerializer;
_config = config;
_logger = logger;
_httpClient = httpClient;
}
/// <summary>
/// Runs the specified progress.
/// </summary>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
{
if (!_config.Configuration.EnableInternetProviders)
{
progress.Report(100);
return;
}
var path = FanArtTvProvider.GetSeriesDataPath(_config.CommonApplicationPaths);
var timestampFile = Path.Combine(path, "time.txt");
var timestampFileInfo = new FileInfo(timestampFile);
if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays)
{
return;
}
// Find out the last time we queried for updates
var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
// If this is our first time, don't do any updates and just record the timestamp
if (!string.IsNullOrEmpty(lastUpdateTime))
{
var seriesToUpdate = await GetSeriesIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false);
progress.Report(5);
await UpdateSeries(seriesToUpdate, path, progress, cancellationToken).ConfigureAwait(false);
}
var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture);
File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8);
progress.Report(100);
}
/// <summary>
/// Gets the series ids to update.
/// </summary>
/// <param name="existingSeriesIds">The existing series ids.</param>
/// <param name="lastUpdateTime">The last update time.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task{IEnumerable{System.String}}.</returns>
private async Task<IEnumerable<string>> GetSeriesIdsToUpdate(IEnumerable<string> existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken)
{
// First get last time
using (var stream = await _httpClient.Get(new HttpRequestOptions
{
Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime),
CancellationToken = cancellationToken,
EnableHttpCompression = true,
ResourcePool = FanartBaseProvider.FanArtResourcePool
}).ConfigureAwait(false))
{
// If empty fanart will return a string of "null", rather than an empty list
using (var reader = new StreamReader(stream))
{
var json = await reader.ReadToEndAsync().ConfigureAwait(false);
if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase))
{
return new List<string>();
}
var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdatesPrescanTask.FanArtUpdate>>(json);
return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase));
}
}
}
/// <summary>
/// Updates the series.
/// </summary>
/// <param name="idList">The id list.</param>
/// <param name="seriesDataPath">The artists data path.</param>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private async Task UpdateSeries(IEnumerable<string> idList, string seriesDataPath, IProgress<double> progress, CancellationToken cancellationToken)
{
var list = idList.ToList();
var numComplete = 0;
foreach (var id in list)
{
try
{
await UpdateSeries(id, seriesDataPath, cancellationToken).ConfigureAwait(false);
}
catch (HttpException ex)
{
// Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout
if (!ex.IsTimedOut)
{
throw;
}
}
numComplete++;
double percent = numComplete;
percent /= list.Count;
percent *= 95;
progress.Report(percent + 5);
}
}
private Task UpdateSeries(string tvdbId, string seriesDataPath, CancellationToken cancellationToken)
{
_logger.Info("Updating series " + tvdbId);
seriesDataPath = Path.Combine(seriesDataPath, tvdbId);
if (!Directory.Exists(seriesDataPath))
{
Directory.CreateDirectory(seriesDataPath);
}
return FanArtTvProvider.Current.DownloadSeriesXml(seriesDataPath, tvdbId, cancellationToken);
}
/// <summary>
/// Dates the time to unix timestamp.
/// </summary>
/// <param name="dateTime">The date time.</param>
/// <returns>System.Double.</returns>
private static double DateTimeToUnixTimestamp(DateTime dateTime)
{
return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds;
}
public class FanArtUpdate
{
public string id { get; set; }
public string name { get; set; }
public string new_images { get; set; }
public string total_images { get; set; }
}
}
}