From 2bf2d5fd769788cade10a84fc7a4a4af23c23cf1 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Sat, 28 Feb 2015 08:42:47 -0500 Subject: [PATCH] cloud sync updates --- MediaBrowser.Api/Images/ImageService.cs | 2 +- .../Playback/TranscodingThrottler.cs | 50 +--- .../Drawing/IImageProcessor.cs | 6 +- .../MediaBrowser.Controller.csproj | 2 +- .../Sync/ICloudSyncProvider.cs | 46 ---- .../Sync/IServerSyncProvider.cs | 54 +++- .../Sync/ISyncDataProvider.cs | 41 +++ MediaBrowser.Controller/Sync/ISyncProvider.cs | 7 - MediaBrowser.Dlna/Didl/DidlBuilder.cs | 2 +- MediaBrowser.Model/Users/UserPolicy.cs | 2 + .../Photos/PhotoProvider.cs | 2 +- .../Drawing/ImageProcessor.cs | 7 +- .../Dto/DtoService.cs | 5 +- .../Localization/Server/server.json | 6 +- ...MediaBrowser.Server.Implementations.csproj | 6 +- .../Sync/AppSyncProvider.cs | 2 +- .../Sync/CloudSyncProfile.cs | 118 +++++++++ .../Sync/CloudSyncProvider.cs | 57 ---- .../Sync/FolderSync/FolderSyncDataProvider.cs | 31 +++ .../Sync/FolderSync/FolderSyncProvider.cs | 142 ++++++++++ .../Sync/IHasSyncProfile.cs | 15 ++ .../Sync/MediaSync.cs | 249 +++++++++++++++--- .../Sync/MultiProviderSync.cs | 69 +++++ .../Sync/SyncManager.cs | 21 +- 24 files changed, 732 insertions(+), 210 deletions(-) delete mode 100644 MediaBrowser.Controller/Sync/ICloudSyncProvider.cs create mode 100644 MediaBrowser.Controller/Sync/ISyncDataProvider.cs create mode 100644 MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs delete mode 100644 MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs create mode 100644 MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncDataProvider.cs create mode 100644 MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncProvider.cs create mode 100644 MediaBrowser.Server.Implementations/Sync/IHasSyncProfile.cs create mode 100644 MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index e29bbf6743..0a39b51c3a 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -318,7 +318,7 @@ namespace MediaBrowser.Api.Images try { - var size = _imageProcessor.GetImageSize(info.Path, info.DateModified); + var size = _imageProcessor.GetImageSize(info); width = Convert.ToInt32(size.Width); height = Convert.ToInt32(size.Height); diff --git a/MediaBrowser.Api/Playback/TranscodingThrottler.cs b/MediaBrowser.Api/Playback/TranscodingThrottler.cs index 81848c0170..50c213655b 100644 --- a/MediaBrowser.Api/Playback/TranscodingThrottler.cs +++ b/MediaBrowser.Api/Playback/TranscodingThrottler.cs @@ -13,11 +13,17 @@ namespace MediaBrowser.Api.Playback public void Start() { - _timer = new Timer(TimerCallback, null, 1000, 1000); + _timer = new Timer(TimerCallback, null, 5000, 5000); } private void TimerCallback(object state) { + if (_job.HasExited) + { + DisposeTimer(); + return; + } + if (IsThrottleAllowed(_job)) { PauseTranscoding(); @@ -50,36 +56,6 @@ namespace MediaBrowser.Api.Playback private bool IsThrottleAllowed(TranscodingJob job) { - //var job = string.IsNullOrEmpty(request.TranscodingJobId) ? - //null : - //ApiEntryPoint.Instance.GetTranscodingJob(request.TranscodingJobId); - - //var limits = new List(); - //if (state.InputBitrate.HasValue) - //{ - // // Bytes per second - // limits.Add((state.InputBitrate.Value / 8)); - //} - //if (state.InputFileSize.HasValue && state.RunTimeTicks.HasValue) - //{ - // var totalSeconds = TimeSpan.FromTicks(state.RunTimeTicks.Value).TotalSeconds; - - // if (totalSeconds > 1) - // { - // var timeBasedLimit = state.InputFileSize.Value / totalSeconds; - // limits.Add(Convert.ToInt64(timeBasedLimit)); - // } - //} - - //// Take the greater of the above to methods, just to be safe - //var throttleLimit = limits.Count > 0 ? limits.First() : 0; - - //// Pad to play it safe - //var bytesPerSecond = Convert.ToInt64(1.05 * throttleLimit); - - //// Don't even start evaluating this until at least two minutes have content have been consumed - //var targetGap = throttleLimit * 120; - var bytesDownloaded = job.BytesDownloaded ?? 0; var transcodingPositionTicks = job.TranscodingPositionTicks ?? 0; var downloadPositionTicks = job.DownloadPositionTicks ?? 0; @@ -95,11 +71,11 @@ namespace MediaBrowser.Api.Playback if (gap < targetGap) { - //Logger.Debug("Not throttling transcoder gap {0} target gap {1}", gap, targetGap); + //_logger.Debug("Not throttling transcoder gap {0} target gap {1}", gap, targetGap); return false; } - //Logger.Debug("Throttling transcoder gap {0} target gap {1}", gap, targetGap); + //_logger.Debug("Throttling transcoder gap {0} target gap {1}", gap, targetGap); return true; } @@ -120,21 +96,21 @@ namespace MediaBrowser.Api.Playback if (gap < targetGap) { - //Logger.Debug("Not throttling transcoder gap {0} target gap {1} bytes downloaded {2}", gap, targetGap, bytesDownloaded); + //_logger.Debug("Not throttling transcoder gap {0} target gap {1} bytes downloaded {2}", gap, targetGap, bytesDownloaded); return false; } - //Logger.Debug("Throttling transcoder gap {0} target gap {1} bytes downloaded {2}", gap, targetGap, bytesDownloaded); + //_logger.Debug("Throttling transcoder gap {0} target gap {1} bytes downloaded {2}", gap, targetGap, bytesDownloaded); return true; } catch { - //Logger.Error("Error getting output size"); + //_logger.Error("Error getting output size"); } } else { - //Logger.Debug("No throttle data for " + path); + //_logger.Debug("No throttle data for " + path); } return false; diff --git a/MediaBrowser.Controller/Drawing/IImageProcessor.cs b/MediaBrowser.Controller/Drawing/IImageProcessor.cs index 8ac7d56d29..6fafc2b464 100644 --- a/MediaBrowser.Controller/Drawing/IImageProcessor.cs +++ b/MediaBrowser.Controller/Drawing/IImageProcessor.cs @@ -2,7 +2,6 @@ using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Entities; -using System; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -30,10 +29,9 @@ namespace MediaBrowser.Controller.Drawing /// /// Gets the size of the image. /// - /// The path. - /// The image date modified. + /// The information. /// ImageSize. - ImageSize GetImageSize(string path, DateTime imageDateModified); + ImageSize GetImageSize(ItemImageInfo info); /// /// Adds the parts. diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index e9531e0571..603cb02e0d 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -341,8 +341,8 @@ - + diff --git a/MediaBrowser.Controller/Sync/ICloudSyncProvider.cs b/MediaBrowser.Controller/Sync/ICloudSyncProvider.cs deleted file mode 100644 index 8f03aea0a0..0000000000 --- a/MediaBrowser.Controller/Sync/ICloudSyncProvider.cs +++ /dev/null @@ -1,46 +0,0 @@ -using MediaBrowser.Model.Sync; -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace MediaBrowser.Controller.Sync -{ - public interface ICloudSyncProvider - { - /// - /// Gets the name. - /// - /// The name. - string Name { get; } - - /// - /// Gets the synchronize targets. - /// - /// The user identifier. - /// IEnumerable<SyncTarget>. - IEnumerable GetSyncTargets(string userId); - - /// - /// Transfers the item file. - /// - /// The input file. - /// The path parts. - /// The target. - /// The progress. - /// The cancellation token. - /// Task. - Task SendFile(string inputFile, string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken); - - /// - /// Gets the file. - /// - /// The path parts. - /// The target. - /// The progress. - /// The cancellation token. - /// Task<Stream>. - Task GetFile(string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken); - } -} diff --git a/MediaBrowser.Controller/Sync/IServerSyncProvider.cs b/MediaBrowser.Controller/Sync/IServerSyncProvider.cs index b3e74ee7db..9ee83acbf7 100644 --- a/MediaBrowser.Controller/Sync/IServerSyncProvider.cs +++ b/MediaBrowser.Controller/Sync/IServerSyncProvider.cs @@ -1,5 +1,6 @@ using MediaBrowser.Model.Sync; using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -12,21 +13,66 @@ namespace MediaBrowser.Controller.Sync /// Transfers the file. /// /// The input file. - /// The path parts. + /// The path. /// The target. /// The progress. /// The cancellation token. /// Task. - Task SendFile(string inputFile, string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken); + Task SendFile(string inputFile, string path, SyncTarget target, IProgress progress, CancellationToken cancellationToken); + + /// + /// Deletes the file. + /// + /// The path. + /// The target. + /// The cancellation token. + /// Task. + Task DeleteFile(string path, SyncTarget target, CancellationToken cancellationToken); /// /// Gets the file. /// - /// The path parts. + /// The path. /// The target. /// The progress. /// The cancellation token. /// Task<Stream>. - Task GetFile(string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken); + Task GetFile(string path, SyncTarget target, IProgress progress, CancellationToken cancellationToken); + + /// + /// Gets the full path. + /// + /// The path. + /// The target. + /// System.String. + string GetFullPath(IEnumerable path, SyncTarget target); + + /// + /// Gets the parent directory path. + /// + /// The path. + /// The target. + /// System.String. + string GetParentDirectoryPath(string path, SyncTarget target); + + /// + /// Gets the file system entries. + /// + /// The path. + /// The target. + /// Task<List<DeviceFileInfo>>. + Task> GetFileSystemEntries(string path, SyncTarget target); + + /// + /// Gets the data provider. + /// + /// ISyncDataProvider. + ISyncDataProvider GetDataProvider(); + + /// + /// Gets all synchronize targets. + /// + /// IEnumerable<SyncTarget>. + IEnumerable GetAllSyncTargets(); } } diff --git a/MediaBrowser.Controller/Sync/ISyncDataProvider.cs b/MediaBrowser.Controller/Sync/ISyncDataProvider.cs new file mode 100644 index 0000000000..cf698dd3c0 --- /dev/null +++ b/MediaBrowser.Controller/Sync/ISyncDataProvider.cs @@ -0,0 +1,41 @@ +using MediaBrowser.Model.Sync; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Sync +{ + public interface ISyncDataProvider + { + /// + /// Gets the server item ids. + /// + /// The target. + /// The server identifier. + /// Task<List<System.String>>. + Task> GetServerItemIds(SyncTarget target, string serverId); + + /// + /// Adds the or update. + /// + /// The target. + /// The item. + /// Task. + Task AddOrUpdate(SyncTarget target, LocalItem item); + + /// + /// Deletes the specified identifier. + /// + /// The target. + /// The identifier. + /// Task. + Task Delete(SyncTarget target, string id); + + /// + /// Gets the specified identifier. + /// + /// The target. + /// The identifier. + /// Task<LocalItem>. + Task Get(SyncTarget target, string id); + } +} diff --git a/MediaBrowser.Controller/Sync/ISyncProvider.cs b/MediaBrowser.Controller/Sync/ISyncProvider.cs index 6f24eac1ae..ba6e54d459 100644 --- a/MediaBrowser.Controller/Sync/ISyncProvider.cs +++ b/MediaBrowser.Controller/Sync/ISyncProvider.cs @@ -18,13 +18,6 @@ namespace MediaBrowser.Controller.Sync /// The user identifier. /// IEnumerable<SyncTarget>. IEnumerable GetSyncTargets(string userId); - - /// - /// Gets the device profile. - /// - /// The target. - /// DeviceProfile. - DeviceProfile GetDeviceProfile(SyncTarget target); } public interface IHasUniqueTargetIds diff --git a/MediaBrowser.Dlna/Didl/DidlBuilder.cs b/MediaBrowser.Dlna/Didl/DidlBuilder.cs index af7c8dbed0..b2eedad7c6 100644 --- a/MediaBrowser.Dlna/Didl/DidlBuilder.cs +++ b/MediaBrowser.Dlna/Didl/DidlBuilder.cs @@ -930,7 +930,7 @@ namespace MediaBrowser.Dlna.Didl try { - var size = _imageProcessor.GetImageSize(imageInfo.Path, imageInfo.DateModified); + var size = _imageProcessor.GetImageSize(imageInfo); width = Convert.ToInt32(size.Width); height = Convert.ToInt32(size.Height); diff --git a/MediaBrowser.Model/Users/UserPolicy.cs b/MediaBrowser.Model/Users/UserPolicy.cs index 410cdc51f3..7efc2cf6f6 100644 --- a/MediaBrowser.Model/Users/UserPolicy.cs +++ b/MediaBrowser.Model/Users/UserPolicy.cs @@ -59,6 +59,8 @@ namespace MediaBrowser.Model.Users public string[] EnabledFolders { get; set; } public bool EnableAllFolders { get; set; } + public int InvalidLoginAttemptCount { get; set; } + public UserPolicy() { EnableLiveTvManagement = true; diff --git a/MediaBrowser.Providers/Photos/PhotoProvider.cs b/MediaBrowser.Providers/Photos/PhotoProvider.cs index 3eaef59fbe..96160dcc46 100644 --- a/MediaBrowser.Providers/Photos/PhotoProvider.cs +++ b/MediaBrowser.Providers/Photos/PhotoProvider.cs @@ -146,7 +146,7 @@ namespace MediaBrowser.Providers.Photos try { - var size = _imageProcessor.GetImageSize(imageInfo.Path, imageInfo.DateModified); + var size = _imageProcessor.GetImageSize(imageInfo); item.Width = Convert.ToInt32(size.Width); item.Height = Convert.ToInt32(size.Height); diff --git a/MediaBrowser.Server.Implementations/Drawing/ImageProcessor.cs b/MediaBrowser.Server.Implementations/Drawing/ImageProcessor.cs index 85eadd73cf..c484a60dba 100644 --- a/MediaBrowser.Server.Implementations/Drawing/ImageProcessor.cs +++ b/MediaBrowser.Server.Implementations/Drawing/ImageProcessor.cs @@ -414,6 +414,11 @@ namespace MediaBrowser.Server.Implementations.Drawing return GetImageSize(path, File.GetLastWriteTimeUtc(path)); } + public ImageSize GetImageSize(ItemImageInfo info) + { + return GetImageSize(info.Path, info.DateModified); + } + /// /// Gets the size of the image. /// @@ -421,7 +426,7 @@ namespace MediaBrowser.Server.Implementations.Drawing /// The image date modified. /// ImageSize. /// path - public ImageSize GetImageSize(string path, DateTime imageDateModified) + private ImageSize GetImageSize(string path, DateTime imageDateModified) { if (string.IsNullOrEmpty(path)) { diff --git a/MediaBrowser.Server.Implementations/Dto/DtoService.cs b/MediaBrowser.Server.Implementations/Dto/DtoService.cs index b158097380..75037159c2 100644 --- a/MediaBrowser.Server.Implementations/Dto/DtoService.cs +++ b/MediaBrowser.Server.Implementations/Dto/DtoService.cs @@ -1598,14 +1598,11 @@ namespace MediaBrowser.Server.Implementations.Dto var path = imageInfo.Path; - // See if we can avoid a file system lookup by looking for the file in ResolveArgs - var dateModified = imageInfo.DateModified; - ImageSize size; try { - size = _imageProcessor.GetImageSize(path, dateModified); + size = _imageProcessor.GetImageSize(imageInfo); } catch (FileNotFoundException) { diff --git a/MediaBrowser.Server.Implementations/Localization/Server/server.json b/MediaBrowser.Server.Implementations/Localization/Server/server.json index dc80778da9..5f221a7bea 100644 --- a/MediaBrowser.Server.Implementations/Localization/Server/server.json +++ b/MediaBrowser.Server.Implementations/Localization/Server/server.json @@ -1114,7 +1114,7 @@ "MessageApplicationUpdated": "Media Browser Server has been updated", "AuthenticationSucceededWithUserName": "{0} successfully authenticated", "FailedLoginAttemptWithUserName": "Failed login attempt from {0}", - "UserDownloadingItemWithValues": "{0} is downloading {1}", + "UserDownloadingItemWithValues": "{0} is downloading {1}", "UserStartedPlayingItemWithValues": "{0} has started playing {1}", "UserStoppedPlayingItemWithValues": "{0} has stopped playing {1}", "AppDeviceValues": "App: {0}, Device: {1}", @@ -1369,5 +1369,7 @@ "TabJobs": "Jobs", "TabSyncJobs": "Sync Jobs", "LabelTagFilterMode": "Mode:", - "LabelTagFilterAllowModeHelp": "If allowed tags are used as part of a deeply nested folder structure, content that is tagged will require parent folders to be tagged as well." + "LabelTagFilterAllowModeHelp": "If allowed tags are used as part of a deeply nested folder structure, content that is tagged will require parent folders to be tagged as well.", + "HeaderThisUserIsCurrentlyDisabled": "This user is currently disabled", + "MessageReenableUser": "See below to reenable" } diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index 5ceaa5f5b7..40aaa299fe 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -303,8 +303,12 @@ - + + + + + diff --git a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs index d35ff8fc46..29ac74e826 100644 --- a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs @@ -8,7 +8,7 @@ using System.Linq; namespace MediaBrowser.Server.Implementations.Sync { - public class AppSyncProvider : ISyncProvider, IHasUniqueTargetIds + public class AppSyncProvider : ISyncProvider, IHasUniqueTargetIds, IHasSyncProfile { private readonly IDeviceManager _deviceManager; diff --git a/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs b/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs new file mode 100644 index 0000000000..babdf3f800 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs @@ -0,0 +1,118 @@ +using MediaBrowser.Model.Dlna; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public class CloudSyncProfile : DeviceProfile + { + public CloudSyncProfile(bool supportsAc3, bool supportsDca) + { + Name = "Cloud Sync"; + + MaxStreamingBitrate = 20000000; + MaxStaticBitrate = 20000000; + + var mkvAudio = "aac,mp3"; + var mp4Audio = "aac"; + + if (supportsAc3) + { + mkvAudio += ",ac3"; + mp4Audio += ",ac3"; + } + + if (supportsDca) + { + mkvAudio += ",dca"; + } + + DirectPlayProfiles = new[] + { + new DirectPlayProfile + { + Container = "mkv", + VideoCodec = "h264,mpeg4", + AudioCodec = mkvAudio, + Type = DlnaProfileType.Video + }, + new DirectPlayProfile + { + Container = "mp4,mov,m4v", + VideoCodec = "h264,mpeg4", + AudioCodec = mp4Audio, + Type = DlnaProfileType.Video + } + }; + + ContainerProfiles = new ContainerProfile[] { }; + + CodecProfiles = new[] + { + new CodecProfile + { + Type = CodecType.Video, + Conditions = new [] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoBitDepth, + Value = "8", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.Height, + Value = "1080", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.RefFrames, + Value = "12", + IsRequired = false + } + } + } + }; + + SubtitleProfiles = new[] + { + new SubtitleProfile + { + Format = "srt", + Method = SubtitleDeliveryMethod.External + } + }; + + TranscodingProfiles = new[] + { + new TranscodingProfile + { + Container = "mp3", + AudioCodec = "mp3", + Type = DlnaProfileType.Audio, + Context = EncodingContext.Static + }, + + new TranscodingProfile + { + Container = "mp4", + Type = DlnaProfileType.Video, + AudioCodec = "aac", + VideoCodec = "h264", + Context = EncodingContext.Static + }, + + new TranscodingProfile + { + Container = "jpeg", + Type = DlnaProfileType.Photo, + Context = EncodingContext.Static + } + }; + + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs deleted file mode 100644 index 1ef9d4b154..0000000000 --- a/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs +++ /dev/null @@ -1,57 +0,0 @@ -using MediaBrowser.Common; -using MediaBrowser.Controller.Sync; -using MediaBrowser.Model.Dlna; -using MediaBrowser.Model.Sync; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace MediaBrowser.Server.Implementations.Sync -{ - public class CloudSyncProvider : IServerSyncProvider - { - private readonly ICloudSyncProvider[] _providers = {}; - - public CloudSyncProvider(IApplicationHost appHost) - { - _providers = appHost.GetExports().ToArray(); - } - - public IEnumerable GetSyncTargets(string userId) - { - return _providers.SelectMany(i => i.GetSyncTargets(userId)); - } - - public DeviceProfile GetDeviceProfile(SyncTarget target) - { - return new DeviceProfile(); - } - - public string Name - { - get { return "Cloud Sync"; } - } - - private ICloudSyncProvider GetProvider(SyncTarget target) - { - return null; - } - - public Task SendFile(string inputFile, string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken) - { - var provider = GetProvider(target); - - return provider.SendFile(inputFile, pathParts, target, progress, cancellationToken); - } - - public Task GetFile(string[] pathParts, SyncTarget target, IProgress progress, CancellationToken cancellationToken) - { - var provider = GetProvider(target); - - return provider.GetFile(pathParts, target, progress, cancellationToken); - } - } -} diff --git a/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncDataProvider.cs b/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncDataProvider.cs new file mode 100644 index 0000000000..b9008d87e5 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncDataProvider.cs @@ -0,0 +1,31 @@ +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Sync.FolderSync +{ + public class FolderSyncDataProvider : ISyncDataProvider + { + public Task> GetServerItemIds(SyncTarget target, string serverId) + { + throw new NotImplementedException(); + } + + public Task AddOrUpdate(SyncTarget target, LocalItem item) + { + throw new NotImplementedException(); + } + + public Task Delete(SyncTarget target, string id) + { + throw new NotImplementedException(); + } + + public Task Get(SyncTarget target, string id) + { + throw new NotImplementedException(); + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncProvider.cs new file mode 100644 index 0000000000..9cf2341064 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/FolderSync/FolderSyncProvider.cs @@ -0,0 +1,142 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Sync.FolderSync +{ + public class FolderSyncProvider : IServerSyncProvider + { + private readonly IApplicationPaths _appPaths; + private readonly IUserManager _userManager; + + public FolderSyncProvider(IApplicationPaths appPaths, IUserManager userManager) + { + _appPaths = appPaths; + _userManager = userManager; + } + + public Task SendFile(string inputFile, string path, SyncTarget target, IProgress progress, CancellationToken cancellationToken) + { + return Task.Run(() => File.Copy(inputFile, path, true), cancellationToken); + } + + public Task DeleteFile(string path, SyncTarget target, CancellationToken cancellationToken) + { + return Task.Run(() => File.Delete(path), cancellationToken); + } + + public Task GetFile(string path, SyncTarget target, IProgress progress, CancellationToken cancellationToken) + { + return Task.FromResult((Stream)File.OpenRead(path)); + } + + public string GetFullPath(IEnumerable paths, SyncTarget target) + { + var account = GetSyncAccounts() + .FirstOrDefault(i => string.Equals(i.Id, target.Id, StringComparison.OrdinalIgnoreCase)); + + if (account == null) + { + throw new ArgumentException("Invalid SyncTarget supplied."); + } + + var list = paths.ToList(); + list.Insert(0, account.Path); + + return Path.Combine(list.ToArray()); + } + + public string GetParentDirectoryPath(string path, SyncTarget target) + { + return Path.GetDirectoryName(path); + } + + public Task> GetFileSystemEntries(string path, SyncTarget target) + { + List files; + + try + { + files = new DirectoryInfo(path).EnumerateFiles("*", SearchOption.TopDirectoryOnly).ToList(); + } + catch (DirectoryNotFoundException) + { + files = new List(); + } + + return Task.FromResult(files.Select(i => new DeviceFileInfo + { + Name = i.Name, + Path = i.FullName + + }).ToList()); + } + + public ISyncDataProvider GetDataProvider() + { + // If single instances are needed, manage them here + return new FolderSyncDataProvider(); + } + + public string Name + { + get { return "Folder Sync"; } + } + + public IEnumerable GetSyncTargets(string userId) + { + return GetSyncAccounts() + .Where(i => i.UserIds.Contains(userId, StringComparer.OrdinalIgnoreCase)) + .Select(GetSyncTarget); + } + + public IEnumerable GetAllSyncTargets() + { + return GetSyncAccounts().Select(GetSyncTarget); + } + + private SyncTarget GetSyncTarget(SyncAccount account) + { + return new SyncTarget + { + Id = account.Id, + Name = account.Name + }; + } + + private IEnumerable GetSyncAccounts() + { + // Dummy this up + return _userManager + .Users + .Select(i => new SyncAccount + { + Id = i.Id.ToString("N"), + UserIds = new List { i.Id.ToString("N") }, + Path = Path.Combine(_appPaths.DataPath, "foldersync", i.Id.ToString("N")), + Name = i.Name + "'s Folder Sync" + }); + } + + // An internal class to manage all configured Folder Sync accounts for differnet users + class SyncAccount + { + public string Id { get; set; } + public string Name { get; set; } + public string Path { get; set; } + public List UserIds { get; set; } + + public SyncAccount() + { + UserIds = new List(); + } + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/IHasSyncProfile.cs b/MediaBrowser.Server.Implementations/Sync/IHasSyncProfile.cs new file mode 100644 index 0000000000..b7e9daf496 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/IHasSyncProfile.cs @@ -0,0 +1,15 @@ +using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Sync; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public interface IHasSyncProfile + { + /// + /// Gets the device profile. + /// + /// The target. + /// DeviceProfile. + DeviceProfile GetDeviceProfile(SyncTarget target); + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs index 764652bbff..0407510a84 100644 --- a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs +++ b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs @@ -1,10 +1,18 @@ -using MediaBrowser.Common.Progress; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Progress; using MediaBrowser.Controller; using MediaBrowser.Controller.Sync; using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; +using MediaBrowser.Model.MediaInfo; using MediaBrowser.Model.Sync; using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -15,22 +23,25 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly ISyncManager _syncManager; private readonly IServerApplicationHost _appHost; private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; - public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost) + public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem) { _logger = logger; _syncManager = syncManager; _appHost = appHost; + _fileSystem = fileSystem; } public async Task Sync(IServerSyncProvider provider, + ISyncDataProvider dataProvider, SyncTarget target, IProgress progress, CancellationToken cancellationToken) { var serverId = _appHost.SystemId; - await SyncData(provider, serverId, target, cancellationToken).ConfigureAwait(false); + await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false); progress.Report(3); var innerProgress = new ActionableProgress(); @@ -40,44 +51,46 @@ namespace MediaBrowser.Server.Implementations.Sync totalProgress += 1; progress.Report(totalProgress); }); - await GetNewMedia(provider, target, serverId, innerProgress, cancellationToken); + await GetNewMedia(provider, dataProvider, target, serverId, innerProgress, cancellationToken); // Do the data sync twice so the server knows what was removed from the device - await SyncData(provider, serverId, target, cancellationToken).ConfigureAwait(false); + await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false); progress.Report(100); } private async Task SyncData(IServerSyncProvider provider, + ISyncDataProvider dataProvider, string serverId, SyncTarget target, CancellationToken cancellationToken) { - //var localIds = await provider.GetServerItemIds(serverId, target, cancellationToken).ConfigureAwait(false); + var localIds = await dataProvider.GetServerItemIds(target, serverId).ConfigureAwait(false); - //var result = await _syncManager.SyncData(new SyncDataRequest - //{ - // TargetId = target.Id, - // LocalItemIds = localIds + var result = await _syncManager.SyncData(new SyncDataRequest + { + TargetId = target.Id, + LocalItemIds = localIds - //}).ConfigureAwait(false); + }).ConfigureAwait(false); - //cancellationToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); - //foreach (var itemIdToRemove in result.ItemIdsToRemove) - //{ - // try - // { - // await RemoveItem(provider, serverId, itemIdToRemove, target, cancellationToken).ConfigureAwait(false); - // } - // catch (Exception ex) - // { - // _logger.ErrorException("Error deleting item from sync target. Id: {0}", ex, itemIdToRemove); - // } - //} + foreach (var itemIdToRemove in result.ItemIdsToRemove) + { + try + { + await RemoveItem(provider, dataProvider, serverId, itemIdToRemove, target, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting item from device. Id: {0}", ex, itemIdToRemove); + } + } } private async Task GetNewMedia(IServerSyncProvider provider, + ISyncDataProvider dataProvider, SyncTarget target, string serverId, IProgress progress, @@ -106,7 +119,7 @@ namespace MediaBrowser.Server.Implementations.Sync progress.Report(totalProgress); }); - await GetItem(provider, target, serverId, jobItem, innerProgress, cancellationToken).ConfigureAwait(false); + await GetItem(provider, dataProvider, target, serverId, jobItem, innerProgress, cancellationToken).ConfigureAwait(false); numComplete++; startingPercent = numComplete; @@ -117,6 +130,7 @@ namespace MediaBrowser.Server.Implementations.Sync } private async Task GetItem(IServerSyncProvider provider, + ISyncDataProvider dataProvider, SyncTarget target, string serverId, SyncedItem jobItem, @@ -129,6 +143,8 @@ namespace MediaBrowser.Server.Implementations.Sync var fileTransferProgress = new ActionableProgress(); fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92)); + var localItem = CreateLocalItem(provider, target, libraryItem, serverId, jobItem.OriginalFileName); + await _syncManager.ReportSyncJobItemTransferBeginning(internalSyncJobItem.Id); var transferSuccess = false; @@ -136,9 +152,10 @@ namespace MediaBrowser.Server.Implementations.Sync try { - string[] pathParts = GetPathParts(serverId, libraryItem); + await SendFile(provider, internalSyncJobItem.OutputPath, localItem, target, cancellationToken).ConfigureAwait(false); - await SendFile(provider, internalSyncJobItem.OutputPath, pathParts, target, cancellationToken).ConfigureAwait(false); + // Create db record + await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false); progress.Report(92); @@ -164,25 +181,189 @@ namespace MediaBrowser.Server.Implementations.Sync } } - private Task RemoveItem(IServerSyncProvider provider, + private async Task RemoveItem(IServerSyncProvider provider, + ISyncDataProvider dataProvider, string serverId, string itemId, SyncTarget target, CancellationToken cancellationToken) { - return Task.FromResult(true); - //return provider.DeleteItem(serverId, itemId, target, cancellationToken); + var localId = GetLocalId(serverId, itemId); + var localItem = await dataProvider.Get(target, localId); + + if (localItem == null) + { + return; + } + + var files = await GetFiles(provider, localItem, target); + + foreach (var file in files) + { + await provider.DeleteFile(file.Path, target, cancellationToken).ConfigureAwait(false); + } + + await dataProvider.Delete(target, localId).ConfigureAwait(false); } - private string[] GetPathParts(string serverId, BaseItemDto item) + private Task SendFile(IServerSyncProvider provider, string inputPath, LocalItem item, SyncTarget target, CancellationToken cancellationToken) { - return null; + return provider.SendFile(inputPath, item.LocalPath, target, new Progress(), cancellationToken); } - private async Task SendFile(IServerSyncProvider provider, string inputPath, string[] path, SyncTarget target, CancellationToken cancellationToken) + private string GetLocalId(string serverId, string itemId) { - await provider.SendFile(inputPath, path, target, new Progress(), cancellationToken) - .ConfigureAwait(false); + var bytes = Encoding.UTF8.GetBytes(serverId + itemId); + bytes = CreateMD5(bytes); + return BitConverter.ToString(bytes, 0, bytes.Length).Replace("-", string.Empty); + } + + private byte[] CreateMD5(byte[] value) + { + using (var provider = MD5.Create()) + { + return provider.ComputeHash(value); + } + } + + public LocalItem CreateLocalItem(IServerSyncProvider provider, SyncTarget target, BaseItemDto libraryItem, string serverId, string originalFileName) + { + var path = GetDirectoryPath(provider, libraryItem, serverId); + path.Add(GetLocalFileName(provider, libraryItem, originalFileName)); + + var localPath = provider.GetFullPath(path, target); + + foreach (var mediaSource in libraryItem.MediaSources) + { + mediaSource.Path = localPath; + mediaSource.Protocol = MediaProtocol.File; + } + + return new LocalItem + { + Item = libraryItem, + ItemId = libraryItem.Id, + ServerId = serverId, + LocalPath = localPath, + Id = GetLocalId(serverId, libraryItem.Id) + }; + } + + private List GetDirectoryPath(IServerSyncProvider provider, BaseItemDto item, string serverId) + { + var parts = new List + { + serverId + }; + + if (item.IsType("episode")) + { + parts.Add("TV"); + parts.Add(item.SeriesName); + + if (!string.IsNullOrWhiteSpace(item.SeasonName)) + { + parts.Add(item.SeasonName); + } + } + else if (item.IsVideo) + { + parts.Add("Videos"); + parts.Add(item.Name); + } + else if (item.IsAudio) + { + parts.Add("Music"); + + if (!string.IsNullOrWhiteSpace(item.AlbumArtist)) + { + parts.Add(item.AlbumArtist); + } + + if (!string.IsNullOrWhiteSpace(item.Album)) + { + parts.Add(item.Album); + } + } + else if (string.Equals(item.MediaType, MediaType.Photo, StringComparison.OrdinalIgnoreCase)) + { + parts.Add("Photos"); + + if (!string.IsNullOrWhiteSpace(item.Album)) + { + parts.Add(item.Album); + } + } + + return parts.Select(i => GetValidFilename(provider, i)).ToList(); + } + + private string GetLocalFileName(IServerSyncProvider provider, BaseItemDto item, string originalFileName) + { + var filename = originalFileName; + + if (string.IsNullOrEmpty(filename)) + { + filename = item.Name; + } + + return GetValidFilename(provider, filename); + } + + private string GetValidFilename(IServerSyncProvider provider, string filename) + { + // We can always add this method to the sync provider if it's really needed + return _fileSystem.GetValidFilename(filename); + } + + private async Task> GetFiles(IServerSyncProvider provider, LocalItem item, SyncTarget target) + { + var path = item.LocalPath; + path = provider.GetParentDirectoryPath(path, target); + + var list = await provider.GetFileSystemEntries(path, target).ConfigureAwait(false); + + var itemFiles = new List(); + + var name = Path.GetFileNameWithoutExtension(item.LocalPath); + + foreach (var file in list.Where(f => f.Name.Contains(name))) + { + var itemFile = new ItemFileInfo + { + Path = file.Path, + Name = file.Name + }; + + if (IsSubtitleFile(file.Name)) + { + itemFile.Type = ItemFileType.Subtitles; + } + else if (!IsImageFile(file.Name)) + { + itemFile.Type = ItemFileType.Media; + } + + itemFiles.Add(itemFile); + } + + return itemFiles; + } + + private static readonly string[] SupportedImageExtensions = { ".png", ".jpg", ".jpeg", ".webp" }; + private bool IsImageFile(string path) + { + var ext = Path.GetExtension(path) ?? string.Empty; + + return SupportedImageExtensions.Contains(ext, StringComparer.OrdinalIgnoreCase); + } + + private static readonly string[] SupportedSubtitleExtensions = { ".srt", ".vtt" }; + private bool IsSubtitleFile(string path) + { + var ext = Path.GetExtension(path) ?? string.Empty; + + return SupportedSubtitleExtensions.Contains(ext, StringComparer.OrdinalIgnoreCase); } } } diff --git a/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs b/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs new file mode 100644 index 0000000000..cbfa82f1d6 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs @@ -0,0 +1,69 @@ +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public class MultiProviderSync + { + private readonly ISyncManager _syncManager; + private readonly IServerApplicationHost _appHost; + private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; + + public MultiProviderSync(ISyncManager syncManager, IServerApplicationHost appHost, ILogger logger, IFileSystem fileSystem) + { + _syncManager = syncManager; + _appHost = appHost; + _logger = logger; + _fileSystem = fileSystem; + } + + public async Task Sync(IEnumerable providers, IProgress progress, CancellationToken cancellationToken) + { + var targets = providers + .SelectMany(i => i.GetAllSyncTargets().Select(t => new Tuple(i, t))) + .ToList(); + + var numComplete = 0; + double startingPercent = 0; + double percentPerItem = 1; + if (targets.Count > 0) + { + percentPerItem /= targets.Count; + } + + foreach (var target in targets) + { + cancellationToken.ThrowIfCancellationRequested(); + + var currentPercent = startingPercent; + var innerProgress = new ActionableProgress(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * percentPerItem; + totalProgress += currentPercent; + progress.Report(totalProgress); + }); + + await new MediaSync(_logger, _syncManager, _appHost, _fileSystem) + .Sync(target.Item1, target.Item1.GetDataProvider(), target.Item2, innerProgress, cancellationToken) + .ConfigureAwait(false); + + numComplete++; + startingPercent = numComplete; + startingPercent /= targets.Count; + startingPercent *= 100; + progress.Report(startingPercent); + } + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs index a2fd92bf50..912967020a 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs @@ -429,13 +429,6 @@ namespace MediaBrowser.Server.Implementations.Sync return (providerId + "-" + target.Id).GetMD5().ToString("N"); } - private ISyncProvider GetSyncProvider(SyncTarget target) - { - var providerId = target.Id.Split(new[] { '-' }, 2).First(); - - return _providers.First(i => string.Equals(providerId, GetSyncProviderId(i))); - } - private string GetSyncProviderId(ISyncProvider provider) { return (provider.GetType().Name).GetMD5().ToString("N"); @@ -547,7 +540,7 @@ namespace MediaBrowser.Server.Implementations.Sync { if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) { - return provider.GetDeviceProfile(target); + return GetDeviceProfile(provider, target); } } } @@ -555,6 +548,18 @@ namespace MediaBrowser.Server.Implementations.Sync return null; } + public DeviceProfile GetDeviceProfile(ISyncProvider provider, SyncTarget target) + { + var hasProfile = provider as IHasSyncProfile; + + if (hasProfile != null) + { + return hasProfile.GetDeviceProfile(target); + } + + return new CloudSyncProfile(true, false); + } + public async Task ReportSyncJobItemTransferred(string id) { var jobItem = _repo.GetJobItem(id);