jellyfin/MediaBrowser.Server.Implementations/Persistence/CleanDatabaseScheduledTask.cs

using MediaBrowser.Common.Progress;
using MediaBrowser.Common.ScheduledTasks;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.LiveTv;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
using CommonIO;
using MediaBrowser.Controller.Channels;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Localization;
using MediaBrowser.Controller.Net;

namespace MediaBrowser.Server.Implementations.Persistence
{
    public class CleanDatabaseScheduledTask : IScheduledTask
    {
        private readonly ILibraryManager _libraryManager;
        private readonly IItemRepository _itemRepo;
        private readonly ILogger _logger;
        private readonly IServerConfigurationManager _config;
        private readonly IFileSystem _fileSystem;
        private readonly IHttpServer _httpServer;
        private readonly ILocalizationManager _localization;

        public const int MigrationVersion = 12;
        public static bool EnableUnavailableMessage = false;

        public CleanDatabaseScheduledTask(ILibraryManager libraryManager, IItemRepository itemRepo, ILogger logger, IServerConfigurationManager config, IFileSystem fileSystem, IHttpServer httpServer, ILocalizationManager localization)
        {
            _libraryManager = libraryManager;
            _itemRepo = itemRepo;
            _logger = logger;
            _config = config;
            _fileSystem = fileSystem;
            _httpServer = httpServer;
            _localization = localization;
        }

        public string Name
        {
            get { return "Clean Database"; }
        }

        public string Description
        {
            get { return "Deletes obsolete content from the database."; }
        }

        public string Category
        {
            get { return "Library"; }
        }

        public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress)
        {
            var innerProgress = new ActionableProgress<double>();
            innerProgress.RegisterAction(p =>
            {
                double newPercentCommplete = .4 * p;
                OnProgress(newPercentCommplete);

                progress.Report(newPercentCommplete);
            });

            await UpdateToLatestSchema(cancellationToken, innerProgress).ConfigureAwait(false);

            innerProgress = new ActionableProgress<double>();
            innerProgress.RegisterAction(p =>
            {
                double newPercentCommplete = 40 + (.05 * p);
                OnProgress(newPercentCommplete);
                progress.Report(newPercentCommplete);
            });
            await CleanDeadItems(cancellationToken, innerProgress).ConfigureAwait(false);
            progress.Report(45);

            innerProgress = new ActionableProgress<double>();
            innerProgress.RegisterAction(p =>
            {
                double newPercentCommplete = 45 + (.55 * p);
                OnProgress(newPercentCommplete);
                progress.Report(newPercentCommplete);
            });
            await CleanDeletedItems(cancellationToken, innerProgress).ConfigureAwait(false);
            progress.Report(100);

            await _itemRepo.UpdateInheritedValues(cancellationToken).ConfigureAwait(false);

            if (EnableUnavailableMessage)
            {
                EnableUnavailableMessage = false;
                _httpServer.GlobalResponse = null;
            }
        }

        private void OnProgress(double newPercentCommplete)
        {
            if (EnableUnavailableMessage)
            {
                var html = "<!doctype html><html><head><title>Emby</title></head><body>";
                var text = _localization.GetLocalizedString("DbUpgradeMessage");
                html += string.Format(text, newPercentCommplete.ToString("N2", CultureInfo.InvariantCulture));

                html += "<script>setTimeout(function(){window.location.reload(true);}, 5000);</script>";
                html += "</body></html>";

                _httpServer.GlobalResponse = html;
            }
        }

        private async Task UpdateToLatestSchema(CancellationToken cancellationToken, IProgress<double> progress)
        {
            var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
            {
                IsCurrentSchema = false
            });

            var numComplete = 0;
            var numItems = itemIds.Count;

            _logger.Debug("Upgrading schema for {0} items", numItems);

            foreach (var itemId in itemIds)
            {
                cancellationToken.ThrowIfCancellationRequested();

                if (itemId != Guid.Empty)
                {
                    // Somehow some invalid data got into the db. It probably predates the boundary checking
                    var item = _libraryManager.GetItemById(itemId);

                    if (item != null)
                    {
                        try
                        {
                            await _itemRepo.SaveItem(item, cancellationToken).ConfigureAwait(false);
                        }
                        catch (OperationCanceledException)
                        {
                            throw;
                        }
                        catch (Exception ex)
                        {
                            _logger.ErrorException("Error saving item", ex);
                        }
                    }
                }

                numComplete++;
                double percent = numComplete;
                percent /= numItems;
                progress.Report(percent * 100);
            }

            if (_config.Configuration.MigrationVersion < MigrationVersion)
            {
                _config.Configuration.MigrationVersion = MigrationVersion;
                _config.SaveConfiguration();
            }

            progress.Report(100);
        }

        private async Task CleanDeadItems(CancellationToken cancellationToken, IProgress<double> progress)
        {
            var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
            {
                HasDeadParentId = true
            });

            var numComplete = 0;
            var numItems = itemIds.Count;

            _logger.Debug("Cleaning {0} items with dead parent links", numItems);

            foreach (var itemId in itemIds)
            {
                cancellationToken.ThrowIfCancellationRequested();

                var item = _libraryManager.GetItemById(itemId);

                if (item != null)
                {
                    _logger.Info("Cleaning item {0} type: {1} path: {2}", item.Name, item.GetType().Name, item.Path ?? string.Empty);

                    await _libraryManager.DeleteItem(item, new DeleteOptions
                    {
                        DeleteFileLocation = false
                    });
                }

                numComplete++;
                double percent = numComplete;
                percent /= numItems;
                progress.Report(percent * 100);
            }

            progress.Report(100);
        }

        private async Task CleanDeletedItems(CancellationToken cancellationToken, IProgress<double> progress)
        {
            var result = _itemRepo.GetItemIdsWithPath(new InternalItemsQuery
            {
                LocationType = LocationType.FileSystem,
                //Limit = limit,

                // These have their own cleanup routines
                ExcludeItemTypes = new[]
                {
                    typeof(Person).Name, 
                    typeof(Genre).Name, 
                    typeof(MusicGenre).Name, 
                    typeof(GameGenre).Name, 
                    typeof(Studio).Name, 
                    typeof(Year).Name, 
                    typeof(Channel).Name, 
                    typeof(AggregateFolder).Name, 
                    typeof(CollectionFolder).Name
                }
            });

            var numComplete = 0;
            var numItems = result.Items.Length;

            foreach (var item in result.Items)
            {
                cancellationToken.ThrowIfCancellationRequested();

                var path = item.Item2;

                try
                {
                    if (_fileSystem.FileExists(path) || _fileSystem.DirectoryExists(path))
                    {
                        continue;
                    }

                    var libraryItem = _libraryManager.GetItemById(item.Item1);

                    if (libraryItem.IsTopParent)
                    {
                        continue;
                    }

                    if (Folder.IsPathOffline(path))
                    {
                        libraryItem.IsOffline = true;
                        await libraryItem.UpdateToRepository(ItemUpdateType.None, cancellationToken).ConfigureAwait(false);
                        continue;
                    }

                    _logger.Info("Deleting item from database {0} because path no longer exists. type: {1} path: {2}", libraryItem.Name, libraryItem.GetType().Name, libraryItem.Path ?? string.Empty);

                    await _libraryManager.DeleteItem(libraryItem, new DeleteOptions
                    {
                        DeleteFileLocation = false
                    });
                }
                catch (OperationCanceledException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    _logger.ErrorException("Error in CleanDeletedItems. File {0}", ex, path);
                }

                numComplete++;
                double percent = numComplete;
                percent /= numItems;
                progress.Report(percent * 100);
            }
        }

        public IEnumerable<ITaskTrigger> GetDefaultTriggers()
        {
            return new ITaskTrigger[] 
            { 
                new IntervalTrigger{ Interval = TimeSpan.FromHours(24)}
            };
        }
    }
}
-												add migrations for new release

											
										
										
											2015-10-16 20:11:11 +02:00
+								using MediaBrowser.Common.Progress;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								using MediaBrowser.Common.ScheduledTasks;
-												update interval trigger

											
										
										
											2015-08-27 03:31:54 +02:00
+								using MediaBrowser.Controller.Configuration;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								using MediaBrowser.Controller.Entities;
 								using MediaBrowser.Controller.Library;
-												update photos

											
										
										
											2015-08-26 20:17:38 +02:00
+								using MediaBrowser.Controller.LiveTv;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								using MediaBrowser.Controller.Persistence;
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								using MediaBrowser.Model.Entities;
-												update interval trigger

											
										
										
											2015-08-27 03:31:54 +02:00
+								using MediaBrowser.Model.Logging;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								using System;
 								using System.Collections.Generic;
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								using System.Globalization;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								using System.Threading;
 								using System.Threading.Tasks;
-												continue file system rework

											
										
										
											2015-10-04 06:23:11 +02:00
+								using CommonIO;
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
+								using MediaBrowser.Controller.Channels;
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								using MediaBrowser.Controller.Entities.Audio;
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								using MediaBrowser.Controller.Localization;
 								using MediaBrowser.Controller.Net;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
 								namespace MediaBrowser.Server.Implementations.Persistence
 								{
-												add migrations for new release

											
										
										
											2015-10-16 20:11:11 +02:00
+								    public class CleanDatabaseScheduledTask : IScheduledTask
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								    {
 								        private readonly ILibraryManager _libraryManager;
 								        private readonly IItemRepository _itemRepo;
 								        private readonly ILogger _logger;
-												update interval trigger

											
										
										
											2015-08-27 03:31:54 +02:00
+								        private readonly IServerConfigurationManager _config;
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								        private readonly IFileSystem _fileSystem;
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								        private readonly IHttpServer _httpServer;
 								        private readonly ILocalizationManager _localization;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								        public const int MigrationVersion = 12;
 								        public static bool EnableUnavailableMessage = false;
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								        public CleanDatabaseScheduledTask(ILibraryManager libraryManager, IItemRepository itemRepo, ILogger logger, IServerConfigurationManager config, IFileSystem fileSystem, IHttpServer httpServer, ILocalizationManager localization)
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								        {
 								            _libraryManager = libraryManager;
 								            _itemRepo = itemRepo;
 								            _logger = logger;
-												update interval trigger

											
										
										
											2015-08-27 03:31:54 +02:00
+								            _config = config;
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								            _fileSystem = fileSystem;
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								            _httpServer = httpServer;
 								            _localization = localization;
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								        }
 								        public string Name
 								        {
 								            get { return "Clean Database"; }
 								        }
 								        public string Description
 								        {
 								            get { return "Deletes obsolete content from the database."; }
 								        }
 								        public string Category
 								        {
 								            get { return "Library"; }
 								        }
 								        public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress)
 								        {
 								            var innerProgress = new ActionableProgress<double>();
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
+								            innerProgress.RegisterAction(p =>
 								            {
 								                double newPercentCommplete = .4 * p;
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								                OnProgress(newPercentCommplete);
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
 								                progress.Report(newPercentCommplete);
 								            });
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
 								            await UpdateToLatestSchema(cancellationToken, innerProgress).ConfigureAwait(false);
-												update live tv image

											
										
										
											2015-08-28 06:19:08 +02:00
 								            innerProgress = new ActionableProgress<double>();
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								            innerProgress.RegisterAction(p =>
 								            {
 								                double newPercentCommplete = 40 + (.05 * p);
 								                OnProgress(newPercentCommplete);
 								                progress.Report(newPercentCommplete);
 								            });
-												fix shortcut setting

											
										
										
											2015-09-03 06:16:31 +02:00
+								            await CleanDeadItems(cancellationToken, innerProgress).ConfigureAwait(false);
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								            progress.Report(45);
-												update live tv image

											
										
										
											2015-08-28 06:19:08 +02:00
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								            innerProgress = new ActionableProgress<double>();
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								            innerProgress.RegisterAction(p =>
 								            {
 								                double newPercentCommplete = 45 + (.55 * p);
 								                OnProgress(newPercentCommplete);
 								                progress.Report(newPercentCommplete);
 								            });
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								            await CleanDeletedItems(cancellationToken, innerProgress).ConfigureAwait(false);
-												update live tv image

											
										
										
											2015-08-28 06:19:08 +02:00
+								            progress.Report(100);
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
 								            await _itemRepo.UpdateInheritedValues(cancellationToken).ConfigureAwait(false);
-												add message for db upgrade

											
										
										
											2016-02-01 20:54:49 +01:00
 								            if (EnableUnavailableMessage)
 								            {
 								                EnableUnavailableMessage = false;
 								                _httpServer.GlobalResponse = null;
 								            }
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								        }
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								        private void OnProgress(double newPercentCommplete)
 								        {
 								            if (EnableUnavailableMessage)
 								            {
 								                var html = "<!doctype html><html><head><title>Emby</title></head><body>";
 								                var text = _localization.GetLocalizedString("DbUpgradeMessage");
 								                html += string.Format(text, newPercentCommplete.ToString("N2", CultureInfo.InvariantCulture));
 								                html += "<script>setTimeout(function(){window.location.reload(true);}, 5000);</script>";
 								                html += "</body></html>";
 								                _httpServer.GlobalResponse = html;
 								            }
 								        }
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								        private async Task UpdateToLatestSchema(CancellationToken cancellationToken, IProgress<double> progress)
 								        {
 								            var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
 								            {
-												update views

											
										
										
											2015-11-18 06:49:20 +01:00
+								                IsCurrentSchema = false
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								            });
 								            var numComplete = 0;
 								            var numItems = itemIds.Count;
 								            _logger.Debug("Upgrading schema for {0} items", numItems);
 								            foreach (var itemId in itemIds)
 								            {
-												update photos

											
										
										
											2015-08-26 20:17:38 +02:00
+								                cancellationToken.ThrowIfCancellationRequested();
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								                if (itemId != Guid.Empty)
-												fix shortcut setting

											
										
										
											2015-09-03 06:16:31 +02:00
+								                {
 								                    // Somehow some invalid data got into the db. It probably predates the boundary checking
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								                    var item = _libraryManager.GetItemById(itemId);
-												fix shortcut setting

											
										
										
											2015-09-03 06:16:31 +02:00
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								                    if (item != null)
-												update select server

											
										
										
											2015-08-30 02:40:52 +02:00
+								                    {
-												update clean db task

											
										
										
											2016-02-03 03:51:00 +01:00
+								                        try
 								                        {
 								                            await _itemRepo.SaveItem(item, cancellationToken).ConfigureAwait(false);
 								                        }
 								                        catch (OperationCanceledException)
 								                        {
 								                            throw;
 								                        }
 								                        catch (Exception ex)
 								                        {
 								                            _logger.ErrorException("Error saving item", ex);
 								                        }
-												update select server

											
										
										
											2015-08-30 02:40:52 +02:00
+								                    }
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								                }
 								                numComplete++;
 								                double percent = numComplete;
 								                percent /= numItems;
 								                progress.Report(percent * 100);
 								            }
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
+								            if (_config.Configuration.MigrationVersion < MigrationVersion)
-												update db queries

											
										
										
											2015-10-29 14:28:05 +01:00
+								            {
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
+								                _config.Configuration.MigrationVersion = MigrationVersion;
-												update db queries

											
										
										
											2015-10-29 14:28:05 +01:00
+								                _config.SaveConfiguration();
 								            }
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								            progress.Report(100);
 								        }
-												update live tv image

											
										
										
											2015-08-28 06:19:08 +02:00
+								        private async Task CleanDeadItems(CancellationToken cancellationToken, IProgress<double> progress)
 								        {
 								            var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
 								            {
 								                HasDeadParentId = true
 								            });
 								            var numComplete = 0;
 								            var numItems = itemIds.Count;
 								            _logger.Debug("Cleaning {0} items with dead parent links", numItems);
 								            foreach (var itemId in itemIds)
 								            {
 								                cancellationToken.ThrowIfCancellationRequested();
 								                var item = _libraryManager.GetItemById(itemId);
 								                if (item != null)
 								                {
-												add master changes

											
										
										
											2015-11-13 21:54:04 +01:00
+								                    _logger.Info("Cleaning item {0} type: {1} path: {2}", item.Name, item.GetType().Name, item.Path ?? string.Empty);
-												update live tv image

											
										
										
											2015-08-28 06:19:08 +02:00
 								                    await _libraryManager.DeleteItem(item, new DeleteOptions
 								                    {
 								                        DeleteFileLocation = false
 								                    });
 								                }
 								                numComplete++;
 								                double percent = numComplete;
 								                percent /= numItems;
 								                progress.Report(percent * 100);
 								            }
 								            progress.Report(100);
 								        }
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								        private async Task CleanDeletedItems(CancellationToken cancellationToken, IProgress<double> progress)
 								        {
 								            var result = _itemRepo.GetItemIdsWithPath(new InternalItemsQuery
 								            {
 								                LocationType = LocationType.FileSystem,
 								                //Limit = limit,
 								                // These have their own cleanup routines
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
+								                ExcludeItemTypes = new[]
 								                {
 								                    typeof(Person).Name,
 								                    typeof(Genre).Name,
 								                    typeof(MusicGenre).Name,
 								                    typeof(GameGenre).Name,
 								                    typeof(Studio).Name,
 								                    typeof(Year).Name,
 								                    typeof(Channel).Name,
 								                    typeof(AggregateFolder).Name,
 								                    typeof(CollectionFolder).Name
 								                }
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								            });
 								            var numComplete = 0;
 								            var numItems = result.Items.Length;
 								            foreach (var item in result.Items)
 								            {
 								                cancellationToken.ThrowIfCancellationRequested();
 								                var path = item.Item2;
 								                try
 								                {
-												update offline detection

											
										
										
											2015-09-21 17:43:10 +02:00
+								                    if (_fileSystem.FileExists(path) || _fileSystem.DirectoryExists(path))
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								                    {
-												update offline detection

											
										
										
											2015-09-21 17:43:10 +02:00
+								                        continue;
 								                    }
 								                    var libraryItem = _libraryManager.GetItemById(item.Item1);
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
-												update subtitles

											
										
										
											2015-11-14 20:56:56 +01:00
+								                    if (libraryItem.IsTopParent)
 								                    {
 								                        continue;
 								                    }
-												update offline detection

											
										
										
											2015-09-21 17:43:10 +02:00
+								                    if (Folder.IsPathOffline(path))
 								                    {
 								                        libraryItem.IsOffline = true;
 								                        await libraryItem.UpdateToRepository(ItemUpdateType.None, cancellationToken).ConfigureAwait(false);
 								                        continue;
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								                    }
-												update offline detection

											
										
										
											2015-09-21 17:43:10 +02:00
-												add master changes

											
										
										
											2015-11-13 21:54:04 +01:00
+								                    _logger.Info("Deleting item from database {0} because path no longer exists. type: {1} path: {2}", libraryItem.Name, libraryItem.GetType().Name, libraryItem.Path ?? string.Empty);
-												update offline detection

											
										
										
											2015-09-21 17:43:10 +02:00
+								                    await _libraryManager.DeleteItem(libraryItem, new DeleteOptions
 								                    {
 								                        DeleteFileLocation = false
 								                    });
-												update item list page

											
										
										
											2015-09-16 19:16:39 +02:00
+								                }
 								                catch (OperationCanceledException)
 								                {
 								                    throw;
 								                }
 								                catch (Exception ex)
 								                {
 								                    _logger.ErrorException("Error in CleanDeletedItems. File {0}", ex, path);
 								                }
 								                numComplete++;
 								                double percent = numComplete;
 								                percent /= numItems;
 								                progress.Report(percent * 100);
 								            }
 								        }
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								        public IEnumerable<ITaskTrigger> GetDefaultTriggers()
 								        {
 								            return new ITaskTrigger[]
 								            {
-												update channel db

											
										
										
											2015-10-04 20:10:50 +02:00
+								                new IntervalTrigger{ Interval = TimeSpan.FromHours(24)}
-												add clean db task

											
										
										
											2015-08-26 07:10:04 +02:00
+								            };
 								        }
 								    }
-												restore changes

											
										
										
											2015-11-13 21:53:29 +01:00
+								}