diff --git a/i18n/en.json b/i18n/en.json
index e8726b3d20..e08c8ec545 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -1082,7 +1082,9 @@
   "remove_url": "Remove URL",
   "remove_user": "Remove user",
   "removed_api_key": "Removed API Key: {name}",
+  "remove_memory": "Remove memory",
   "removed_memory": "Removed memory",
+  "remove_photo_from_memory": "Remove photo from this memory",
   "removed_photo_from_memory": "Removed photo from memory",
   "removed_from_archive": "Removed from archive",
   "removed_from_favorites": "Removed from favorites",
diff --git a/server/src/dtos/memory.dto.ts b/server/src/dtos/memory.dto.ts
index 9eef78d4d0..36f4631ef5 100644
--- a/server/src/dtos/memory.dto.ts
+++ b/server/src/dtos/memory.dto.ts
@@ -2,6 +2,7 @@ import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import { IsEnum, IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
 import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
+import { AuthDto } from 'src/dtos/auth.dto';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { MemoryType } from 'src/enum';
 import { MemoryItem } from 'src/types';
@@ -88,7 +89,7 @@ export class MemoryResponseDto {
   assets!: AssetResponseDto[];
 }
 
-export const mapMemory = (entity: MemoryItem): MemoryResponseDto => {
+export const mapMemory = (entity: MemoryItem, auth: AuthDto): MemoryResponseDto => {
   return {
     id: entity.id,
     createdAt: entity.createdAt,
@@ -102,6 +103,6 @@ export const mapMemory = (entity: MemoryItem): MemoryResponseDto => {
     type: entity.type as MemoryType,
     data: entity.data as unknown as MemoryData,
     isSaved: entity.isSaved,
-    assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset as AssetEntity)),
+    assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset as AssetEntity, { auth })),
   };
 };
diff --git a/server/src/services/memory.service.ts b/server/src/services/memory.service.ts
index 28c90f6576..8ad3c27b4d 100644
--- a/server/src/services/memory.service.ts
+++ b/server/src/services/memory.service.ts
@@ -74,13 +74,13 @@ export class MemoryService extends BaseService {
 
   async search(auth: AuthDto, dto: MemorySearchDto) {
     const memories = await this.memoryRepository.search(auth.user.id, dto);
-    return memories.map((memory) => mapMemory(memory));
+    return memories.map((memory) => mapMemory(memory, auth));
   }
 
   async get(auth: AuthDto, id: string): Promise<MemoryResponseDto> {
     await this.requireAccess({ auth, permission: Permission.MEMORY_READ, ids: [id] });
     const memory = await this.findOrFail(id);
-    return mapMemory(memory);
+    return mapMemory(memory, auth);
   }
 
   async create(auth: AuthDto, dto: MemoryCreateDto) {
@@ -104,7 +104,7 @@ export class MemoryService extends BaseService {
       allowedAssetIds,
     );
 
-    return mapMemory(memory);
+    return mapMemory(memory, auth);
   }
 
   async update(auth: AuthDto, id: string, dto: MemoryUpdateDto): Promise<MemoryResponseDto> {
@@ -116,7 +116,7 @@ export class MemoryService extends BaseService {
       seenAt: dto.seenAt,
     });
 
-    return mapMemory(memory);
+    return mapMemory(memory, auth);
   }
 
   async remove(auth: AuthDto, id: string): Promise<void> {
diff --git a/web/src/lib/components/memory-page/memory-viewer.svelte b/web/src/lib/components/memory-page/memory-viewer.svelte
index 9a6ad628a3..ceaabd8387 100644
--- a/web/src/lib/components/memory-page/memory-viewer.svelte
+++ b/web/src/lib/components/memory-page/memory-viewer.svelte
@@ -1,6 +1,6 @@
 <script lang="ts">
   import { afterNavigate, goto } from '$app/navigation';
-  import { page } from '$app/stores';
+  import { page } from '$app/state';
   import { intersectionObserver } from '$lib/actions/intersection-observer';
   import { resizeObserver } from '$lib/actions/resize-observer';
   import { shortcuts } from '$lib/actions/shortcut';
@@ -27,21 +27,13 @@
   import { AssetInteraction } from '$lib/stores/asset-interaction.svelte';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { type Viewport } from '$lib/stores/assets-store.svelte';
-  import { loadMemories, memoryStore } from '$lib/stores/memory.store';
+  import { type MemoryAsset, memoryStore } from '$lib/stores/memory.store.svelte';
   import { locale, videoViewerMuted } from '$lib/stores/preferences.store';
   import { preferences } from '$lib/stores/user.store';
   import { getAssetPlaybackUrl, getAssetThumbnailUrl, handlePromiseError, memoryLaneTitle } from '$lib/utils';
   import { cancelMultiselect } from '$lib/utils/asset-utils';
   import { fromLocalDateTime } from '$lib/utils/timeline-util';
-  import {
-    AssetMediaSize,
-    AssetTypeEnum,
-    deleteMemory,
-    removeMemoryAssets,
-    updateMemory,
-    type AssetResponseDto,
-    type MemoryResponseDto,
-  } from '@immich/sdk';
+  import { AssetMediaSize, type AssetResponseDto, AssetTypeEnum } from '@immich/sdk';
   import { IconButton } from '@immich/ui';
   import {
     mdiCardsOutline,
@@ -58,105 +50,50 @@
     mdiPlay,
     mdiPlus,
     mdiSelectAll,
-    mdiVolumeOff,
     mdiVolumeHigh,
+    mdiVolumeOff,
   } from '@mdi/js';
-  import type { NavigationTarget } from '@sveltejs/kit';
+  import type { NavigationTarget, Page } from '@sveltejs/kit';
   import { DateTime } from 'luxon';
-  import { onMount } from 'svelte';
   import { t } from 'svelte-i18n';
-  import { tweened } from 'svelte/motion';
-  import { derived as storeDerived } from 'svelte/store';
+  import { Tween } from 'svelte/motion';
   import { fade } from 'svelte/transition';
 
-  type MemoryIndex = {
-    memoryIndex: number;
-    assetIndex: number;
-  };
-
-  type MemoryAsset = MemoryIndex & {
-    memory: MemoryResponseDto;
-    asset: AssetResponseDto;
-    previousMemory?: MemoryResponseDto;
-    previous?: MemoryAsset;
-    next?: MemoryAsset;
-    nextMemory?: MemoryResponseDto;
-  };
-
   let memoryGallery: HTMLElement | undefined = $state();
   let memoryWrapper: HTMLElement | undefined = $state();
   let galleryInView = $state(false);
+  let galleryFirstLoad = $state(true);
+  let playerInitialized = $state(false);
   let paused = $state(false);
   let current = $state<MemoryAsset | undefined>(undefined);
   let isSaved = $derived(current?.memory.isSaved);
-  let resetPromise = $state(Promise.resolve());
 
   const { isViewing } = assetViewingStore;
   const viewport: Viewport = $state({ width: 0, height: 0 });
   const assetInteraction = new AssetInteraction();
-  let progressBarController = tweened<number>(0, {
-    duration: (from: number, to: number) => (to ? 5000 * (to - from) : 0),
-  });
+  let progressBarController: Tween<number> | undefined = $state(undefined);
   let videoPlayer: HTMLVideoElement | undefined = $state();
-  const memories = storeDerived(memoryStore, (memories) => {
-    memories = memories ?? [];
-    const memoryAssets: MemoryAsset[] = [];
-    let previous: MemoryAsset | undefined;
-    for (const [memoryIndex, memory] of memories.entries()) {
-      for (const [assetIndex, asset] of memory.assets.entries()) {
-        const current = {
-          memory,
-          memoryIndex,
-          previousMemory: memories[memoryIndex - 1],
-          nextMemory: memories[memoryIndex + 1],
-          asset,
-          assetIndex,
-          previous,
-        };
-
-        memoryAssets.push(current);
-
-        if (previous) {
-          previous.next = current;
-        }
-
-        previous = current;
-      }
-    }
-
-    return memoryAssets;
-  });
-
-  const loadFromParams = (memories: MemoryAsset[], page: typeof $page | NavigationTarget | null) => {
-    const assetId = page?.params?.assetId ?? page?.url.searchParams.get(QueryParameter.ID) ?? undefined;
-    handlePromiseError(handleAction($isViewing ? 'pause' : 'reset'));
-    return memories.find((memory) => memory.asset.id === assetId) ?? memories[0];
-  };
   const asHref = (asset: AssetResponseDto) => `?${QueryParameter.ID}=${asset.id}`;
   const handleNavigate = async (asset?: AssetResponseDto) => {
     if ($isViewing) {
       return asset;
     }
 
-    await handleAction('reset');
     if (!asset) {
       return;
     }
 
-    // Adjust the progress bar duration to the video length
-    setProgressDuration(asset);
-
     await goto(asHref(asset));
   };
   const setProgressDuration = (asset: AssetResponseDto) => {
     if (asset.type === AssetTypeEnum.Video) {
       const timeParts = asset.duration.split(':').map(Number);
       const durationInMilliseconds = (timeParts[0] * 3600 + timeParts[1] * 60 + timeParts[2]) * 1000;
-      progressBarController = tweened<number>(0, {
+      progressBarController = new Tween<number>(0, {
         duration: (from: number, to: number) => (to ? durationInMilliseconds * (to - from) : 0),
       });
     } else {
-      progressBarController = tweened<number>(0, {
+      progressBarController = new Tween<number>(0, {
         duration: (from: number, to: number) => (to ? 5000 * (to - from) : 0),
       });
     }
@@ -167,159 +104,187 @@
   const handlePreviousMemory = () => handleNavigate(current?.previousMemory?.assets[0]);
   const handleEscape = async () => goto(AppRoute.PHOTOS);
   const handleSelectAll = () => assetInteraction.selectAssets(current?.memory.assets || []);
-  const handleAction = async (action: 'reset' | 'pause' | 'play') => {
+  const handleAction = async (callingContext: string, action: 'reset' | 'pause' | 'play') => {
+    // leaving these log statements here as comments. Very useful to figure out what's going on during dev!
+    // console.log(`handleAction[${callingContext}] called with: ${action}`);
+    if (!progressBarController) {
+      // console.log(`handleAction[${callingContext}] NOT READY!`);
+      return;
+    }
+
     switch (action) {
       case 'play': {
-        paused = false;
-        await videoPlayer?.play();
-        await progressBarController.set(1);
+        try {
+          paused = false;
+          await videoPlayer?.play();
+          await progressBarController.set(1);
+        } catch (error) {
+          // this may happen if browser blocks auto-play of the video on first page load. This can either be a setting
+          // or just defaut in certain browsers on page load without any DOM interaction by user.
+          console.error(`handleAction[${callingContext}] videoPlayer play problem: ${error}`);
+          paused = true;
+          await progressBarController.set(0);
+        }
         break;
       }
 
       case 'pause': {
         paused = true;
         videoPlayer?.pause();
-        await progressBarController.set($progressBarController);
+        await progressBarController.set(progressBarController.current);
         break;
       }
 
       case 'reset': {
         paused = false;
         videoPlayer?.pause();
-        resetPromise = progressBarController.set(0);
+        await progressBarController.set(0);
         break;
       }
     }
   };
   const handleProgress = async (progress: number) => {
-    if (progress === 0 && !paused) {
-      await handleAction('play');
+    if (!progressBarController) {
       return;
     }
 
-    if (progress === 1) {
-      await progressBarController.set(0);
-      await (current?.next ? handleNextAsset() : handleAction('pause'));
+    if (progress === 1 && !paused) {
+      await (current?.next ? handleNextAsset() : handlePromiseError(handleAction('handleProgressLast', 'pause')));
     }
   };
-  const handleUpdate = () => {
+
+  const toProgressPercentage = (index: number) => {
+    if (!progressBarController || current?.assetIndex === undefined) {
+      return 0;
+    }
+    if (index < current?.assetIndex) {
+      return 100;
+    }
+    if (index > current?.assetIndex) {
+      return 0;
+    }
+    return progressBarController.current * 100;
+  };
+
+  const handleDeleteOrArchiveAssets = (ids: string[]) => {
     if (!current) {
       return;
     }
-    // eslint-disable-next-line no-self-assign
-    current.memory.assets = current.memory.assets;
+    memoryStore.hideAssetsFromMemory(ids);
+    init(page);
   };
-
-  const handleRemove = (ids: string[]) => {
+  const handleDeleteMemoryAsset = async () => {
     if (!current) {
       return;
     }
-    const idSet = new Set(ids);
-    current.memory.assets = current.memory.assets.filter((asset) => !idSet.has(asset.id));
-    init();
+
+    await memoryStore.deleteAssetFromMemory(current.asset.id);
+    init(page);
+  };
+  const handleDeleteMemory = async () => {
+    if (!current) {
+      return;
+    }
+
+    await memoryStore.deleteMemory(current.memory.id);
+    notificationController.show({ message: $t('removed_memory'), type: NotificationType.Info });
+    init(page);
+  };
+  const handleSaveMemory = async () => {
+    if (!current) {
+      return;
+    }
+
+    const newSavedState = !current.memory.isSaved;
+    await memoryStore.updateMemorySaved(current.memory.id, newSavedState);
+    notificationController.show({
+      message: newSavedState ? $t('added_to_favorites') : $t('removed_from_favorites'),
+      type: NotificationType.Info,
+    });
+    init(page);
+  };
+  const handleGalleryScrollsIntoView = () => {
+    galleryInView = true;
+    handlePromiseError(handleAction('galleryInView', 'pause'));
+  };
+  const handleGalleryScrollsOutOfView = () => {
+    galleryInView = false;
+    // only call play after the first page load. When page first loads the gallery will not be visible
+    // and calling play here will result in duplicate invocation.
+    if (!galleryFirstLoad) {
+      handlePromiseError(handleAction('galleryOutOfView', 'play'));
+    }
+    galleryFirstLoad = false;
   };
 
-  const init = () => {
-    $memoryStore = $memoryStore.filter((memory) => memory.assets.length > 0);
-    if ($memoryStore.length === 0) {
+  const loadFromParams = (page: Page | NavigationTarget | null) => {
+    const assetId = page?.params?.assetId ?? page?.url.searchParams.get(QueryParameter.ID) ?? undefined;
+    return memoryStore.getMemoryAsset(assetId);
+  };
+
+  const init = (target: Page | NavigationTarget | null) => {
+    if (memoryStore.memories.length === 0) {
       return handlePromiseError(goto(AppRoute.PHOTOS));
     }
 
-    current = loadFromParams($memories, $page);
-
+    current = loadFromParams(target);
     // Adjust the progress bar duration to the video length
-    setProgressDuration(current.asset);
+    if (current) {
+      setProgressDuration(current.asset);
+    }
+    playerInitialized = false;
   };
 
-  const handleDeleteMemoryAsset = async (current?: MemoryAsset) => {
-    if (!current) {
+  const initPlayer = () => {
+    const isVideoAssetButPlayerHasNotLoadedYet = current && current.asset.type === AssetTypeEnum.Video && !videoPlayer;
+    if (playerInitialized || isVideoAssetButPlayerHasNotLoadedYet) {
       return;
     }
-
-    if (current.memory.assets.length === 1) {
-      return handleDeleteMemory(current);
-    }
-
-    if (current.previous) {
-      current.previous.next = current.next;
-    }
-    if (current.next) {
-      current.next.previous = current.previous;
-    }
-
-    current.memory.assets = current.memory.assets.filter((asset) => asset.id !== current.asset.id);
-
-    // eslint-disable-next-line no-self-assign
-    $memoryStore = $memoryStore;
-
-    await removeMemoryAssets({ id: current.memory.id, bulkIdsDto: { ids: [current.asset.id] } });
-  };
-
-  const handleDeleteMemory = async (current?: MemoryAsset) => {
-    if (!current) {
-      return;
-    }
-
-    await deleteMemory({ id: current.memory.id });
-
-    notificationController.show({ message: $t('removed_memory'), type: NotificationType.Info });
-
-    await loadMemories();
-    init();
-  };
-
-  const handleSaveMemory = async (current?: MemoryAsset) => {
-    if (!current) {
-      return;
-    }
-
-    current.memory.isSaved = !current.memory.isSaved;
-
-    await updateMemory({
-      id: current.memory.id,
-      memoryUpdateDto: {
-        isSaved: current.memory.isSaved,
-      },
-    });
-
-    notificationController.show({
-      message: current.memory.isSaved ? $t('added_to_favorites') : $t('removed_from_favorites'),
-      type: NotificationType.Info,
-    });
-  };
-
-  onMount(async () => {
-    if (!$memoryStore) {
-      await loadMemories();
-    }
-
-    init();
-  });
-
-  afterNavigate(({ from, to }) => {
-    let target = null;
-    if (to?.params?.assetId) {
-      target = to;
-    } else if (from?.params?.assetId) {
-      target = from;
+    if ($isViewing) {
+      handlePromiseError(handleAction('initPlayer[AssetViewOpen]', 'pause'));
     } else {
-      target = $page;
+      handlePromiseError(handleAction('initPlayer[AssetViewClosed]', 'reset'));
+      handlePromiseError(handleAction('initPlayer[AssetViewClosed]', 'play'));
     }
+    playerInitialized = true;
+  };
 
-    current = loadFromParams($memories, target);
+  afterNavigate(({ from, to, type }) => {
+    if (type === 'enter') {
+      // afterNavigate triggers twice on first page load (once when mounted with 'enter' and then a second time
+      // with the actual 'goto' to URL).
+      return;
+    }
+    memoryStore.initialize().then(
+      () => {
+        let target = null;
+        if (to?.params?.assetId) {
+          target = to;
+        } else if (from?.params?.assetId) {
+          target = from;
+        } else {
+          target = page;
+        }
+
+        init(target);
+        initPlayer();
+      },
+      (error) => {
+        console.error(`Error loading memories: ${error}`);
+      },
+    );
   });
 
   $effect(() => {
-    handlePromiseError(handleProgress($progressBarController));
-  });
-
-  $effect(() => {
-    handlePromiseError(handleAction(galleryInView ? 'pause' : 'play'));
+    if (progressBarController) {
+      handlePromiseError(handleProgress(progressBarController.current));
+    }
   });
 
   $effect(() => {
     if (videoPlayer) {
       videoPlayer.muted = $videoViewerMuted;
+      initPlayer();
     }
   });
 </script>
@@ -350,24 +315,24 @@
         <AddToAlbum shared />
       </ButtonContextMenu>
 
-      <FavoriteAction removeFavorite={assetInteraction.isAllFavorite} onFavorite={handleUpdate} />
+      <FavoriteAction removeFavorite={assetInteraction.isAllFavorite} />
 
       <ButtonContextMenu icon={mdiDotsVertical} title={$t('menu')}>
         <DownloadAction menuItem />
         <ChangeDate menuItem />
         <ChangeLocation menuItem />
-        <ArchiveAction menuItem unarchive={assetInteraction.isAllArchived} onArchive={handleRemove} />
+        <ArchiveAction menuItem unarchive={assetInteraction.isAllArchived} onArchive={handleDeleteOrArchiveAssets} />
         {#if $preferences.tags.enabled && assetInteraction.isAllUserOwned}
           <TagAction menuItem />
         {/if}
-        <DeleteAssets menuItem onAssetDelete={handleRemove} />
+        <DeleteAssets menuItem onAssetDelete={handleDeleteOrArchiveAssets} />
       </ButtonContextMenu>
     </AssetSelectControlBar>
   </div>
 {/if}
 
 <section id="memory-viewer" class="w-full bg-immich-dark-gray" bind:this={memoryWrapper}>
-  {#if current && current.memory.assets.length > 0}
+  {#if current}
     <ControlAppBar onClose={() => goto(AppRoute.PHOTOS)} forceDark multiRow>
       {#snippet leading()}
         {#if current}
@@ -381,22 +346,14 @@
         <CircleIconButton
           title={paused ? $t('play_memories') : $t('pause_memories')}
           icon={paused ? mdiPlay : mdiPause}
-          onclick={() => handleAction(paused ? 'play' : 'pause')}
+          onclick={() => handlePromiseError(handleAction('PlayPauseButtonClick', paused ? 'play' : 'pause'))}
           class="hover:text-black"
         />
 
         {#each current.memory.assets as asset, index (asset.id)}
-          <a class="relative w-full py-2" href={asHref(asset)}>
+          <a class="relative w-full py-2" href={asHref(asset)} aria-label={$t('view')}>
             <span class="absolute left-0 h-[2px] w-full bg-gray-500"></span>
-            {#await resetPromise}
-              <span class="absolute left-0 h-[2px] bg-white" style:width={`${index < current.assetIndex ? 100 : 0}%`}
-              ></span>
-            {:then}
-              <span
-                class="absolute left-0 h-[2px] bg-white"
-                style:width={`${index < current.assetIndex ? 100 : index > current.assetIndex ? 0 : $progressBarController * 100}%`}
-              ></span>
-            {/await}
+            <span class="absolute left-0 h-[2px] bg-white" style:width={`${toProgressPercentage(index)}%`}></span>
           </a>
         {/each}
 
@@ -474,7 +431,7 @@
           <div class="relative h-full w-full rounded-2xl bg-black">
             {#key current.asset.id}
               <div transition:fade class="h-full w-full">
-                {#if current.asset.type == AssetTypeEnum.Video}
+                {#if current.asset.type === AssetTypeEnum.Video}
                   <video
                     bind:this={videoPlayer}
                     autoplay
@@ -510,8 +467,8 @@
                   variant="ghost"
                   color="secondary"
                   aria-label={isSaved ? $t('unfavorite') : $t('favorite')}
-                  onclick={() => handleSaveMemory(current)}
-                  class="text-white dark:text-white"
+                  onclick={() => handleSaveMemory()}
+                  class="text-white dark:text-white w-[48px] h-[48px]"
                 />
                 <!-- <IconButton
                   icon={mdiShareVariantOutline}
@@ -525,16 +482,16 @@
                   icon={mdiDotsVertical}
                   padding="3"
                   title={$t('menu')}
-                  onclick={() => handleAction('pause')}
+                  onclick={() => handlePromiseError(handleAction('ContextMenuClick', 'pause'))}
                   direction="left"
                   size="20"
                   align="bottom-right"
                   class="text-white dark:text-white"
                 >
-                  <MenuOption onClick={() => handleDeleteMemory(current)} text="Remove memory" icon={mdiCardsOutline} />
+                  <MenuOption onClick={() => handleDeleteMemory()} text={$t('remove_memory')} icon={mdiCardsOutline} />
                   <MenuOption
-                    onClick={() => handleDeleteMemoryAsset(current)}
-                    text="Remove photo from this memory"
+                    onClick={() => handleDeleteMemoryAsset()}
+                    text={$t('remove_photo_from_memory')}
                     icon={mdiImageMinusOutline}
                   />
                   <!-- shortcut={{ key: 'l', shift: shared }} -->
@@ -642,8 +599,8 @@
       <div
         id="gallery-memory"
         use:intersectionObserver={{
-          onIntersect: () => (galleryInView = true),
-          onSeparate: () => (galleryInView = false),
+          onIntersect: handleGalleryScrollsIntoView,
+          onSeparate: handleGalleryScrollsOutOfView,
           bottom: '-200px',
         }}
         use:resizeObserver={({ height, width }) => ((viewport.height = height), (viewport.width = width))}
diff --git a/web/src/lib/components/photos-page/memory-lane.svelte b/web/src/lib/components/photos-page/memory-lane.svelte
index 77c556834e..9536aaf746 100644
--- a/web/src/lib/components/photos-page/memory-lane.svelte
+++ b/web/src/lib/components/photos-page/memory-lane.svelte
@@ -2,7 +2,7 @@
   import { resizeObserver } from '$lib/actions/resize-observer';
   import Icon from '$lib/components/elements/icon.svelte';
   import { AppRoute, QueryParameter } from '$lib/constants';
-  import { loadMemories, memoryStore } from '$lib/stores/memory.store';
+  import { memoryStore } from '$lib/stores/memory.store.svelte';
   import { getAssetThumbnailUrl, memoryLaneTitle } from '$lib/utils';
   import { getAltText } from '$lib/utils/thumbnail-util';
   import { mdiChevronLeft, mdiChevronRight } from '@mdi/js';
@@ -10,10 +10,10 @@
   import { t } from 'svelte-i18n';
   import { fade } from 'svelte/transition';
 
-  let shouldRender = $derived($memoryStore?.length > 0);
+  let shouldRender = $derived(memoryStore.memories?.length > 0);
 
   onMount(async () => {
-    await loadMemories();
+    await memoryStore.initialize();
   });
 
   let memoryLaneElement: HTMLElement | undefined = $state();
@@ -74,26 +74,24 @@
       </div>
     {/if}
     <div class="inline-block" use:resizeObserver={({ width }) => (innerWidth = width)}>
-      {#each $memoryStore as memory (memory.id)}
-        {#if memory.assets.length > 0}
-          <a
-            class="memory-card relative mr-8 last:mr-0 inline-block aspect-[3/4] md:aspect-[4/3] xl:aspect-video h-[215px] rounded-xl"
-            href="{AppRoute.MEMORY}?{QueryParameter.ID}={memory.assets[0].id}"
-          >
-            <img
-              class="h-full w-full rounded-xl object-cover"
-              src={getAssetThumbnailUrl(memory.assets[0].id)}
-              alt={$t('memory_lane_title', { values: { title: $getAltText(memory.assets[0]) } })}
-              draggable="false"
-            />
-            <p class="absolute bottom-2 left-4 z-10 text-lg text-white">
-              {$memoryLaneTitle(memory)}
-            </p>
-            <div
-              class="absolute left-0 top-0 z-0 h-full w-full rounded-xl bg-gradient-to-t from-black/40 via-transparent to-transparent transition-all hover:bg-black/20"
-            ></div>
-          </a>
-        {/if}
+      {#each memoryStore.memories as memory (memory.id)}
+        <a
+          class="memory-card relative mr-8 last:mr-0 inline-block aspect-[3/4] md:aspect-[4/3] xl:aspect-video h-[215px] rounded-xl"
+          href="{AppRoute.MEMORY}?{QueryParameter.ID}={memory.assets[0].id}"
+        >
+          <img
+            class="h-full w-full rounded-xl object-cover"
+            src={getAssetThumbnailUrl(memory.assets[0].id)}
+            alt={$t('memory_lane_title', { values: { title: $getAltText(memory.assets[0]) } })}
+            draggable="false"
+          />
+          <p class="absolute bottom-2 left-4 z-10 text-lg text-white">
+            {$memoryLaneTitle(memory)}
+          </p>
+          <div
+            class="absolute left-0 top-0 z-0 h-full w-full rounded-xl bg-gradient-to-t from-black/40 via-transparent to-transparent transition-all hover:bg-black/20"
+          ></div>
+        </a>
       {/each}
     </div>
   </section>
diff --git a/web/src/lib/stores/memory.store.svelte.ts b/web/src/lib/stores/memory.store.svelte.ts
new file mode 100644
index 0000000000..76e406682d
--- /dev/null
+++ b/web/src/lib/stores/memory.store.svelte.ts
@@ -0,0 +1,119 @@
+import { asLocalTimeISO } from '$lib/utils/date-time';
+import {
+  type AssetResponseDto,
+  deleteMemory,
+  type MemoryResponseDto,
+  removeMemoryAssets,
+  searchMemories,
+  updateMemory,
+} from '@immich/sdk';
+import { DateTime } from 'luxon';
+
+type MemoryIndex = {
+  memoryIndex: number;
+  assetIndex: number;
+};
+
+export type MemoryAsset = MemoryIndex & {
+  memory: MemoryResponseDto;
+  asset: AssetResponseDto;
+  previousMemory?: MemoryResponseDto;
+  previous?: MemoryAsset;
+  next?: MemoryAsset;
+  nextMemory?: MemoryResponseDto;
+};
+
+class MemoryStoreSvelte {
+  memories = $state<MemoryResponseDto[]>([]);
+  private initialized = false;
+  private memoryAssets = $derived.by(() => {
+    const memoryAssets: MemoryAsset[] = [];
+    let previous: MemoryAsset | undefined;
+    for (const [memoryIndex, memory] of this.memories.entries()) {
+      for (const [assetIndex, asset] of memory.assets.entries()) {
+        const current = {
+          memory,
+          memoryIndex,
+          previousMemory: this.memories[memoryIndex - 1],
+          nextMemory: this.memories[memoryIndex + 1],
+          asset,
+          assetIndex,
+          previous,
+        };
+
+        memoryAssets.push(current);
+
+        if (previous) {
+          previous.next = current;
+        }
+
+        previous = current;
+      }
+    }
+
+    return memoryAssets;
+  });
+
+  getMemoryAsset(assetId: string | undefined) {
+    return this.memoryAssets.find((memoryAsset) => memoryAsset.asset.id === assetId) ?? this.memoryAssets[0];
+  }
+
+  hideAssetsFromMemory(ids: string[]) {
+    const idSet = new Set<string>(ids);
+    for (const memory of this.memories) {
+      memory.assets = memory.assets.filter((asset) => !idSet.has(asset.id));
+    }
+    // if we removed all assets from a memory, then lets remove those memories (we don't show memories with 0 assets)
+    this.memories = this.memories.filter((memory) => memory.assets.length > 0);
+  }
+
+  async deleteMemory(id: string) {
+    const memory = this.memories.find((memory) => memory.id === id);
+    if (memory) {
+      await deleteMemory({ id: memory.id });
+      this.memories = this.memories.filter((memory) => memory.id !== id);
+    }
+  }
+
+  async deleteAssetFromMemory(assetId: string) {
+    const memoryWithAsset = this.memories.find((memory) => memory.assets.some((asset) => asset.id === assetId));
+
+    if (memoryWithAsset) {
+      if (memoryWithAsset.assets.length === 1) {
+        await this.deleteMemory(memoryWithAsset.id);
+      } else {
+        await removeMemoryAssets({ id: memoryWithAsset.id, bulkIdsDto: { ids: [assetId] } });
+        memoryWithAsset.assets = memoryWithAsset.assets.filter((asset) => asset.id !== assetId);
+      }
+    }
+  }
+
+  async updateMemorySaved(id: string, isSaved: boolean) {
+    const memory = this.memories.find((memory) => memory.id === id);
+    if (memory) {
+      await updateMemory({
+        id,
+        memoryUpdateDto: {
+          isSaved,
+        },
+      });
+      memory.isSaved = isSaved;
+    }
+  }
+
+  async initialize() {
+    if (this.initialized) {
+      return;
+    }
+    this.initialized = true;
+
+    await this.loadAllMemories();
+  }
+
+  private async loadAllMemories() {
+    const memories = await searchMemories({ $for: asLocalTimeISO(DateTime.now()) });
+    this.memories = memories.filter((memory) => memory.assets.length > 0);
+  }
+}
+
+export const memoryStore = new MemoryStoreSvelte();
diff --git a/web/src/lib/stores/memory.store.ts b/web/src/lib/stores/memory.store.ts
deleted file mode 100644
index a927ab648a..0000000000
--- a/web/src/lib/stores/memory.store.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { asLocalTimeISO } from '$lib/utils/date-time';
-import { searchMemories, type MemoryResponseDto } from '@immich/sdk';
-import { DateTime } from 'luxon';
-import { writable } from 'svelte/store';
-
-export const memoryStore = writable<MemoryResponseDto[]>();
-
-export const loadMemories = async () => {
-  const memories = await searchMemories({ $for: asLocalTimeISO(DateTime.now()) });
-  memoryStore.set(memories);
-};