diff --git a/server/src/queries/memory.repository.sql b/server/src/queries/memory.repository.sql
index e9e7340bf6..a3243025b4 100644
--- a/server/src/queries/memory.repository.sql
+++ b/server/src/queries/memory.repository.sql
@@ -15,6 +15,7 @@ select
           inner join "memories_assets_assets" on "assets"."id" = "memories_assets_assets"."assetsId"
         where
           "memories_assets_assets"."memoriesId" = "memories"."id"
+          and "assets"."visibility" = 'timeline'
           and "assets"."deletedAt" is null
         order by
           "assets"."fileCreatedAt" asc
@@ -43,6 +44,7 @@ select
           inner join "memories_assets_assets" on "assets"."id" = "memories_assets_assets"."assetsId"
         where
           "memories_assets_assets"."memoriesId" = "memories"."id"
+          and "assets"."visibility" = 'timeline'
           and "assets"."deletedAt" is null
         order by
           "assets"."fileCreatedAt" asc
@@ -79,6 +81,7 @@ select
           inner join "memories_assets_assets" on "assets"."id" = "memories_assets_assets"."assetsId"
         where
           "memories_assets_assets"."memoriesId" = "memories"."id"
+          and "assets"."visibility" = 'timeline'
           and "assets"."deletedAt" is null
         order by
           "assets"."fileCreatedAt" asc
@@ -111,6 +114,7 @@ select
           inner join "memories_assets_assets" on "assets"."id" = "memories_assets_assets"."assetsId"
         where
           "memories_assets_assets"."memoriesId" = "memories"."id"
+          and "assets"."visibility" = 'timeline'
           and "assets"."deletedAt" is null
         order by
           "assets"."fileCreatedAt" asc
diff --git a/server/src/repositories/memory.repository.ts b/server/src/repositories/memory.repository.ts
index 1a1ea2827b..96eb78e6d6 100644
--- a/server/src/repositories/memory.repository.ts
+++ b/server/src/repositories/memory.repository.ts
@@ -1,18 +1,26 @@
 import { Injectable } from '@nestjs/common';
-import { Insertable, Kysely, Updateable } from 'kysely';
+import { Insertable, Kysely, sql, Updateable } from 'kysely';
 import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { DateTime } from 'luxon';
 import { InjectKysely } from 'nestjs-kysely';
 import { DB, Memories } from 'src/db';
 import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators';
 import { MemorySearchDto } from 'src/dtos/memory.dto';
+import { AssetVisibility } from 'src/enum';
 import { IBulkAsset } from 'src/types';
 
 @Injectable()
 export class MemoryRepository implements IBulkAsset {
   constructor(@InjectKysely() private db: Kysely<DB>) {}
 
-  cleanup() {
+  async cleanup() {
+    await this.db
+      .deleteFrom('memories_assets_assets')
+      .using('assets')
+      .whereRef('memories_assets_assets.assetsId', '=', 'assets.id')
+      .where('assets.visibility', '!=', AssetVisibility.TIMELINE)
+      .execute();
+
     return this.db
       .deleteFrom('memories')
       .where('createdAt', '<', DateTime.now().minus({ days: 30 }).toJSDate())
@@ -36,6 +44,7 @@ export class MemoryRepository implements IBulkAsset {
             .innerJoin('memories_assets_assets', 'assets.id', 'memories_assets_assets.assetsId')
             .whereRef('memories_assets_assets.memoriesId', '=', 'memories.id')
             .orderBy('assets.fileCreatedAt', 'asc')
+            .where('assets.visibility', '=', sql.lit(AssetVisibility.TIMELINE))
             .where('assets.deletedAt', 'is', null),
         ).as('assets'),
       )
@@ -138,6 +147,7 @@ export class MemoryRepository implements IBulkAsset {
             .innerJoin('memories_assets_assets', 'assets.id', 'memories_assets_assets.assetsId')
             .whereRef('memories_assets_assets.memoriesId', '=', 'memories.id')
             .orderBy('assets.fileCreatedAt', 'asc')
+            .where('assets.visibility', '=', sql.lit(AssetVisibility.TIMELINE))
             .where('assets.deletedAt', 'is', null),
         ).as('assets'),
       )