Backupppert/immich
1
1
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-05-19 21:01:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions6

WIP

Browse source
This commit is contained in:
Jason Rasmussen 2025-05-07 12:17:58 -04:00
parent 867f6e64f9
commit ac86975b04
No known key found for this signature in database
GPG key ID: 2EF24B77EAFA4A41
14 changed files with 352 additions and 16 deletions

68
open-api/immich-openapi-specs.json
View file

@ -12215,6 +12215,67 @@
],
"type": "object"
},
"SyncAlbumDeleteV1": {
"properties": {
"albumId": {
"type": "string"
}
},
"required": [
"albumId"
],
"type": "object"
},
"SyncAlbumV1": {
"properties": {
"createdAt": {
"format": "date-time",
"type": "string"
},
"description": {
"type": "string"
},
"id": {
"type": "string"
},
"isActivityEnabled": {
"type": "boolean"
},
"name": {
"type": "string"
},
"order": {
"allOf": [
{
"$ref": "#/components/schemas/AssetOrder"
}
]
},
"ownerId": {
"type": "string"
},
"thumbnailAssetId": {
"nullable": true,
"type": "string"
},
"updatedAt": {
"format": "date-time",
"type": "string"
}
},
"required": [
"createdAt",
"description",
"id",
"isActivityEnabled",
"name",
"order",
"ownerId",
"thumbnailAssetId",
"updatedAt"
],
"type": "object"
},
"SyncAssetDeleteV1": {
"properties": {
"assetId": {
@ -12441,7 +12502,9 @@
"AssetExifV1",
"PartnerAssetV1",
"PartnerAssetDeleteV1",
"PartnerAssetExifV1"
"PartnerAssetExifV1",
"AlbumV1",
"AlbumDeleteV1"
],
"type": "string"
},
@ -12486,7 +12549,8 @@
"AssetsV1",
"AssetExifsV1",
"PartnerAssetsV1",
"PartnerAssetExifsV1"
"PartnerAssetExifsV1",
"AlbumsV1"
],
"type": "string"
},

8
server/src/db.d.ts vendored
View file

@ -74,6 +74,13 @@ export interface Albums {
updateId: Generated<string>;
}
export interface AlbumsAudit {
deletedAt: Generated<Timestamp>;
id: Generated<string>;
albumId: string;
ownerId: string;
}
export interface AlbumsAssetsAssets {
albumsId: string;
assetsId: string;
@ -463,6 +470,7 @@ export interface VersionHistory {
export interface DB {
activity: Activity;
albums: Albums;
albums_audit: AlbumsAudit;
albums_assets_assets: AlbumsAssetsAssets;
albums_shared_users_users: AlbumsSharedUsersUsers;
api_keys: ApiKeys;

24
server/src/dtos/sync.dto.ts
View file

@ -1,7 +1,7 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsEnum, IsInt, IsPositive, IsString } from 'class-validator';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { AssetType, AssetVisibility, SyncEntityType, SyncRequestType } from 'src/enum';
import { AssetOrder, AssetType, AssetVisibility, SyncEntityType, SyncRequestType } from 'src/enum';
import { Optional, ValidateDate, ValidateUUID } from 'src/validation';
export class AssetFullSyncDto {
@ -112,6 +112,23 @@ export class SyncAssetExifV1 {
fps!: number | null;
}
export class SyncAlbumDeleteV1 {
albumId!: string;
}
export class SyncAlbumV1 {
id!: string;
ownerId!: string;
name!: string;
description!: string;
createdAt!: Date;
updatedAt!: Date;
thumbnailAssetId!: string | null;
isActivityEnabled!: boolean;
@ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder })
order!: AssetOrder;
}
export type SyncItem = {
[SyncEntityType.UserV1]: SyncUserV1;
[SyncEntityType.UserDeleteV1]: SyncUserDeleteV1;
@ -123,10 +140,11 @@ export type SyncItem = {
[SyncEntityType.PartnerAssetV1]: SyncAssetV1;
[SyncEntityType.PartnerAssetDeleteV1]: SyncAssetDeleteV1;
[SyncEntityType.PartnerAssetExifV1]: SyncAssetExifV1;
[SyncEntityType.AlbumV1]: SyncAlbumV1;
[SyncEntityType.AlbumDeleteV1]: SyncAlbumDeleteV1;
};
const responseDtos = [
//
SyncUserV1,
SyncUserDeleteV1,
SyncPartnerV1,
@ -134,6 +152,8 @@ const responseDtos = [
SyncAssetV1,
SyncAssetDeleteV1,
SyncAssetExifV1,
SyncAlbumV1,
SyncAlbumDeleteV1,
];
export const extraSyncModels = responseDtos;

4
server/src/enum.ts
View file

@ -574,6 +574,7 @@ export enum SyncRequestType {
AssetExifsV1 = 'AssetExifsV1',
PartnerAssetsV1 = 'PartnerAssetsV1',
PartnerAssetExifsV1 = 'PartnerAssetExifsV1',
AlbumsV1 = 'AlbumsV1',
}
export enum SyncEntityType {
@ -590,6 +591,9 @@ export enum SyncEntityType {
PartnerAssetV1 = 'PartnerAssetV1',
PartnerAssetDeleteV1 = 'PartnerAssetDeleteV1',
PartnerAssetExifV1 = 'PartnerAssetExifV1',
AlbumV1 = 'AlbumV1',
AlbumDeleteV1 = 'AlbumDeleteV1',
}
export enum NotificationLevel {

45
server/src/repositories/sync.repository.ts
View file

@ -7,8 +7,8 @@ import { DummyValue, GenerateSql } from 'src/decorators';
import { SyncEntityType } from 'src/enum';
import { SyncAck } from 'src/types';
type auditTables = 'users_audit' | 'partners_audit' | 'assets_audit';
type upsertTables = 'users' | 'partners' | 'assets' | 'exif';
type AuditTables = 'users_audit' | 'partners_audit' | 'assets_audit' | 'albums_audit';
type UpsertTables = 'users' | 'partners' | 'assets' | 'exif' | 'albums';
@Injectable()
export class SyncRepository {
@ -154,19 +154,52 @@ export class SyncRepository {
.stream();
}
private auditTableFilters<T extends keyof Pick<DB, auditTables>, D>(qb: SelectQueryBuilder<DB, T, D>, ack?: SyncAck) {
const builder = qb as SelectQueryBuilder<DB, auditTables, D>;
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_audit')
.select(['id', 'albumId'])
.where('ownerId', '=', userId)
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId))
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums')
.$call((qb) => this.upsertTableFilters(qb, ack))
.leftJoin('albums_shared_users_users as album_users', 'albums.id', 'album_users.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.select([
'albums.id',
'albums.ownerId',
'albums.albumName as name',
'albums.description',
'albums.createdAt',
'albums.updatedAt',
'albums.albumThumbnailAssetId as thumbnailAssetId',
'albums.isActivityEnabled',
'albums.order',
'albums.updateId',
])
.stream();
}
private auditTableFilters<T extends keyof Pick<DB, AuditTables>, D>(qb: SelectQueryBuilder<DB, T, D>, ack?: SyncAck) {
const builder = qb as SelectQueryBuilder<DB, AuditTables, D>;
return builder
.where('deletedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId))
.orderBy('id', 'asc') as SelectQueryBuilder<DB, T, D>;
}
private upsertTableFilters<T extends keyof Pick<DB, upsertTables>, D>(
private upsertTableFilters<T extends keyof Pick<DB, UpsertTables>, D>(
qb: SelectQueryBuilder<DB, T, D>,
ack?: SyncAck,
) {
const builder = qb as SelectQueryBuilder<DB, upsertTables, D>;
const builder = qb as SelectQueryBuilder<DB, UpsertTables, D>;
return builder
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId))

13
server/src/schema/functions.ts
View file

@ -23,6 +23,19 @@ export const immich_uuid_v7 = registerFunction({
synchronize: false,
});
export const album_after_insert = registerFunction({
name: 'album_after_insert',
returnType: 'TRIGGER',
language: 'PLPGSQL',
body: `
BEGIN
INSERT INTO users_audit ("userId")
SELECT "id"
FROM OLD;
RETURN NULL;
END`,
});
export const updated_at = registerFunction({
name: 'updated_at',
returnType: 'TRIGGER',

5
server/src/schema/index.ts
View file

@ -1,6 +1,7 @@
import { AssetVisibility } from 'src/enum';
import { asset_face_source_type, assets_status_enum } from 'src/schema/enums';
import {
album_after_insert,
assets_delete_audit,
f_concat_ws,
f_unaccent,
@ -46,7 +47,7 @@ import { UserAuditTable } from 'src/schema/tables/user-audit.table';
import { UserMetadataTable } from 'src/schema/tables/user-metadata.table';
import { UserTable } from 'src/schema/tables/user.table';
import { VersionHistoryTable } from 'src/schema/tables/version-history.table';
import { ConfigurationParameter, Database, Extensions, registerEnum } from 'src/sql-tools';
import { Database, Extensions, registerEnum } from 'src/sql-tools';
export const asset_visibility_enum = registerEnum({
name: 'asset_visibility_enum',
@ -54,7 +55,6 @@ export const asset_visibility_enum = registerEnum({
});
@Extensions(['uuid-ossp', 'unaccent', 'cube', 'earthdistance', 'pg_trgm', 'plpgsql'])
@ConfigurationParameter({ name: 'search_path', value: () => '"$user", public, vectors', scope: 'database' })
@Database({ name: 'immich' })
export class ImmichDatabase {
tables = [
@ -105,6 +105,7 @@ export class ImmichDatabase {
users_delete_audit,
partners_delete_audit,
assets_delete_audit,
album_after_insert,
];
enum = [assets_status_enum, asset_face_source_type];

17
server/src/schema/tables/album-audit.table.ts Normal file
View file

@ -0,0 +1,17 @@
import { PrimaryGeneratedUuidV7Column } from 'src/decorators';
import { Column, CreateDateColumn, Table } from 'src/sql-tools';
@Table('albums_audit')
export class AlbumAuditTable {
@PrimaryGeneratedUuidV7Column()
id!: string;
@Column({ type: 'uuid', indexName: 'IDX_albums_audit_album_id' })
albumId!: string;
@Column({ type: 'uuid', indexName: 'IDX_albums_audit_owner_id' })
ownerId!: string;
@CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_albums_audit_deleted_at' })
deletedAt!: Date;
}

8
server/src/schema/tables/album-user.table.ts
View file

@ -1,12 +1,18 @@
import { AlbumUserRole } from 'src/enum';
import { album_after_insert } from 'src/schema/functions';
import { AlbumTable } from 'src/schema/tables/album.table';
import { UserTable } from 'src/schema/tables/user.table';
import { Column, ForeignKeyColumn, Index, Table } from 'src/sql-tools';
import { AfterInsertTrigger, Column, ForeignKeyColumn, Index, Table } from 'src/sql-tools';
@Table({ name: 'albums_shared_users_users', primaryConstraintName: 'PK_7df55657e0b2e8b626330a0ebc8' })
// Pre-existing indices from original album <--> user ManyToMany mapping
@Index({ name: 'IDX_427c350ad49bd3935a50baab73', columns: ['albumsId'] })
@Index({ name: 'IDX_f48513bf9bccefd6ff3ad30bd0', columns: ['usersId'] })
@AfterInsertTrigger({
name: 'albums_after_insert',
scope: 'statement',
function: album_after_insert,
})
export class AlbumUserTable {
@ForeignKeyColumn(() => AlbumTable, {
onDelete: 'CASCADE',

19
server/src/services/sync.service.ts
View file

@ -23,13 +23,13 @@ import { fromAck, serialize } from 'src/utils/sync';
const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] };
export const SYNC_TYPES_ORDER = [
//
SyncRequestType.UsersV1,
SyncRequestType.PartnersV1,
SyncRequestType.AssetsV1,
SyncRequestType.AssetExifsV1,
SyncRequestType.PartnerAssetsV1,
SyncRequestType.PartnerAssetExifsV1,
SyncRequestType.AlbumsV1,
];
const throwSessionRequired = () => {
@ -205,6 +205,23 @@ export class SyncService extends BaseService {
break;
}
case SyncRequestType.AlbumsV1: {
// const deletes = this.syncRepository.getAlbumDeletes(
// auth.user.id,
// checkpointMap[SyncEntityType.AlbumDeleteV1],
// );
// for await (const { id, ...data } of deletes) {
// response.write(serialize({ type: SyncEntityType.AlbumDeleteV1, updateId: id, data }));
// }
const upserts = this.syncRepository.getAlbumUpserts(auth.user.id, checkpointMap[SyncEntityType.AlbumV1]);
for await (const { updateId, ...data } of upserts) {
response.write(serialize({ type: SyncEntityType.AlbumV1, updateId, data }));
}
break;
}
default: {
this.logger.warn(`Unsupported sync type: ${type}`);
break;

8
server/src/sql-tools/from-code/decorators/after-insert.decorator.ts Normal file
View file

@ -0,0 +1,8 @@
import { TriggerFunction, TriggerFunctionOptions } from 'src/sql-tools/from-code/decorators/trigger-function.decorator';
export const AfterInsertTrigger = (options: Omit<TriggerFunctionOptions, 'timing' | 'actions'>) =>
TriggerFunction({
timing: 'after',
actions: ['insert'],
...options,
});

1
server/src/sql-tools/public_api.ts
View file

@ -1,6 +1,7 @@
export { schemaDiff } from 'src/sql-tools/diff';
export { schemaFromCode } from 'src/sql-tools/from-code';
export * from 'src/sql-tools/from-code/decorators/after-delete.decorator';
export * from 'src/sql-tools/from-code/decorators/after-insert.decorator';
export * from 'src/sql-tools/from-code/decorators/before-update.decorator';
export * from 'src/sql-tools/from-code/decorators/check.decorator';
export * from 'src/sql-tools/from-code/decorators/column.decorator';

26
server/test/medium.factory.ts
View file

@ -4,9 +4,10 @@ import { DateTime } from 'luxon';
import { createHash, randomBytes } from 'node:crypto';
import { Writable } from 'node:stream';
import { AssetFace } from 'src/database';
import { AssetJobStatus, Assets, DB, FaceSearch, Person, Sessions } from 'src/db';
import { Albums, AssetJobStatus, Assets, DB, FaceSearch, Person, Sessions } from 'src/db';
import { AssetType, AssetVisibility, SourceType } from 'src/enum';
import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
@ -39,6 +40,7 @@ const sha256 = (value: string) => createHash('sha256').update(value).digest('bas
type RepositoriesTypes = {
activity: ActivityRepository;
album: AlbumRepository;
albumUser: AlbumUserRepository;
asset: AssetRepository;
assetJob: AssetJobRepository;
config: ConfigRepository;
@ -125,6 +127,14 @@ export const getRepository = <K extends keyof RepositoriesTypes>(key: K, db: Kys
return new ActivityRepository(db);
}
case 'album': {
return new AlbumRepository(db);
}
case 'albumUser': {
return new AlbumUserRepository(db);
}
case 'asset': {
return new AssetRepository(db);
}
@ -380,6 +390,19 @@ const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {
};
};
const albumInsert = (album: Partial<Insertable<Albums>> & { ownerId: string }) => {
const id = album.id || newUuid();
const defaults: Omit<Insertable<Albums>, 'ownerId'> = {
albumName: 'Album',
};
return {
...defaults,
...album,
id,
};
};
const faceInsert = (face: Partial<Insertable<FaceSearch>> & { faceId: string }) => {
const defaults = {
faceId: face.faceId,
@ -502,6 +525,7 @@ export const mediumFactory = {
assetInsert,
assetFaceInsert,
assetJobStatusInsert,
albumInsert,
faceInsert,
personInsert,
sessionInsert,

122
server/test/medium/specs/services/sync.service.spec.ts
View file

@ -1,5 +1,5 @@
import { AuthDto } from 'src/dtos/auth.dto';
import { SyncEntityType, SyncRequestType } from 'src/enum';
import { AlbumUserRole, SyncEntityType, SyncRequestType } from 'src/enum';
import { SYNC_TYPES_ORDER, SyncService } from 'src/services/sync.service';
import { mediumFactory, newMediumService } from 'test/medium.factory';
import { factory } from 'test/small.factory';
@ -907,4 +907,124 @@ describe(SyncService.name, () => {
await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0);
});
});
describe.concurrent(SyncRequestType.AlbumsV1, () => {
it('should sync an album with the correct properties', async () => {
const { auth, getRepository, testSync } = await setup();
const albumRepo = getRepository('album');
const album = mediumFactory.albumInsert({ ownerId: auth.user.id });
await albumRepo.create(album, [], []);
await expect(testSync(auth, [SyncRequestType.AlbumsV1])).resolves.toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({
id: album.id,
name: album.albumName,
ownerId: album.ownerId,
}),
type: SyncEntityType.AlbumV1,
},
]);
});
it('should detect and sync a new album', async () => {
const { auth, getRepository, testSync } = await setup();
const albumRepo = getRepository('album');
const album = mediumFactory.albumInsert({ ownerId: auth.user.id });
await albumRepo.create(album, [], []);
await expect(testSync(auth, [SyncRequestType.AlbumsV1])).resolves.toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({
id: album.id,
}),
type: SyncEntityType.AlbumV1,
},
]);
});
describe('shared albums', () => {
it('should detect and sync an album create', async () => {
const { auth, getRepository, testSync } = await setup();
const albumRepo = getRepository('album');
const userRepo = getRepository('user');
const user2 = mediumFactory.userInsert();
await userRepo.create(user2);
const album = mediumFactory.albumInsert({ ownerId: user2.id });
await albumRepo.create(album, [], [{ userId: auth.user.id, role: AlbumUserRole.EDITOR }]);
await expect(testSync(auth, [SyncRequestType.AlbumsV1])).resolves.toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({ id: album.id }),
type: SyncEntityType.AlbumV1,
},
]);
});
it('should detect and sync an recently shared album', async () => {
const { auth, getRepository, testSync } = await setup();
const albumRepo = getRepository('album');
const albumUserRepo = getRepository('albumUser');
const userRepo = getRepository('user');
const user2 = mediumFactory.userInsert();
await userRepo.create(user2);
const album = mediumFactory.albumInsert({ ownerId: user2.id });
await albumRepo.create(album, [], []);
await albumUserRepo.create({ usersId: auth.user.id, albumsId: album.id, role: AlbumUserRole.EDITOR });
await expect(testSync(auth, [SyncRequestType.AlbumsV1])).resolves.toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({ id: album.id }),
type: SyncEntityType.AlbumV1,
},
]);
});
it('should handle sharing an album that was created before the last the sync', async () => {
const { auth, getRepository, sut, testSync } = await setup();
const albumRepo = getRepository('album');
const albumUserRepo = getRepository('albumUser');
const userRepo = getRepository('user');
const user2 = mediumFactory.userInsert();
await userRepo.create(user2);
const userAlbum = mediumFactory.albumInsert({ ownerId: auth.user.id });
const user2Album = mediumFactory.albumInsert({ ownerId: user2.id });
await Promise.all([albumRepo.create(user2Album, [], []), albumRepo.create(userAlbum, [], [])]);
const initialSyncResponse = await testSync(auth, [SyncRequestType.AlbumsV1]);
expect(initialSyncResponse).toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({ id: userAlbum.id }),
type: SyncEntityType.AlbumV1,
},
]);
const acks = [initialSyncResponse[0].ack];
await sut.setAcks(auth, { acks });
await albumUserRepo.create({ usersId: auth.user.id, albumsId: user2Album.id, role: AlbumUserRole.EDITOR });
await expect(testSync(auth, [SyncRequestType.AlbumsV1])).resolves.toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({ id: user2Album.id }),
type: SyncEntityType.AlbumV1,
},
]);
});
});
});
// describe.concurrent(SyncRequestType.AlbumsV1, () => {
// });
});