diff --git a/server/src/domain/album/album.service.spec.ts b/server/src/domain/album/album.service.spec.ts index fa421342c8..10b6dde5e5 100644 --- a/server/src/domain/album/album.service.spec.ts +++ b/server/src/domain/album/album.service.spec.ts @@ -679,7 +679,7 @@ describe(AlbumService.name, () => { ]); expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) }); - expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' }); + expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']); }); it('should skip assets not in the album', async () => { diff --git a/server/src/domain/album/album.service.ts b/server/src/domain/album/album.service.ts index 941fcf4c5c..898e3f5263 100644 --- a/server/src/domain/album/album.service.ts +++ b/server/src/domain/album/album.service.ts @@ -231,7 +231,7 @@ export class AlbumService { const removedIds = results.filter(({ success }) => success).map(({ id }) => id); if (removedIds.length > 0) { - await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds }); + await this.albumRepository.removeAssets(id, removedIds); await this.albumRepository.update({ id, updatedAt: new Date() }); if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) { await this.albumRepository.updateThumbnails(); diff --git a/server/src/domain/domain.util.ts b/server/src/domain/domain.util.ts index e6bc41e7b2..79a1913b8e 100644 --- a/server/src/domain/domain.util.ts +++ b/server/src/domain/domain.util.ts @@ -13,6 +13,7 @@ import { ValidationOptions, } from 'class-validator'; import { CronJob } from 'cron'; +import _ from 'lodash'; import { basename, extname } from 'node:path'; import sanitize from 'sanitize-filename'; @@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = { return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions); } +/** + * Chunks an array or set into smaller arrays of the specified size. + * + * @param collection The collection to chunk. + * @param size The size of each chunk. + */ +export function chunks(collection: Array | Set, size: number): T[][] { + if (collection instanceof Set) { + const result = []; + let chunk = []; + for (const elem of collection) { + chunk.push(elem); + if (chunk.length === size) { + result.push(chunk); + chunk = []; + } + } + if (chunk.length > 0) { + result.push(chunk); + } + return result; + } else { + return _.chunk(collection, size); + } +} + // NOTE: The following Set utils have been added here, to easily determine where they are used. // They should be replaced with native Set operations, when they are added to the language. // Proposal reference: https://github.com/tc39/proposal-set-methods diff --git a/server/src/domain/repositories/album.repository.ts b/server/src/domain/repositories/album.repository.ts index 10b789b4b3..eb4d4bf3d4 100644 --- a/server/src/domain/repositories/album.repository.ts +++ b/server/src/domain/repositories/album.repository.ts @@ -31,7 +31,7 @@ export interface IAlbumRepository { getAssetIds(albumId: string, assetIds?: string[]): Promise>; hasAsset(asset: AlbumAsset): Promise; removeAsset(assetId: string): Promise; - removeAssets(assets: AlbumAssets): Promise; + removeAssets(albumId: string, assetIds: string[]): Promise; getMetadataForIds(ids: string[]): Promise; getInvalidThumbnail(): Promise; getOwned(ownerId: string): Promise; diff --git a/server/src/infra/infra.util.ts b/server/src/infra/infra.util.ts index 35ed7c4b4a..8dcf6bf1ac 100644 --- a/server/src/infra/infra.util.ts +++ b/server/src/infra/infra.util.ts @@ -19,3 +19,9 @@ export const DummyValue = { DATE: new Date(), TIME_BUCKET: '2024-01-01T00:00:00.000Z', }; + +// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the +// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching +// by a list of IDs) requires splitting the query into multiple chunks. +// We are rounding down this limit, as queries commonly include other filters and parameters. +export const DATABASE_PARAMETER_CHUNK_SIZE = 65500; diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index 6956b2fbd7..608f844088 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -1,5 +1,8 @@ import { Paginated, PaginationOptions } from '@app/domain'; +import _ from 'lodash'; import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm'; +import { chunks, setUnion } from '../domain/domain.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util'; /** * Allows optional values unlike the regular Between and uses MoreThanOrEqual @@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options; return Number.isInteger(value) && value >= min && value <= max; }; + +/** + * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection, + * to overcome the maximum number of parameters allowed by the database driver. + * + * @param options.paramIndex The index of the function parameter to chunk. Defaults to 0. + * @param options.flatten Whether to flatten the results. Defaults to false. + */ +export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator { + return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { + const originalMethod = descriptor.value; + const paramIndex = options.paramIndex ?? 0; + descriptor.value = async function (...args: any[]) { + const arg = args[paramIndex]; + + // Early return if argument length is less than or equal to the chunk size. + if ( + (arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) || + (arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE) + ) { + return await originalMethod.apply(this, args); + } + + return Promise.all( + chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => { + await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]); + }), + ).then((results) => (options.mergeFn ? options.mergeFn(results) : results)); + }; + }; +} + +export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: _.flatten }); +} + +export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: setUnion }); +} diff --git a/server/src/infra/repositories/access.repository.ts b/server/src/infra/repositories/access.repository.ts index 6c5b24aaca..359dca3943 100644 --- a/server/src/infra/repositories/access.repository.ts +++ b/server/src/infra/repositories/access.repository.ts @@ -1,6 +1,7 @@ import { IAccessRepository } from '@app/domain'; import { InjectRepository } from '@nestjs/typeorm'; import { Brackets, In, Repository } from 'typeorm'; +import { chunks, setUnion } from '../../domain/domain.util'; import { ActivityEntity, AlbumEntity, @@ -12,6 +13,7 @@ import { SharedLinkEntity, UserTokenEntity, } from '../entities'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util'; export class AccessRepository implements IAccessRepository { constructor( @@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - userId, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + userId, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkAlbumOwnerAccess: async (userId: string, activityIds: Set): Promise> => { @@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - album: { - ownerId: userId, - }, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + album: { + ownerId: userId, + }, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkCreateAccess: async (userId: string, albumIds: Set): Promise> => { @@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .select('album.id') - .leftJoin('album.sharedUsers', 'sharedUsers') - .where('album.id IN (:...albumIds)', { albumIds: [...albumIds] }) - .andWhere('album.isActivityEnabled = true') - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getMany() - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .select('album.id') + .leftJoin('album.sharedUsers', 'sharedUsers') + .where('album.id IN (:...albumIds)', { albumIds: idChunk }) + .andWhere('album.isActivityEnabled = true') + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getMany() + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.libraryRepository - .find({ - select: { id: true }, - where: { - id: In([...libraryIds]), - ownerId: userId, - }, - }) - .then((libraries) => new Set(libraries.map((library) => library.id))); + return Promise.all( + chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.libraryRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((libraries) => new Set(libraries.map((library) => library.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, partnerIds: Set): Promise> => { @@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .innerJoin('album.assets', 'asset') - .leftJoin('album.sharedUsers', 'sharedUsers') - .select('asset.id', 'assetId') - .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') - .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { - assetIds: [...assetIds], - }) - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { - allowedIds.add(row.livePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .innerJoin('album.assets', 'asset') + .leftJoin('album.sharedUsers', 'sharedUsers') + .select('asset.id', 'assetId') + .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') + .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { + assetIds: idChunk, + }) + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { + allowedIds.add(row.livePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, checkOwnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.assetRepository - .find({ - select: { id: true }, - where: { - id: In([...assetIds]), - ownerId: userId, - }, - withDeleted: true, - }) - .then((assets) => new Set(assets.map((asset) => asset.id))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + withDeleted: true, + }) + .then((assets) => new Set(assets.map((asset) => asset.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .innerJoin('partner.sharedBy', 'sharedBy') - .innerJoin('sharedBy.assets', 'asset') - .select('asset.id', 'assetId') - .where('partner.sharedWithId = :userId', { userId }) - .andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] }) - .getRawMany() - .then((rows) => new Set(rows.map((row) => row.assetId))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .innerJoin('partner.sharedBy', 'sharedBy') + .innerJoin('sharedBy.assets', 'asset') + .select('asset.id', 'assetId') + .where('partner.sharedWithId = :userId', { userId }) + .andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((rows) => new Set(rows.map((row) => row.assetId))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set): Promise> => { @@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .createQueryBuilder('sharedLink') - .leftJoin('sharedLink.album', 'album') - .leftJoin('sharedLink.assets', 'assets') - .leftJoin('album.assets', 'albumAssets') - .select('assets.id', 'assetId') - .addSelect('albumAssets.id', 'albumAssetId') - .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') - .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') - .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) - .andWhere( - 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', - { - assetIds: [...assetIds], - }, - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { - allowedIds.add(row.assetLivePhotoVideoId); - } - if (row.albumAssetId && assetIds.has(row.albumAssetId)) { - allowedIds.add(row.albumAssetId); - } - if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { - allowedIds.add(row.albumAssetLivePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .createQueryBuilder('sharedLink') + .leftJoin('sharedLink.album', 'album') + .leftJoin('sharedLink.assets', 'assets') + .leftJoin('album.assets', 'albumAssets') + .select('assets.id', 'assetId') + .addSelect('albumAssets.id', 'albumAssetId') + .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') + .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') + .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) + .andWhere( + 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', + { + assetIds: idChunk, + }, + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { + allowedIds.add(row.assetLivePhotoVideoId); + } + if (row.albumAssetId && assetIds.has(row.albumAssetId)) { + allowedIds.add(row.albumAssetId); + } + if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { + allowedIds.add(row.albumAssetLivePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, }; @@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.tokenRepository - .find({ - select: { id: true }, - where: { - userId, - id: In([...deviceIds]), - }, - }) - .then((tokens) => new Set(tokens.map((token) => token.id))); + return Promise.all( + chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.tokenRepository + .find({ + select: { id: true }, + where: { + userId, + id: In(idChunk), + }, + }) + .then((tokens) => new Set(tokens.map((token) => token.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - ownerId: userId, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedAlbumAccess: async (userId: string, albumIds: Set): Promise> => { @@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - sharedUsers: { - id: userId, - }, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + sharedUsers: { + id: userId, + }, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set): Promise> => { @@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .find({ - select: { albumId: true }, - where: { - id: sharedLinkId, - albumId: In([...albumIds]), - }, - }) - .then( - (sharedLinks) => - new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), - ); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .find({ + select: { albumId: true }, + where: { + id: sharedLinkId, + albumId: In(idChunk), + }, + }) + .then( + (sharedLinks) => + new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), + ), + ), + ).then((results) => setUnion(...results)); }, }; @@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.personRepository - .find({ - select: { id: true }, - where: { - id: In([...personIds]), - ownerId: userId, - }, - }) - .then((persons) => new Set(persons.map((person) => person.id))); + return Promise.all( + chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.personRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((persons) => new Set(persons.map((person) => person.id))), + ), + ).then((results) => setUnion(...results)); }, checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set): Promise> => { if (assetFaceIds.size === 0) { return new Set(); } - return this.assetFaceRepository - .find({ - select: { id: true }, - where: { - id: In([...assetFaceIds]), - asset: { - ownerId: userId, - }, - }, - }) - .then((faces) => new Set(faces.map((face) => face.id))); + + return Promise.all( + chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetFaceRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + asset: { + ownerId: userId, + }, + }, + }) + .then((faces) => new Set(faces.map((face) => face.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; } diff --git a/server/src/infra/repositories/album.repository.ts b/server/src/infra/repositories/album.repository.ts index ed92bbfe15..aa66ba2dc8 100644 --- a/server/src/infra/repositories/album.repository.ts +++ b/server/src/infra/repositories/album.repository.ts @@ -1,10 +1,13 @@ import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain'; import { Injectable } from '@nestjs/common'; import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; +import _ from 'lodash'; import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm'; +import { setUnion } from '../../domain/domain.util'; import { dataSource } from '../database.config'; import { AlbumEntity, AssetEntity } from '../entities'; -import { DummyValue, GenerateSql } from '../infra.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util'; +import { Chunked, ChunkedArray } from '../infra.utils'; @Injectable() export class AlbumRepository implements IAlbumRepository { @@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[]): Promise { return this.repository.find({ where: { @@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() async getMetadataForIds(ids: string[]): Promise { // Guard against running invalid query when ids list is empty. if (!ids.length) { @@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository { .execute(); } - @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] }) - async removeAssets(asset: AlbumAssets): Promise { + @GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] }) + @Chunked({ paramIndex: 1 }) + async removeAssets(albumId: string, assetIds: string[]): Promise { await this.dataSource .createQueryBuilder() .delete() .from('albums_assets_assets') .where({ - albumsId: asset.albumId, - assetsId: In(asset.assetIds), + albumsId: albumId, + assetsId: In(assetIds), }) .execute(); } @@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository { .from('albums_assets_assets', 'albums_assets') .where('"albums_assets"."albumsId" = :albumId', { albumId }); - if (assetIds?.length) { - query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds }); + if (!assetIds?.length) { + const result = await query.getRawMany(); + return new Set(result.map((row) => row['assetId'])); } - const result = await query.getRawMany(); - return new Set(result.map((row) => row['assetId'])); + return Promise.all( + _.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + query + .andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((result) => new Set(result.map((row) => row['assetId']))), + ), + ).then((results) => setUnion(...results)); } @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] }) diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index 31bf554861..f250dbeda8 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -27,7 +27,7 @@ import { DateTime } from 'luxon'; import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm'; import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { OptionalBetween, paginate } from '../infra.utils'; +import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils'; const DEFAULT_SEARCH_SIZE = 250; @@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[], relations?: FindOptionsRelations): Promise { if (!relations) { relations = { @@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByLibraryId(libraryIds: string[]): Promise { return this.repository.find({ where: { library: { id: In(libraryIds) } }, @@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) + @Chunked() async updateAll(ids: string[], options: Partial): Promise { await this.repository.update({ id: In(ids) }, options); } + @Chunked() async softDeleteAll(ids: string[]): Promise { await this.repository.softDelete({ id: In(ids), isExternal: false }); } + @Chunked() async restoreAll(ids: string[]): Promise { await this.repository.restore({ id: In(ids) }); } diff --git a/server/src/infra/repositories/person.repository.ts b/server/src/infra/repositories/person.repository.ts index 64fe71d1fe..b49859483a 100644 --- a/server/src/infra/repositories/person.repository.ts +++ b/server/src/infra/repositories/person.repository.ts @@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { In, Repository } from 'typeorm'; import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { asVector } from '../infra.utils'; +import { Chunked, ChunkedArray, asVector } from '../infra.utils'; export class PersonRepository implements IPersonRepository { constructor( @@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository { .getRawMany(); const assetIds = results.map(({ assetId }) => assetId); - - await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) }); + await this.deletePersonFromAssets(oldPersonId, assetIds); return assetIds; } + @Chunked({ paramIndex: 1 }) + async deletePersonFromAssets(personId: string, assetIds: string[]): Promise { + await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) }); + } + @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] }) async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise { const result = await this.assetFaceRepository @@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository { } @GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] }) + @ChunkedArray() async getFacesByIds(ids: AssetFaceId[]): Promise { return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true }); } diff --git a/server/src/infra/repositories/system-config.repository.ts b/server/src/infra/repositories/system-config.repository.ts index 57eb6c1fd4..4ab35b4d63 100644 --- a/server/src/infra/repositories/system-config.repository.ts +++ b/server/src/infra/repositories/system-config.repository.ts @@ -5,6 +5,7 @@ import { readFile } from 'fs/promises'; import { In, Repository } from 'typeorm'; import { SystemConfigEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; +import { Chunked } from '../infra.utils'; export class SystemConfigRepository implements ISystemConfigRepository { constructor( @@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository { } @GenerateSql({ params: [DummyValue.STRING] }) + @Chunked() async deleteKeys(keys: string[]): Promise { await this.repository.delete({ key: In(keys) }); }