From e2622980905bd119a8ae2dcbc352ebefcbd8d212 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Sat, 6 Jan 2024 20:36:12 -0500 Subject: [PATCH] fix(server): Split database queries based on PostgreSQL bound params limit (#6034) * fix(server): Split database queries based on PostgreSQL bound params limit PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the maximum number of parameters for any query is 65535. Any query that tries to bind more than that (e.g. searching by a list of IDs) requires splitting the query into multiple chunks. This change includes refactoring every Repository that runs queries using a list of ids, and either flattening or merging results. Fixes #5788, #5997. Also, potentially a fix for #4648 (at least based on [this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)). References: * https://github.com/typeorm/typeorm/issues/7565 * [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND) * misc: Create Chunked decorator to simplify implementation * feat: Add ChunkedArray/ChunkedSet decorators --- server/src/domain/album/album.service.spec.ts | 2 +- server/src/domain/album/album.service.ts | 2 +- server/src/domain/domain.util.ts | 27 + .../domain/repositories/album.repository.ts | 2 +- server/src/infra/infra.util.ts | 6 + server/src/infra/infra.utils.ts | 42 ++ .../infra/repositories/access.repository.ts | 481 ++++++++++-------- .../infra/repositories/album.repository.ts | 31 +- .../infra/repositories/asset.repository.ts | 7 +- .../infra/repositories/person.repository.ts | 11 +- .../repositories/system-config.repository.ts | 2 + 11 files changed, 392 insertions(+), 221 deletions(-) diff --git a/server/src/domain/album/album.service.spec.ts b/server/src/domain/album/album.service.spec.ts index fa421342c8..10b6dde5e5 100644 --- a/server/src/domain/album/album.service.spec.ts +++ b/server/src/domain/album/album.service.spec.ts @@ -679,7 +679,7 @@ describe(AlbumService.name, () => { ]); expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) }); - expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' }); + expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']); }); it('should skip assets not in the album', async () => { diff --git a/server/src/domain/album/album.service.ts b/server/src/domain/album/album.service.ts index 941fcf4c5c..898e3f5263 100644 --- a/server/src/domain/album/album.service.ts +++ b/server/src/domain/album/album.service.ts @@ -231,7 +231,7 @@ export class AlbumService { const removedIds = results.filter(({ success }) => success).map(({ id }) => id); if (removedIds.length > 0) { - await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds }); + await this.albumRepository.removeAssets(id, removedIds); await this.albumRepository.update({ id, updatedAt: new Date() }); if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) { await this.albumRepository.updateThumbnails(); diff --git a/server/src/domain/domain.util.ts b/server/src/domain/domain.util.ts index e6bc41e7b2..79a1913b8e 100644 --- a/server/src/domain/domain.util.ts +++ b/server/src/domain/domain.util.ts @@ -13,6 +13,7 @@ import { ValidationOptions, } from 'class-validator'; import { CronJob } from 'cron'; +import _ from 'lodash'; import { basename, extname } from 'node:path'; import sanitize from 'sanitize-filename'; @@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = { return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions); } +/** + * Chunks an array or set into smaller arrays of the specified size. + * + * @param collection The collection to chunk. + * @param size The size of each chunk. + */ +export function chunks(collection: Array | Set, size: number): T[][] { + if (collection instanceof Set) { + const result = []; + let chunk = []; + for (const elem of collection) { + chunk.push(elem); + if (chunk.length === size) { + result.push(chunk); + chunk = []; + } + } + if (chunk.length > 0) { + result.push(chunk); + } + return result; + } else { + return _.chunk(collection, size); + } +} + // NOTE: The following Set utils have been added here, to easily determine where they are used. // They should be replaced with native Set operations, when they are added to the language. // Proposal reference: https://github.com/tc39/proposal-set-methods diff --git a/server/src/domain/repositories/album.repository.ts b/server/src/domain/repositories/album.repository.ts index 10b789b4b3..eb4d4bf3d4 100644 --- a/server/src/domain/repositories/album.repository.ts +++ b/server/src/domain/repositories/album.repository.ts @@ -31,7 +31,7 @@ export interface IAlbumRepository { getAssetIds(albumId: string, assetIds?: string[]): Promise>; hasAsset(asset: AlbumAsset): Promise; removeAsset(assetId: string): Promise; - removeAssets(assets: AlbumAssets): Promise; + removeAssets(albumId: string, assetIds: string[]): Promise; getMetadataForIds(ids: string[]): Promise; getInvalidThumbnail(): Promise; getOwned(ownerId: string): Promise; diff --git a/server/src/infra/infra.util.ts b/server/src/infra/infra.util.ts index 35ed7c4b4a..8dcf6bf1ac 100644 --- a/server/src/infra/infra.util.ts +++ b/server/src/infra/infra.util.ts @@ -19,3 +19,9 @@ export const DummyValue = { DATE: new Date(), TIME_BUCKET: '2024-01-01T00:00:00.000Z', }; + +// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the +// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching +// by a list of IDs) requires splitting the query into multiple chunks. +// We are rounding down this limit, as queries commonly include other filters and parameters. +export const DATABASE_PARAMETER_CHUNK_SIZE = 65500; diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index 6956b2fbd7..608f844088 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -1,5 +1,8 @@ import { Paginated, PaginationOptions } from '@app/domain'; +import _ from 'lodash'; import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm'; +import { chunks, setUnion } from '../domain/domain.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util'; /** * Allows optional values unlike the regular Between and uses MoreThanOrEqual @@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options; return Number.isInteger(value) && value >= min && value <= max; }; + +/** + * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection, + * to overcome the maximum number of parameters allowed by the database driver. + * + * @param options.paramIndex The index of the function parameter to chunk. Defaults to 0. + * @param options.flatten Whether to flatten the results. Defaults to false. + */ +export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator { + return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { + const originalMethod = descriptor.value; + const paramIndex = options.paramIndex ?? 0; + descriptor.value = async function (...args: any[]) { + const arg = args[paramIndex]; + + // Early return if argument length is less than or equal to the chunk size. + if ( + (arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) || + (arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE) + ) { + return await originalMethod.apply(this, args); + } + + return Promise.all( + chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => { + await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]); + }), + ).then((results) => (options.mergeFn ? options.mergeFn(results) : results)); + }; + }; +} + +export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: _.flatten }); +} + +export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: setUnion }); +} diff --git a/server/src/infra/repositories/access.repository.ts b/server/src/infra/repositories/access.repository.ts index 6c5b24aaca..359dca3943 100644 --- a/server/src/infra/repositories/access.repository.ts +++ b/server/src/infra/repositories/access.repository.ts @@ -1,6 +1,7 @@ import { IAccessRepository } from '@app/domain'; import { InjectRepository } from '@nestjs/typeorm'; import { Brackets, In, Repository } from 'typeorm'; +import { chunks, setUnion } from '../../domain/domain.util'; import { ActivityEntity, AlbumEntity, @@ -12,6 +13,7 @@ import { SharedLinkEntity, UserTokenEntity, } from '../entities'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util'; export class AccessRepository implements IAccessRepository { constructor( @@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - userId, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + userId, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkAlbumOwnerAccess: async (userId: string, activityIds: Set): Promise> => { @@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - album: { - ownerId: userId, - }, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + album: { + ownerId: userId, + }, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkCreateAccess: async (userId: string, albumIds: Set): Promise> => { @@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .select('album.id') - .leftJoin('album.sharedUsers', 'sharedUsers') - .where('album.id IN (:...albumIds)', { albumIds: [...albumIds] }) - .andWhere('album.isActivityEnabled = true') - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getMany() - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .select('album.id') + .leftJoin('album.sharedUsers', 'sharedUsers') + .where('album.id IN (:...albumIds)', { albumIds: idChunk }) + .andWhere('album.isActivityEnabled = true') + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getMany() + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.libraryRepository - .find({ - select: { id: true }, - where: { - id: In([...libraryIds]), - ownerId: userId, - }, - }) - .then((libraries) => new Set(libraries.map((library) => library.id))); + return Promise.all( + chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.libraryRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((libraries) => new Set(libraries.map((library) => library.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, partnerIds: Set): Promise> => { @@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .innerJoin('album.assets', 'asset') - .leftJoin('album.sharedUsers', 'sharedUsers') - .select('asset.id', 'assetId') - .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') - .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { - assetIds: [...assetIds], - }) - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { - allowedIds.add(row.livePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .innerJoin('album.assets', 'asset') + .leftJoin('album.sharedUsers', 'sharedUsers') + .select('asset.id', 'assetId') + .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') + .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { + assetIds: idChunk, + }) + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { + allowedIds.add(row.livePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, checkOwnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.assetRepository - .find({ - select: { id: true }, - where: { - id: In([...assetIds]), - ownerId: userId, - }, - withDeleted: true, - }) - .then((assets) => new Set(assets.map((asset) => asset.id))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + withDeleted: true, + }) + .then((assets) => new Set(assets.map((asset) => asset.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .innerJoin('partner.sharedBy', 'sharedBy') - .innerJoin('sharedBy.assets', 'asset') - .select('asset.id', 'assetId') - .where('partner.sharedWithId = :userId', { userId }) - .andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] }) - .getRawMany() - .then((rows) => new Set(rows.map((row) => row.assetId))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .innerJoin('partner.sharedBy', 'sharedBy') + .innerJoin('sharedBy.assets', 'asset') + .select('asset.id', 'assetId') + .where('partner.sharedWithId = :userId', { userId }) + .andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((rows) => new Set(rows.map((row) => row.assetId))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set): Promise> => { @@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .createQueryBuilder('sharedLink') - .leftJoin('sharedLink.album', 'album') - .leftJoin('sharedLink.assets', 'assets') - .leftJoin('album.assets', 'albumAssets') - .select('assets.id', 'assetId') - .addSelect('albumAssets.id', 'albumAssetId') - .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') - .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') - .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) - .andWhere( - 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', - { - assetIds: [...assetIds], - }, - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { - allowedIds.add(row.assetLivePhotoVideoId); - } - if (row.albumAssetId && assetIds.has(row.albumAssetId)) { - allowedIds.add(row.albumAssetId); - } - if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { - allowedIds.add(row.albumAssetLivePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .createQueryBuilder('sharedLink') + .leftJoin('sharedLink.album', 'album') + .leftJoin('sharedLink.assets', 'assets') + .leftJoin('album.assets', 'albumAssets') + .select('assets.id', 'assetId') + .addSelect('albumAssets.id', 'albumAssetId') + .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') + .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') + .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) + .andWhere( + 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', + { + assetIds: idChunk, + }, + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { + allowedIds.add(row.assetLivePhotoVideoId); + } + if (row.albumAssetId && assetIds.has(row.albumAssetId)) { + allowedIds.add(row.albumAssetId); + } + if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { + allowedIds.add(row.albumAssetLivePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, }; @@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.tokenRepository - .find({ - select: { id: true }, - where: { - userId, - id: In([...deviceIds]), - }, - }) - .then((tokens) => new Set(tokens.map((token) => token.id))); + return Promise.all( + chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.tokenRepository + .find({ + select: { id: true }, + where: { + userId, + id: In(idChunk), + }, + }) + .then((tokens) => new Set(tokens.map((token) => token.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - ownerId: userId, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedAlbumAccess: async (userId: string, albumIds: Set): Promise> => { @@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - sharedUsers: { - id: userId, - }, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + sharedUsers: { + id: userId, + }, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set): Promise> => { @@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .find({ - select: { albumId: true }, - where: { - id: sharedLinkId, - albumId: In([...albumIds]), - }, - }) - .then( - (sharedLinks) => - new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), - ); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .find({ + select: { albumId: true }, + where: { + id: sharedLinkId, + albumId: In(idChunk), + }, + }) + .then( + (sharedLinks) => + new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), + ), + ), + ).then((results) => setUnion(...results)); }, }; @@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.personRepository - .find({ - select: { id: true }, - where: { - id: In([...personIds]), - ownerId: userId, - }, - }) - .then((persons) => new Set(persons.map((person) => person.id))); + return Promise.all( + chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.personRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((persons) => new Set(persons.map((person) => person.id))), + ), + ).then((results) => setUnion(...results)); }, checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set): Promise> => { if (assetFaceIds.size === 0) { return new Set(); } - return this.assetFaceRepository - .find({ - select: { id: true }, - where: { - id: In([...assetFaceIds]), - asset: { - ownerId: userId, - }, - }, - }) - .then((faces) => new Set(faces.map((face) => face.id))); + + return Promise.all( + chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetFaceRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + asset: { + ownerId: userId, + }, + }, + }) + .then((faces) => new Set(faces.map((face) => face.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; } diff --git a/server/src/infra/repositories/album.repository.ts b/server/src/infra/repositories/album.repository.ts index ed92bbfe15..aa66ba2dc8 100644 --- a/server/src/infra/repositories/album.repository.ts +++ b/server/src/infra/repositories/album.repository.ts @@ -1,10 +1,13 @@ import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain'; import { Injectable } from '@nestjs/common'; import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; +import _ from 'lodash'; import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm'; +import { setUnion } from '../../domain/domain.util'; import { dataSource } from '../database.config'; import { AlbumEntity, AssetEntity } from '../entities'; -import { DummyValue, GenerateSql } from '../infra.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util'; +import { Chunked, ChunkedArray } from '../infra.utils'; @Injectable() export class AlbumRepository implements IAlbumRepository { @@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[]): Promise { return this.repository.find({ where: { @@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() async getMetadataForIds(ids: string[]): Promise { // Guard against running invalid query when ids list is empty. if (!ids.length) { @@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository { .execute(); } - @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] }) - async removeAssets(asset: AlbumAssets): Promise { + @GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] }) + @Chunked({ paramIndex: 1 }) + async removeAssets(albumId: string, assetIds: string[]): Promise { await this.dataSource .createQueryBuilder() .delete() .from('albums_assets_assets') .where({ - albumsId: asset.albumId, - assetsId: In(asset.assetIds), + albumsId: albumId, + assetsId: In(assetIds), }) .execute(); } @@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository { .from('albums_assets_assets', 'albums_assets') .where('"albums_assets"."albumsId" = :albumId', { albumId }); - if (assetIds?.length) { - query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds }); + if (!assetIds?.length) { + const result = await query.getRawMany(); + return new Set(result.map((row) => row['assetId'])); } - const result = await query.getRawMany(); - return new Set(result.map((row) => row['assetId'])); + return Promise.all( + _.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + query + .andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((result) => new Set(result.map((row) => row['assetId']))), + ), + ).then((results) => setUnion(...results)); } @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] }) diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index 31bf554861..f250dbeda8 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -27,7 +27,7 @@ import { DateTime } from 'luxon'; import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm'; import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { OptionalBetween, paginate } from '../infra.utils'; +import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils'; const DEFAULT_SEARCH_SIZE = 250; @@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[], relations?: FindOptionsRelations): Promise { if (!relations) { relations = { @@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByLibraryId(libraryIds: string[]): Promise { return this.repository.find({ where: { library: { id: In(libraryIds) } }, @@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) + @Chunked() async updateAll(ids: string[], options: Partial): Promise { await this.repository.update({ id: In(ids) }, options); } + @Chunked() async softDeleteAll(ids: string[]): Promise { await this.repository.softDelete({ id: In(ids), isExternal: false }); } + @Chunked() async restoreAll(ids: string[]): Promise { await this.repository.restore({ id: In(ids) }); } diff --git a/server/src/infra/repositories/person.repository.ts b/server/src/infra/repositories/person.repository.ts index 64fe71d1fe..b49859483a 100644 --- a/server/src/infra/repositories/person.repository.ts +++ b/server/src/infra/repositories/person.repository.ts @@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { In, Repository } from 'typeorm'; import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { asVector } from '../infra.utils'; +import { Chunked, ChunkedArray, asVector } from '../infra.utils'; export class PersonRepository implements IPersonRepository { constructor( @@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository { .getRawMany(); const assetIds = results.map(({ assetId }) => assetId); - - await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) }); + await this.deletePersonFromAssets(oldPersonId, assetIds); return assetIds; } + @Chunked({ paramIndex: 1 }) + async deletePersonFromAssets(personId: string, assetIds: string[]): Promise { + await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) }); + } + @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] }) async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise { const result = await this.assetFaceRepository @@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository { } @GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] }) + @ChunkedArray() async getFacesByIds(ids: AssetFaceId[]): Promise { return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true }); } diff --git a/server/src/infra/repositories/system-config.repository.ts b/server/src/infra/repositories/system-config.repository.ts index 57eb6c1fd4..4ab35b4d63 100644 --- a/server/src/infra/repositories/system-config.repository.ts +++ b/server/src/infra/repositories/system-config.repository.ts @@ -5,6 +5,7 @@ import { readFile } from 'fs/promises'; import { In, Repository } from 'typeorm'; import { SystemConfigEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; +import { Chunked } from '../infra.utils'; export class SystemConfigRepository implements ISystemConfigRepository { constructor( @@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository { } @GenerateSql({ params: [DummyValue.STRING] }) + @Chunked() async deleteKeys(keys: string[]): Promise { await this.repository.delete({ key: In(keys) }); }