fix(server): Split database queries based on PostgreSQL bound params limit (#6034)

* fix(server): Split database queries based on PostgreSQL bound params limit

PostgreSQL uses a 16-bit integer to indicate the number of bound
parameters.

This means that the maximum number of parameters for any query is 65535.
Any query that tries to bind more than that (e.g. searching by a list of
IDs) requires splitting the query into multiple chunks.

This change includes refactoring every Repository that runs queries
using a list of ids, and either flattening or merging results.

Fixes #5788, #5997.

Also, potentially a fix for #4648 (at least based on
[this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)).

References:

* https://github.com/typeorm/typeorm/issues/7565
* [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND)

* misc: Create Chunked decorator to simplify implementation

* feat: Add ChunkedArray/ChunkedSet decorators

server/src/domain/album/album.service.spec.ts

@@ -679,7 +679,7 @@ describe(AlbumService.name, () => {
       ]);
 
       expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) });
-      expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' });
+      expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']);
     });
 
     it('should skip assets not in the album', async () => {

server/src/domain/album/album.service.ts

@@ -231,7 +231,7 @@ export class AlbumService {
 
     const removedIds = results.filter(({ success }) => success).map(({ id }) => id);
     if (removedIds.length > 0) {
-      await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds });
+      await this.albumRepository.removeAssets(id, removedIds);
       await this.albumRepository.update({ id, updatedAt: new Date() });
       if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) {
         await this.albumRepository.updateThumbnails();

server/src/domain/domain.util.ts

@@ -13,6 +13,7 @@ import {
   ValidationOptions,
 } from 'class-validator';
 import { CronJob } from 'cron';
+import _ from 'lodash';
 import { basename, extname } from 'node:path';
 import sanitize from 'sanitize-filename';
 
@@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = {
   return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions);
 }
 
+/**
+ * Chunks an array or set into smaller arrays of the specified size.
+ *
+ * @param collection The collection to chunk.
+ * @param size The size of each chunk.
+ */
+export function chunks<T>(collection: Array<T> | Set<T>, size: number): T[][] {
+  if (collection instanceof Set) {
+    const result = [];
+    let chunk = [];
+    for (const elem of collection) {
+      chunk.push(elem);
+      if (chunk.length === size) {
+        result.push(chunk);
+        chunk = [];
+      }
+    }
+    if (chunk.length > 0) {
+      result.push(chunk);
+    }
+    return result;
+  } else {
+    return _.chunk(collection, size);
+  }
+}
+
 // NOTE: The following Set utils have been added here, to easily determine where they are used.
 //       They should be replaced with native Set operations, when they are added to the language.
 //       Proposal reference: https://github.com/tc39/proposal-set-methods

server/src/domain/repositories/album.repository.ts

@@ -31,7 +31,7 @@ export interface IAlbumRepository {
   getAssetIds(albumId: string, assetIds?: string[]): Promise<Set<string>>;
   hasAsset(asset: AlbumAsset): Promise<boolean>;
   removeAsset(assetId: string): Promise<void>;
-  removeAssets(assets: AlbumAssets): Promise<void>;
+  removeAssets(albumId: string, assetIds: string[]): Promise<void>;
   getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]>;
   getInvalidThumbnail(): Promise<string[]>;
   getOwned(ownerId: string): Promise<AlbumEntity[]>;

server/src/infra/infra.util.ts

@@ -19,3 +19,9 @@ export const DummyValue = {
   DATE: new Date(),
   TIME_BUCKET: '2024-01-01T00:00:00.000Z',
 };
+
+// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
+// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
+// by a list of IDs) requires splitting the query into multiple chunks.
+// We are rounding down this limit, as queries commonly include other filters and parameters.
+export const DATABASE_PARAMETER_CHUNK_SIZE = 65500;

server/src/infra/infra.utils.ts

@@ -1,5 +1,8 @@
 import { Paginated, PaginationOptions } from '@app/domain';
+import _ from 'lodash';
 import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
+import { chunks, setUnion } from '../domain/domain.util';
+import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
 
 /**
  * Allows optional values unlike the regular Between and uses MoreThanOrEqual
@@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num
   const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
   return Number.isInteger(value) && value >= min && value <= max;
 };
+
+/**
+ * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
+ * to overcome the maximum number of parameters allowed by the database driver.
+ *
+ * @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
+ * @param options.flatten Whether to flatten the results. Defaults to false.
+ */
+export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
+  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
+    const originalMethod = descriptor.value;
+    const paramIndex = options.paramIndex ?? 0;
+    descriptor.value = async function (...args: any[]) {
+      const arg = args[paramIndex];
+
+      // Early return if argument length is less than or equal to the chunk size.
+      if (
+        (arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
+        (arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
+      ) {
+        return await originalMethod.apply(this, args);
+      }
+
+      return Promise.all(
+        chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
+          await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
+        }),
+      ).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
+    };
+  };
+}
+
+export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
+  return Chunked({ ...options, mergeFn: _.flatten });
+}
+
+export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
+  return Chunked({ ...options, mergeFn: setUnion });
+}

server/src/infra/repositories/access.repository.ts

@@ -1,6 +1,7 @@
 import { IAccessRepository } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Brackets, In, Repository } from 'typeorm';
+import { chunks, setUnion } from '../../domain/domain.util';
 import {
   ActivityEntity,
   AlbumEntity,
@@ -12,6 +13,7 @@ import {
   SharedLinkEntity,
   UserTokenEntity,
 } from '../entities';
+import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util';
 
 export class AccessRepository implements IAccessRepository {
   constructor(
@@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.activityRepository
+      return Promise.all(
-        .find({
+        chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.activityRepository
-          where: {
+            .find({
-            id: In([...activityIds]),
+              select: { id: true },
-            userId,
+              where: {
-          },
+                id: In(idChunk),
-        })
+                userId,
-        .then((activities) => new Set(activities.map((activity) => activity.id)));
+              },
+            })
+            .then((activities) => new Set(activities.map((activity) => activity.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkAlbumOwnerAccess: async (userId: string, activityIds: Set<string>): Promise<Set<string>> => {
@@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.activityRepository
+      return Promise.all(
-        .find({
+        chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.activityRepository
-          where: {
+            .find({
-            id: In([...activityIds]),
+              select: { id: true },
-            album: {
+              where: {
-              ownerId: userId,
+                id: In(idChunk),
-            },
+                album: {
-          },
+                  ownerId: userId,
-        })
+                },
-        .then((activities) => new Set(activities.map((activity) => activity.id)));
+              },
+            })
+            .then((activities) => new Set(activities.map((activity) => activity.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkCreateAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
@@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.albumRepository
+      return Promise.all(
-        .createQueryBuilder('album')
+        chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .select('album.id')
+          this.albumRepository
-        .leftJoin('album.sharedUsers', 'sharedUsers')
+            .createQueryBuilder('album')
-        .where('album.id IN (:...albumIds)', { albumIds: [...albumIds] })
+            .select('album.id')
-        .andWhere('album.isActivityEnabled = true')
+            .leftJoin('album.sharedUsers', 'sharedUsers')
-        .andWhere(
+            .where('album.id IN (:...albumIds)', { albumIds: idChunk })
-          new Brackets((qb) => {
+            .andWhere('album.isActivityEnabled = true')
-            qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
+            .andWhere(
-          }),
+              new Brackets((qb) => {
-        )
+                qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
-        .getMany()
+              }),
-        .then((albums) => new Set(albums.map((album) => album.id)));
+            )
+            .getMany()
+            .then((albums) => new Set(albums.map((album) => album.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.libraryRepository
+      return Promise.all(
-        .find({
+        chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.libraryRepository
-          where: {
+            .find({
-            id: In([...libraryIds]),
+              select: { id: true },
-            ownerId: userId,
+              where: {
-          },
+                id: In(idChunk),
-        })
+                ownerId: userId,
-        .then((libraries) => new Set(libraries.map((library) => library.id)));
+              },
+            })
+            .then((libraries) => new Set(libraries.map((library) => library.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkPartnerAccess: async (userId: string, partnerIds: Set<string>): Promise<Set<string>> => {
@@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.partnerRepository
+      return Promise.all(
-        .createQueryBuilder('partner')
+        chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .select('partner.sharedById')
+          this.partnerRepository
-        .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
+            .createQueryBuilder('partner')
-        .andWhere('partner.sharedWithId = :userId', { userId })
+            .select('partner.sharedById')
-        .getMany()
+            .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
-        .then((partners) => new Set(partners.map((partner) => partner.sharedById)));
+            .andWhere('partner.sharedWithId = :userId', { userId })
+            .getMany()
+            .then((partners) => new Set(partners.map((partner) => partner.sharedById))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.partnerRepository
+      return Promise.all(
-        .createQueryBuilder('partner')
+        chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .select('partner.sharedById')
+          this.partnerRepository
-        .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
+            .createQueryBuilder('partner')
-        .andWhere('partner.sharedWithId = :userId', { userId })
+            .select('partner.sharedById')
-        .getMany()
+            .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
-        .then((partners) => new Set(partners.map((partner) => partner.sharedById)));
+            .andWhere('partner.sharedWithId = :userId', { userId })
+            .getMany()
+            .then((partners) => new Set(partners.map((partner) => partner.sharedById))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.albumRepository
+      return Promise.all(
-        .createQueryBuilder('album')
+        chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .innerJoin('album.assets', 'asset')
+          this.albumRepository
-        .leftJoin('album.sharedUsers', 'sharedUsers')
+            .createQueryBuilder('album')
-        .select('asset.id', 'assetId')
+            .innerJoin('album.assets', 'asset')
-        .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
+            .leftJoin('album.sharedUsers', 'sharedUsers')
-        .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
+            .select('asset.id', 'assetId')
-          assetIds: [...assetIds],
+            .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
-        })
+            .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
-        .andWhere(
+              assetIds: idChunk,
-          new Brackets((qb) => {
+            })
-            qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
+            .andWhere(
-          }),
+              new Brackets((qb) => {
-        )
+                qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
-        .getRawMany()
+              }),
-        .then((rows) => {
+            )
-          const allowedIds = new Set<string>();
+            .getRawMany()
-          for (const row of rows) {
+            .then((rows) => {
-            if (row.assetId && assetIds.has(row.assetId)) {
+              const allowedIds = new Set<string>();
-              allowedIds.add(row.assetId);
+              for (const row of rows) {
-            }
+                if (row.assetId && assetIds.has(row.assetId)) {
-            if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
+                  allowedIds.add(row.assetId);
-              allowedIds.add(row.livePhotoVideoId);
+                }
-            }
+                if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
-          }
+                  allowedIds.add(row.livePhotoVideoId);
-          return allowedIds;
+                }
-        });
+              }
+              return allowedIds;
+            }),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkOwnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
@@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.assetRepository
+      return Promise.all(
-        .find({
+        chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.assetRepository
-          where: {
+            .find({
-            id: In([...assetIds]),
+              select: { id: true },
-            ownerId: userId,
+              where: {
-          },
+                id: In(idChunk),
-          withDeleted: true,
+                ownerId: userId,
-        })
+              },
-        .then((assets) => new Set(assets.map((asset) => asset.id)));
+              withDeleted: true,
+            })
+            .then((assets) => new Set(assets.map((asset) => asset.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkPartnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
@@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.partnerRepository
+      return Promise.all(
-        .createQueryBuilder('partner')
+        chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .innerJoin('partner.sharedBy', 'sharedBy')
+          this.partnerRepository
-        .innerJoin('sharedBy.assets', 'asset')
+            .createQueryBuilder('partner')
-        .select('asset.id', 'assetId')
+            .innerJoin('partner.sharedBy', 'sharedBy')
-        .where('partner.sharedWithId = :userId', { userId })
+            .innerJoin('sharedBy.assets', 'asset')
-        .andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] })
+            .select('asset.id', 'assetId')
-        .getRawMany()
+            .where('partner.sharedWithId = :userId', { userId })
-        .then((rows) => new Set(rows.map((row) => row.assetId)));
+            .andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk })
+            .getRawMany()
+            .then((rows) => new Set(rows.map((row) => row.assetId))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set<string>): Promise<Set<string>> => {
@@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.sharedLinkRepository
+      return Promise.all(
-        .createQueryBuilder('sharedLink')
+        chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .leftJoin('sharedLink.album', 'album')
+          this.sharedLinkRepository
-        .leftJoin('sharedLink.assets', 'assets')
+            .createQueryBuilder('sharedLink')
-        .leftJoin('album.assets', 'albumAssets')
+            .leftJoin('sharedLink.album', 'album')
-        .select('assets.id', 'assetId')
+            .leftJoin('sharedLink.assets', 'assets')
-        .addSelect('albumAssets.id', 'albumAssetId')
+            .leftJoin('album.assets', 'albumAssets')
-        .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
+            .select('assets.id', 'assetId')
-        .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
+            .addSelect('albumAssets.id', 'albumAssetId')
-        .where('sharedLink.id = :sharedLinkId', { sharedLinkId })
+            .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
-        .andWhere(
+            .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
-          'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
+            .where('sharedLink.id = :sharedLinkId', { sharedLinkId })
-          {
+            .andWhere(
-            assetIds: [...assetIds],
+              'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
-          },
+              {
-        )
+                assetIds: idChunk,
-        .getRawMany()
+              },
-        .then((rows) => {
+            )
-          const allowedIds = new Set<string>();
+            .getRawMany()
-          for (const row of rows) {
+            .then((rows) => {
-            if (row.assetId && assetIds.has(row.assetId)) {
+              const allowedIds = new Set<string>();
-              allowedIds.add(row.assetId);
+              for (const row of rows) {
-            }
+                if (row.assetId && assetIds.has(row.assetId)) {
-            if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
+                  allowedIds.add(row.assetId);
-              allowedIds.add(row.assetLivePhotoVideoId);
+                }
-            }
+                if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
-            if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
+                  allowedIds.add(row.assetLivePhotoVideoId);
-              allowedIds.add(row.albumAssetId);
+                }
-            }
+                if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
-            if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
+                  allowedIds.add(row.albumAssetId);
-              allowedIds.add(row.albumAssetLivePhotoVideoId);
+                }
-            }
+                if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
-          }
+                  allowedIds.add(row.albumAssetLivePhotoVideoId);
-          return allowedIds;
+                }
-        });
+              }
+              return allowedIds;
+            }),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.tokenRepository
+      return Promise.all(
-        .find({
+        chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.tokenRepository
-          where: {
+            .find({
-            userId,
+              select: { id: true },
-            id: In([...deviceIds]),
+              where: {
-          },
+                userId,
-        })
+                id: In(idChunk),
-        .then((tokens) => new Set(tokens.map((token) => token.id)));
+              },
+            })
+            .then((tokens) => new Set(tokens.map((token) => token.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.albumRepository
+      return Promise.all(
-        .find({
+        chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.albumRepository
-          where: {
+            .find({
-            id: In([...albumIds]),
+              select: { id: true },
-            ownerId: userId,
+              where: {
-          },
+                id: In(idChunk),
-        })
+                ownerId: userId,
-        .then((albums) => new Set(albums.map((album) => album.id)));
+              },
+            })
+            .then((albums) => new Set(albums.map((album) => album.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkSharedAlbumAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
@@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.albumRepository
+      return Promise.all(
-        .find({
+        chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.albumRepository
-          where: {
+            .find({
-            id: In([...albumIds]),
+              select: { id: true },
-            sharedUsers: {
+              where: {
-              id: userId,
+                id: In(idChunk),
-            },
+                sharedUsers: {
-          },
+                  id: userId,
-        })
+                },
-        .then((albums) => new Set(albums.map((album) => album.id)));
+              },
+            })
+            .then((albums) => new Set(albums.map((album) => album.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set<string>): Promise<Set<string>> => {
@@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.sharedLinkRepository
+      return Promise.all(
-        .find({
+        chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { albumId: true },
+          this.sharedLinkRepository
-          where: {
+            .find({
-            id: sharedLinkId,
+              select: { albumId: true },
-            albumId: In([...albumIds]),
+              where: {
-          },
+                id: sharedLinkId,
-        })
+                albumId: In(idChunk),
-        .then(
+              },
-          (sharedLinks) =>
+            })
-            new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
+            .then(
-        );
+              (sharedLinks) =>
+                new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
+            ),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.personRepository
+      return Promise.all(
-        .find({
+        chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          select: { id: true },
+          this.personRepository
-          where: {
+            .find({
-            id: In([...personIds]),
+              select: { id: true },
-            ownerId: userId,
+              where: {
-          },
+                id: In(idChunk),
-        })
+                ownerId: userId,
-        .then((persons) => new Set(persons.map((person) => person.id)));
+              },
+            })
+            .then((persons) => new Set(persons.map((person) => person.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
 
     checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set<string>): Promise<Set<string>> => {
       if (assetFaceIds.size === 0) {
         return new Set();
       }
-      return this.assetFaceRepository
+
-        .find({
+      return Promise.all(
-          select: { id: true },
+        chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-          where: {
+          this.assetFaceRepository
-            id: In([...assetFaceIds]),
+            .find({
-            asset: {
+              select: { id: true },
-              ownerId: userId,
+              where: {
-            },
+                id: In(idChunk),
-          },
+                asset: {
-        })
+                  ownerId: userId,
-        .then((faces) => new Set(faces.map((face) => face.id)));
+                },
+              },
+            })
+            .then((faces) => new Set(faces.map((face) => face.id))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 
@@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository {
         return new Set();
       }
 
-      return this.partnerRepository
+      return Promise.all(
-        .createQueryBuilder('partner')
+        chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
-        .select('partner.sharedById')
+          this.partnerRepository
-        .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
+            .createQueryBuilder('partner')
-        .andWhere('partner.sharedWithId = :userId', { userId })
+            .select('partner.sharedById')
-        .getMany()
+            .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
-        .then((partners) => new Set(partners.map((partner) => partner.sharedById)));
+            .andWhere('partner.sharedWithId = :userId', { userId })
+            .getMany()
+            .then((partners) => new Set(partners.map((partner) => partner.sharedById))),
+        ),
+      ).then((results) => setUnion(...results));
     },
   };
 }

server/src/infra/repositories/album.repository.ts

@@ -1,10 +1,13 @@
 import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+import _ from 'lodash';
 import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm';
+import { setUnion } from '../../domain/domain.util';
 import { dataSource } from '../database.config';
 import { AlbumEntity, AssetEntity } from '../entities';
-import { DummyValue, GenerateSql } from '../infra.util';
+import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util';
+import { Chunked, ChunkedArray } from '../infra.utils';
 
 @Injectable()
 export class AlbumRepository implements IAlbumRepository {
@@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
+  @ChunkedArray()
   getByIds(ids: string[]): Promise<AlbumEntity[]> {
     return this.repository.find({
       where: {
@@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
+  @ChunkedArray()
   async getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]> {
     // Guard against running invalid query when ids list is empty.
     if (!ids.length) {
@@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository {
       .execute();
   }
 
-  @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] })
+  @GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
-  async removeAssets(asset: AlbumAssets): Promise<void> {
+  @Chunked({ paramIndex: 1 })
+  async removeAssets(albumId: string, assetIds: string[]): Promise<void> {
     await this.dataSource
       .createQueryBuilder()
       .delete()
       .from('albums_assets_assets')
       .where({
-        albumsId: asset.albumId,
+        albumsId: albumId,
-        assetsId: In(asset.assetIds),
+        assetsId: In(assetIds),
       })
       .execute();
   }
@@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository {
       .from('albums_assets_assets', 'albums_assets')
       .where('"albums_assets"."albumsId" = :albumId', { albumId });
 
-    if (assetIds?.length) {
+    if (!assetIds?.length) {
-      query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds });
+      const result = await query.getRawMany();
+      return new Set(result.map((row) => row['assetId']));
     }
 
-    const result = await query.getRawMany();
+    return Promise.all(
-    return new Set(result.map((row) => row['assetId']));
+      _.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
+        query
+          .andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk })
+          .getRawMany()
+          .then((result) => new Set(result.map((row) => row['assetId']))),
+      ),
+    ).then((results) => setUnion(...results));
   }
 
   @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] })

server/src/infra/repositories/asset.repository.ts

@@ -27,7 +27,7 @@ import { DateTime } from 'luxon';
 import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm';
 import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
-import { OptionalBetween, paginate } from '../infra.utils';
+import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
 
 const DEFAULT_SEARCH_SIZE = 250;
 
@@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
+  @ChunkedArray()
   getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]> {
     if (!relations) {
       relations = {
@@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
+  @ChunkedArray()
   getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
     return this.repository.find({
       where: { library: { id: In(libraryIds) } },
@@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] })
+  @Chunked()
   async updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void> {
     await this.repository.update({ id: In(ids) }, options);
   }
 
+  @Chunked()
   async softDeleteAll(ids: string[]): Promise<void> {
     await this.repository.softDelete({ id: In(ids), isExternal: false });
   }
 
+  @Chunked()
   async restoreAll(ids: string[]): Promise<void> {
     await this.repository.restore({ id: In(ids) });
   }

server/src/infra/repositories/person.repository.ts

@@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { In, Repository } from 'typeorm';
 import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
-import { asVector } from '../infra.utils';
+import { Chunked, ChunkedArray, asVector } from '../infra.utils';
 
 export class PersonRepository implements IPersonRepository {
   constructor(
@@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository {
       .getRawMany();
 
     const assetIds = results.map(({ assetId }) => assetId);
-
+    await this.deletePersonFromAssets(oldPersonId, assetIds);
-    await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) });
 
     return assetIds;
   }
 
+  @Chunked({ paramIndex: 1 })
+  async deletePersonFromAssets(personId: string, assetIds: string[]): Promise<void> {
+    await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) });
+  }
+
   @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
   async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise<number> {
     const result = await this.assetFaceRepository
@@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository {
   }
 
   @GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] })
+  @ChunkedArray()
   async getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
     return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true });
   }

server/src/infra/repositories/system-config.repository.ts

@@ -5,6 +5,7 @@ import { readFile } from 'fs/promises';
 import { In, Repository } from 'typeorm';
 import { SystemConfigEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Chunked } from '../infra.utils';
 
 export class SystemConfigRepository implements ISystemConfigRepository {
   constructor(
@@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
   }
 
   @GenerateSql({ params: [DummyValue.STRING] })
+  @Chunked()
   async deleteKeys(keys: string[]): Promise<void> {
     await this.repository.delete({ key: In(keys) });
   }