cotugno/immich
2
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-15 09:59:00 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): Split database queries based on PostgreSQL bound params limit (#6034)

Browse Source 
* fix(server): Split database queries based on PostgreSQL bound params limit

PostgreSQL uses a 16-bit integer to indicate the number of bound
parameters.

This means that the maximum number of parameters for any query is 65535.
Any query that tries to bind more than that (e.g. searching by a list of
IDs) requires splitting the query into multiple chunks.

This change includes refactoring every Repository that runs queries
using a list of ids, and either flattening or merging results.

Fixes #5788, #5997.

Also, potentially a fix for #4648 (at least based on
[this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)).

References:

* https://github.com/typeorm/typeorm/issues/7565
* [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND)

* misc: Create Chunked decorator to simplify implementation

* feat: Add ChunkedArray/ChunkedSet decorators
This commit is contained in:
Michael Manganiello 2024-01-06 20:36:12 -05:00 committed by GitHub
parent 6835d4519a
commit e262298090
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 392 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/domain/album/album.service.spec.ts
View File

@ -679,7 +679,7 @@ describe(AlbumService.name, () => {
]);
expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) });
expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' });
expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']);
});
it('should skip assets not in the album', async () => {

2
server/src/domain/album/album.service.ts
View File

@ -231,7 +231,7 @@ export class AlbumService {
const removedIds = results.filter(({ success }) => success).map(({ id }) => id);
if (removedIds.length > 0) {
await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds });
await this.albumRepository.removeAssets(id, removedIds);
await this.albumRepository.update({ id, updatedAt: new Date() });
if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) {
await this.albumRepository.updateThumbnails();

27
server/src/domain/domain.util.ts
View File

@ -13,6 +13,7 @@ import {
ValidationOptions,
} from 'class-validator';
import { CronJob } from 'cron';
import _ from 'lodash';
import { basename, extname } from 'node:path';
import sanitize from 'sanitize-filename';
@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = {
return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions);
}
/**
* Chunks an array or set into smaller arrays of the specified size.
*
* @param collection The collection to chunk.
* @param size The size of each chunk.
*/
export function chunks<T>(collection: Array<T> | Set<T>, size: number): T[][] {
if (collection instanceof Set) {
const result = [];
let chunk = [];
for (const elem of collection) {
chunk.push(elem);
if (chunk.length === size) {
result.push(chunk);
chunk = [];
}
}
if (chunk.length > 0) {
result.push(chunk);
}
return result;
} else {
return _.chunk(collection, size);
}
}
// NOTE: The following Set utils have been added here, to easily determine where they are used.
// They should be replaced with native Set operations, when they are added to the language.
// Proposal reference: https://github.com/tc39/proposal-set-methods

2
server/src/domain/repositories/album.repository.ts
View File

@ -31,7 +31,7 @@ export interface IAlbumRepository {
getAssetIds(albumId: string, assetIds?: string[]): Promise<Set<string>>;
hasAsset(asset: AlbumAsset): Promise<boolean>;
removeAsset(assetId: string): Promise<void>;
removeAssets(assets: AlbumAssets): Promise<void>;
removeAssets(albumId: string, assetIds: string[]): Promise<void>;
getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]>;
getInvalidThumbnail(): Promise<string[]>;
getOwned(ownerId: string): Promise<AlbumEntity[]>;

6
server/src/infra/infra.util.ts
View File

@ -19,3 +19,9 @@ export const DummyValue = {
DATE: new Date(),
TIME_BUCKET: '2024-01-01T00:00:00.000Z',
};
// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
// by a list of IDs) requires splitting the query into multiple chunks.
// We are rounding down this limit, as queries commonly include other filters and parameters.
export const DATABASE_PARAMETER_CHUNK_SIZE = 65500;

42
server/src/infra/infra.utils.ts
View File

@ -1,5 +1,8 @@
import { Paginated, PaginationOptions } from '@app/domain';
import _ from 'lodash';
import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
import { chunks, setUnion } from '../domain/domain.util';
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
/**
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
* to overcome the maximum number of parameters allowed by the database driver.
*
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
* @param options.flatten Whether to flatten the results. Defaults to false.
*/
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
const originalMethod = descriptor.value;
const paramIndex = options.paramIndex ?? 0;
descriptor.value = async function (...args: any[]) {
const arg = args[paramIndex];
// Early return if argument length is less than or equal to the chunk size.
if (
(arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
) {
return await originalMethod.apply(this, args);
}
return Promise.all(
chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
}),
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
};
};
}
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: _.flatten });
}
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}

481
server/src/infra/repositories/access.repository.ts
View File

@ -1,6 +1,7 @@
import { IAccessRepository } from '@app/domain';
import { InjectRepository } from '@nestjs/typeorm';
import { Brackets, In, Repository } from 'typeorm';
import { chunks, setUnion } from '../../domain/domain.util';
import {
ActivityEntity,
AlbumEntity,
@ -12,6 +13,7 @@ import {
SharedLinkEntity,
UserTokenEntity,
} from '../entities';
import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util';
export class AccessRepository implements IAccessRepository {
constructor(
@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.activityRepository
.find({
select: { id: true },
where: {
id: In([...activityIds]),
userId,
},
})
.then((activities) => new Set(activities.map((activity) => activity.id)));
return Promise.all(
chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.activityRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
userId,
},
})
.then((activities) => new Set(activities.map((activity) => activity.id))),
),
).then((results) => setUnion(...results));
},
checkAlbumOwnerAccess: async (userId: string, activityIds: Set<string>): Promise<Set<string>> => {
@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.activityRepository
.find({
select: { id: true },
where: {
id: In([...activityIds]),
album: {
ownerId: userId,
},
},
})
.then((activities) => new Set(activities.map((activity) => activity.id)));
return Promise.all(
chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.activityRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
album: {
ownerId: userId,
},
},
})
.then((activities) => new Set(activities.map((activity) => activity.id))),
),
).then((results) => setUnion(...results));
},
checkCreateAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.albumRepository
.createQueryBuilder('album')
.select('album.id')
.leftJoin('album.sharedUsers', 'sharedUsers')
.where('album.id IN (:...albumIds)', { albumIds: [...albumIds] })
.andWhere('album.isActivityEnabled = true')
.andWhere(
new Brackets((qb) => {
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
}),
)
.getMany()
.then((albums) => new Set(albums.map((album) => album.id)));
return Promise.all(
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.albumRepository
.createQueryBuilder('album')
.select('album.id')
.leftJoin('album.sharedUsers', 'sharedUsers')
.where('album.id IN (:...albumIds)', { albumIds: idChunk })
.andWhere('album.isActivityEnabled = true')
.andWhere(
new Brackets((qb) => {
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
}),
)
.getMany()
.then((albums) => new Set(albums.map((album) => album.id))),
),
).then((results) => setUnion(...results));
},
};
@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.libraryRepository
.find({
select: { id: true },
where: {
id: In([...libraryIds]),
ownerId: userId,
},
})
.then((libraries) => new Set(libraries.map((library) => library.id)));
return Promise.all(
chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.libraryRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
ownerId: userId,
},
})
.then((libraries) => new Set(libraries.map((library) => library.id))),
),
).then((results) => setUnion(...results));
},
checkPartnerAccess: async (userId: string, partnerIds: Set<string>): Promise<Set<string>> => {
@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
return Promise.all(
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
),
).then((results) => setUnion(...results));
},
};
@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
return Promise.all(
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
),
).then((results) => setUnion(...results));
},
};
@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.albumRepository
.createQueryBuilder('album')
.innerJoin('album.assets', 'asset')
.leftJoin('album.sharedUsers', 'sharedUsers')
.select('asset.id', 'assetId')
.addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
.where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
assetIds: [...assetIds],
})
.andWhere(
new Brackets((qb) => {
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
}),
)
.getRawMany()
.then((rows) => {
const allowedIds = new Set<string>();
for (const row of rows) {
if (row.assetId && assetIds.has(row.assetId)) {
allowedIds.add(row.assetId);
}
if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
allowedIds.add(row.livePhotoVideoId);
}
}
return allowedIds;
});
return Promise.all(
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.albumRepository
.createQueryBuilder('album')
.innerJoin('album.assets', 'asset')
.leftJoin('album.sharedUsers', 'sharedUsers')
.select('asset.id', 'assetId')
.addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
.where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
assetIds: idChunk,
})
.andWhere(
new Brackets((qb) => {
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
}),
)
.getRawMany()
.then((rows) => {
const allowedIds = new Set<string>();
for (const row of rows) {
if (row.assetId && assetIds.has(row.assetId)) {
allowedIds.add(row.assetId);
}
if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
allowedIds.add(row.livePhotoVideoId);
}
}
return allowedIds;
}),
),
).then((results) => setUnion(...results));
},
checkOwnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.assetRepository
.find({
select: { id: true },
where: {
id: In([...assetIds]),
ownerId: userId,
},
withDeleted: true,
})
.then((assets) => new Set(assets.map((asset) => asset.id)));
return Promise.all(
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.assetRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
ownerId: userId,
},
withDeleted: true,
})
.then((assets) => new Set(assets.map((asset) => asset.id))),
),
).then((results) => setUnion(...results));
},
checkPartnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.partnerRepository
.createQueryBuilder('partner')
.innerJoin('partner.sharedBy', 'sharedBy')
.innerJoin('sharedBy.assets', 'asset')
.select('asset.id', 'assetId')
.where('partner.sharedWithId = :userId', { userId })
.andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] })
.getRawMany()
.then((rows) => new Set(rows.map((row) => row.assetId)));
return Promise.all(
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.partnerRepository
.createQueryBuilder('partner')
.innerJoin('partner.sharedBy', 'sharedBy')
.innerJoin('sharedBy.assets', 'asset')
.select('asset.id', 'assetId')
.where('partner.sharedWithId = :userId', { userId })
.andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk })
.getRawMany()
.then((rows) => new Set(rows.map((row) => row.assetId))),
),
).then((results) => setUnion(...results));
},
checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set<string>): Promise<Set<string>> => {
@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.sharedLinkRepository
.createQueryBuilder('sharedLink')
.leftJoin('sharedLink.album', 'album')
.leftJoin('sharedLink.assets', 'assets')
.leftJoin('album.assets', 'albumAssets')
.select('assets.id', 'assetId')
.addSelect('albumAssets.id', 'albumAssetId')
.addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
.addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
.where('sharedLink.id = :sharedLinkId', { sharedLinkId })
.andWhere(
'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
{
assetIds: [...assetIds],
},
)
.getRawMany()
.then((rows) => {
const allowedIds = new Set<string>();
for (const row of rows) {
if (row.assetId && assetIds.has(row.assetId)) {
allowedIds.add(row.assetId);
}
if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
allowedIds.add(row.assetLivePhotoVideoId);
}
if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
allowedIds.add(row.albumAssetId);
}
if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
allowedIds.add(row.albumAssetLivePhotoVideoId);
}
}
return allowedIds;
});
return Promise.all(
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.sharedLinkRepository
.createQueryBuilder('sharedLink')
.leftJoin('sharedLink.album', 'album')
.leftJoin('sharedLink.assets', 'assets')
.leftJoin('album.assets', 'albumAssets')
.select('assets.id', 'assetId')
.addSelect('albumAssets.id', 'albumAssetId')
.addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
.addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
.where('sharedLink.id = :sharedLinkId', { sharedLinkId })
.andWhere(
'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
{
assetIds: idChunk,
},
)
.getRawMany()
.then((rows) => {
const allowedIds = new Set<string>();
for (const row of rows) {
if (row.assetId && assetIds.has(row.assetId)) {
allowedIds.add(row.assetId);
}
if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
allowedIds.add(row.assetLivePhotoVideoId);
}
if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
allowedIds.add(row.albumAssetId);
}
if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
allowedIds.add(row.albumAssetLivePhotoVideoId);
}
}
return allowedIds;
}),
),
).then((results) => setUnion(...results));
},
};
@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.tokenRepository
.find({
select: { id: true },
where: {
userId,
id: In([...deviceIds]),
},
})
.then((tokens) => new Set(tokens.map((token) => token.id)));
return Promise.all(
chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.tokenRepository
.find({
select: { id: true },
where: {
userId,
id: In(idChunk),
},
})
.then((tokens) => new Set(tokens.map((token) => token.id))),
),
).then((results) => setUnion(...results));
},
};
@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.albumRepository
.find({
select: { id: true },
where: {
id: In([...albumIds]),
ownerId: userId,
},
})
.then((albums) => new Set(albums.map((album) => album.id)));
return Promise.all(
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.albumRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
ownerId: userId,
},
})
.then((albums) => new Set(albums.map((album) => album.id))),
),
).then((results) => setUnion(...results));
},
checkSharedAlbumAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.albumRepository
.find({
select: { id: true },
where: {
id: In([...albumIds]),
sharedUsers: {
id: userId,
},
},
})
.then((albums) => new Set(albums.map((album) => album.id)));
return Promise.all(
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.albumRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
sharedUsers: {
id: userId,
},
},
})
.then((albums) => new Set(albums.map((album) => album.id))),
),
).then((results) => setUnion(...results));
},
checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set<string>): Promise<Set<string>> => {
@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.sharedLinkRepository
.find({
select: { albumId: true },
where: {
id: sharedLinkId,
albumId: In([...albumIds]),
},
})
.then(
(sharedLinks) =>
new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
);
return Promise.all(
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.sharedLinkRepository
.find({
select: { albumId: true },
where: {
id: sharedLinkId,
albumId: In(idChunk),
},
})
.then(
(sharedLinks) =>
new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
),
),
).then((results) => setUnion(...results));
},
};
@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.personRepository
.find({
select: { id: true },
where: {
id: In([...personIds]),
ownerId: userId,
},
})
.then((persons) => new Set(persons.map((person) => person.id)));
return Promise.all(
chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.personRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
ownerId: userId,
},
})
.then((persons) => new Set(persons.map((person) => person.id))),
),
).then((results) => setUnion(...results));
},
checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set<string>): Promise<Set<string>> => {
if (assetFaceIds.size === 0) {
return new Set();
}
return this.assetFaceRepository
.find({
select: { id: true },
where: {
id: In([...assetFaceIds]),
asset: {
ownerId: userId,
},
},
})
.then((faces) => new Set(faces.map((face) => face.id)));
return Promise.all(
chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.assetFaceRepository
.find({
select: { id: true },
where: {
id: In(idChunk),
asset: {
ownerId: userId,
},
},
})
.then((faces) => new Set(faces.map((face) => face.id))),
),
).then((results) => setUnion(...results));
},
};
@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository {
return new Set();
}
return this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
return Promise.all(
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
this.partnerRepository
.createQueryBuilder('partner')
.select('partner.sharedById')
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
.andWhere('partner.sharedWithId = :userId', { userId })
.getMany()
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
),
).then((results) => setUnion(...results));
},
};
}

31
server/src/infra/repositories/album.repository.ts
View File

@ -1,10 +1,13 @@
import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import _ from 'lodash';
import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm';
import { setUnion } from '../../domain/domain.util';
import { dataSource } from '../database.config';
import { AlbumEntity, AssetEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util';
import { Chunked, ChunkedArray } from '../infra.utils';
@Injectable()
export class AlbumRepository implements IAlbumRepository {
@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository {
}
@GenerateSql({ params: [[DummyValue.UUID]] })
@ChunkedArray()
getByIds(ids: string[]): Promise<AlbumEntity[]> {
return this.repository.find({
where: {
@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository {
}
@GenerateSql({ params: [[DummyValue.UUID]] })
@ChunkedArray()
async getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]> {
// Guard against running invalid query when ids list is empty.
if (!ids.length) {
@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository {
.execute();
}
@GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] })
async removeAssets(asset: AlbumAssets): Promise<void> {
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
@Chunked({ paramIndex: 1 })
async removeAssets(albumId: string, assetIds: string[]): Promise<void> {
await this.dataSource
.createQueryBuilder()
.delete()
.from('albums_assets_assets')
.where({
albumsId: asset.albumId,
assetsId: In(asset.assetIds),
albumsId: albumId,
assetsId: In(assetIds),
})
.execute();
}
@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository {
.from('albums_assets_assets', 'albums_assets')
.where('"albums_assets"."albumsId" = :albumId', { albumId });
if (assetIds?.length) {
query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds });
if (!assetIds?.length) {
const result = await query.getRawMany();
return new Set(result.map((row) => row['assetId']));
}
const result = await query.getRawMany();
return new Set(result.map((row) => row['assetId']));
return Promise.all(
_.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
query
.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk })
.getRawMany()
.then((result) => new Set(result.map((row) => row['assetId']))),
),
).then((results) => setUnion(...results));
}
@GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] })

7
server/src/infra/repositories/asset.repository.ts
View File

@ -27,7 +27,7 @@ import { DateTime } from 'luxon';
import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { OptionalBetween, paginate } from '../infra.utils';
import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
const DEFAULT_SEARCH_SIZE = 250;
@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository {
}
@GenerateSql({ params: [[DummyValue.UUID]] })
@ChunkedArray()
getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]> {
if (!relations) {
relations = {
@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository {
}
@GenerateSql({ params: [[DummyValue.UUID]] })
@ChunkedArray()
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
return this.repository.find({
where: { library: { id: In(libraryIds) } },
@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository {
}
@GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] })
@Chunked()
async updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void> {
await this.repository.update({ id: In(ids) }, options);
}
@Chunked()
async softDeleteAll(ids: string[]): Promise<void> {
await this.repository.softDelete({ id: In(ids), isExternal: false });
}
@Chunked()
async restoreAll(ids: string[]): Promise<void> {
await this.repository.restore({ id: In(ids) });
}

11
server/src/infra/repositories/person.repository.ts
View File

@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { In, Repository } from 'typeorm';
import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { asVector } from '../infra.utils';
import { Chunked, ChunkedArray, asVector } from '../infra.utils';
export class PersonRepository implements IPersonRepository {
constructor(
@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository {
.getRawMany();
const assetIds = results.map(({ assetId }) => assetId);
await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) });
await this.deletePersonFromAssets(oldPersonId, assetIds);
return assetIds;
}
@Chunked({ paramIndex: 1 })
async deletePersonFromAssets(personId: string, assetIds: string[]): Promise<void> {
await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) });
}
@GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise<number> {
const result = await this.assetFaceRepository
@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository {
}
@GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] })
@ChunkedArray()
async getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true });
}

2
server/src/infra/repositories/system-config.repository.ts
View File

@ -5,6 +5,7 @@ import { readFile } from 'fs/promises';
import { In, Repository } from 'typeorm';
import { SystemConfigEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { Chunked } from '../infra.utils';
export class SystemConfigRepository implements ISystemConfigRepository {
constructor(
@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
}
@GenerateSql({ params: [DummyValue.STRING] })
@Chunked()
async deleteKeys(keys: string[]): Promise<void> {
await this.repository.delete({ key: In(keys) });
}