cotugno/immich
2
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-16 10:28:54 -07:00
Code Issues Packages Projects Releases Wiki Activity

perf: cache getConfig (#9377)

Browse Source 
* cache `getConfig`

* critical fix

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Mert 2024-05-10 14:15:25 -04:00 committed by GitHub
parent 35ef82ab6b
commit f3fbb9b588
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 73 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
server/src/cores/system-config.core.ts
View File

@ -1,5 +1,6 @@
import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common';
import { CronExpression } from '@nestjs/schedule';
import AsyncLock from 'async-lock';
import { plainToInstance } from 'class-transformer';
import { validate } from 'class-validator';
import { load as loadYaml } from 'js-yaml';
@ -21,6 +22,7 @@ import {
TranscodePolicy,
VideoCodec,
} from 'src/entities/system-config.entity';
import { DatabaseLock } from 'src/interfaces/database.interface';
import { QueueName } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { ISystemConfigRepository } from 'src/interfaces/system-config.interface';
@ -186,7 +188,9 @@ let instance: SystemConfigCore | null;
@Injectable()
export class SystemConfigCore {
private configCache: SystemConfigEntity<SystemConfigValue>[] | null = null;
private readonly asyncLock = new AsyncLock();
private config: SystemConfig | null = null;
private lastUpdated: number | null = null;
public config$ = new Subject<SystemConfig>();
@ -268,32 +272,17 @@ export class SystemConfigCore {
}
public async getConfig(force = false): Promise<SystemConfig> {
const configFilePath = process.env.IMMICH_CONFIG_FILE;
const config = _.cloneDeep(defaults);
const overrides = configFilePath ? await this.loadFromFile(configFilePath, force) : await this.repository.load();
for (const { key, value } of overrides) {
// set via dot notation
_.set(config, key, value);
if (force || !this.config) {
const lastUpdated = this.lastUpdated;
await this.asyncLock.acquire(DatabaseLock[DatabaseLock.GetSystemConfig], async () => {
if (lastUpdated === this.lastUpdated) {
this.config = await this.buildConfig();
this.lastUpdated = Date.now();
}
});
}
const errors = await validate(plainToInstance(SystemConfigDto, config));
if (errors.length > 0) {
this.logger.error('Validation error', errors);
if (configFilePath) {
throw new Error(`Invalid value(s) in file: ${errors}`);
}
}
if (!config.ffmpeg.acceptedVideoCodecs.includes(config.ffmpeg.targetVideoCodec)) {
config.ffmpeg.acceptedVideoCodecs.unshift(config.ffmpeg.targetVideoCodec);
}
if (!config.ffmpeg.acceptedAudioCodecs.includes(config.ffmpeg.targetAudioCodec)) {
config.ffmpeg.acceptedAudioCodecs.unshift(config.ffmpeg.targetAudioCodec);
}
return config;
return this.config!;
}
public async updateConfig(newConfig: SystemConfig): Promise<SystemConfig> {
@ -345,33 +334,60 @@ export class SystemConfigCore {
this.config$.next(newConfig);
}
private async loadFromFile(filepath: string, force = false) {
if (force || !this.configCache) {
try {
const file = await this.repository.readFile(filepath);
const config = loadYaml(file.toString()) as any;
const overrides: SystemConfigEntity<SystemConfigValue>[] = [];
private async buildConfig() {
const config = _.cloneDeep(defaults);
const overrides = process.env.IMMICH_CONFIG_FILE
? await this.loadFromFile(process.env.IMMICH_CONFIG_FILE)
: await this.repository.load();
for (const key of Object.values(SystemConfigKey)) {
const value = _.get(config, key);
this.unsetDeep(config, key);
if (value !== undefined) {
overrides.push({ key, value });
}
}
for (const { key, value } of overrides) {
// set via dot notation
_.set(config, key, value);
}
if (!_.isEmpty(config)) {
this.logger.warn(`Unknown keys found: ${JSON.stringify(config, null, 2)}`);
}
this.configCache = overrides;
} catch (error: Error | any) {
this.logger.error(`Unable to load configuration file: ${filepath}`);
throw error;
const errors = await validate(plainToInstance(SystemConfigDto, config));
if (errors.length > 0) {
if (process.env.IMMICH_CONFIG_FILE) {
throw new Error(`Invalid value(s) in file: ${errors}`);
} else {
this.logger.error('Validation error', errors);
}
}
return this.configCache;
if (!config.ffmpeg.acceptedVideoCodecs.includes(config.ffmpeg.targetVideoCodec)) {
config.ffmpeg.acceptedVideoCodecs.push(config.ffmpeg.targetVideoCodec);
}
if (!config.ffmpeg.acceptedAudioCodecs.includes(config.ffmpeg.targetAudioCodec)) {
config.ffmpeg.acceptedAudioCodecs.push(config.ffmpeg.targetAudioCodec);
}
return config;
}
private async loadFromFile(filepath: string) {
try {
const file = await this.repository.readFile(filepath);
const config = loadYaml(file.toString()) as any;
const overrides: SystemConfigEntity<SystemConfigValue>[] = [];
for (const key of Object.values(SystemConfigKey)) {
const value = _.get(config, key);
this.unsetDeep(config, key);
if (value !== undefined) {
overrides.push({ key, value });
}
}
if (!_.isEmpty(config)) {
this.logger.warn(`Unknown keys found: ${JSON.stringify(config, null, 2)}`);
}
return overrides;
} catch (error: Error | any) {
this.logger.error(`Unable to load configuration file: ${filepath}`);
throw error;
}
}
private unsetDeep(object: object, key: string) {

1
server/src/interfaces/database.interface.ts
View File

@ -20,6 +20,7 @@ export enum DatabaseLock {
StorageTemplateMigration = 420,
CLIPDimSize = 512,
LibraryWatch = 1337,
GetSystemConfig = 69,
}
export const extName: Record<DatabaseExtension, string> = {

15
server/src/services/metadata.service.spec.ts
View File

@ -99,19 +99,22 @@ describe(MetadataService.name, () => {
});
describe('init', () => {
beforeEach(async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
it('should pause and resume queue during init', async () => {
await sut.init();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should return if reverse geocoding is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
await sut.init();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
expect(jobMock.pause).not.toHaveBeenCalled();
expect(metadataMock.init).not.toHaveBeenCalled();
expect(jobMock.resume).not.toHaveBeenCalled();
});
});