From 3ac00b0ffaa514f6c406cc63d99f992742725f7a Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 3 Oct 2024 17:48:40 -0400 Subject: [PATCH] refactor(server): db env (#13167) --- server/src/database.config.ts | 18 +++++++-------- server/src/interfaces/config.interface.ts | 6 +++++ .../migrations/1700713871511-UsePgVectors.ts | 6 +++-- .../1700713994428-AddCLIPEmbeddingIndex.ts | 6 +++-- .../1700714033632-AddFaceEmbeddingIndex.ts | 6 +++-- .../1718486162779-AddFaceSearchRelation.ts | 15 ++++++++----- server/src/repositories/config.repository.ts | 12 ++++++++-- .../src/repositories/database.repository.ts | 9 +++++--- server/src/repositories/search.repository.ts | 9 +++++--- server/src/services/database.service.spec.ts | 22 ++++++++++++++++++- .../repositories/config.repository.mock.ts | 6 +++++ 11 files changed, 84 insertions(+), 31 deletions(-) diff --git a/server/src/database.config.ts b/server/src/database.config.ts index 9cc317a734..2a46067bc1 100644 --- a/server/src/database.config.ts +++ b/server/src/database.config.ts @@ -1,16 +1,17 @@ -import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { DataSource } from 'typeorm'; import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js'; -const url = process.env.DB_URL; +const { database } = new ConfigRepository().getEnv(); +const { url, host, port, username, password, name } = database; const urlOrParts = url ? { url } : { - host: process.env.DB_HOSTNAME || 'database', - port: Number.parseInt(process.env.DB_PORT || '5432'), - username: process.env.DB_USERNAME || 'postgres', - password: process.env.DB_PASSWORD || 'postgres', - database: process.env.DB_DATABASE_NAME || 'immich', + host, + port, + username, + password, + database: name, }; /* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/ @@ -32,6 +33,3 @@ export const databaseConfig: PostgresConnectionOptions = { * this export is ONLY to be used for TypeORM commands in package.json#scripts */ export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' }); - -export const getVectorExtension = () => - process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS; diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index bad4a83222..23a3803284 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -26,6 +26,12 @@ export interface EnvData { }; database: { + url?: string; + host: string; + port: number; + username: string; + password: string; + name: string; skipMigrations: boolean; vectorExtension: VectorExtension; }; diff --git a/server/src/migrations/1700713871511-UsePgVectors.ts b/server/src/migrations/1700713871511-UsePgVectors.ts index 033e2ba9ad..e67c7275a7 100644 --- a/server/src/migrations/1700713871511-UsePgVectors.ts +++ b/server/src/migrations/1700713871511-UsePgVectors.ts @@ -1,13 +1,15 @@ -import { getVectorExtension } from 'src/database.config'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { getCLIPModelInfo } from 'src/utils/misc'; import { MigrationInterface, QueryRunner } from 'typeorm'; +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + export class UsePgVectors1700713871511 implements MigrationInterface { name = 'UsePgVectors1700713871511'; public async up(queryRunner: QueryRunner): Promise { await queryRunner.query(`SET search_path TO "$user", public, vectors`); - await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`); + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`); const faceDimQuery = await queryRunner.query(` SELECT CARDINALITY(embedding::real[]) as dimsize FROM asset_faces diff --git a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts index e325f270fd..f9ea5a0dc3 100644 --- a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts +++ b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts @@ -1,12 +1,14 @@ -import { getVectorExtension } from 'src/database.config'; import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { MigrationInterface, QueryRunner } from 'typeorm'; +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface { name = 'AddCLIPEmbeddingIndex1700713994428'; public async up(queryRunner: QueryRunner): Promise { - if (getVectorExtension() === DatabaseExtension.VECTORS) { + if (vectorExtension === DatabaseExtension.VECTORS) { await queryRunner.query(`SET vectors.pgvector_compatibility=on`); } await queryRunner.query(`SET search_path TO "$user", public, vectors`); diff --git a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts index bc6bad6dbd..d11e7b921e 100644 --- a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts +++ b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts @@ -1,12 +1,14 @@ -import { getVectorExtension } from 'src/database.config'; import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { MigrationInterface, QueryRunner } from 'typeorm'; +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface { name = 'AddFaceEmbeddingIndex1700714033632'; public async up(queryRunner: QueryRunner): Promise { - if (getVectorExtension() === DatabaseExtension.VECTORS) { + if (vectorExtension === DatabaseExtension.VECTORS) { await queryRunner.query(`SET vectors.pgvector_compatibility=on`); } await queryRunner.query(`SET search_path TO "$user", public, vectors`); diff --git a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts index c8e02ec0c5..ae6d752c65 100644 --- a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts +++ b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts @@ -1,10 +1,12 @@ -import { getVectorExtension } from 'src/database.config'; import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { MigrationInterface, QueryRunner } from 'typeorm'; +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + export class AddFaceSearchRelation1718486162779 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - if (getVectorExtension() === DatabaseExtension.VECTORS) { + if (vectorExtension === DatabaseExtension.VECTORS) { await queryRunner.query(`SET search_path TO "$user", public, vectors`); await queryRunner.query(`SET vectors.pgvector_compatibility=on`); } @@ -13,9 +15,10 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface { const columns = await queryRunner.query( `SELECT column_name as name FROM information_schema.columns - WHERE table_name = '${tableName}'`); + WHERE table_name = '${tableName}'`, + ); return columns.some((column: { name: string }) => column.name === 'embedding'); - } + }; const hasAssetEmbeddings = await hasEmbeddings('smart_search'); if (!hasAssetEmbeddings) { @@ -31,7 +34,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface { await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`); await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`); - const hasFaceEmbeddings = await hasEmbeddings('asset_faces') + const hasFaceEmbeddings = await hasEmbeddings('asset_faces'); if (hasFaceEmbeddings) { await queryRunner.query(` INSERT INTO face_search("faceId", embedding) @@ -56,7 +59,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface { } public async down(queryRunner: QueryRunner): Promise { - if (getVectorExtension() === DatabaseExtension.VECTORS) { + if (vectorExtension === DatabaseExtension.VECTORS) { await queryRunner.query(`SET search_path TO "$user", public, vectors`); await queryRunner.query(`SET vectors.pgvector_compatibility=on`); } diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 37013d8512..ed9d80a980 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; -import { getVectorExtension } from 'src/database.config'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; +import { DatabaseExtension } from 'src/interfaces/database.interface'; import { setDifference } from 'src/utils/set'; // TODO replace src/config validation with class-validator, here @@ -65,8 +65,16 @@ export class ConfigRepository implements IConfigRepository { }, database: { + url: process.env.DB_URL, + host: process.env.DB_HOSTNAME || 'database', + port: Number(process.env.DB_PORT) || 5432, + username: process.env.DB_USERNAME || 'postgres', + password: process.env.DB_PASSWORD || 'postgres', + name: process.env.DB_DATABASE_NAME || 'immich', + skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true', - vectorExtension: getVectorExtension(), + vectorExtension: + process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS, }, licensePublicKey: isProd ? productionKeys : stagingKeys, diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index 0453421a39..76998b5239 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -3,7 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm'; import AsyncLock from 'async-lock'; import semver from 'semver'; import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants'; -import { getVectorExtension } from 'src/database.config'; +import { IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension, DatabaseLock, @@ -22,12 +22,15 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm'; @Instrumentation() @Injectable() export class DatabaseRepository implements IDatabaseRepository { + private vectorExtension: VectorExtension; readonly asyncLock = new AsyncLock(); constructor( @InjectDataSource() private dataSource: DataSource, @Inject(ILoggerRepository) private logger: ILoggerRepository, + @Inject(IConfigRepository) configRepository: IConfigRepository, ) { + this.vectorExtension = configRepository.getEnv().database.vectorExtension; this.logger.setContext(DatabaseRepository.name); } @@ -119,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository { try { await this.dataSource.query(`REINDEX INDEX ${index}`); } catch (error) { - if (getVectorExtension() !== DatabaseExtension.VECTORS) { + if (this.vectorExtension !== DatabaseExtension.VECTORS) { throw error; } this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`); @@ -141,7 +144,7 @@ export class DatabaseRepository implements IDatabaseRepository { } async shouldReindex(name: VectorIndex): Promise { - if (getVectorExtension() !== DatabaseExtension.VECTORS) { + if (this.vectorExtension !== DatabaseExtension.VECTORS) { return false; } diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index cb80c8d2f1..1e4b32357c 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -1,7 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { randomUUID } from 'node:crypto'; -import { getVectorExtension } from 'src/database.config'; import { DummyValue, GenerateSql } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; @@ -10,7 +9,8 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { SmartInfoEntity } from 'src/entities/smart-info.entity'; import { SmartSearchEntity } from 'src/entities/smart-search.entity'; import { AssetType, PaginationMode } from 'src/enum'; -import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { IConfigRepository } from 'src/interfaces/config.interface'; +import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { AssetDuplicateResult, @@ -31,6 +31,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm'; @Instrumentation() @Injectable() export class SearchRepository implements ISearchRepository { + private vectorExtension: VectorExtension; private faceColumns: string[]; private assetsByCityQuery: string; @@ -42,7 +43,9 @@ export class SearchRepository implements ISearchRepository { @InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository, @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository, @Inject(ILoggerRepository) private logger: ILoggerRepository, + @Inject(IConfigRepository) configRepository: IConfigRepository, ) { + this.vectorExtension = configRepository.getEnv().database.vectorExtension; this.logger.setContext(SearchRepository.name); this.faceColumns = this.assetFaceRepository.manager.connection .getMetadata(AssetFaceEntity) @@ -440,7 +443,7 @@ export class SearchRepository implements ISearchRepository { } private getRuntimeConfig(numResults?: number): string { - if (getVectorExtension() === DatabaseExtension.VECTOR) { + if (this.vectorExtension === DatabaseExtension.VECTOR) { return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall } diff --git a/server/src/services/database.service.spec.ts b/server/src/services/database.service.spec.ts index e28bf1649a..0bf851f41d 100644 --- a/server/src/services/database.service.spec.ts +++ b/server/src/services/database.service.spec.ts @@ -57,7 +57,17 @@ describe(DatabaseService.name, () => { ])('should work with $extensionName', ({ extension, extensionName }) => { beforeEach(() => { configMock.getEnv.mockReturnValue( - mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }), + mockEnvData({ + database: { + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + name: 'immich', + skipMigrations: false, + vectorExtension: extension, + }, + }), ); }); @@ -245,6 +255,11 @@ describe(DatabaseService.name, () => { configMock.getEnv.mockReturnValue( mockEnvData({ database: { + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + name: 'immich', skipMigrations: true, vectorExtension: DatabaseExtension.VECTORS, }, @@ -260,6 +275,11 @@ describe(DatabaseService.name, () => { configMock.getEnv.mockReturnValue( mockEnvData({ database: { + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + name: 'immich', skipMigrations: true, vectorExtension: DatabaseExtension.VECTOR, }, diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 42a113534e..960a7c1e83 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -10,6 +10,12 @@ const envData: EnvData = { buildMetadata: {}, database: { + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + name: 'immich', + skipMigrations: false, vectorExtension: DatabaseExtension.VECTORS, },