cotugno/immich
2
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-15 18:08:48 -07:00
Code Issues Packages Projects Releases Wiki Activity

chore(server) Add job for storage migration (#1117)

Browse Source
This commit is contained in:
Alex 2022-12-19 12:13:10 -06:00 committed by GitHub
parent 8998a79ff9
commit de69d0031e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 398 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mobile/openapi/doc/AllJobStatusResponseDto.md generated
View File

@ -12,10 +12,12 @@ Name | Type | Description | Notes
**metadataExtractionQueueCount** | [**JobCounts**](JobCounts.md) | |
**videoConversionQueueCount** | [**JobCounts**](JobCounts.md) | |
**machineLearningQueueCount** | [**JobCounts**](JobCounts.md) | |
**storageMigrationQueueCount** | [**JobCounts**](JobCounts.md) | |
**isThumbnailGenerationActive** | **bool** | |
**isMetadataExtractionActive** | **bool** | |
**isVideoConversionActive** | **bool** | |
**isMachineLearningActive** | **bool** | |
**isStorageMigrationActive** | **bool** | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

22
mobile/openapi/lib/model/all_job_status_response_dto.dart generated
View File

@ -17,10 +17,12 @@ class AllJobStatusResponseDto {
required this.metadataExtractionQueueCount,
required this.videoConversionQueueCount,
required this.machineLearningQueueCount,
required this.storageMigrationQueueCount,
required this.isThumbnailGenerationActive,
required this.isMetadataExtractionActive,
required this.isVideoConversionActive,
required this.isMachineLearningActive,
required this.isStorageMigrationActive,
});
JobCounts thumbnailGenerationQueueCount;
@ -31,6 +33,8 @@ class AllJobStatusResponseDto {
JobCounts machineLearningQueueCount;
JobCounts storageMigrationQueueCount;
bool isThumbnailGenerationActive;
bool isMetadataExtractionActive;
@ -39,16 +43,20 @@ class AllJobStatusResponseDto {
bool isMachineLearningActive;
bool isStorageMigrationActive;
@override
bool operator ==(Object other) => identical(this, other) || other is AllJobStatusResponseDto &&
other.thumbnailGenerationQueueCount == thumbnailGenerationQueueCount &&
other.metadataExtractionQueueCount == metadataExtractionQueueCount &&
other.videoConversionQueueCount == videoConversionQueueCount &&
other.machineLearningQueueCount == machineLearningQueueCount &&
other.storageMigrationQueueCount == storageMigrationQueueCount &&
other.isThumbnailGenerationActive == isThumbnailGenerationActive &&
other.isMetadataExtractionActive == isMetadataExtractionActive &&
other.isVideoConversionActive == isVideoConversionActive &&
other.isMachineLearningActive == isMachineLearningActive;
other.isMachineLearningActive == isMachineLearningActive &&
other.isStorageMigrationActive == isStorageMigrationActive;
@override
int get hashCode =>
@ -57,13 +65,15 @@ class AllJobStatusResponseDto {
(metadataExtractionQueueCount.hashCode) +
(videoConversionQueueCount.hashCode) +
(machineLearningQueueCount.hashCode) +
(storageMigrationQueueCount.hashCode) +
(isThumbnailGenerationActive.hashCode) +
(isMetadataExtractionActive.hashCode) +
(isVideoConversionActive.hashCode) +
(isMachineLearningActive.hashCode);
(isMachineLearningActive.hashCode) +
(isStorageMigrationActive.hashCode);
@override
String toString() => 'AllJobStatusResponseDto[thumbnailGenerationQueueCount=$thumbnailGenerationQueueCount, metadataExtractionQueueCount=$metadataExtractionQueueCount, videoConversionQueueCount=$videoConversionQueueCount, machineLearningQueueCount=$machineLearningQueueCount, isThumbnailGenerationActive=$isThumbnailGenerationActive, isMetadataExtractionActive=$isMetadataExtractionActive, isVideoConversionActive=$isVideoConversionActive, isMachineLearningActive=$isMachineLearningActive]';
String toString() => 'AllJobStatusResponseDto[thumbnailGenerationQueueCount=$thumbnailGenerationQueueCount, metadataExtractionQueueCount=$metadataExtractionQueueCount, videoConversionQueueCount=$videoConversionQueueCount, machineLearningQueueCount=$machineLearningQueueCount, storageMigrationQueueCount=$storageMigrationQueueCount, isThumbnailGenerationActive=$isThumbnailGenerationActive, isMetadataExtractionActive=$isMetadataExtractionActive, isVideoConversionActive=$isVideoConversionActive, isMachineLearningActive=$isMachineLearningActive, isStorageMigrationActive=$isStorageMigrationActive]';
Map<String, dynamic> toJson() {
final _json = <String, dynamic>{};
@ -71,10 +81,12 @@ class AllJobStatusResponseDto {
_json[r'metadataExtractionQueueCount'] = metadataExtractionQueueCount;
_json[r'videoConversionQueueCount'] = videoConversionQueueCount;
_json[r'machineLearningQueueCount'] = machineLearningQueueCount;
_json[r'storageMigrationQueueCount'] = storageMigrationQueueCount;
_json[r'isThumbnailGenerationActive'] = isThumbnailGenerationActive;
_json[r'isMetadataExtractionActive'] = isMetadataExtractionActive;
_json[r'isVideoConversionActive'] = isVideoConversionActive;
_json[r'isMachineLearningActive'] = isMachineLearningActive;
_json[r'isStorageMigrationActive'] = isStorageMigrationActive;
return _json;
}
@ -101,10 +113,12 @@ class AllJobStatusResponseDto {
metadataExtractionQueueCount: JobCounts.fromJson(json[r'metadataExtractionQueueCount'])!,
videoConversionQueueCount: JobCounts.fromJson(json[r'videoConversionQueueCount'])!,
machineLearningQueueCount: JobCounts.fromJson(json[r'machineLearningQueueCount'])!,
storageMigrationQueueCount: JobCounts.fromJson(json[r'storageMigrationQueueCount'])!,
isThumbnailGenerationActive: mapValueOfType<bool>(json, r'isThumbnailGenerationActive')!,
isMetadataExtractionActive: mapValueOfType<bool>(json, r'isMetadataExtractionActive')!,
isVideoConversionActive: mapValueOfType<bool>(json, r'isVideoConversionActive')!,
isMachineLearningActive: mapValueOfType<bool>(json, r'isMachineLearningActive')!,
isStorageMigrationActive: mapValueOfType<bool>(json, r'isStorageMigrationActive')!,
);
}
return null;
@ -158,10 +172,12 @@ class AllJobStatusResponseDto {
'metadataExtractionQueueCount',
'videoConversionQueueCount',
'machineLearningQueueCount',
'storageMigrationQueueCount',
'isThumbnailGenerationActive',
'isMetadataExtractionActive',
'isVideoConversionActive',
'isMachineLearningActive',
'isStorageMigrationActive',
};
}

3
mobile/openapi/lib/model/job_id.dart generated
View File

@ -27,6 +27,7 @@ class JobId {
static const metadataExtraction = JobId._(r'metadata-extraction');
static const videoConversion = JobId._(r'video-conversion');
static const machineLearning = JobId._(r'machine-learning');
static const storageTemplateMigration = JobId._(r'storage-template-migration');
/// List of all possible values in this [enum][JobId].
static const values = <JobId>[
@ -34,6 +35,7 @@ class JobId {
metadataExtraction,
videoConversion,
machineLearning,
storageTemplateMigration,
];
static JobId? fromJson(dynamic value) => JobIdTypeTransformer().decode(value);
@ -76,6 +78,7 @@ class JobIdTypeTransformer {
case r'metadata-extraction': return JobId.metadataExtraction;
case r'video-conversion': return JobId.videoConversion;
case r'machine-learning': return JobId.machineLearning;
case r'storage-template-migration': return JobId.storageTemplateMigration;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');

10
mobile/openapi/test/all_job_status_response_dto_test.dart generated
View File

@ -36,6 +36,11 @@ void main() {
// TODO
});
// JobCounts storageMigrationQueueCount
test('to test the property `storageMigrationQueueCount`', () async {
// TODO
});
// bool isThumbnailGenerationActive
test('to test the property `isThumbnailGenerationActive`', () async {
// TODO
@ -56,6 +61,11 @@ void main() {
// TODO
});
// bool isStorageMigrationActive
test('to test the property `isStorageMigrationActive`', () async {
// TODO
});
});

17
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@ -15,6 +15,7 @@ import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-as
import { In } from 'typeorm/find-options/operator/In';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { ITagRepository, TAG_REPOSITORY } from '../tag/tag.repository';
import { IsNull } from 'typeorm';
export interface IAssetRepository {
create(
@ -69,14 +70,14 @@ export class AssetRepository implements IAssetRepository {
}
async getAssetWithNoThumbnail(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')
.where('asset.resizePath IS NULL')
.andWhere('asset.isVisible = true')
.orWhere('asset.resizePath = :resizePath', { resizePath: '' })
.orWhere('asset.webpPath IS NULL')
.orWhere('asset.webpPath = :webpPath', { webpPath: '' })
.getMany();
return await this.assetRepository.find({
where: [
{ resizePath: IsNull(), isVisible: true },
{ resizePath: '', isVisible: true },
{ webpPath: IsNull(), isVisible: true },
{ webpPath: '', isVisible: true },
],
});
}
async getAssetWithNoEXIF(): Promise<AssetEntity[]> {

19
server/apps/immich/src/api-v1/asset/asset.module.ts
View File

@ -7,13 +7,13 @@ import { BullModule } from '@nestjs/bull';
import { BackgroundTaskModule } from '../../modules/background-task/background-task.module';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { CommunicationModule } from '../communication/communication.module';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { AssetRepository, ASSET_REPOSITORY } from './asset-repository';
import { DownloadModule } from '../../modules/download/download.module';
import { TagModule } from '../tag/tag.module';
import { AlbumModule } from '../album/album.module';
import { UserModule } from '../user/user.module';
import { StorageModule } from '@app/storage';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
const ASSET_REPOSITORY_PROVIDER = {
provide: ASSET_REPOSITORY,
@ -31,22 +31,7 @@ const ASSET_REPOSITORY_PROVIDER = {
TagModule,
StorageModule,
forwardRef(() => AlbumModule),
BullModule.registerQueue({
name: QueueNameEnum.ASSET_UPLOADED,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue({
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue(...immichSharedQueues),
],
controllers: [AssetController],
providers: [AssetService, BackgroundTaskService, ASSET_REPOSITORY_PROVIDER],

1
server/apps/immich/src/api-v1/job/dto/get-job.dto.ts
View File

@ -6,6 +6,7 @@ export enum JobId {
METADATA_EXTRACTION = 'metadata-extraction',
VIDEO_CONVERSION = 'video-conversion',
MACHINE_LEARNING = 'machine-learning',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
}
export class GetJobDto {

58
server/apps/immich/src/api-v1/job/job.module.ts
View File

@ -6,13 +6,15 @@ import { ImmichJwtModule } from '../../modules/immich-jwt/immich-jwt.module';
import { JwtModule } from '@nestjs/jwt';
import { jwtConfig } from '../../config/jwt.config';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BullModule } from '@nestjs/bull';
import { QueueNameEnum } from '@app/job';
import { ExifEntity } from '@app/database/entities/exif.entity';
import { TagModule } from '../tag/tag.module';
import { AssetModule } from '../asset/asset.module';
import { UserModule } from '../user/user.module';
import { StorageModule } from '@app/storage';
import { BullModule } from '@nestjs/bull';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
@Module({
imports: [
TypeOrmModule.forFeature([ExifEntity]),
@ -21,56 +23,8 @@ import { UserModule } from '../user/user.module';
AssetModule,
UserModule,
JwtModule.register(jwtConfig),
BullModule.registerQueue(
{
name: QueueNameEnum.THUMBNAIL_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.ASSET_UPLOADED,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.METADATA_EXTRACTION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.CHECKSUM_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.MACHINE_LEARNING,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
),
StorageModule,
BullModule.registerQueue(...immichSharedQueues),
],
controllers: [JobController],
providers: [JobService, ImmichJwtService],

34
server/apps/immich/src/api-v1/job/job.service.ts
View File

@ -6,6 +6,7 @@ import {
IVideoTranscodeJob,
MachineLearningJobNameEnum,
QueueNameEnum,
templateMigrationProcessorName,
videoMetadataExtractionProcessorName,
} from '@app/job';
import { InjectQueue } from '@nestjs/bull';
@ -18,6 +19,7 @@ import { AssetType } from '@app/database/entities/asset.entity';
import { GetJobDto, JobId } from './dto/get-job.dto';
import { JobStatusResponseDto } from './response-dto/job-status-response.dto';
import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interface';
import { StorageService } from '@app/storage';
@Injectable()
export class JobService {
@ -34,12 +36,18 @@ export class JobService {
@InjectQueue(QueueNameEnum.MACHINE_LEARNING)
private machineLearningQueue: Queue<IMachineLearningJob>,
@InjectQueue(QueueNameEnum.STORAGE_MIGRATION)
private storageMigrationQueue: Queue,
@Inject(ASSET_REPOSITORY)
private _assetRepository: IAssetRepository,
private storageService: StorageService,
) {
this.thumbnailGeneratorQueue.empty();
this.metadataExtractionQueue.empty();
this.videoConversionQueue.empty();
this.storageMigrationQueue.empty();
}
async startJob(jobDto: GetJobDto): Promise<number> {
@ -52,6 +60,8 @@ export class JobService {
return 0;
case JobId.MACHINE_LEARNING:
return this.runMachineLearningPipeline();
case JobId.STORAGE_TEMPLATE_MIGRATION:
return this.runStorageMigration();
default:
throw new BadRequestException('Invalid job id');
}
@ -62,6 +72,7 @@ export class JobService {
const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts();
const videoConversionJobCount = await this.videoConversionQueue.getJobCounts();
const machineLearningJobCount = await this.machineLearningQueue.getJobCounts();
const storageMigrationJobCount = await this.storageMigrationQueue.getJobCounts();
const response = new AllJobStatusResponseDto();
response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting);
@ -73,6 +84,9 @@ export class JobService {
response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
response.machineLearningQueueCount = machineLearningJobCount;
response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
response.storageMigrationQueueCount = storageMigrationJobCount;
return response;
}
@ -93,6 +107,11 @@ export class JobService {
response.queueCount = await this.videoConversionQueue.getJobCounts();
}
if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) {
response.isActive = Boolean((await this.storageMigrationQueue.getJobCounts()).waiting);
response.queueCount = await this.storageMigrationQueue.getJobCounts();
}
return response;
}
@ -110,6 +129,9 @@ export class JobService {
case JobId.MACHINE_LEARNING:
this.machineLearningQueue.empty();
return 0;
case JobId.STORAGE_TEMPLATE_MIGRATION:
this.storageMigrationQueue.empty();
return 0;
default:
throw new BadRequestException('Invalid job id');
}
@ -177,4 +199,16 @@ export class JobService {
return assetWithNoSmartInfo.length;
}
async runStorageMigration() {
const jobCount = await this.storageMigrationQueue.getJobCounts();
if (jobCount.active > 0) {
throw new BadRequestException('Storage migration job is already running');
}
await this.storageMigrationQueue.add(templateMigrationProcessorName, {}, { jobId: randomUUID() });
return 1;
}
}

6
server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts
View File

@ -17,6 +17,7 @@ export class AllJobStatusResponseDto {
isMetadataExtractionActive!: boolean;
isVideoConversionActive!: boolean;
isMachineLearningActive!: boolean;
isStorageMigrationActive!: boolean;
@ApiProperty({
type: JobCounts,
@ -37,4 +38,9 @@ export class AllJobStatusResponseDto {
type: JobCounts,
})
machineLearningQueueCount!: JobCounts;
@ApiProperty({
type: JobCounts,
})
storageMigrationQueueCount!: JobCounts;
}

9
server/apps/immich/src/api-v1/system-config/system-config.module.ts
View File

@ -1,4 +1,6 @@
import { SystemConfigEntity } from '@app/database/entities/system-config.entity';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ImmichConfigModule } from 'libs/immich-config/src';
@ -7,7 +9,12 @@ import { SystemConfigController } from './system-config.controller';
import { SystemConfigService } from './system-config.service';
@Module({
imports: [ImmichJwtModule, ImmichConfigModule, TypeOrmModule.forFeature([SystemConfigEntity])],
imports: [
ImmichJwtModule,
ImmichConfigModule,
TypeOrmModule.forFeature([SystemConfigEntity]),
BullModule.registerQueue(...immichSharedQueues),
],
controllers: [SystemConfigController],
providers: [SystemConfigService],
})

11
server/apps/immich/src/api-v1/system-config/system-config.service.ts
View File

@ -1,3 +1,4 @@
import { QueueNameEnum, updateTemplateProcessorName } from '@app/job';
import {
supportedDayTokens,
supportedHourTokens,
@ -7,14 +8,21 @@ import {
supportedSecondTokens,
supportedYearTokens,
} from '@app/storage/constants/supported-datetime-template';
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { randomUUID } from 'crypto';
import { ImmichConfigService } from 'libs/immich-config/src';
import { mapConfig, SystemConfigDto } from './dto/system-config.dto';
import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
@Injectable()
export class SystemConfigService {
constructor(private immichConfigService: ImmichConfigService) {}
constructor(
private immichConfigService: ImmichConfigService,
@InjectQueue(QueueNameEnum.STORAGE_MIGRATION)
private storageMigrationQueue: Queue,
) {}
public async getConfig(): Promise<SystemConfigDto> {
const config = await this.immichConfigService.getConfig();
@ -28,6 +36,7 @@ export class SystemConfigService {
public async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
const config = await this.immichConfigService.updateConfig(dto);
this.storageMigrationQueue.add(updateTemplateProcessorName, {}, { jobId: randomUUID() });
return mapConfig(config);
}

5
server/apps/immich/src/api-v1/user/user.service.ts
View File

@ -10,7 +10,7 @@ import {
UnauthorizedException,
} from '@nestjs/common';
import { Response as Res } from 'express';
import { createReadStream } from 'fs';
import { constants, createReadStream } from 'fs';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { CreateUserDto } from './dto/create-user.dto';
import { UpdateUserDto } from './dto/update-user.dto';
@ -22,6 +22,7 @@ import {
import { mapUserCountResponse, UserCountResponseDto } from './response-dto/user-count-response.dto';
import { mapUser, UserResponseDto } from './response-dto/user-response.dto';
import { IUserRepository, USER_REPOSITORY } from './user-repository';
import fs from 'fs/promises';
@Injectable()
export class UserService {
@ -196,6 +197,8 @@ export class UserService {
throw new NotFoundException('User does not have a profile image');
}
await fs.access(user.profileImagePath, constants.R_OK | constants.W_OK);
res.set({
'Content-Type': 'image/jpeg',
});

15
server/apps/immich/src/app.module.ts
View File

@ -1,4 +1,4 @@
import { immichAppConfig } from '@app/common/config';
import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config';
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
import { UserModule } from './api-v1/user/user.module';
import { AssetModule } from './api-v1/asset/asset.module';
@ -36,18 +36,7 @@ import { TagModule } from './api-v1/tag/tag.module';
DeviceInfoModule,
BullModule.forRootAsync({
useFactory: async () => ({
prefix: 'immich_bull',
redis: {
host: process.env.REDIS_HOSTNAME || 'immich_redis',
port: parseInt(process.env.REDIS_PORT || '6379'),
db: parseInt(process.env.REDIS_DBINDEX || '0'),
password: process.env.REDIS_PASSWORD || undefined,
path: process.env.REDIS_SOCKET || undefined,
},
}),
}),
BullModule.forRootAsync(immichBullAsyncConfig),
ServerInfoModule,

5
server/apps/immich/src/modules/background-task/background-task.module.ts
View File

@ -11,11 +11,6 @@ import { BackgroundTaskService } from './background-task.service';
imports: [
BullModule.registerQueue({
name: 'background-task',
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
],

36
server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts
View File

@ -3,46 +3,14 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ScheduleTasksService } from './schedule-tasks.service';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { ExifEntity } from '@app/database/entities/exif.entity';
import { UserEntity } from '@app/database/entities/user.entity';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
@Module({
imports: [
TypeOrmModule.forFeature([AssetEntity, ExifEntity, UserEntity]),
BullModule.registerQueue({
name: QueueNameEnum.USER_DELETION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue({
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue({
name: QueueNameEnum.THUMBNAIL_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue({
name: QueueNameEnum.METADATA_EXTRACTION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
BullModule.registerQueue(...immichSharedQueues),
],
providers: [ScheduleTasksService],
})

82
server/apps/microservices/src/microservices.module.ts
View File

@ -1,10 +1,10 @@
import { immichAppConfig } from '@app/common/config';
import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config';
import { DatabaseModule } from '@app/database';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ExifEntity } from '@app/database/entities/exif.entity';
import { SmartInfoEntity } from '@app/database/entities/smart-info.entity';
import { UserEntity } from '@app/database/entities/user.entity';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { StorageModule } from '@app/storage';
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
@ -16,9 +16,11 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
import { GenerateChecksumProcessor } from './processors/generate-checksum.processor';
import { MachineLearningProcessor } from './processors/machine-learning.processor';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
import { StorageMigrationProcessor } from './processors/storage-migration.processor';
import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
import { UserDeletionProcessor } from './processors/user-deletion.processor';
import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
@Module({
imports: [
@ -26,76 +28,9 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
DatabaseModule,
ImmichConfigModule,
TypeOrmModule.forFeature([UserEntity, ExifEntity, AssetEntity, SmartInfoEntity]),
BullModule.forRootAsync({
useFactory: async () => ({
prefix: 'immich_bull',
redis: {
host: process.env.REDIS_HOSTNAME || 'immich_redis',
port: parseInt(process.env.REDIS_PORT || '6379'),
db: parseInt(process.env.REDIS_DBINDEX || '0'),
password: process.env.REDIS_PASSWORD || undefined,
path: process.env.REDIS_SOCKET || undefined,
},
}),
}),
BullModule.registerQueue(
{
name: QueueNameEnum.USER_DELETION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.THUMBNAIL_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.ASSET_UPLOADED,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.METADATA_EXTRACTION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.CHECKSUM_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.MACHINE_LEARNING,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
),
StorageModule,
BullModule.forRootAsync(immichBullAsyncConfig),
BullModule.registerQueue(...immichSharedQueues),
CommunicationModule,
],
controllers: [],
@ -108,7 +43,8 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
GenerateChecksumProcessor,
MachineLearningProcessor,
UserDeletionProcessor,
StorageMigrationProcessor,
],
exports: [],
exports: [BullModule],
})
export class MicroservicesModule {}

61
server/apps/microservices/src/processors/storage-migration.processor.ts Normal file
View File

@ -0,0 +1,61 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ImmichConfigService } from '@app/immich-config';
import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job';
import { StorageService } from '@app/storage';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
@Processor(QueueNameEnum.STORAGE_MIGRATION)
export class StorageMigrationProcessor {
readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
constructor(
private storageService: StorageService,
private immichConfigService: ImmichConfigService,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) {}
/**
* Migration process when a new user set a new storage template.
* @param job
*/
@Process({ name: templateMigrationProcessorName, concurrency: 100 })
async templateMigration() {
console.time('migrating-time');
const assets = await this.assetRepository.find({
relations: ['exifInfo'],
});
const livePhotoMap: Record<string, AssetEntity> = {};
for (const asset of assets) {
if (asset.livePhotoVideoId) {
livePhotoMap[asset.livePhotoVideoId] = asset;
}
}
for (const asset of assets) {
const livePhotoParentAsset = livePhotoMap[asset.id];
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
await this.storageService.moveAsset(asset, filename);
}
await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
console.timeEnd('migrating-time');
}
/**
* Update config when a new storage template is set.
* This is to ensure the synchronization between processes.
* @param job
*/
@Process({ name: updateTemplateProcessorName, concurrency: 1 })
async updateTemplate() {
await this.immichConfigService.refreshConfig();
}
}

27
server/apps/microservices/src/processors/thumbnail.processor.ts
View File

@ -1,5 +1,4 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { ImmichLogLevel } from '@app/common/constants/log-level.constant';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import {
WebpGeneratorProcessor,
@ -11,7 +10,6 @@ import {
} from '@app/job';
import { InjectQueue, Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { mapAsset } from 'apps/immich/src/api-v1/asset/response-dto/asset-response.dto';
import { Job, Queue } from 'bull';
@ -27,7 +25,7 @@ import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interf
@Processor(QueueNameEnum.THUMBNAIL_GENERATION)
export class ThumbnailGeneratorProcessor {
private logLevel: ImmichLogLevel;
readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
constructor(
@InjectRepository(AssetEntity)
@ -40,12 +38,7 @@ export class ThumbnailGeneratorProcessor {
@InjectQueue(QueueNameEnum.MACHINE_LEARNING)
private machineLearningQueue: Queue<IMachineLearningJob>,
private configService: ConfigService,
) {
this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE;
// TODO - Add observable paterrn to listen to the config change
}
) {}
@Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
@ -70,12 +63,8 @@ export class ThumbnailGeneratorProcessor {
.rotate()
.toFile(jpegThumbnailPath);
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
} catch (error) {
Logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id);
if (this.logLevel == ImmichLogLevel.VERBOSE) {
console.trace('Failed to generate jpeg thumbnail for asset', error);
}
} catch (error: any) {
this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
}
// Update resize path to send to generate webp queue
@ -140,12 +129,8 @@ export class ThumbnailGeneratorProcessor {
try {
await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
} catch (error) {
Logger.error('Failed to generate webp thumbnail for asset: ' + asset.id);
if (this.logLevel == ImmichLogLevel.VERBOSE) {
console.trace('Failed to generate webp thumbnail for asset', error);
}
} catch (error: any) {
this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
}
}
}

13
server/immich-openapi-specs.json
View File

@ -3562,6 +3562,9 @@
"machineLearningQueueCount": {
"$ref": "#/components/schemas/JobCounts"
},
"storageMigrationQueueCount": {
"$ref": "#/components/schemas/JobCounts"
},
"isThumbnailGenerationActive": {
"type": "boolean"
},
@ -3573,6 +3576,9 @@
},
"isMachineLearningActive": {
"type": "boolean"
},
"isStorageMigrationActive": {
"type": "boolean"
}
},
"required": [
@ -3580,10 +3586,12 @@
"metadataExtractionQueueCount",
"videoConversionQueueCount",
"machineLearningQueueCount",
"storageMigrationQueueCount",
"isThumbnailGenerationActive",
"isMetadataExtractionActive",
"isVideoConversionActive",
"isMachineLearningActive"
"isMachineLearningActive",
"isStorageMigrationActive"
]
},
"JobId": {
@ -3592,7 +3600,8 @@
"thumbnail-generation",
"metadata-extraction",
"video-conversion",
"machine-learning"
"machine-learning",
"storage-template-migration"
]
},
"JobStatusResponseDto": {

19
server/libs/common/src/config/bull-queue.config.ts Normal file
View File

@ -0,0 +1,19 @@
import { SharedBullAsyncConfiguration } from '@nestjs/bull';
export const immichBullAsyncConfig: SharedBullAsyncConfiguration = {
useFactory: async () => ({
prefix: 'immich_bull',
redis: {
host: process.env.REDIS_HOSTNAME || 'immich_redis',
port: parseInt(process.env.REDIS_PORT || '6379'),
db: parseInt(process.env.REDIS_DBINDEX || '0'),
password: process.env.REDIS_PASSWORD || undefined,
path: process.env.REDIS_SOCKET || undefined,
},
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
};

1
server/libs/common/src/config/index.ts
View File

@ -1 +1,2 @@
export * from './app.config';
export * from './bull-queue.config';

6
server/libs/immich-config/src/immich-config.service.ts
View File

@ -102,4 +102,10 @@ export class ImmichConfigService {
return newConfig;
}
public async refreshConfig() {
const newConfig = await this.getConfig();
this.config$.next(newConfig);
}
}

32
server/libs/job/src/constants/bull-queue-registration.constant.ts Normal file
View File

@ -0,0 +1,32 @@
import { BullModuleOptions } from '@nestjs/bull';
import { QueueNameEnum } from './queue-name.constant';
/**
* Shared queues between apps and microservices
*/
export const immichSharedQueues: BullModuleOptions[] = [
{
name: QueueNameEnum.USER_DELETION,
},
{
name: QueueNameEnum.THUMBNAIL_GENERATION,
},
{
name: QueueNameEnum.ASSET_UPLOADED,
},
{
name: QueueNameEnum.METADATA_EXTRACTION,
},
{
name: QueueNameEnum.VIDEO_CONVERSION,
},
{
name: QueueNameEnum.CHECKSUM_GENERATION,
},
{
name: QueueNameEnum.MACHINE_LEARNING,
},
{
name: QueueNameEnum.STORAGE_MIGRATION,
},
];

6
server/libs/job/src/constants/job-name.constant.ts
View File

@ -34,3 +34,9 @@ export enum MachineLearningJobNameEnum {
* User deletion Queue Jobs
*/
export const userDeletionProcessorName = 'user-deletion';
/**
* Storage Template Migration Queue Jobs
*/
export const templateMigrationProcessorName = 'template-migration';
export const updateTemplateProcessorName = 'update-template';

1
server/libs/job/src/constants/queue-name.constant.ts
View File

@ -6,4 +6,5 @@ export enum QueueNameEnum {
ASSET_UPLOADED = 'asset-uploaded-queue',
MACHINE_LEARNING = 'machine-learning-queue',
USER_DELETION = 'user-deletion-queue',
STORAGE_MIGRATION = 'storage-template-migration',
}

63
server/libs/storage/src/storage.service.ts
View File

@ -26,7 +26,7 @@ const moveFile = promisify<string, string, mv.Options>(mv);
@Injectable()
export class StorageService {
readonly log = new Logger(StorageService.name);
readonly logger = new Logger(StorageService.name);
private storageTemplate: HandlebarsTemplateDelegate<any>;
@ -41,7 +41,7 @@ export class StorageService {
this.immichConfigService.addValidator((config) => this.validateConfig(config));
this.immichConfigService.config$.subscribe((config) => {
this.log.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
this.storageTemplate = this.compile(config.storageTemplate.template);
});
}
@ -54,14 +54,40 @@ export class StorageService {
const rootPath = path.join(APP_UPLOAD_LOCATION, asset.userId);
const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
const fullPath = path.normalize(path.join(rootPath, storagePath));
let destination = `${fullPath}.${ext}`;
if (!fullPath.startsWith(rootPath)) {
this.log.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
return asset;
}
if (source === destination) {
return asset;
}
/**
* In case of migrating duplicate filename to a new path, we need to check if it is already migrated
* Due to the mechanism of appending +1, +2, +3, etc to the filename
*
* Example:
* Source = upload/abc/def/FullSizeRender+7.heic
* Expected Destination = upload/abc/def/FullSizeRender.heic
*
* The file is already at the correct location, but since there are other FullSizeRender.heic files in the
* destination, it was renamed to FullSizeRender+7.heic.
*
* The lines below will be used to check if the differences between the source and destination is only the
* +7 suffix, and if so, it will be considered as already migrated.
*/
if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
if (hasDuplicationAnnotation) {
return asset;
}
}
let duplicateCount = 0;
let destination = `${fullPath}.${ext}`;
while (true) {
const exists = await this.checkFileExist(destination);
@ -70,7 +96,7 @@ export class StorageService {
}
duplicateCount++;
destination = `${fullPath}_${duplicateCount}.${ext}`;
destination = `${fullPath}+${duplicateCount}.${ext}`;
}
await this.safeMove(source, destination);
@ -78,7 +104,7 @@ export class StorageService {
asset.originalPath = destination;
return await this.assetRepository.save(asset);
} catch (error: any) {
this.log.error(error, error.stack);
this.logger.error(error);
return asset;
}
}
@ -115,7 +141,7 @@ export class StorageService {
'jpg',
);
} catch (e) {
this.log.warn(`Storage template validation failed: ${e}`);
this.logger.warn(`Storage template validation failed: ${e}`);
throw new Error(`Invalid storage template: ${e}`);
}
}
@ -150,4 +176,27 @@ export class StorageService {
return template(substitutions);
}
public async removeEmptyDirectories(directory: string) {
// lstat does not follow symlinks (in contrast to stat)
const fileStats = await fsPromise.lstat(directory);
if (!fileStats.isDirectory()) {
return;
}
let fileNames = await fsPromise.readdir(directory);
if (fileNames.length > 0) {
const recursiveRemovalPromises = fileNames.map((fileName) =>
this.removeEmptyDirectories(path.join(directory, fileName)),
);
await Promise.all(recursiveRemovalPromises);
// re-evaluate fileNames; after deleting subdirectory
// we may have parent directory empty now
fileNames = await fsPromise.readdir(directory);
}
if (fileNames.length === 0) {
await fsPromise.rmdir(directory);
}
}
}

15
web/src/api/open-api/api.ts generated
View File

@ -225,6 +225,12 @@ export interface AllJobStatusResponseDto {
* @memberof AllJobStatusResponseDto
*/
'machineLearningQueueCount': JobCounts;
/**
*
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'storageMigrationQueueCount': JobCounts;
/**
*
* @type {boolean}
@ -249,6 +255,12 @@ export interface AllJobStatusResponseDto {
* @memberof AllJobStatusResponseDto
*/
'isMachineLearningActive': boolean;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isStorageMigrationActive': boolean;
}
/**
*
@ -1038,7 +1050,8 @@ export const JobId = {
ThumbnailGeneration: 'thumbnail-generation',
MetadataExtraction: 'metadata-extraction',
VideoConversion: 'video-conversion',
MachineLearning: 'machine-learning'
MachineLearning: 'machine-learning',
StorageTemplateMigration: 'storage-template-migration'
} as const;
export type JobId = typeof JobId[keyof typeof JobId];

44
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View File

@ -9,6 +9,7 @@
let allJobsStatus: AllJobStatusResponseDto;
let setIntervalHandler: NodeJS.Timer;
onMount(async () => {
const { data } = await api.jobApi.getAllJobsStatus();
allJobsStatus = data;
@ -104,6 +105,33 @@
});
}
};
const runTemplateMigration = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.StorageTemplateMigration, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Storage migration started`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `All files have been migrated to the new storage template`,
type: NotificationType.Info
});
}
} catch (e) {
console.log('[ERROR] runTemplateMigration', e);
notificationController.show({
message: `Error running template migration job, check console for more detail`,
type: NotificationType.Error
});
}
};
</script>
<div class="flex flex-col gap-10">
@ -135,4 +163,20 @@
>
Note that some asset does not have any object detected, this is normal.
</JobTile>
<JobTile
title={'Storage migration'}
subtitle={''}
on:click={runTemplateMigration}
jobStatus={allJobsStatus?.isStorageMigrationActive}
waitingJobCount={allJobsStatus?.storageMigrationQueueCount.waiting}
activeJobCount={allJobsStatus?.storageMigrationQueueCount.active}
>
Apply the current
<a
href="/admin/system-settings?open=storage-template"
class="text-immich-primary dark:text-immich-dark-primary">Storage template</a
>
to previously uploaded assets
</JobTile>
</div>

2
web/src/lib/components/admin-page/settings/setting-accordion.svelte
View File

@ -3,7 +3,7 @@
export let title: string;
export let subtitle = '';
let isOpen = false;
export let isOpen = false;
const toggle = () => (isOpen = !isOpen);
</script>

10
web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte
View File

@ -214,6 +214,16 @@
</div>
</div>
<div id="migration-info" class="text-sm mt-4">
<p>
Template changes will only apply to new assets. To retroactively apply the template to
previously uploaded assets, run the <a
href="/admin/jobs-status"
class="text-immich-primary dark:text-immich-dark-primary">Storage Migration Job</a
>
</p>
</div>
<SettingButtonsRow
on:reset={reset}
on:save={saveSetting}

2
web/src/routes/admin/+layout.svelte
View File

@ -73,7 +73,7 @@
</div>
<section id="setting-content" class="pt-[85px] flex place-content-center">
<section class="w-[800px] pt-5">
<section class="w-[800px] pt-5 pb-28">
<slot />
</section>
</section>

2
web/src/routes/admin/system-settings/+page.svelte
View File

@ -6,6 +6,7 @@
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import { api, SystemConfigDto } from '@api';
import type { PageData } from './$types';
import { page } from '$app/stores';
let systemConfig: SystemConfigDto;
export let data: PageData;
@ -39,6 +40,7 @@
<SettingAccordion
title="Storage Template"
subtitle="Manage the folder structure and file name of the upload asset"
isOpen={$page.url.searchParams.get('open') === 'storage-template'}
>
<StorageTemplateSettings storageConfig={configs.storageTemplate} user={data.user} />
</SettingAccordion>