cotugno/immich
2
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-16 10:28:54 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:immich-app/immich

Browse Source
This commit is contained in:
Alex Tran 2023-03-28 15:07:53 -05:00
commit 2f64af9cb2
10 changed files with 54 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
server/apps/immich/src/api-v1/asset/asset.core.ts
View File

@ -1,29 +1,10 @@
import {
AuthUserDto,
IJobRepository,
IStorageRepository,
ISystemConfigRepository,
JobName,
StorageTemplateCore,
} from '@app/domain';
import { AssetEntity, SystemConfig, UserEntity } from '@app/infra/db/entities';
import { Logger } from '@nestjs/common';
import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
import { AssetEntity, UserEntity } from '@app/infra/db/entities';
import { IAssetRepository } from './asset-repository';
import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
export class AssetCore {
private templateCore: StorageTemplateCore;
private logger = new Logger(AssetCore.name);
constructor(
private repository: IAssetRepository,
private jobRepository: IJobRepository,
configRepository: ISystemConfigRepository,
config: SystemConfig,
private storageRepository: IStorageRepository,
) {
this.templateCore = new StorageTemplateCore(configRepository, config, storageRepository);
}
constructor(private repository: IAssetRepository, private jobRepository: IJobRepository) {}
async create(
authUser: AuthUserDto,
@ -31,7 +12,7 @@ export class AssetCore {
file: UploadFile,
livePhotoAssetId?: string,
): Promise<AssetEntity> {
let asset = await this.repository.create({
const asset = await this.repository.create({
owner: { id: authUser.id } as UserEntity,
mimeType: file.mimeType,
@ -56,31 +37,8 @@ export class AssetCore {
sharedLinks: [],
});
asset = await this.moveAsset(asset, file.originalName);
await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
return asset;
}
async moveAsset(asset: AssetEntity, originalName: string) {
const destination = await this.templateCore.getTemplatePath(asset, originalName);
if (asset.originalPath !== destination) {
const source = asset.originalPath;
try {
await this.storageRepository.moveFile(asset.originalPath, destination);
try {
await this.repository.save({ id: asset.id, originalPath: destination });
asset.originalPath = destination;
} catch (error: any) {
this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
await this.storageRepository.moveFile(destination, source);
}
} catch (error: any) {
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
}
}
return asset;
}
}

19
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@ -8,14 +8,7 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { DownloadService } from '../../modules/download/download.service';
import { AlbumRepository, IAlbumRepository } from '../album/album-repository';
import {
ICryptoRepository,
IJobRepository,
ISharedLinkRepository,
IStorageRepository,
ISystemConfigRepository,
JobName,
} from '@app/domain';
import { ICryptoRepository, IJobRepository, ISharedLinkRepository, IStorageRepository, JobName } from '@app/domain';
import {
assetEntityStub,
authStub,
@ -24,10 +17,8 @@ import {
newJobRepositoryMock,
newSharedLinkRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
sharedLinkResponseStub,
sharedLinkStub,
systemConfigStub,
} from '@app/domain/../test';
import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
import { BadRequestException, ForbiddenException } from '@nestjs/common';
@ -121,7 +112,6 @@ describe('AssetService', () => {
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
@ -160,7 +150,6 @@ describe('AssetService', () => {
sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
jobMock = newJobRepositoryMock();
configMock = newSystemConfigRepositoryMock();
cryptoMock = newCryptoRepositoryMock();
storageMock = newStorageRepositoryMock();
@ -171,8 +160,6 @@ describe('AssetService', () => {
downloadServiceMock as DownloadService,
sharedLinkRepositoryMock,
jobMock,
configMock,
systemConfigStub.defaults,
cryptoMock,
storageMock,
);
@ -273,10 +260,6 @@ describe('AssetService', () => {
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' });
expect(assetRepositoryMock.create).toHaveBeenCalled();
expect(assetRepositoryMock.save).toHaveBeenCalledWith({
id: 'id_1',
originalPath: 'upload/library/user_id_1/2022/2022-06-19/asset_1.jpeg',
});
});
it('should handle a duplicate', async () => {

13
server/apps/immich/src/api-v1/asset/asset.service.ts
View File

@ -12,7 +12,7 @@ import {
import { InjectRepository } from '@nestjs/typeorm';
import { QueryFailedError, Repository } from 'typeorm';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { AssetEntity, AssetType, SharedLinkType, SystemConfig } from '@app/infra/db/entities';
import { AssetEntity, AssetType, SharedLinkType } from '@app/infra/db/entities';
import { constants, createReadStream, stat } from 'fs';
import { ServeFileDto } from './dto/serve-file.dto';
import { Response as Res } from 'express';
@ -24,10 +24,9 @@ import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
import {
AssetResponseDto,
getLivePhotoMotionFilename,
ImmichReadStream,
INITIAL_SYSTEM_CONFIG,
IStorageRepository,
ISystemConfigRepository,
JobName,
mapAsset,
mapAssetWithoutExif,
@ -62,8 +61,6 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
import { AssetSearchDto } from './dto/asset-search.dto';
import { AddAssetsDto } from '../album/dto/add-assets.dto';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import path from 'path';
import { getFileNameWithoutExtension } from '@app/domain';
const fileInfo = promisify(stat);
@ -86,12 +83,10 @@ export class AssetService {
private downloadService: DownloadService,
@Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.assetCore = new AssetCore(_assetRepository, jobRepository, configRepository, config, storageRepository);
this.assetCore = new AssetCore(_assetRepository, jobRepository);
this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
}
@ -104,7 +99,7 @@ export class AssetService {
if (livePhotoFile) {
livePhotoFile = {
...livePhotoFile,
originalName: getFileNameWithoutExtension(file.originalName) + path.extname(livePhotoFile.originalName),
originalName: getLivePhotoMotionFilename(file.originalName, livePhotoFile.originalName),
};
}

5
server/apps/microservices/src/processors.ts
View File

@ -127,6 +127,11 @@ export class StorageTemplateMigrationProcessor {
async onTemplateMigration() {
await this.storageTemplateService.handleTemplateMigration();
}
@Process({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE })
async onTemplateMigrationSingle(job: Job<IAssetJob>) {
await this.storageTemplateService.handleTemplateMigrationSingle(job.data);
}
}
@Processor(QueueName.THUMBNAIL_GENERATION)

15
server/apps/microservices/src/processors/metadata-extraction.processor.ts
View File

@ -10,7 +10,7 @@ import {
QueueName,
WithoutProperty,
} from '@app/domain';
import { AssetEntity, AssetType, ExifEntity } from '@app/infra/db/entities';
import { AssetType, ExifEntity } from '@app/infra/db/entities';
import { Process, Processor } from '@nestjs/bull';
import { Inject, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
@ -173,7 +173,8 @@ export class MetadataExtractionProcessor {
@Process(JobName.EXIF_EXTRACTION)
async extractExifInfo(job: Job<IAssetUploadedJob>) {
try {
const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
let asset = job.data.asset;
const fileName = job.data.fileName;
const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
this.logger.warn(`The exifData parsing failed due to: ${e} on file ${asset.originalPath}`);
return null;
@ -256,7 +257,8 @@ export class MetadataExtractionProcessor {
}
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() });
asset = await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() });
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } });
} catch (error: any) {
this.logger.error(`Error extracting EXIF ${error}`, error?.stack);
}
@ -273,7 +275,8 @@ export class MetadataExtractionProcessor {
@Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 })
async extractVideoMetadata(job: Job<IAssetUploadedJob>) {
const { asset, fileName } = job.data;
let asset = job.data.asset;
const fileName = job.data.fileName;
if (!asset.isVisible) {
return;
@ -318,6 +321,7 @@ export class MetadataExtractionProcessor {
if (photoAsset) {
await this.assetCore.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
await this.assetCore.save({ id: asset.id, isVisible: false });
newExif.imageName = (photoAsset.exifInfo as ExifEntity).imageName;
}
}
@ -373,7 +377,8 @@ export class MetadataExtractionProcessor {
}
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt });
asset = await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt });
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } });
} catch (err) {
``;
// do nothing

4
server/libs/domain/src/domain.util.ts
View File

@ -4,6 +4,10 @@ export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}
export function getLivePhotoMotionFilename(stillName: string, motionName: string) {
return getFileNameWithoutExtension(stillName) + extname(motionName);
}
const KiB = Math.pow(1024, 1);
const MiB = Math.pow(1024, 2);
const GiB = Math.pow(1024, 3);

1
server/libs/domain/src/job/job.constants.ts
View File

@ -41,6 +41,7 @@ export enum JobName {
// storage template
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single',
SYSTEM_CONFIG_CHANGE = 'system-config-change',
// object tagging

1
server/libs/domain/src/job/job.repository.ts
View File

@ -36,6 +36,7 @@ export type JobItem =
// Storage Template
| { name: JobName.STORAGE_TEMPLATE_MIGRATION }
| { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IAssetJob }
| { name: JobName.SYSTEM_CONFIG_CHANGE }
// Metadata Extraction

20
server/libs/domain/src/storage-template/storage-template.service.ts
View File

@ -2,6 +2,8 @@ import { AssetEntity, SystemConfig } from '@app/infra/db/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IAssetRepository } from '../asset/asset.repository';
import { APP_MEDIA_LOCATION } from '../domain.constant';
import { getLivePhotoMotionFilename } from '../domain.util';
import { IAssetJob } from '../job';
import { IStorageRepository } from '../storage/storage.repository';
import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
import { StorageTemplateCore } from './storage-template.core';
@ -20,6 +22,24 @@ export class StorageTemplateService {
this.core = new StorageTemplateCore(configRepository, config, storageRepository);
}
async handleTemplateMigrationSingle(data: IAssetJob) {
const { asset } = data;
try {
const filename = asset.exifInfo?.imageName || asset.id;
await this.moveAsset(asset, filename);
// move motion part of live photo
if (asset.livePhotoVideoId) {
const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
await this.moveAsset(livePhotoVideo, motionFilename);
}
} catch (error: any) {
this.logger.error('Error running single template migration', error);
}
}
async handleTemplateMigration() {
try {
console.time('migrating-time');

4
server/libs/infra/src/job/job.repository.ts
View File

@ -99,6 +99,10 @@ export class JobRepository implements IJobRepository {
await this.storageTemplateMigration.add(item.name);
break;
case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE:
await this.storageTemplateMigration.add(item.name, item.data);
break;
case JobName.SYSTEM_CONFIG_CHANGE:
await this.backgroundTask.add(item.name, {});
break;