refactor(server): media service (#2051)

* refactor(server): media service

* merge main

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/libs/domain/src/media/media.service.spec.ts

@@ -0,0 +1,173 @@
+import _ from 'lodash';
+import {
+  assetEntityStub,
+  newAssetRepositoryMock,
+  newCommunicationRepositoryMock,
+  newJobRepositoryMock,
+  newMediaRepositoryMock,
+  newStorageRepositoryMock,
+} from '../../test';
+import { IAssetRepository, WithoutProperty } from '../asset';
+import { ICommunicationRepository } from '../communication';
+import { IJobRepository, JobName } from '../job';
+import { IStorageRepository } from '../storage';
+import { IMediaRepository } from './media.repository';
+import { MediaService } from './media.service';
+
+describe(MediaService.name, () => {
+  let sut: MediaService;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let communicationMock: jest.Mocked<ICommunicationRepository>;
+  let jobMock: jest.Mocked<IJobRepository>;
+  let mediaMock: jest.Mocked<IMediaRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
+
+  beforeEach(async () => {
+    assetMock = newAssetRepositoryMock();
+    communicationMock = newCommunicationRepositoryMock();
+    jobMock = newJobRepositoryMock();
+    mediaMock = newMediaRepositoryMock();
+    storageMock = newStorageRepositoryMock();
+    sut = new MediaService(assetMock, communicationMock, jobMock, mediaMock, storageMock);
+  });
+
+  it('should be defined', () => {
+    expect(sut).toBeDefined();
+  });
+
+  describe('handleQueueGenerateThumbnails', () => {
+    it('should queue all assets', async () => {
+      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+
+      await sut.handleQueueGenerateThumbnails({ force: true });
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(assetMock.getWithout).not.toHaveBeenCalled();
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_JPEG_THUMBNAIL,
+        data: { asset: assetEntityStub.image },
+      });
+    });
+
+    it('should queue all assets with missing thumbnails', async () => {
+      assetMock.getWithout.mockResolvedValue([assetEntityStub.image]);
+
+      await sut.handleQueueGenerateThumbnails({ force: false });
+
+      expect(assetMock.getAll).not.toHaveBeenCalled();
+      expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.THUMBNAIL);
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_JPEG_THUMBNAIL,
+        data: { asset: assetEntityStub.image },
+      });
+    });
+
+    it('should log an error', async () => {
+      assetMock.getAll.mockRejectedValue(new Error('database unavailable'));
+
+      await sut.handleQueueGenerateThumbnails({ force: true });
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+    });
+  });
+
+  describe('handleGenerateJpegThumbnail', () => {
+    it('should generate a thumbnail for an image', async () => {
+      await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) });
+
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
+      expect(mediaMock.resize).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/thumb/device-id/asset-id.jpeg',
+        { size: 1440, format: 'jpeg' },
+      );
+      expect(mediaMock.extractThumbnailFromExif).not.toHaveBeenCalled();
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: 'asset-id',
+        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+      });
+    });
+
+    it('should generate a thumbnail for an image from exif', async () => {
+      mediaMock.resize.mockRejectedValue(new Error('unsupported format'));
+
+      await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) });
+
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
+      expect(mediaMock.resize).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/thumb/device-id/asset-id.jpeg',
+        { size: 1440, format: 'jpeg' },
+      );
+      expect(mediaMock.extractThumbnailFromExif).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/thumb/device-id/asset-id.jpeg',
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: 'asset-id',
+        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+      });
+    });
+
+    it('should generate a thumbnail for a video', async () => {
+      await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.video) });
+
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
+      expect(mediaMock.extractVideoThumbnail).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/thumb/device-id/asset-id.jpeg',
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: 'asset-id',
+        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+      });
+    });
+
+    it('should queue some jobs', async () => {
+      const asset = _.cloneDeep(assetEntityStub.image);
+
+      await sut.handleGenerateJpegThumbnail({ asset });
+
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.CLASSIFY_IMAGE, data: { asset } });
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.DETECT_OBJECTS, data: { asset } });
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.ENCODE_CLIP, data: { asset } });
+    });
+
+    it('should log an error', async () => {
+      mediaMock.resize.mockRejectedValue(new Error('unsupported format'));
+      mediaMock.extractThumbnailFromExif.mockRejectedValue(new Error('unsupported format'));
+
+      await sut.handleGenerateJpegThumbnail({ asset: assetEntityStub.image });
+
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('handleGenerateWebpThumbnail', () => {
+    it('should skip thumbnail generate if resize path is missing', async () => {
+      await sut.handleGenerateWepbThumbnail({ asset: assetEntityStub.noResizePath });
+
+      expect(mediaMock.resize).not.toHaveBeenCalled();
+    });
+
+    it('should generate a thumbnail', async () => {
+      await sut.handleGenerateWepbThumbnail({ asset: assetEntityStub.image });
+
+      expect(mediaMock.resize).toHaveBeenCalledWith(
+        '/uploads/user-id/thumbs/path.ext',
+        '/uploads/user-id/thumbs/path.ext',
+        { format: 'webp', size: 250 },
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.ext' });
+    });
+
+    it('should log an error', async () => {
+      mediaMock.resize.mockRejectedValue(new Error('service unavailable'));
+
+      await sut.handleGenerateWepbThumbnail({ asset: assetEntityStub.image });
+
+      expect(mediaMock.resize).toHaveBeenCalled();
+    });
+  });
+});

server/libs/domain/src/media/media.service.ts

@@ -40,31 +40,32 @@ export class MediaService {
   async handleGenerateJpegThumbnail(data: IAssetJob): Promise<void> {
     const { asset } = data;
 
-    const basePath = APP_UPLOAD_LOCATION;
+    try {
-    const sanitizedDeviceId = sanitize(String(asset.deviceId));
+      const basePath = APP_UPLOAD_LOCATION;
-    const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
+      const sanitizedDeviceId = sanitize(String(asset.deviceId));
-    const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
+      const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
+      const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
+      this.storageRepository.mkdirSync(resizePath);
 
-    this.storageRepository.mkdirSync(resizePath);
+      if (asset.type == AssetType.IMAGE) {
-
+        try {
-    if (asset.type == AssetType.IMAGE) {
+          await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, { size: 1440, format: 'jpeg' });
-      try {
+        } catch (error) {
-        await this.mediaRepository
+          this.logger.warn(
-          .resize(asset.originalPath, jpegThumbnailPath, { size: 1440, format: 'jpeg' })
+            `Failed to generate jpeg thumbnail using sharp, trying with exiftool-vendored (asset=${asset.id})`,
-          .catch(() => {
+          );
-            this.logger.warn(
+          await this.mediaRepository.extractThumbnailFromExif(asset.originalPath, jpegThumbnailPath);
-              'Failed to generate jpeg thumbnail for asset: ' +
+        }
-                asset.id +
-                ' using sharp, failing over to exiftool-vendored',
-            );
-            return this.mediaRepository.extractThumbnailFromExif(asset.originalPath, jpegThumbnailPath);
-          });
-        await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
-      } catch (error: any) {
-        this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
       }
 
-      // Update resize path to send to generate webp queue
+      if (asset.type == AssetType.VIDEO) {
+        this.logger.log('Start Generating Video Thumbnail');
+        await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath);
+        this.logger.log(`Generating Video Thumbnail Success ${asset.id}`);
+      }
+
+      await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
+
       asset.resizePath = jpegThumbnailPath;
 
       await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
@@ -73,28 +74,8 @@ export class MediaService {
       await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
 
       this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
-    }
+    } catch (error: any) {
-
+      this.logger.error(`Failed to generate thumbnail for asset: ${asset.id}/${asset.type}`, error.stack);
-    if (asset.type == AssetType.VIDEO) {
-      try {
-        this.logger.log('Start Generating Video Thumbnail');
-        await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath);
-        this.logger.log(`Generating Video Thumbnail Success ${asset.id}`);
-
-        await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
-
-        // Update resize path to send to generate webp queue
-        asset.resizePath = jpegThumbnailPath;
-
-        await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
-        await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { asset } });
-        await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: { asset } });
-        await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
-
-        this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
-      } catch (error: any) {
-        this.logger.error(`Cannot Generate Video Thumbnail: ${asset.id}`, error?.stack);
-      }
     }
   }
 

server/libs/domain/test/communication.repository.mock.ts

@@ -0,0 +1,7 @@
+import { ICommunicationRepository } from '../src';
+
+export const newCommunicationRepositoryMock = (): jest.Mocked<ICommunicationRepository> => {
+  return {
+    send: jest.fn(),
+  };
+};

server/libs/domain/test/fixtures.ts

@@ -111,7 +111,7 @@ export const fileStub = {
 };
 
 export const assetEntityStub = {
-  image: Object.freeze<AssetEntity>({
+  noResizePath: Object.freeze<AssetEntity>({
     id: 'asset-id',
     deviceAssetId: 'device-asset-id',
     fileModifiedAt: '2023-02-23T05:06:29.716Z',
@@ -135,6 +135,54 @@ export const assetEntityStub = {
     tags: [],
     sharedLinks: [],
   }),
+  image: Object.freeze<AssetEntity>({
+    id: 'asset-id',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: '2023-02-23T05:06:29.716Z',
+    fileCreatedAt: '2023-02-23T05:06:29.716Z',
+    owner: userEntityStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.ext',
+    resizePath: '/uploads/user-id/thumbs/path.ext',
+    type: AssetType.IMAGE,
+    webpPath: null,
+    encodedVideoPath: null,
+    createdAt: '2023-02-23T05:06:29.716Z',
+    updatedAt: '2023-02-23T05:06:29.716Z',
+    mimeType: null,
+    isFavorite: true,
+    duration: null,
+    isVisible: true,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    tags: [],
+    sharedLinks: [],
+  }),
+  video: Object.freeze<AssetEntity>({
+    id: 'asset-id',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: '2023-02-23T05:06:29.716Z',
+    fileCreatedAt: '2023-02-23T05:06:29.716Z',
+    owner: userEntityStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.ext',
+    resizePath: '/uploads/user-id/thumbs/path.ext',
+    type: AssetType.VIDEO,
+    webpPath: null,
+    encodedVideoPath: null,
+    createdAt: '2023-02-23T05:06:29.716Z',
+    updatedAt: '2023-02-23T05:06:29.716Z',
+    mimeType: null,
+    isFavorite: true,
+    duration: null,
+    isVisible: true,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    tags: [],
+    sharedLinks: [],
+  }),
   livePhotoMotionAsset: Object.freeze({
     id: 'live-photo-motion-asset',
     originalPath: fileStub.livePhotoMotion.originalPath,

server/libs/domain/test/index.ts

@@ -1,11 +1,13 @@
 export * from './album.repository.mock';
 export * from './api-key.repository.mock';
 export * from './asset.repository.mock';
+export * from './communication.repository.mock';
 export * from './crypto.repository.mock';
 export * from './device-info.repository.mock';
 export * from './fixtures';
 export * from './job.repository.mock';
 export * from './machine-learning.repository.mock';
+export * from './media.repository.mock';
 export * from './search.repository.mock';
 export * from './shared-link.repository.mock';
 export * from './smart-info.repository.mock';

server/libs/domain/test/media.repository.mock.ts

@@ -0,0 +1,9 @@
+import { IMediaRepository } from '../src';
+
+export const newMediaRepositoryMock = (): jest.Mocked<IMediaRepository> => {
+  return {
+    extractThumbnailFromExif: jest.fn(),
+    extractVideoThumbnail: jest.fn(),
+    resize: jest.fn(),
+  };
+};