From 77e6a6d78b05e731e33dce461dd08f3459f9a2d2 Mon Sep 17 00:00:00 2001 From: BugFest Date: Thu, 5 Sep 2024 00:23:58 +0200 Subject: [PATCH] feat(server): Import face regions from metadata (#6455) * feat: faces-from-metadata - Import face regions from metadata Implements immich-app#1692. - OpenAPI spec changes to accomodate metadata face import configs. New settings to enable the feature. - Updates admin UI compoments - ML faces detection/recognition & Exif/Metadata faces compatibility Signed-off-by: BugFest * chore(web): remove unused file confirm-enable-import-faces * chore(web): format metadata-settings * fix(server): faces-from-metadata tests and format * fix(server): code refinements, nullable face asset sourceType * fix(server): Add RegionInfo to ImmichTags interface * fix(server): deleteAllFaces sourceType param can be undefined * fix(server): exiftool-vendored 27.0.0 moves readArgs into ExifToolOptions * fix(server): rename isImportFacesFromMetadataEnabled to isFaceImportEnabled * fix(server): simplify sourceType conditional * fix(server): small fixes * fix(server): handling sourceType * fix(server): sourceType enum * fix(server): refactor metadata applyTaggedFaces * fix(server): create/update signature changes * fix(server): reduce computational cost of Person.getManyByName * fix(server): use faceList instead of faceSet * fix(server): Skip regions without Name defined * fix(mobile): Update open-api (face assets feature changes) * fix(server): Face-Person reconciliation with map/index * fix(server): tags.RegionInfo.AppliedToDimensions must be defined to process face-region * fix(server): fix shared-link.service.ts format * fix(mobile): Update open-api after branch update * simplify * fix(server): minor fixes * fix(server): person create/update methods type enforcement * fix(server): style fixes * fix(server): remove unused metadata code * fix(server): metadata faces unit tests * fix(server): top level config metadata category * fix(server): rename upsertFaces to replaceFaces * fix(server): remove sourceType when unnecessary * fix(server): sourceType as ENUM * fix(server): format fixes * fix(server): fix tests after sourceType ENUM change * fix(server): remove unnecessary JobItem cast * fix(server): fix asset enum imports * fix(open-api): add metadata config * fix(mobile): update open-api after metadata open-api spec changes * fix(web): update web/api metadata config * fix(server): remove duplicated sourceType def * fix(server): update generated sql queries * fix(e2e): tests for metadata face import feature * fix(web): Fix check:typescript * fix(e2e): update subproject ref * fix(server): revert format changes to pass format checks after ci * fix(mobile): update open-api * fix(server,movile,open-api,mobile): sourceType as DB data type * fix(e2e): upload face asset after enabling metadata face import * fix(web): simplify metadata admin settings and i18n keys * Update person.repository.ts Co-authored-by: Jason Rasmussen * fix(server): asset_faces.sourceType column not nullable * fix(server): simplified syntax * fix(e2e): use SDK for everything except the endpoint being tested * fix(e2e): fix test format * chore: clean up * chore: clean up * chore: update e2e/test-assets --------- Signed-off-by: BugFest Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Jason Rasmussen --- e2e/src/api/specs/asset.e2e-spec.ts | 64 +++++++++ e2e/src/api/specs/server-info.e2e-spec.ts | 1 + e2e/src/api/specs/server.e2e-spec.ts | 1 + e2e/test-assets | 2 +- mobile/openapi/README.md | 3 + mobile/openapi/lib/api.dart | 3 + mobile/openapi/lib/api_client.dart | 6 + mobile/openapi/lib/api_helper.dart | 3 + .../lib/model/asset_face_response_dto.dart | 23 +++- ...sset_face_without_person_response_dto.dart | 23 +++- .../lib/model/server_features_dto.dart | 10 +- mobile/openapi/lib/model/source_type.dart | 85 ++++++++++++ .../openapi/lib/model/system_config_dto.dart | 10 +- .../lib/model/system_config_faces_dto.dart | 98 ++++++++++++++ .../lib/model/system_config_metadata_dto.dart | 98 ++++++++++++++ open-api/immich-openapi-specs.json | 43 +++++++ open-api/typescript-sdk/src/fetch-client.ts | 14 ++ server/src/config.ts | 10 ++ server/src/cores/storage.core.ts | 2 +- server/src/dtos/person.dto.ts | 4 + server/src/dtos/server.dto.ts | 1 + server/src/dtos/system-config.dto.ts | 17 +++ server/src/entities/asset-face.entity.ts | 4 + server/src/enum.ts | 5 + server/src/interfaces/metadata.interface.ts | 25 +++- server/src/interfaces/person.interface.ts | 17 ++- ...721249222549-AddSourceColumnToAssetFace.ts | 16 +++ server/src/queries/asset.repository.sql | 1 + server/src/queries/person.repository.sql | 18 ++- server/src/queries/search.repository.sql | 1 + server/src/repositories/person.repository.ts | 50 ++++++-- server/src/services/audit.service.ts | 2 +- server/src/services/media.service.ts | 2 +- server/src/services/metadata.service.spec.ts | 121 +++++++++++++++++- server/src/services/metadata.service.ts | 71 +++++++++- server/src/services/person.service.spec.ts | 93 ++++++++------ server/src/services/person.service.ts | 62 ++++----- server/src/services/server.service.spec.ts | 1 + server/src/services/server.service.ts | 3 +- .../services/system-config.service.spec.ts | 5 + server/src/utils/misc.ts | 1 + server/test/fixtures/face.stub.ts | 10 ++ server/test/fixtures/metadata.stub.ts | 71 ++++++++++ .../repositories/person.repository.mock.ts | 2 + .../metadata-settings.svelte | 38 ++++++ web/src/lib/i18n/en.json | 6 +- web/src/lib/stores/server-config.store.ts | 1 + .../routes/admin/system-settings/+page.svelte | 7 + 48 files changed, 1058 insertions(+), 96 deletions(-) create mode 100644 mobile/openapi/lib/model/source_type.dart create mode 100644 mobile/openapi/lib/model/system_config_faces_dto.dart create mode 100644 mobile/openapi/lib/model/system_config_metadata_dto.dart create mode 100644 server/src/migrations/1721249222549-AddSourceColumnToAssetFace.ts create mode 100644 server/test/fixtures/metadata.stub.ts create mode 100644 web/src/lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte diff --git a/e2e/src/api/specs/asset.e2e-spec.ts b/e2e/src/api/specs/asset.e2e-spec.ts index 82ce17865a..7d3c3c6e59 100644 --- a/e2e/src/api/specs/asset.e2e-spec.ts +++ b/e2e/src/api/specs/asset.e2e-spec.ts @@ -6,7 +6,9 @@ import { LoginResponseDto, SharedLinkType, getAssetInfo, + getConfig, getMyUser, + updateConfig, } from '@immich/sdk'; import { exiftool } from 'exiftool-vendored'; import { DateTime } from 'luxon'; @@ -43,6 +45,9 @@ const TEN_TIMES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; const locationAssetFilepath = `${testAssetDir}/metadata/gps-position/thompson-springs.jpg`; const ratingAssetFilepath = `${testAssetDir}/metadata/rating/mongolels.jpg`; +const facesAssetFilepath = `${testAssetDir}/metadata/faces/portrait.jpg`; + +const getSystemConfig = (accessToken: string) => getConfig({ headers: asBearerAuth(accessToken) }); const readTags = async (bytes: Buffer, filename: string) => { const filepath = join(tempDir, filename); @@ -71,6 +76,7 @@ describe('/asset', () => { let user2Assets: AssetMediaResponseDto[]; let locationAsset: AssetMediaResponseDto; let ratingAsset: AssetMediaResponseDto; + let facesAsset: AssetMediaResponseDto; const setupTests = async () => { await utils.resetDatabase(); @@ -224,6 +230,64 @@ describe('/asset', () => { }); }); + it('should get the asset faces', async () => { + const config = await getSystemConfig(admin.accessToken); + config.metadata.faces.import = true; + await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) }); + + // asset faces + facesAsset = await utils.createAsset(admin.accessToken, { + assetData: { + filename: 'portrait.jpg', + bytes: await readFile(facesAssetFilepath), + }, + }); + + await utils.waitForWebsocketEvent({ event: 'assetUpload', id: facesAsset.id }); + + const { status, body } = await request(app) + .get(`/assets/${facesAsset.id}`) + .set('Authorization', `Bearer ${admin.accessToken}`); + expect(status).toBe(200); + expect(body.id).toEqual(facesAsset.id); + expect(body.people).toMatchObject([ + { + name: 'Marie Curie', + birthDate: null, + thumbnailPath: '', + isHidden: false, + faces: [ + { + imageHeight: 700, + imageWidth: 840, + boundingBoxX1: 261, + boundingBoxX2: 356, + boundingBoxY1: 146, + boundingBoxY2: 284, + sourceType: 'exif', + }, + ], + }, + { + name: 'Pierre Curie', + birthDate: null, + thumbnailPath: '', + isHidden: false, + faces: [ + { + imageHeight: 700, + imageWidth: 840, + boundingBoxX1: 536, + boundingBoxX2: 618, + boundingBoxY1: 83, + boundingBoxY2: 252, + sourceType: 'exif', + }, + ], + }, + ]); + }); + it('should work with a shared link', async () => { const sharedLink = await utils.createSharedLink(user1.accessToken, { type: SharedLinkType.Individual, diff --git a/e2e/src/api/specs/server-info.e2e-spec.ts b/e2e/src/api/specs/server-info.e2e-spec.ts index 092eab3ec5..571d98cda7 100644 --- a/e2e/src/api/specs/server-info.e2e-spec.ts +++ b/e2e/src/api/specs/server-info.e2e-spec.ts @@ -102,6 +102,7 @@ describe('/server-info', () => { configFile: false, duplicateDetection: false, facialRecognition: false, + importFaces: false, map: true, reverseGeocoding: true, oauth: false, diff --git a/e2e/src/api/specs/server.e2e-spec.ts b/e2e/src/api/specs/server.e2e-spec.ts index d19744674f..b19e6d85c4 100644 --- a/e2e/src/api/specs/server.e2e-spec.ts +++ b/e2e/src/api/specs/server.e2e-spec.ts @@ -110,6 +110,7 @@ describe('/server', () => { facialRecognition: false, map: true, reverseGeocoding: true, + importFaces: false, oauth: false, oauthAutoLaunch: false, passwordLogin: true, diff --git a/e2e/test-assets b/e2e/test-assets index 4e9731d3fc..3e057d2f58 160000 --- a/e2e/test-assets +++ b/e2e/test-assets @@ -1 +1 @@ -Subproject commit 4e9731d3fc270fe25901f72a6b6f57277cdb8a30 +Subproject commit 3e057d2f58750acdf7ff281a3938e34a86cfef4d diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index c7201d1d24..68cf2e7a99 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -407,11 +407,13 @@ Class | Method | HTTP request | Description - [SignUpDto](doc//SignUpDto.md) - [SmartInfoResponseDto](doc//SmartInfoResponseDto.md) - [SmartSearchDto](doc//SmartSearchDto.md) + - [SourceType](doc//SourceType.md) - [StackCreateDto](doc//StackCreateDto.md) - [StackResponseDto](doc//StackResponseDto.md) - [StackUpdateDto](doc//StackUpdateDto.md) - [SystemConfigDto](doc//SystemConfigDto.md) - [SystemConfigFFmpegDto](doc//SystemConfigFFmpegDto.md) + - [SystemConfigFacesDto](doc//SystemConfigFacesDto.md) - [SystemConfigImageDto](doc//SystemConfigImageDto.md) - [SystemConfigJobDto](doc//SystemConfigJobDto.md) - [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md) @@ -420,6 +422,7 @@ Class | Method | HTTP request | Description - [SystemConfigLoggingDto](doc//SystemConfigLoggingDto.md) - [SystemConfigMachineLearningDto](doc//SystemConfigMachineLearningDto.md) - [SystemConfigMapDto](doc//SystemConfigMapDto.md) + - [SystemConfigMetadataDto](doc//SystemConfigMetadataDto.md) - [SystemConfigNewVersionCheckDto](doc//SystemConfigNewVersionCheckDto.md) - [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md) - [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index d6ce89624c..091e900145 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -221,11 +221,13 @@ part 'model/shared_link_type.dart'; part 'model/sign_up_dto.dart'; part 'model/smart_info_response_dto.dart'; part 'model/smart_search_dto.dart'; +part 'model/source_type.dart'; part 'model/stack_create_dto.dart'; part 'model/stack_response_dto.dart'; part 'model/stack_update_dto.dart'; part 'model/system_config_dto.dart'; part 'model/system_config_f_fmpeg_dto.dart'; +part 'model/system_config_faces_dto.dart'; part 'model/system_config_image_dto.dart'; part 'model/system_config_job_dto.dart'; part 'model/system_config_library_dto.dart'; @@ -234,6 +236,7 @@ part 'model/system_config_library_watch_dto.dart'; part 'model/system_config_logging_dto.dart'; part 'model/system_config_machine_learning_dto.dart'; part 'model/system_config_map_dto.dart'; +part 'model/system_config_metadata_dto.dart'; part 'model/system_config_new_version_check_dto.dart'; part 'model/system_config_notifications_dto.dart'; part 'model/system_config_o_auth_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 47375f0b50..9ec00aecc8 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -497,6 +497,8 @@ class ApiClient { return SmartInfoResponseDto.fromJson(value); case 'SmartSearchDto': return SmartSearchDto.fromJson(value); + case 'SourceType': + return SourceTypeTypeTransformer().decode(value); case 'StackCreateDto': return StackCreateDto.fromJson(value); case 'StackResponseDto': @@ -507,6 +509,8 @@ class ApiClient { return SystemConfigDto.fromJson(value); case 'SystemConfigFFmpegDto': return SystemConfigFFmpegDto.fromJson(value); + case 'SystemConfigFacesDto': + return SystemConfigFacesDto.fromJson(value); case 'SystemConfigImageDto': return SystemConfigImageDto.fromJson(value); case 'SystemConfigJobDto': @@ -523,6 +527,8 @@ class ApiClient { return SystemConfigMachineLearningDto.fromJson(value); case 'SystemConfigMapDto': return SystemConfigMapDto.fromJson(value); + case 'SystemConfigMetadataDto': + return SystemConfigMetadataDto.fromJson(value); case 'SystemConfigNewVersionCheckDto': return SystemConfigNewVersionCheckDto.fromJson(value); case 'SystemConfigNotificationsDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index a486551cc5..8dcef880f5 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -127,6 +127,9 @@ String parameterToString(dynamic value) { if (value is SharedLinkType) { return SharedLinkTypeTypeTransformer().encode(value).toString(); } + if (value is SourceType) { + return SourceTypeTypeTransformer().encode(value).toString(); + } if (value is TimeBucketSize) { return TimeBucketSizeTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/asset_face_response_dto.dart b/mobile/openapi/lib/model/asset_face_response_dto.dart index 812b165caa..7a8588ce5c 100644 --- a/mobile/openapi/lib/model/asset_face_response_dto.dart +++ b/mobile/openapi/lib/model/asset_face_response_dto.dart @@ -21,6 +21,7 @@ class AssetFaceResponseDto { required this.imageHeight, required this.imageWidth, required this.person, + this.sourceType, }); int boundingBoxX1; @@ -39,6 +40,14 @@ class AssetFaceResponseDto { PersonResponseDto? person; + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + SourceType? sourceType; + @override bool operator ==(Object other) => identical(this, other) || other is AssetFaceResponseDto && other.boundingBoxX1 == boundingBoxX1 && @@ -48,7 +57,8 @@ class AssetFaceResponseDto { other.id == id && other.imageHeight == imageHeight && other.imageWidth == imageWidth && - other.person == person; + other.person == person && + other.sourceType == sourceType; @override int get hashCode => @@ -60,10 +70,11 @@ class AssetFaceResponseDto { (id.hashCode) + (imageHeight.hashCode) + (imageWidth.hashCode) + - (person == null ? 0 : person!.hashCode); + (person == null ? 0 : person!.hashCode) + + (sourceType == null ? 0 : sourceType!.hashCode); @override - String toString() => 'AssetFaceResponseDto[boundingBoxX1=$boundingBoxX1, boundingBoxX2=$boundingBoxX2, boundingBoxY1=$boundingBoxY1, boundingBoxY2=$boundingBoxY2, id=$id, imageHeight=$imageHeight, imageWidth=$imageWidth, person=$person]'; + String toString() => 'AssetFaceResponseDto[boundingBoxX1=$boundingBoxX1, boundingBoxX2=$boundingBoxX2, boundingBoxY1=$boundingBoxY1, boundingBoxY2=$boundingBoxY2, id=$id, imageHeight=$imageHeight, imageWidth=$imageWidth, person=$person, sourceType=$sourceType]'; Map toJson() { final json = {}; @@ -79,6 +90,11 @@ class AssetFaceResponseDto { } else { // json[r'person'] = null; } + if (this.sourceType != null) { + json[r'sourceType'] = this.sourceType; + } else { + // json[r'sourceType'] = null; + } return json; } @@ -98,6 +114,7 @@ class AssetFaceResponseDto { imageHeight: mapValueOfType(json, r'imageHeight')!, imageWidth: mapValueOfType(json, r'imageWidth')!, person: PersonResponseDto.fromJson(json[r'person']), + sourceType: SourceType.fromJson(json[r'sourceType']), ); } return null; diff --git a/mobile/openapi/lib/model/asset_face_without_person_response_dto.dart b/mobile/openapi/lib/model/asset_face_without_person_response_dto.dart index 893f8ff353..ecfe06bd7d 100644 --- a/mobile/openapi/lib/model/asset_face_without_person_response_dto.dart +++ b/mobile/openapi/lib/model/asset_face_without_person_response_dto.dart @@ -20,6 +20,7 @@ class AssetFaceWithoutPersonResponseDto { required this.id, required this.imageHeight, required this.imageWidth, + this.sourceType, }); int boundingBoxX1; @@ -36,6 +37,14 @@ class AssetFaceWithoutPersonResponseDto { int imageWidth; + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + SourceType? sourceType; + @override bool operator ==(Object other) => identical(this, other) || other is AssetFaceWithoutPersonResponseDto && other.boundingBoxX1 == boundingBoxX1 && @@ -44,7 +53,8 @@ class AssetFaceWithoutPersonResponseDto { other.boundingBoxY2 == boundingBoxY2 && other.id == id && other.imageHeight == imageHeight && - other.imageWidth == imageWidth; + other.imageWidth == imageWidth && + other.sourceType == sourceType; @override int get hashCode => @@ -55,10 +65,11 @@ class AssetFaceWithoutPersonResponseDto { (boundingBoxY2.hashCode) + (id.hashCode) + (imageHeight.hashCode) + - (imageWidth.hashCode); + (imageWidth.hashCode) + + (sourceType == null ? 0 : sourceType!.hashCode); @override - String toString() => 'AssetFaceWithoutPersonResponseDto[boundingBoxX1=$boundingBoxX1, boundingBoxX2=$boundingBoxX2, boundingBoxY1=$boundingBoxY1, boundingBoxY2=$boundingBoxY2, id=$id, imageHeight=$imageHeight, imageWidth=$imageWidth]'; + String toString() => 'AssetFaceWithoutPersonResponseDto[boundingBoxX1=$boundingBoxX1, boundingBoxX2=$boundingBoxX2, boundingBoxY1=$boundingBoxY1, boundingBoxY2=$boundingBoxY2, id=$id, imageHeight=$imageHeight, imageWidth=$imageWidth, sourceType=$sourceType]'; Map toJson() { final json = {}; @@ -69,6 +80,11 @@ class AssetFaceWithoutPersonResponseDto { json[r'id'] = this.id; json[r'imageHeight'] = this.imageHeight; json[r'imageWidth'] = this.imageWidth; + if (this.sourceType != null) { + json[r'sourceType'] = this.sourceType; + } else { + // json[r'sourceType'] = null; + } return json; } @@ -87,6 +103,7 @@ class AssetFaceWithoutPersonResponseDto { id: mapValueOfType(json, r'id')!, imageHeight: mapValueOfType(json, r'imageHeight')!, imageWidth: mapValueOfType(json, r'imageWidth')!, + sourceType: SourceType.fromJson(json[r'sourceType']), ); } return null; diff --git a/mobile/openapi/lib/model/server_features_dto.dart b/mobile/openapi/lib/model/server_features_dto.dart index 3e5466237a..0a7d8a4b47 100644 --- a/mobile/openapi/lib/model/server_features_dto.dart +++ b/mobile/openapi/lib/model/server_features_dto.dart @@ -17,6 +17,7 @@ class ServerFeaturesDto { required this.duplicateDetection, required this.email, required this.facialRecognition, + required this.importFaces, required this.map, required this.oauth, required this.oauthAutoLaunch, @@ -36,6 +37,8 @@ class ServerFeaturesDto { bool facialRecognition; + bool importFaces; + bool map; bool oauth; @@ -60,6 +63,7 @@ class ServerFeaturesDto { other.duplicateDetection == duplicateDetection && other.email == email && other.facialRecognition == facialRecognition && + other.importFaces == importFaces && other.map == map && other.oauth == oauth && other.oauthAutoLaunch == oauthAutoLaunch && @@ -77,6 +81,7 @@ class ServerFeaturesDto { (duplicateDetection.hashCode) + (email.hashCode) + (facialRecognition.hashCode) + + (importFaces.hashCode) + (map.hashCode) + (oauth.hashCode) + (oauthAutoLaunch.hashCode) + @@ -88,7 +93,7 @@ class ServerFeaturesDto { (trash.hashCode); @override - String toString() => 'ServerFeaturesDto[configFile=$configFile, duplicateDetection=$duplicateDetection, email=$email, facialRecognition=$facialRecognition, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]'; + String toString() => 'ServerFeaturesDto[configFile=$configFile, duplicateDetection=$duplicateDetection, email=$email, facialRecognition=$facialRecognition, importFaces=$importFaces, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]'; Map toJson() { final json = {}; @@ -96,6 +101,7 @@ class ServerFeaturesDto { json[r'duplicateDetection'] = this.duplicateDetection; json[r'email'] = this.email; json[r'facialRecognition'] = this.facialRecognition; + json[r'importFaces'] = this.importFaces; json[r'map'] = this.map; json[r'oauth'] = this.oauth; json[r'oauthAutoLaunch'] = this.oauthAutoLaunch; @@ -120,6 +126,7 @@ class ServerFeaturesDto { duplicateDetection: mapValueOfType(json, r'duplicateDetection')!, email: mapValueOfType(json, r'email')!, facialRecognition: mapValueOfType(json, r'facialRecognition')!, + importFaces: mapValueOfType(json, r'importFaces')!, map: mapValueOfType(json, r'map')!, oauth: mapValueOfType(json, r'oauth')!, oauthAutoLaunch: mapValueOfType(json, r'oauthAutoLaunch')!, @@ -180,6 +187,7 @@ class ServerFeaturesDto { 'duplicateDetection', 'email', 'facialRecognition', + 'importFaces', 'map', 'oauth', 'oauthAutoLaunch', diff --git a/mobile/openapi/lib/model/source_type.dart b/mobile/openapi/lib/model/source_type.dart new file mode 100644 index 0000000000..13c450b010 --- /dev/null +++ b/mobile/openapi/lib/model/source_type.dart @@ -0,0 +1,85 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SourceType { + /// Instantiate a new enum with the provided [value]. + const SourceType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const machineLearning = SourceType._(r'machine-learning'); + static const exif = SourceType._(r'exif'); + + /// List of all possible values in this [enum][SourceType]. + static const values = [ + machineLearning, + exif, + ]; + + static SourceType? fromJson(dynamic value) => SourceTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SourceType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [SourceType] to String, +/// and [decode] dynamic data back to [SourceType]. +class SourceTypeTypeTransformer { + factory SourceTypeTypeTransformer() => _instance ??= const SourceTypeTypeTransformer._(); + + const SourceTypeTypeTransformer._(); + + String encode(SourceType data) => data.value; + + /// Decodes a [dynamic value][data] to a SourceType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + SourceType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'machine-learning': return SourceType.machineLearning; + case r'exif': return SourceType.exif; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [SourceTypeTypeTransformer] instance. + static SourceTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index e56169742a..aff8062c8a 100644 --- a/mobile/openapi/lib/model/system_config_dto.dart +++ b/mobile/openapi/lib/model/system_config_dto.dart @@ -20,6 +20,7 @@ class SystemConfigDto { required this.logging, required this.machineLearning, required this.map, + required this.metadata, required this.newVersionCheck, required this.notifications, required this.oauth, @@ -46,6 +47,8 @@ class SystemConfigDto { SystemConfigMapDto map; + SystemConfigMetadataDto metadata; + SystemConfigNewVersionCheckDto newVersionCheck; SystemConfigNotificationsDto notifications; @@ -75,6 +78,7 @@ class SystemConfigDto { other.logging == logging && other.machineLearning == machineLearning && other.map == map && + other.metadata == metadata && other.newVersionCheck == newVersionCheck && other.notifications == notifications && other.oauth == oauth && @@ -96,6 +100,7 @@ class SystemConfigDto { (logging.hashCode) + (machineLearning.hashCode) + (map.hashCode) + + (metadata.hashCode) + (newVersionCheck.hashCode) + (notifications.hashCode) + (oauth.hashCode) + @@ -108,7 +113,7 @@ class SystemConfigDto { (user.hashCode); @override - String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, newVersionCheck=$newVersionCheck, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, theme=$theme, trash=$trash, user=$user]'; + String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, theme=$theme, trash=$trash, user=$user]'; Map toJson() { final json = {}; @@ -119,6 +124,7 @@ class SystemConfigDto { json[r'logging'] = this.logging; json[r'machineLearning'] = this.machineLearning; json[r'map'] = this.map; + json[r'metadata'] = this.metadata; json[r'newVersionCheck'] = this.newVersionCheck; json[r'notifications'] = this.notifications; json[r'oauth'] = this.oauth; @@ -147,6 +153,7 @@ class SystemConfigDto { logging: SystemConfigLoggingDto.fromJson(json[r'logging'])!, machineLearning: SystemConfigMachineLearningDto.fromJson(json[r'machineLearning'])!, map: SystemConfigMapDto.fromJson(json[r'map'])!, + metadata: SystemConfigMetadataDto.fromJson(json[r'metadata'])!, newVersionCheck: SystemConfigNewVersionCheckDto.fromJson(json[r'newVersionCheck'])!, notifications: SystemConfigNotificationsDto.fromJson(json[r'notifications'])!, oauth: SystemConfigOAuthDto.fromJson(json[r'oauth'])!, @@ -211,6 +218,7 @@ class SystemConfigDto { 'logging', 'machineLearning', 'map', + 'metadata', 'newVersionCheck', 'notifications', 'oauth', diff --git a/mobile/openapi/lib/model/system_config_faces_dto.dart b/mobile/openapi/lib/model/system_config_faces_dto.dart new file mode 100644 index 0000000000..980e494fb7 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_faces_dto.dart @@ -0,0 +1,98 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigFacesDto { + /// Returns a new [SystemConfigFacesDto] instance. + SystemConfigFacesDto({ + required this.import_, + }); + + bool import_; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigFacesDto && + other.import_ == import_; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (import_.hashCode); + + @override + String toString() => 'SystemConfigFacesDto[import_=$import_]'; + + Map toJson() { + final json = {}; + json[r'import'] = this.import_; + return json; + } + + /// Returns a new [SystemConfigFacesDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigFacesDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return SystemConfigFacesDto( + import_: mapValueOfType(json, r'import')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigFacesDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigFacesDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigFacesDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigFacesDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'import', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_metadata_dto.dart b/mobile/openapi/lib/model/system_config_metadata_dto.dart new file mode 100644 index 0000000000..60ca35c835 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_metadata_dto.dart @@ -0,0 +1,98 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigMetadataDto { + /// Returns a new [SystemConfigMetadataDto] instance. + SystemConfigMetadataDto({ + required this.faces, + }); + + SystemConfigFacesDto faces; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigMetadataDto && + other.faces == faces; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (faces.hashCode); + + @override + String toString() => 'SystemConfigMetadataDto[faces=$faces]'; + + Map toJson() { + final json = {}; + json[r'faces'] = this.faces; + return json; + } + + /// Returns a new [SystemConfigMetadataDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigMetadataDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return SystemConfigMetadataDto( + faces: SystemConfigFacesDto.fromJson(json[r'faces'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigMetadataDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigMetadataDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigMetadataDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigMetadataDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'faces', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index fbec7be7e1..bbfabfe1d7 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -8018,6 +8018,9 @@ } ], "nullable": true + }, + "sourceType": { + "$ref": "#/components/schemas/SourceType" } }, "required": [ @@ -8086,6 +8089,9 @@ }, "imageWidth": { "type": "integer" + }, + "sourceType": { + "$ref": "#/components/schemas/SourceType" } }, "required": [ @@ -10688,6 +10694,9 @@ "facialRecognition": { "type": "boolean" }, + "importFaces": { + "type": "boolean" + }, "map": { "type": "boolean" }, @@ -10721,6 +10730,7 @@ "duplicateDetection", "email", "facialRecognition", + "importFaces", "map", "oauth", "oauthAutoLaunch", @@ -11229,6 +11239,13 @@ ], "type": "object" }, + "SourceType": { + "enum": [ + "machine-learning", + "exif" + ], + "type": "string" + }, "StackCreateDto": { "properties": { "assetIds": { @@ -11299,6 +11316,9 @@ "map": { "$ref": "#/components/schemas/SystemConfigMapDto" }, + "metadata": { + "$ref": "#/components/schemas/SystemConfigMetadataDto" + }, "newVersionCheck": { "$ref": "#/components/schemas/SystemConfigNewVersionCheckDto" }, @@ -11338,6 +11358,7 @@ "logging", "machineLearning", "map", + "metadata", "newVersionCheck", "notifications", "oauth", @@ -11464,6 +11485,17 @@ ], "type": "object" }, + "SystemConfigFacesDto": { + "properties": { + "import": { + "type": "boolean" + } + }, + "required": [ + "import" + ], + "type": "object" + }, "SystemConfigImageDto": { "properties": { "colorspace": { @@ -11656,6 +11688,17 @@ ], "type": "object" }, + "SystemConfigMetadataDto": { + "properties": { + "faces": { + "$ref": "#/components/schemas/SystemConfigFacesDto" + } + }, + "required": [ + "faces" + ], + "type": "object" + }, "SystemConfigNewVersionCheckDto": { "properties": { "enabled": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 277aa57141..9e74ae88a0 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -207,6 +207,7 @@ export type AssetFaceWithoutPersonResponseDto = { id: string; imageHeight: number; imageWidth: number; + sourceType?: SourceType; }; export type PersonWithFacesResponseDto = { birthDate: string | null; @@ -508,6 +509,7 @@ export type AssetFaceResponseDto = { imageHeight: number; imageWidth: number; person: (PersonResponseDto) | null; + sourceType?: SourceType; }; export type FaceDto = { id: string; @@ -893,6 +895,7 @@ export type ServerFeaturesDto = { duplicateDetection: boolean; email: boolean; facialRecognition: boolean; + importFaces: boolean; map: boolean; oauth: boolean; oauthAutoLaunch: boolean; @@ -1122,6 +1125,12 @@ export type SystemConfigMapDto = { enabled: boolean; lightStyle: string; }; +export type SystemConfigFacesDto = { + "import": boolean; +}; +export type SystemConfigMetadataDto = { + faces: SystemConfigFacesDto; +}; export type SystemConfigNewVersionCheckDto = { enabled: boolean; }; @@ -1178,6 +1187,7 @@ export type SystemConfigDto = { logging: SystemConfigLoggingDto; machineLearning: SystemConfigMachineLearningDto; map: SystemConfigMapDto; + metadata: SystemConfigMetadataDto; newVersionCheck: SystemConfigNewVersionCheckDto; notifications: SystemConfigNotificationsDto; oauth: SystemConfigOAuthDto; @@ -3226,6 +3236,10 @@ export enum AlbumUserRole { Editor = "editor", Viewer = "viewer" } +export enum SourceType { + MachineLearning = "machine-learning", + Exif = "exif" +} export enum AssetTypeEnum { Image = "IMAGE", Video = "VIDEO", diff --git a/server/src/config.ts b/server/src/config.ts index 96ce63cf45..057c9a69e2 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -141,6 +141,11 @@ export interface SystemConfig { reverseGeocoding: { enabled: boolean; }; + metadata: { + faces: { + import: boolean; + }; + }; oauth: { autoLaunch: boolean; autoRegister: boolean; @@ -286,6 +291,11 @@ export const defaults = Object.freeze({ reverseGeocoding: { enabled: true, }, + metadata: { + faces: { + import: false, + }, + }, oauth: { autoLaunch: false, autoRegister: true, diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts index e20a0c658d..a4d0d06152 100644 --- a/server/src/cores/storage.core.ts +++ b/server/src/cores/storage.core.ts @@ -301,7 +301,7 @@ export class StorageCore { return this.assetRepository.update({ id, sidecarPath: newPath }); } case PersonPathType.FACE: { - return this.personRepository.update({ id, thumbnailPath: newPath }); + return this.personRepository.update([{ id, thumbnailPath: newPath }]); } } } diff --git a/server/src/dtos/person.dto.ts b/server/src/dtos/person.dto.ts index 3833e4f3e7..94ee52d916 100644 --- a/server/src/dtos/person.dto.ts +++ b/server/src/dtos/person.dto.ts @@ -6,6 +6,7 @@ import { PropertyLifecycle } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { PersonEntity } from 'src/entities/person.entity'; +import { SourceType } from 'src/enum'; import { IsDateStringFormat, MaxDateString, Optional, ValidateBoolean, ValidateUUID } from 'src/validation'; export class PersonCreateDto { @@ -113,6 +114,8 @@ export class AssetFaceWithoutPersonResponseDto { boundingBoxY1!: number; @ApiProperty({ type: 'integer' }) boundingBoxY2!: number; + @ApiProperty({ enum: SourceType, enumName: 'SourceType' }) + sourceType?: SourceType; } export class AssetFaceResponseDto extends AssetFaceWithoutPersonResponseDto { @@ -176,6 +179,7 @@ export function mapFacesWithoutPerson(face: AssetFaceEntity): AssetFaceWithoutPe boundingBoxX2: face.boundingBoxX2, boundingBoxY1: face.boundingBoxY1, boundingBoxY2: face.boundingBoxY2, + sourceType: face.sourceType, }; } diff --git a/server/src/dtos/server.dto.ts b/server/src/dtos/server.dto.ts index 9c18b0b4fe..78e59e4d1a 100644 --- a/server/src/dtos/server.dto.ts +++ b/server/src/dtos/server.dto.ts @@ -131,6 +131,7 @@ export class ServerFeaturesDto { map!: boolean; trash!: boolean; reverseGeocoding!: boolean; + importFaces!: boolean; oauth!: boolean; oauthAutoLaunch!: boolean; passwordLogin!: boolean; diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index e2255223d0..14027aa16a 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -375,6 +375,18 @@ class SystemConfigReverseGeocodingDto { enabled!: boolean; } +class SystemConfigFacesDto { + @IsBoolean() + import!: boolean; +} + +class SystemConfigMetadataDto { + @Type(() => SystemConfigFacesDto) + @ValidateNested() + @IsObject() + faces!: SystemConfigFacesDto; +} + class SystemConfigServerDto { @ValidateIf((_, value: string) => value !== '') @IsUrl({ require_tld: false, require_protocol: true, protocols: ['http', 'https'] }) @@ -555,6 +567,11 @@ export class SystemConfigDto implements SystemConfig { @IsObject() reverseGeocoding!: SystemConfigReverseGeocodingDto; + @Type(() => SystemConfigMetadataDto) + @ValidateNested() + @IsObject() + metadata!: SystemConfigMetadataDto; + @Type(() => SystemConfigStorageTemplateDto) @ValidateNested() @IsObject() diff --git a/server/src/entities/asset-face.entity.ts b/server/src/entities/asset-face.entity.ts index c21aacfcd1..3a4e916cba 100644 --- a/server/src/entities/asset-face.entity.ts +++ b/server/src/entities/asset-face.entity.ts @@ -1,6 +1,7 @@ import { AssetEntity } from 'src/entities/asset.entity'; import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { PersonEntity } from 'src/entities/person.entity'; +import { SourceType } from 'src/enum'; import { Column, Entity, Index, ManyToOne, OneToOne, PrimaryGeneratedColumn } from 'typeorm'; @Entity('asset_faces', { synchronize: false }) @@ -37,6 +38,9 @@ export class AssetFaceEntity { @Column({ default: 0, type: 'int' }) boundingBoxY2!: number; + @Column({ default: SourceType.MACHINE_LEARNING, type: 'enum', enum: SourceType }) + sourceType!: SourceType; + @ManyToOne(() => AssetEntity, (asset) => asset.faces, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) asset!: AssetEntity; diff --git a/server/src/enum.ts b/server/src/enum.ts index 9cd5c189e8..28973e0205 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -180,3 +180,8 @@ export enum UserStatus { REMOVING = 'removing', DELETED = 'deleted', } + +export enum SourceType { + MACHINE_LEARNING = 'machine-learning', + EXIF = 'exif', +} diff --git a/server/src/interfaces/metadata.interface.ts b/server/src/interfaces/metadata.interface.ts index 386f69a9e7..1f808cbd21 100644 --- a/server/src/interfaces/metadata.interface.ts +++ b/server/src/interfaces/metadata.interface.ts @@ -7,7 +7,8 @@ export interface ExifDuration { Scale?: number; } -export interface ImmichTags extends Omit { +type TagsWithWrongTypes = 'FocalLength' | 'Duration' | 'Description' | 'ImageDescription' | 'RegionInfo'; +export interface ImmichTags extends Omit { ContentIdentifier?: string; MotionPhoto?: number; MotionPhotoVersion?: number; @@ -23,6 +24,28 @@ export interface ImmichTags extends Omit): Paginated; getAllForUser(pagination: PaginationOptions, userId: string, options: PersonSearchOptions): Paginated; getAllWithoutFaces(): Promise; getById(personId: string): Promise; getByName(userId: string, personName: string, options: PersonNameSearchOptions): Promise; + getDistinctNames(userId: string, options: PersonNameSearchOptions): Promise; getAssets(personId: string): Promise; - create(entity: Partial): Promise; + create(entities: Partial[]): Promise; createFaces(entities: Partial[]): Promise; delete(entities: PersonEntity[]): Promise; deleteAll(): Promise; - deleteAllFaces(): Promise; + deleteAllFaces(options: DeleteAllFacesOptions): Promise; + replaceFaces(assetId: string, entities: Partial[], sourceType?: string): Promise; getAllFaces(pagination: PaginationOptions, options?: FindManyOptions): Paginated; getFaceById(id: string): Promise; getFaceByIdWithAssets( @@ -63,6 +74,6 @@ export interface IPersonRepository { reassignFace(assetFaceId: string, newPersonId: string): Promise; getNumberOfPeople(userId: string): Promise; reassignFaces(data: UpdateFacesData): Promise; - update(entity: Partial): Promise; + update(entities: Partial[]): Promise; getLatestFaceDate(): Promise; } diff --git a/server/src/migrations/1721249222549-AddSourceColumnToAssetFace.ts b/server/src/migrations/1721249222549-AddSourceColumnToAssetFace.ts new file mode 100644 index 0000000000..7f185077ff --- /dev/null +++ b/server/src/migrations/1721249222549-AddSourceColumnToAssetFace.ts @@ -0,0 +1,16 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddSourceColumnToAssetFace1721249222549 implements MigrationInterface { + name = 'AddSourceColumnToAssetFace1721249222549' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TYPE sourceType AS ENUM ('machine-learning', 'exif');`); + await queryRunner.query(`ALTER TABLE "asset_faces" ADD "sourceType" sourceType NOT NULL DEFAULT 'machine-learning'`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "sourceType"`); + await queryRunner.query(`DROP TYPE sourceType`); + } + +} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index 3852439936..9b4b17425c 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -199,6 +199,7 @@ SELECT "AssetEntity__AssetEntity_faces"."boundingBoxY1" AS "AssetEntity__AssetEntity_faces_boundingBoxY1", "AssetEntity__AssetEntity_faces"."boundingBoxX2" AS "AssetEntity__AssetEntity_faces_boundingBoxX2", "AssetEntity__AssetEntity_faces"."boundingBoxY2" AS "AssetEntity__AssetEntity_faces_boundingBoxY2", + "AssetEntity__AssetEntity_faces"."sourceType" AS "AssetEntity__AssetEntity_faces_sourceType", "8258e303a73a72cf6abb13d73fb592dde0d68280"."id" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_id", "8258e303a73a72cf6abb13d73fb592dde0d68280"."createdAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_createdAt", "8258e303a73a72cf6abb13d73fb592dde0d68280"."updatedAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_updatedAt", diff --git a/server/src/queries/person.repository.sql b/server/src/queries/person.repository.sql index 9c94232d20..57969e4989 100644 --- a/server/src/queries/person.repository.sql +++ b/server/src/queries/person.repository.sql @@ -74,6 +74,7 @@ SELECT "AssetFaceEntity"."boundingBoxY1" AS "AssetFaceEntity_boundingBoxY1", "AssetFaceEntity"."boundingBoxX2" AS "AssetFaceEntity_boundingBoxX2", "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2", + "AssetFaceEntity"."sourceType" AS "AssetFaceEntity_sourceType", "AssetFaceEntity__AssetFaceEntity_person"."id" AS "AssetFaceEntity__AssetFaceEntity_person_id", "AssetFaceEntity__AssetFaceEntity_person"."createdAt" AS "AssetFaceEntity__AssetFaceEntity_person_createdAt", "AssetFaceEntity__AssetFaceEntity_person"."updatedAt" AS "AssetFaceEntity__AssetFaceEntity_person_updatedAt", @@ -106,6 +107,7 @@ FROM "AssetFaceEntity"."boundingBoxY1" AS "AssetFaceEntity_boundingBoxY1", "AssetFaceEntity"."boundingBoxX2" AS "AssetFaceEntity_boundingBoxX2", "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2", + "AssetFaceEntity"."sourceType" AS "AssetFaceEntity_sourceType", "AssetFaceEntity__AssetFaceEntity_person"."id" AS "AssetFaceEntity__AssetFaceEntity_person_id", "AssetFaceEntity__AssetFaceEntity_person"."createdAt" AS "AssetFaceEntity__AssetFaceEntity_person_createdAt", "AssetFaceEntity__AssetFaceEntity_person"."updatedAt" AS "AssetFaceEntity__AssetFaceEntity_person_updatedAt", @@ -141,6 +143,7 @@ FROM "AssetFaceEntity"."boundingBoxY1" AS "AssetFaceEntity_boundingBoxY1", "AssetFaceEntity"."boundingBoxX2" AS "AssetFaceEntity_boundingBoxX2", "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2", + "AssetFaceEntity"."sourceType" AS "AssetFaceEntity_sourceType", "AssetFaceEntity__AssetFaceEntity_person"."id" AS "AssetFaceEntity__AssetFaceEntity_person_id", "AssetFaceEntity__AssetFaceEntity_person"."createdAt" AS "AssetFaceEntity__AssetFaceEntity_person_createdAt", "AssetFaceEntity__AssetFaceEntity_person"."updatedAt" AS "AssetFaceEntity__AssetFaceEntity_person_updatedAt", @@ -226,6 +229,16 @@ ORDER BY LIMIT 20 +-- PersonRepository.getDistinctNames +SELECT DISTINCT + ON (lower("person"."name")) "person"."id" AS "person_id", + "person"."name" AS "person_name" +FROM + "person" "person" +WHERE + "person"."ownerId" = $1 + AND "person"."name" != '' + -- PersonRepository.getStatistics SELECT COUNT(DISTINCT ("asset"."id")) AS "count" @@ -282,6 +295,7 @@ FROM "AssetEntity__AssetEntity_faces"."boundingBoxY1" AS "AssetEntity__AssetEntity_faces_boundingBoxY1", "AssetEntity__AssetEntity_faces"."boundingBoxX2" AS "AssetEntity__AssetEntity_faces_boundingBoxX2", "AssetEntity__AssetEntity_faces"."boundingBoxY2" AS "AssetEntity__AssetEntity_faces_boundingBoxY2", + "AssetEntity__AssetEntity_faces"."sourceType" AS "AssetEntity__AssetEntity_faces_sourceType", "8258e303a73a72cf6abb13d73fb592dde0d68280"."id" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_id", "8258e303a73a72cf6abb13d73fb592dde0d68280"."createdAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_createdAt", "8258e303a73a72cf6abb13d73fb592dde0d68280"."updatedAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_updatedAt", @@ -375,6 +389,7 @@ SELECT "AssetFaceEntity"."boundingBoxY1" AS "AssetFaceEntity_boundingBoxY1", "AssetFaceEntity"."boundingBoxX2" AS "AssetFaceEntity_boundingBoxX2", "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2", + "AssetFaceEntity"."sourceType" AS "AssetFaceEntity_sourceType", "AssetFaceEntity__AssetFaceEntity_asset"."id" AS "AssetFaceEntity__AssetFaceEntity_asset_id", "AssetFaceEntity__AssetFaceEntity_asset"."deviceAssetId" AS "AssetFaceEntity__AssetFaceEntity_asset_deviceAssetId", "AssetFaceEntity__AssetFaceEntity_asset"."ownerId" AS "AssetFaceEntity__AssetFaceEntity_asset_ownerId", @@ -425,7 +440,8 @@ SELECT "AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1", "AssetFaceEntity"."boundingBoxY1" AS "AssetFaceEntity_boundingBoxY1", "AssetFaceEntity"."boundingBoxX2" AS "AssetFaceEntity_boundingBoxX2", - "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2" + "AssetFaceEntity"."boundingBoxY2" AS "AssetFaceEntity_boundingBoxY2", + "AssetFaceEntity"."sourceType" AS "AssetFaceEntity_sourceType" FROM "asset_faces" "AssetFaceEntity" WHERE diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index e9e94400ad..dd2e3ae75c 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -235,6 +235,7 @@ WITH "faces"."boundingBoxY1" AS "boundingBoxY1", "faces"."boundingBoxX2" AS "boundingBoxX2", "faces"."boundingBoxY2" AS "boundingBoxY2", + "faces"."sourceType" AS "sourceType", "search"."embedding" <= > $1 AS "distance" FROM "asset_faces" "faces" diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index 876ed369f6..7459ca3183 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import _ from 'lodash'; import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; @@ -8,8 +8,10 @@ import { AssetEntity } from 'src/entities/asset.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { AssetFaceId, + DeleteAllFacesOptions, IPersonRepository, PeopleStatistics, + PersonNameResponse, PersonNameSearchOptions, PersonSearchOptions, PersonStatistics, @@ -17,12 +19,13 @@ import { } from 'src/interfaces/person.interface'; import { Instrumentation } from 'src/utils/instrumentation'; import { Paginated, PaginationMode, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination'; -import { FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm'; +import { DataSource, FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm'; @Instrumentation() @Injectable() export class PersonRepository implements IPersonRepository { constructor( + @InjectDataSource() private dataSource: DataSource, @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(PersonEntity) private personRepository: Repository, @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, @@ -49,7 +52,16 @@ export class PersonRepository implements IPersonRepository { await this.personRepository.clear(); } - async deleteAllFaces(): Promise { + async deleteAllFaces({ sourceType }: DeleteAllFacesOptions): Promise { + if (sourceType) { + await this.assetFaceRepository + .createQueryBuilder('asset_faces') + .delete() + .andWhere('sourceType = :sourceType', { sourceType }) + .execute(); + return; + } + await this.assetFaceRepository.query('TRUNCATE TABLE asset_faces CASCADE'); } @@ -182,6 +194,21 @@ export class PersonRepository implements IPersonRepository { return queryBuilder.getMany(); } + @GenerateSql({ params: [DummyValue.UUID, { withHidden: true }] }) + getDistinctNames(userId: string, { withHidden }: PersonNameSearchOptions): Promise { + const queryBuilder = this.personRepository + .createQueryBuilder('person') + .select(['person.id', 'person.name']) + .distinctOn(['lower(person.name)']) + .where(`person.ownerId = :userId AND person.name != ''`, { userId }); + + if (!withHidden) { + queryBuilder.andWhere('person.isHidden = false'); + } + + return queryBuilder.getMany(); + } + @GenerateSql({ params: [DummyValue.UUID] }) async getStatistics(personId: string): Promise { const items = await this.assetFaceRepository @@ -248,8 +275,8 @@ export class PersonRepository implements IPersonRepository { return result; } - create(entity: Partial): Promise { - return this.personRepository.save(entity); + create(entities: Partial[]): Promise { + return this.personRepository.save(entities); } async createFaces(entities: AssetFaceEntity[]): Promise { @@ -257,9 +284,16 @@ export class PersonRepository implements IPersonRepository { return res.map((row) => row.id); } - async update(entity: Partial): Promise { - const { id } = await this.personRepository.save(entity); - return this.personRepository.findOneByOrFail({ id }); + async replaceFaces(assetId: string, entities: AssetFaceEntity[], sourceType: string): Promise { + return this.dataSource.transaction(async (manager) => { + await manager.delete(AssetFaceEntity, { assetId, sourceType }); + const assetFaces = await manager.save(AssetFaceEntity, entities); + return assetFaces.map(({ id }) => id); + }); + } + + async update(entities: Partial[]): Promise { + return await this.personRepository.save(entities); } @GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] }) diff --git a/server/src/services/audit.service.ts b/server/src/services/audit.service.ts index 72db2b6eb5..4f292f7cc1 100644 --- a/server/src/services/audit.service.ts +++ b/server/src/services/audit.service.ts @@ -115,7 +115,7 @@ export class AuditService { } case PersonPathType.FACE: { - await this.personRepository.update({ id, thumbnailPath: pathValue }); + await this.personRepository.update([{ id, thumbnailPath: pathValue }]); break; } diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index e74335bdc3..919348b53e 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -117,7 +117,7 @@ export class MediaService { continue; } - await this.personRepository.update({ id: person.id, faceAssetId: face.id }); + await this.personRepository.update([{ id: person.id, faceAssetId: face.id }]); } jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } }); diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 834fd16afc..967511633a 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto'; import { Stats } from 'node:fs'; import { constants } from 'node:fs/promises'; import { ExifEntity } from 'src/entities/exif.entity'; -import { AssetType } from 'src/enum'; +import { AssetType, SourceType } from 'src/enum'; import { IAlbumRepository } from 'src/interfaces/album.interface'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; @@ -24,6 +24,8 @@ import { MetadataService, Orientation } from 'src/services/metadata.service'; import { assetStub } from 'test/fixtures/asset.stub'; import { fileStub } from 'test/fixtures/file.stub'; import { probeStub } from 'test/fixtures/media.stub'; +import { metadataStub } from 'test/fixtures/metadata.stub'; +import { personStub } from 'test/fixtures/person.stub'; import { tagStub } from 'test/fixtures/tag.stub'; import { newAlbumRepositoryMock } from 'test/repositories/album.repository.mock'; import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock'; @@ -956,6 +958,123 @@ describe(MetadataService.name, () => { }), ); }); + + it('should skip importing metadata when the feature is disabled', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: false } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + await sut.handleMetadataExtraction({ id: assetStub.image.id }); + expect(personMock.getDistinctNames).not.toHaveBeenCalled(); + }); + + it('should skip importing metadata face for assets without tags.RegionInfo', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.empty); + await sut.handleMetadataExtraction({ id: assetStub.image.id }); + expect(personMock.getDistinctNames).not.toHaveBeenCalled(); + }); + + it('should skip importing faces without name', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName); + personMock.getDistinctNames.mockResolvedValue([]); + personMock.create.mockResolvedValue([]); + personMock.replaceFaces.mockResolvedValue([]); + personMock.update.mockResolvedValue([]); + await sut.handleMetadataExtraction({ id: assetStub.image.id }); + expect(personMock.create).toHaveBeenCalledWith([]); + expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); + expect(personMock.update).toHaveBeenCalledWith([]); + }); + + it('should skip importing faces with empty name', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName); + personMock.getDistinctNames.mockResolvedValue([]); + personMock.create.mockResolvedValue([]); + personMock.replaceFaces.mockResolvedValue([]); + personMock.update.mockResolvedValue([]); + await sut.handleMetadataExtraction({ id: assetStub.image.id }); + expect(personMock.create).toHaveBeenCalledWith([]); + expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); + expect(personMock.update).toHaveBeenCalledWith([]); + }); + + it('should apply metadata face tags creating new persons', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + personMock.getDistinctNames.mockResolvedValue([]); + personMock.create.mockResolvedValue([personStub.withName]); + personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']); + personMock.update.mockResolvedValue([personStub.withName]); + await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id }); + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]); + expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true }); + expect(personMock.create).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]); + expect(personMock.replaceFaces).toHaveBeenCalledWith( + assetStub.primaryImage.id, + [ + { + id: 'random-uuid', + assetId: assetStub.primaryImage.id, + personId: 'random-uuid', + imageHeight: 100, + imageWidth: 100, + boundingBoxX1: 0, + boundingBoxX2: 10, + boundingBoxY1: 0, + boundingBoxY2: 10, + sourceType: SourceType.EXIF, + }, + ], + SourceType.EXIF, + ); + expect(personMock.update).toHaveBeenCalledWith([{ id: 'random-uuid', faceAssetId: 'random-uuid' }]); + expect(jobMock.queueAll).toHaveBeenCalledWith([ + { + name: JobName.GENERATE_PERSON_THUMBNAIL, + data: { id: personStub.withName.id }, + }, + ]); + }); + + it('should assign metadata face tags to existing persons', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); + systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); + metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]); + personMock.create.mockResolvedValue([]); + personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']); + personMock.update.mockResolvedValue([personStub.withName]); + await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id }); + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]); + expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true }); + expect(personMock.create).toHaveBeenCalledWith([]); + expect(personMock.replaceFaces).toHaveBeenCalledWith( + assetStub.primaryImage.id, + [ + { + id: 'random-uuid', + assetId: assetStub.primaryImage.id, + personId: personStub.withName.id, + imageHeight: 100, + imageWidth: 100, + boundingBoxX1: 0, + boundingBoxX2: 10, + boundingBoxY1: 0, + boundingBoxY2: 10, + sourceType: SourceType.EXIF, + }, + ], + SourceType.EXIF, + ); + expect(personMock.update).toHaveBeenCalledWith([]); + expect(jobMock.queueAll).toHaveBeenCalledWith([]); + }); }); describe('handleQueueSidecar', () => { diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 7eab4702ad..5978d0eb74 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -9,9 +9,11 @@ import { SystemConfig } from 'src/config'; import { StorageCore } from 'src/cores/storage.core'; import { SystemConfigCore } from 'src/cores/system-config.core'; import { OnEmit } from 'src/decorators'; +import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; -import { AssetType } from 'src/enum'; +import { PersonEntity } from 'src/entities/person.entity'; +import { AssetType, SourceType } from 'src/enum'; import { IAlbumRepository } from 'src/interfaces/album.interface'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; @@ -37,6 +39,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { ITagRepository } from 'src/interfaces/tag.interface'; import { IUserRepository } from 'src/interfaces/user.interface'; +import { isFaceImportEnabled } from 'src/utils/misc'; import { usePagination } from 'src/utils/pagination'; import { upsertTags } from 'src/utils/tag'; @@ -104,7 +107,7 @@ export class MetadataService { @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IMetadataRepository) private repository: IMetadataRepository, @Inject(IMoveRepository) moveRepository: IMoveRepository, - @Inject(IPersonRepository) personRepository: IPersonRepository, + @Inject(IPersonRepository) private personRepository: IPersonRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository, @Inject(ITagRepository) private tagRepository: ITagRepository, @@ -215,6 +218,7 @@ export class MetadataService { } async handleMetadataExtraction({ id }: IEntityJob): Promise { + const { metadata } = await this.configCore.getConfig({ withCache: true }); const [asset] = await this.assetRepository.getByIds([id]); if (!asset) { return JobStatus.FAILED; @@ -253,6 +257,10 @@ export class MetadataService { metadataExtractedAt: new Date(), }); + if (isFaceImportEnabled(metadata)) { + await this.applyTaggedFaces(asset, exifTags); + } + return JobStatus.SUCCESS; } @@ -512,6 +520,65 @@ export class MetadataService { } } + private async applyTaggedFaces(asset: AssetEntity, tags: ImmichTags) { + if (!tags.RegionInfo?.AppliedToDimensions || tags.RegionInfo.RegionList.length === 0) { + return; + } + + const discoveredFaces: Partial[] = []; + const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true }); + const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id])); + const missing: Partial[] = []; + const missingWithFaceAsset: Partial[] = []; + for (const region of tags.RegionInfo.RegionList) { + if (!region.Name) { + continue; + } + + const imageWidth = tags.RegionInfo.AppliedToDimensions.W; + const imageHeight = tags.RegionInfo.AppliedToDimensions.H; + const loweredName = region.Name.toLowerCase(); + const personId = existingNameMap.get(loweredName) || this.cryptoRepository.randomUUID(); + + const face = { + id: this.cryptoRepository.randomUUID(), + personId, + assetId: asset.id, + imageWidth, + imageHeight, + boundingBoxX1: Math.floor((region.Area.X - region.Area.W / 2) * imageWidth), + boundingBoxY1: Math.floor((region.Area.Y - region.Area.H / 2) * imageHeight), + boundingBoxX2: Math.floor((region.Area.X + region.Area.W / 2) * imageWidth), + boundingBoxY2: Math.floor((region.Area.Y + region.Area.H / 2) * imageHeight), + sourceType: SourceType.EXIF, + }; + + discoveredFaces.push(face); + if (!existingNameMap.has(loweredName)) { + missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name }); + missingWithFaceAsset.push({ id: personId, faceAssetId: face.id }); + } + } + + if (missing.length > 0) { + this.logger.debug(`Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`); + } + + const newPersons = await this.personRepository.create(missing); + + const faceIds = await this.personRepository.replaceFaces(asset.id, discoveredFaces, SourceType.EXIF); + this.logger.debug(`Created ${faceIds.length} faces for asset ${asset.id}`); + + await this.personRepository.update(missingWithFaceAsset); + + await this.jobRepository.queueAll( + newPersons.map((person) => ({ + name: JobName.GENERATE_PERSON_THUMBNAIL, + data: { id: person.id }, + })), + ); + } + private async exifData( asset: AssetEntity, ): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> { diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index f8608243ae..51598b93d0 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -3,7 +3,7 @@ import { Colorspace } from 'src/config'; import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto'; import { PersonResponseDto, mapFaces, mapPerson } from 'src/dtos/person.dto'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; -import { SystemMetadataKey } from 'src/enum'; +import { SourceType, SystemMetadataKey } from 'src/enum'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; @@ -241,18 +241,18 @@ describe(PersonService.name, () => { }); it("should update a person's name", async () => { - personMock.update.mockResolvedValue(personStub.withName); + personMock.update.mockResolvedValue([personStub.withName]); personMock.getAssets.mockResolvedValue([assetStub.image]); accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1'])); await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).resolves.toEqual(responseDto); - expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' }); + expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', name: 'Person 1' }]); expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1'])); }); it("should update a person's date of birth", async () => { - personMock.update.mockResolvedValue(personStub.withBirthDate); + personMock.update.mockResolvedValue([personStub.withBirthDate]); personMock.getAssets.mockResolvedValue([assetStub.image]); accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1'])); @@ -264,25 +264,25 @@ describe(PersonService.name, () => { isHidden: false, updatedAt: expect.any(Date), }); - expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: '1976-06-30' }); + expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', birthDate: '1976-06-30' }]); expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queueAll).not.toHaveBeenCalled(); expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1'])); }); it('should update a person visibility', async () => { - personMock.update.mockResolvedValue(personStub.withName); + personMock.update.mockResolvedValue([personStub.withName]); personMock.getAssets.mockResolvedValue([assetStub.image]); accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1'])); await expect(sut.update(authStub.admin, 'person-1', { isHidden: false })).resolves.toEqual(responseDto); - expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false }); + expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', isHidden: false }]); expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1'])); }); it("should update a person's thumbnailPath", async () => { - personMock.update.mockResolvedValue(personStub.withName); + personMock.update.mockResolvedValue([personStub.withName]); personMock.getFacesByIds.mockResolvedValue([faceStub.face1]); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1'])); @@ -291,7 +291,7 @@ describe(PersonService.name, () => { sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }), ).resolves.toEqual(responseDto); - expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', faceAssetId: faceStub.face1.id }); + expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', faceAssetId: faceStub.face1.id }]); expect(personMock.getFacesByIds).toHaveBeenCalledWith([ { assetId: faceStub.face1.assetId, @@ -441,11 +441,11 @@ describe(PersonService.name, () => { describe('createPerson', () => { it('should create a new person', async () => { - personMock.create.mockResolvedValue(personStub.primaryPerson); + personMock.create.mockResolvedValue([personStub.primaryPerson]); await expect(sut.create(authStub.admin, {})).resolves.toBe(personStub.primaryPerson); - expect(personMock.create).toHaveBeenCalledWith({ ownerId: authStub.admin.user.id }); + expect(personMock.create).toHaveBeenCalledWith([{ ownerId: authStub.admin.user.id }]); }); }); @@ -496,6 +496,7 @@ describe(PersonService.name, () => { items: [personStub.withName], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([]); await sut.handleQueueDetectFaces({ force: true }); @@ -510,7 +511,7 @@ describe(PersonService.name, () => { it('should delete existing people and faces if forced', async () => { personMock.getAll.mockResolvedValue({ - items: [faceStub.face1.person], + items: [faceStub.face1.person, personStub.randomPerson], hasNextPage: false, }); personMock.getAllFaces.mockResolvedValue({ @@ -521,6 +522,7 @@ describe(PersonService.name, () => { items: [assetStub.image], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([personStub.randomPerson]); await sut.handleQueueDetectFaces({ force: true }); @@ -531,8 +533,8 @@ describe(PersonService.name, () => { data: { id: assetStub.image.id }, }, ]); - expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]); - expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath); + expect(personMock.delete).toHaveBeenCalledWith([personStub.randomPerson]); + expect(storageMock.unlink).toHaveBeenCalledWith(personStub.randomPerson.thumbnailPath); }); }); @@ -561,10 +563,14 @@ describe(PersonService.name, () => { items: [faceStub.face1], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([]); await sut.handleQueueRecognizeFaces({}); - expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { personId: IsNull() } }); + expect(personMock.getAllFaces).toHaveBeenCalledWith( + { skip: 0, take: 1000 }, + { where: { personId: IsNull(), sourceType: IsNull() } }, + ); expect(jobMock.queueAll).toHaveBeenCalledWith([ { name: JobName.FACIAL_RECOGNITION, @@ -586,6 +592,7 @@ describe(PersonService.name, () => { items: [faceStub.face1], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([]); await sut.handleQueueRecognizeFaces({ force: true }); @@ -616,6 +623,8 @@ describe(PersonService.name, () => { items: [faceStub.face1], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([]); + await sut.handleQueueRecognizeFaces({ force: true, nightly: true }); expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE); @@ -641,6 +650,7 @@ describe(PersonService.name, () => { items: [faceStub.face1], hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([]); await sut.handleQueueRecognizeFaces({ force: true, nightly: true }); @@ -654,7 +664,7 @@ describe(PersonService.name, () => { it('should delete existing people and faces if forced', async () => { jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 }); personMock.getAll.mockResolvedValue({ - items: [faceStub.face1.person], + items: [faceStub.face1.person, personStub.randomPerson], hasNextPage: false, }); personMock.getAllFaces.mockResolvedValue({ @@ -662,17 +672,19 @@ describe(PersonService.name, () => { hasNextPage: false, }); + personMock.getAllWithoutFaces.mockResolvedValue([personStub.randomPerson]); + await sut.handleQueueRecognizeFaces({ force: true }); - expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {}); + expect(personMock.deleteAllFaces).toHaveBeenCalledWith({ sourceType: SourceType.MACHINE_LEARNING }); expect(jobMock.queueAll).toHaveBeenCalledWith([ { name: JobName.FACIAL_RECOGNITION, data: { id: faceStub.face1.id, deferred: false }, }, ]); - expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]); - expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath); + expect(personMock.delete).toHaveBeenCalledWith([personStub.randomPerson]); + expect(storageMock.unlink).toHaveBeenCalledWith(personStub.randomPerson.thumbnailPath); }); }); @@ -807,7 +819,7 @@ describe(PersonService.name, () => { systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); searchMock.searchFaces.mockResolvedValue(faces); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); - personMock.create.mockResolvedValue(faceStub.primaryFace1.person); + personMock.create.mockResolvedValue([faceStub.primaryFace1.person]); await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id }); @@ -832,14 +844,16 @@ describe(PersonService.name, () => { systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); searchMock.searchFaces.mockResolvedValue(faces); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); - personMock.create.mockResolvedValue(personStub.withName); + personMock.create.mockResolvedValue([personStub.withName]); await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id }); - expect(personMock.create).toHaveBeenCalledWith({ - ownerId: faceStub.noPerson1.asset.ownerId, - faceAssetId: faceStub.noPerson1.id, - }); + expect(personMock.create).toHaveBeenCalledWith([ + { + ownerId: faceStub.noPerson1.asset.ownerId, + faceAssetId: faceStub.noPerson1.id, + }, + ]); expect(personMock.reassignFaces).toHaveBeenCalledWith({ faceIds: [faceStub.noPerson1.id], newPersonId: personStub.withName.id, @@ -851,7 +865,7 @@ describe(PersonService.name, () => { searchMock.searchFaces.mockResolvedValue(faces); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); - personMock.create.mockResolvedValue(personStub.withName); + personMock.create.mockResolvedValue([personStub.withName]); await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id }); @@ -870,7 +884,7 @@ describe(PersonService.name, () => { systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); searchMock.searchFaces.mockResolvedValue(faces); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); - personMock.create.mockResolvedValue(personStub.withName); + personMock.create.mockResolvedValue([personStub.withName]); await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id }); @@ -892,7 +906,7 @@ describe(PersonService.name, () => { systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); searchMock.searchFaces.mockResolvedValueOnce(faces).mockResolvedValueOnce([]); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); - personMock.create.mockResolvedValue(personStub.withName); + personMock.create.mockResolvedValue([personStub.withName]); await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id, deferred: true }); @@ -965,10 +979,12 @@ describe(PersonService.name, () => { processInvalidImages: false, }, ); - expect(personMock.update).toHaveBeenCalledWith({ - id: 'person-1', - thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', - }); + expect(personMock.update).toHaveBeenCalledWith([ + { + id: 'person-1', + thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', + }, + ]); }); it('should generate a thumbnail without going negative', async () => { @@ -1087,7 +1103,7 @@ describe(PersonService.name, () => { it('should merge two people with smart merge', async () => { personMock.getById.mockResolvedValueOnce(personStub.randomPerson); personMock.getById.mockResolvedValueOnce(personStub.primaryPerson); - personMock.update.mockResolvedValue({ ...personStub.randomPerson, name: personStub.primaryPerson.name }); + personMock.update.mockResolvedValue([{ ...personStub.randomPerson, name: personStub.primaryPerson.name }]); accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-3'])); accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1'])); @@ -1100,10 +1116,12 @@ describe(PersonService.name, () => { oldPersonId: personStub.primaryPerson.id, }); - expect(personMock.update).toHaveBeenCalledWith({ - id: personStub.randomPerson.id, - name: personStub.primaryPerson.name, - }); + expect(personMock.update).toHaveBeenCalledWith([ + { + id: personStub.randomPerson.id, + name: personStub.primaryPerson.name, + }, + ]); expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1'])); }); @@ -1177,6 +1195,7 @@ describe(PersonService.name, () => { id: faceStub.face1.id, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, person: mapPerson(personStub.withName), }); }); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 6f2283b72c..c4b5df5719 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -25,7 +25,7 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { PersonPathType } from 'src/entities/move.entity'; import { PersonEntity } from 'src/entities/person.entity'; -import { AssetType, Permission, SystemMetadataKey } from 'src/enum'; +import { AssetType, Permission, SourceType, SystemMetadataKey } from 'src/enum'; import { IAccessRepository } from 'src/interfaces/access.interface'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; @@ -53,7 +53,7 @@ import { checkAccess, requireAccess } from 'src/utils/access'; import { getAssetFiles } from 'src/utils/asset.util'; import { CacheControl, ImmichFileResponse } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; -import { isFacialRecognitionEnabled } from 'src/utils/misc'; +import { isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc'; import { usePagination } from 'src/utils/pagination'; import { IsNull } from 'typeorm'; @@ -173,10 +173,7 @@ export class PersonService { const assetFace = await this.repository.getRandomFace(personId); if (assetFace !== null) { - await this.repository.update({ - id: personId, - faceAssetId: assetFace.id, - }); + await this.repository.update([{ id: personId, faceAssetId: assetFace.id }]); jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: personId } }); } } @@ -214,13 +211,16 @@ export class PersonService { return assets.map((asset) => mapAsset(asset)); } - create(auth: AuthDto, dto: PersonCreateDto): Promise { - return this.repository.create({ - ownerId: auth.user.id, - name: dto.name, - birthDate: dto.birthDate, - isHidden: dto.isHidden, - }); + async create(auth: AuthDto, dto: PersonCreateDto): Promise { + const [created] = await this.repository.create([ + { + ownerId: auth.user.id, + name: dto.name, + birthDate: dto.birthDate, + isHidden: dto.isHidden, + }, + ]); + return created; } async update(auth: AuthDto, id: string, dto: PersonUpdateDto): Promise { @@ -239,7 +239,7 @@ export class PersonService { faceId = face.id; } - const person = await this.repository.update({ id, faceAssetId: faceId, name, birthDate, isHidden }); + const [person] = await this.repository.update([{ id, faceAssetId: faceId, name, birthDate, isHidden }]); if (assetId) { await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id } }); @@ -296,8 +296,8 @@ export class PersonService { } if (force) { - await this.deleteAllPeople(); - await this.repository.deleteAllFaces(); + await this.repository.deleteAllFaces({ sourceType: SourceType.MACHINE_LEARNING }); + await this.handlePersonCleanup(); } const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { @@ -339,11 +339,7 @@ export class PersonService { return JobStatus.FAILED; } - if (!asset.isVisible) { - return JobStatus.SKIPPED; - } - - if (!asset.isVisible) { + if (!asset.isVisible || asset.faces.length > 0) { return JobStatus.SKIPPED; } @@ -408,7 +404,8 @@ export class PersonService { const { waiting } = await this.jobRepository.getJobCounts(QueueName.FACIAL_RECOGNITION); if (force) { - await this.deleteAllPeople(); + await this.repository.deleteAllFaces({ sourceType: SourceType.MACHINE_LEARNING }); + await this.handlePersonCleanup(); } else if (waiting) { this.logger.debug( `Skipping facial recognition queueing because ${waiting} job${waiting > 1 ? 's are' : ' is'} already queued`, @@ -418,7 +415,9 @@ export class PersonService { const lastRun = new Date().toISOString(); const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => - this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }), + this.repository.getAllFaces(pagination, { + where: force ? undefined : { personId: IsNull(), sourceType: IsNull() }, + }), ); for await (const page of facePagination) { @@ -441,13 +440,18 @@ export class PersonService { const face = await this.repository.getFaceByIdWithAssets( id, { person: true, asset: true, faceSearch: true }, - { id: true, personId: true, faceSearch: { embedding: true } }, + { id: true, personId: true, sourceType: true, faceSearch: { embedding: true } }, ); if (!face || !face.asset) { this.logger.warn(`Face ${id} not found`); return JobStatus.FAILED; } + if (face.sourceType !== SourceType.MACHINE_LEARNING) { + this.logger.warn(`Skipping face ${id} due to source ${face.sourceType}`); + return JobStatus.SKIPPED; + } + if (!face.faceSearch?.embedding) { this.logger.warn(`Face ${id} does not have an embedding`); return JobStatus.FAILED; @@ -497,7 +501,7 @@ export class PersonService { if (isCore && !personId) { this.logger.log(`Creating new person for face ${id}`); - const newPerson = await this.repository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id }); + const [newPerson] = await this.repository.create([{ ownerId: face.asset.ownerId, faceAssetId: face.id }]); await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } }); personId = newPerson.id; } @@ -522,8 +526,8 @@ export class PersonService { } async handleGeneratePersonThumbnail(data: IEntityJob): Promise { - const { machineLearning, image } = await this.configCore.getConfig({ withCache: true }); - if (!isFacialRecognitionEnabled(machineLearning)) { + const { machineLearning, metadata, image } = await this.configCore.getConfig({ withCache: true }); + if (!isFacialRecognitionEnabled(machineLearning) && !isFaceImportEnabled(metadata)) { return JobStatus.SKIPPED; } @@ -573,7 +577,7 @@ export class PersonService { } as const; await this.mediaRepository.generateThumbnail(inputPath, thumbnailPath, thumbnailOptions); - await this.repository.update({ id: person.id, thumbnailPath }); + await this.repository.update([{ id: person.id, thumbnailPath }]); return JobStatus.SUCCESS; } @@ -620,7 +624,7 @@ export class PersonService { } if (Object.keys(update).length > 0) { - primaryPerson = await this.repository.update({ id: primaryPerson.id, ...update }); + [primaryPerson] = await this.repository.update([{ id: primaryPerson.id, ...update }]); } const mergeName = mergePerson.name || mergePerson.id; diff --git a/server/src/services/server.service.spec.ts b/server/src/services/server.service.spec.ts index 799ec2c5a3..ac899f7b13 100644 --- a/server/src/services/server.service.spec.ts +++ b/server/src/services/server.service.spec.ts @@ -160,6 +160,7 @@ describe(ServerService.name, () => { smartSearch: true, duplicateDetection: true, facialRecognition: true, + importFaces: false, map: true, reverseGeocoding: true, oauth: false, diff --git a/server/src/services/server.service.ts b/server/src/services/server.service.ts index 5ea8a3e459..e57a206765 100644 --- a/server/src/services/server.service.ts +++ b/server/src/services/server.service.ts @@ -90,7 +90,7 @@ export class ServerService { } async getFeatures(): Promise { - const { reverseGeocoding, map, machineLearning, trash, oauth, passwordLogin, notifications } = + const { reverseGeocoding, metadata, map, machineLearning, trash, oauth, passwordLogin, notifications } = await this.configCore.getConfig({ withCache: false }); return { @@ -99,6 +99,7 @@ export class ServerService { duplicateDetection: isDuplicateDetectionEnabled(machineLearning), map: map.enabled, reverseGeocoding: reverseGeocoding.enabled, + importFaces: metadata.faces.import, sidecar: true, search: true, trash: trash.enabled, diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index bb0e706d61..af2b564ab2 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -74,6 +74,11 @@ const updatedConfig = Object.freeze({ enabled: true, level: LogLevel.LOG, }, + metadata: { + faces: { + import: false, + }, + }, machineLearning: { enabled: true, url: 'http://immich-machine-learning:3003', diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 6063b4925c..47f3f552c4 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -64,6 +64,7 @@ export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machin isMachineLearningEnabled(machineLearning) && machineLearning.facialRecognition.enabled; export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) => isSmartSearchEnabled(machineLearning) && machineLearning.duplicateDetection.enabled; +export const isFaceImportEnabled = (metadata: SystemConfig['metadata']) => metadata.faces.import; export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED'; diff --git a/server/test/fixtures/face.stub.ts b/server/test/fixtures/face.stub.ts index 82935dd345..27ca2a4356 100644 --- a/server/test/fixtures/face.stub.ts +++ b/server/test/fixtures/face.stub.ts @@ -1,4 +1,5 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity'; +import { SourceType } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { personStub } from 'test/fixtures/person.stub'; @@ -17,6 +18,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId1', embedding: [1, 2, 3, 4] }, }), primaryFace1: Object.freeze>({ @@ -31,6 +33,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId2', embedding: [1, 2, 3, 4] }, }), mergeFace1: Object.freeze>({ @@ -45,6 +48,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId3', embedding: [1, 2, 3, 4] }, }), mergeFace2: Object.freeze>({ @@ -59,6 +63,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId4', embedding: [1, 2, 3, 4] }, }), start: Object.freeze>({ @@ -73,6 +78,7 @@ export const faceStub = { boundingBoxY2: 505, imageHeight: 2880, imageWidth: 2160, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId5', embedding: [1, 2, 3, 4] }, }), middle: Object.freeze>({ @@ -87,6 +93,7 @@ export const faceStub = { boundingBoxY2: 200, imageHeight: 500, imageWidth: 400, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId6', embedding: [1, 2, 3, 4] }, }), end: Object.freeze>({ @@ -101,6 +108,7 @@ export const faceStub = { boundingBoxY2: 495, imageHeight: 500, imageWidth: 500, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId7', embedding: [1, 2, 3, 4] }, }), noPerson1: Object.freeze({ @@ -115,6 +123,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId8', embedding: [1, 2, 3, 4] }, }), noPerson2: Object.freeze({ @@ -129,6 +138,7 @@ export const faceStub = { boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + sourceType: SourceType.MACHINE_LEARNING, faceSearch: { faceId: 'assetFaceId9', embedding: [1, 2, 3, 4] }, }), }; diff --git a/server/test/fixtures/metadata.stub.ts b/server/test/fixtures/metadata.stub.ts new file mode 100644 index 0000000000..05535303e4 --- /dev/null +++ b/server/test/fixtures/metadata.stub.ts @@ -0,0 +1,71 @@ +import { ImmichTags } from 'src/interfaces/metadata.interface'; +import { personStub } from 'test/fixtures/person.stub'; + +export const metadataStub = { + empty: Object.freeze({}), + withFace: Object.freeze({ + RegionInfo: { + AppliedToDimensions: { + W: 100, + H: 100, + Unit: 'normalized', + }, + RegionList: [ + { + Type: 'face', + Name: personStub.withName.name, + Area: { + X: 0.05, + Y: 0.05, + W: 0.1, + H: 0.1, + Unit: 'normalized', + }, + }, + ], + }, + }), + withFaceEmptyName: Object.freeze({ + RegionInfo: { + AppliedToDimensions: { + W: 100, + H: 100, + Unit: 'normalized', + }, + RegionList: [ + { + Type: 'face', + Name: '', + Area: { + X: 0.05, + Y: 0.05, + W: 0.1, + H: 0.1, + Unit: 'normalized', + }, + }, + ], + }, + }), + withFaceNoName: Object.freeze({ + RegionInfo: { + AppliedToDimensions: { + W: 100, + H: 100, + Unit: 'normalized', + }, + RegionList: [ + { + Type: 'face', + Area: { + X: 0.05, + Y: 0.05, + W: 0.1, + H: 0.1, + Unit: 'normalized', + }, + }, + ], + }, + }), +}; diff --git a/server/test/repositories/person.repository.mock.ts b/server/test/repositories/person.repository.mock.ts index 94a4486c81..6547a54339 100644 --- a/server/test/repositories/person.repository.mock.ts +++ b/server/test/repositories/person.repository.mock.ts @@ -10,6 +10,7 @@ export const newPersonRepositoryMock = (): Mocked => { getAllWithoutFaces: vitest.fn(), getByName: vitest.fn(), + getDistinctNames: vitest.fn(), create: vitest.fn(), update: vitest.fn(), @@ -24,6 +25,7 @@ export const newPersonRepositoryMock = (): Mocked => { reassignFaces: vitest.fn(), createFaces: vitest.fn(), + replaceFaces: vitest.fn(), getFaces: vitest.fn(), reassignFace: vitest.fn(), getFaceById: vitest.fn(), diff --git a/web/src/lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte b/web/src/lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte new file mode 100644 index 0000000000..c28050e022 --- /dev/null +++ b/web/src/lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte @@ -0,0 +1,38 @@ + + +
+
+
+
+ +
+ + onReset({ ...options, configKeys: ['metadata'] })} + onSave={() => onSave({ metadata: config.metadata })} + showResetToDefault={!isEqual(savedConfig.metadata.faces.import, defaultConfig.metadata.faces.import)} + {disabled} + /> + +
+
diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.json index 25f3b6ea2f..113998dc89 100644 --- a/web/src/lib/i18n/en.json +++ b/web/src/lib/i18n/en.json @@ -137,7 +137,11 @@ "map_settings_description": "Manage map settings", "map_style_description": "URL to a style.json map theme", "metadata_extraction_job": "Extract metadata", - "metadata_extraction_job_description": "Extract metadata information from each asset, such as GPS and resolution", + "metadata_extraction_job_description": "Extract metadata information from each asset, such as GPS, faces and resolution", + "metadata_faces_import_setting": "Enable face import", + "metadata_faces_import_setting_description": "Import faces from image EXIF data and sidecar files", + "metadata_settings": "Metadata Settings", + "metadata_settings_description": "Manage metadata settings", "migration_job": "Migration", "migration_job_description": "Migrate thumbnails for assets and faces to the latest folder structure", "no_paths_added": "No paths added", diff --git a/web/src/lib/stores/server-config.store.ts b/web/src/lib/stores/server-config.store.ts index 1d3c4bc00e..14d1e4e66e 100644 --- a/web/src/lib/stores/server-config.store.ts +++ b/web/src/lib/stores/server-config.store.ts @@ -8,6 +8,7 @@ export const featureFlags = writable({ smartSearch: true, duplicateDetection: false, facialRecognition: true, + importFaces: false, sidecar: true, map: true, reverseGeocoding: true, diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index 0555bab256..d03865cb39 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -4,6 +4,7 @@ import FFmpegSettings from '$lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte'; import ImageSettings from '$lib/components/admin-page/settings/image/image-settings.svelte'; import JobSettings from '$lib/components/admin-page/settings/job-settings/job-settings.svelte'; + import MetadataSettings from '$lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte'; import LibrarySettings from '$lib/components/admin-page/settings/library-settings/library-settings.svelte'; import LoggingSettings from '$lib/components/admin-page/settings/logging-settings/logging-settings.svelte'; import MachineLearningSettings from '$lib/components/admin-page/settings/machine-learning-settings/machine-learning-settings.svelte'; @@ -86,6 +87,12 @@ subtitle: $t('admin.job_settings_description'), key: 'job', }, + { + component: MetadataSettings, + title: $t('admin.metadata_settings'), + subtitle: $t('admin.metadata_settings_description'), + key: 'metadata', + }, { component: LibrarySettings, title: $t('admin.library_settings'),