feat(server): custom library scanning interval (#4390)

* add automatic library scan config options

* add validation

* open api

* use CronJob instead of cron-validator

* fix tests

* catch potential error of the library scan initialization

* better description for input field

* move library scan job initialization to server app service

* fix tests

* add comments to all parameters of cronjob contructor

* make scan a child of a more general library object

* open api

* chore: cleanup

* move cronjob handling to job repoistory

* web: select for common cron expressions

* fix open api

* fix tests

* put scanning settings in nested accordion

* fix system config validation

* refactor, tests

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Daniel Dietzler 2023-10-31 21:19:12 +01:00 committed by GitHub
parent 088d5addf2
commit cd375a976e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 786 additions and 114 deletions

View File

@ -3283,6 +3283,12 @@ export interface SystemConfigDto {
* @memberof SystemConfigDto
*/
'job': SystemConfigJobDto;
/**
*
* @type {SystemConfigLibraryDto}
* @memberof SystemConfigDto
*/
'library': SystemConfigLibraryDto;
/**
*
* @type {SystemConfigMachineLearningDto}
@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto {
*/
'videoConversion': JobSettingsDto;
}
/**
*
* @export
* @interface SystemConfigLibraryDto
*/
export interface SystemConfigLibraryDto {
/**
*
* @type {SystemConfigLibraryScanDto}
* @memberof SystemConfigLibraryDto
*/
'scan': SystemConfigLibraryScanDto;
}
/**
*
* @export
* @interface SystemConfigLibraryScanDto
*/
export interface SystemConfigLibraryScanDto {
/**
*
* @type {string}
* @memberof SystemConfigLibraryScanDto
*/
'cronExpression': string;
/**
*
* @type {boolean}
* @memberof SystemConfigLibraryScanDto
*/
'enabled': boolean;
}
/**
*
* @export

View File

@ -128,6 +128,8 @@ doc/SystemConfigApi.md
doc/SystemConfigDto.md
doc/SystemConfigFFmpegDto.md
doc/SystemConfigJobDto.md
doc/SystemConfigLibraryDto.md
doc/SystemConfigLibraryScanDto.md
doc/SystemConfigMachineLearningDto.md
doc/SystemConfigMapDto.md
doc/SystemConfigNewVersionCheckDto.md
@ -296,6 +298,8 @@ lib/model/smart_info_response_dto.dart
lib/model/system_config_dto.dart
lib/model/system_config_f_fmpeg_dto.dart
lib/model/system_config_job_dto.dart
lib/model/system_config_library_dto.dart
lib/model/system_config_library_scan_dto.dart
lib/model/system_config_machine_learning_dto.dart
lib/model/system_config_map_dto.dart
lib/model/system_config_new_version_check_dto.dart
@ -451,6 +455,8 @@ test/system_config_api_test.dart
test/system_config_dto_test.dart
test/system_config_f_fmpeg_dto_test.dart
test/system_config_job_dto_test.dart
test/system_config_library_dto_test.dart
test/system_config_library_scan_dto_test.dart
test/system_config_machine_learning_dto_test.dart
test/system_config_map_dto_test.dart
test/system_config_new_version_check_dto_test.dart

View File

@ -311,6 +311,8 @@ Class | Method | HTTP request | Description
- [SystemConfigDto](doc//SystemConfigDto.md)
- [SystemConfigFFmpegDto](doc//SystemConfigFFmpegDto.md)
- [SystemConfigJobDto](doc//SystemConfigJobDto.md)
- [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md)
- [SystemConfigLibraryScanDto](doc//SystemConfigLibraryScanDto.md)
- [SystemConfigMachineLearningDto](doc//SystemConfigMachineLearningDto.md)
- [SystemConfigMapDto](doc//SystemConfigMapDto.md)
- [SystemConfigNewVersionCheckDto](doc//SystemConfigNewVersionCheckDto.md)

View File

@ -10,6 +10,7 @@ Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**ffmpeg** | [**SystemConfigFFmpegDto**](SystemConfigFFmpegDto.md) | |
**job** | [**SystemConfigJobDto**](SystemConfigJobDto.md) | |
**library_** | [**SystemConfigLibraryDto**](SystemConfigLibraryDto.md) | |
**machineLearning** | [**SystemConfigMachineLearningDto**](SystemConfigMachineLearningDto.md) | |
**map** | [**SystemConfigMapDto**](SystemConfigMapDto.md) | |
**newVersionCheck** | [**SystemConfigNewVersionCheckDto**](SystemConfigNewVersionCheckDto.md) | |

View File

@ -0,0 +1,15 @@
# openapi.model.SystemConfigLibraryDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**scan** | [**SystemConfigLibraryScanDto**](SystemConfigLibraryScanDto.md) | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

View File

@ -0,0 +1,16 @@
# openapi.model.SystemConfigLibraryScanDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**cronExpression** | **String** | |
**enabled** | **bool** | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

View File

@ -156,6 +156,8 @@ part 'model/smart_info_response_dto.dart';
part 'model/system_config_dto.dart';
part 'model/system_config_f_fmpeg_dto.dart';
part 'model/system_config_job_dto.dart';
part 'model/system_config_library_dto.dart';
part 'model/system_config_library_scan_dto.dart';
part 'model/system_config_machine_learning_dto.dart';
part 'model/system_config_map_dto.dart';
part 'model/system_config_new_version_check_dto.dart';

View File

@ -403,6 +403,10 @@ class ApiClient {
return SystemConfigFFmpegDto.fromJson(value);
case 'SystemConfigJobDto':
return SystemConfigJobDto.fromJson(value);
case 'SystemConfigLibraryDto':
return SystemConfigLibraryDto.fromJson(value);
case 'SystemConfigLibraryScanDto':
return SystemConfigLibraryScanDto.fromJson(value);
case 'SystemConfigMachineLearningDto':
return SystemConfigMachineLearningDto.fromJson(value);
case 'SystemConfigMapDto':

View File

@ -15,6 +15,7 @@ class SystemConfigDto {
SystemConfigDto({
required this.ffmpeg,
required this.job,
required this.library_,
required this.machineLearning,
required this.map,
required this.newVersionCheck,
@ -31,6 +32,8 @@ class SystemConfigDto {
SystemConfigJobDto job;
SystemConfigLibraryDto library_;
SystemConfigMachineLearningDto machineLearning;
SystemConfigMapDto map;
@ -55,6 +58,7 @@ class SystemConfigDto {
bool operator ==(Object other) => identical(this, other) || other is SystemConfigDto &&
other.ffmpeg == ffmpeg &&
other.job == job &&
other.library_ == library_ &&
other.machineLearning == machineLearning &&
other.map == map &&
other.newVersionCheck == newVersionCheck &&
@ -71,6 +75,7 @@ class SystemConfigDto {
// ignore: unnecessary_parenthesis
(ffmpeg.hashCode) +
(job.hashCode) +
(library_.hashCode) +
(machineLearning.hashCode) +
(map.hashCode) +
(newVersionCheck.hashCode) +
@ -83,12 +88,13 @@ class SystemConfigDto {
(trash.hashCode);
@override
String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, job=$job, machineLearning=$machineLearning, map=$map, newVersionCheck=$newVersionCheck, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, storageTemplate=$storageTemplate, theme=$theme, thumbnail=$thumbnail, trash=$trash]';
String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, job=$job, library_=$library_, machineLearning=$machineLearning, map=$map, newVersionCheck=$newVersionCheck, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, storageTemplate=$storageTemplate, theme=$theme, thumbnail=$thumbnail, trash=$trash]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'ffmpeg'] = this.ffmpeg;
json[r'job'] = this.job;
json[r'library'] = this.library_;
json[r'machineLearning'] = this.machineLearning;
json[r'map'] = this.map;
json[r'newVersionCheck'] = this.newVersionCheck;
@ -112,6 +118,7 @@ class SystemConfigDto {
return SystemConfigDto(
ffmpeg: SystemConfigFFmpegDto.fromJson(json[r'ffmpeg'])!,
job: SystemConfigJobDto.fromJson(json[r'job'])!,
library_: SystemConfigLibraryDto.fromJson(json[r'library'])!,
machineLearning: SystemConfigMachineLearningDto.fromJson(json[r'machineLearning'])!,
map: SystemConfigMapDto.fromJson(json[r'map'])!,
newVersionCheck: SystemConfigNewVersionCheckDto.fromJson(json[r'newVersionCheck'])!,
@ -171,6 +178,7 @@ class SystemConfigDto {
static const requiredKeys = <String>{
'ffmpeg',
'job',
'library',
'machineLearning',
'map',
'newVersionCheck',

View File

@ -0,0 +1,98 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class SystemConfigLibraryDto {
/// Returns a new [SystemConfigLibraryDto] instance.
SystemConfigLibraryDto({
required this.scan,
});
SystemConfigLibraryScanDto scan;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigLibraryDto &&
other.scan == scan;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(scan.hashCode);
@override
String toString() => 'SystemConfigLibraryDto[scan=$scan]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'scan'] = this.scan;
return json;
}
/// Returns a new [SystemConfigLibraryDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static SystemConfigLibraryDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
return SystemConfigLibraryDto(
scan: SystemConfigLibraryScanDto.fromJson(json[r'scan'])!,
);
}
return null;
}
static List<SystemConfigLibraryDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <SystemConfigLibraryDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = SystemConfigLibraryDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, SystemConfigLibraryDto> mapFromJson(dynamic json) {
final map = <String, SystemConfigLibraryDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = SystemConfigLibraryDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of SystemConfigLibraryDto-objects as value to a dart map
static Map<String, List<SystemConfigLibraryDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<SystemConfigLibraryDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = SystemConfigLibraryDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'scan',
};
}

View File

@ -0,0 +1,106 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class SystemConfigLibraryScanDto {
/// Returns a new [SystemConfigLibraryScanDto] instance.
SystemConfigLibraryScanDto({
required this.cronExpression,
required this.enabled,
});
String cronExpression;
bool enabled;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigLibraryScanDto &&
other.cronExpression == cronExpression &&
other.enabled == enabled;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(cronExpression.hashCode) +
(enabled.hashCode);
@override
String toString() => 'SystemConfigLibraryScanDto[cronExpression=$cronExpression, enabled=$enabled]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'cronExpression'] = this.cronExpression;
json[r'enabled'] = this.enabled;
return json;
}
/// Returns a new [SystemConfigLibraryScanDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static SystemConfigLibraryScanDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
return SystemConfigLibraryScanDto(
cronExpression: mapValueOfType<String>(json, r'cronExpression')!,
enabled: mapValueOfType<bool>(json, r'enabled')!,
);
}
return null;
}
static List<SystemConfigLibraryScanDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <SystemConfigLibraryScanDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = SystemConfigLibraryScanDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, SystemConfigLibraryScanDto> mapFromJson(dynamic json) {
final map = <String, SystemConfigLibraryScanDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = SystemConfigLibraryScanDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of SystemConfigLibraryScanDto-objects as value to a dart map
static Map<String, List<SystemConfigLibraryScanDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<SystemConfigLibraryScanDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = SystemConfigLibraryScanDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'cronExpression',
'enabled',
};
}

View File

@ -26,6 +26,11 @@ void main() {
// TODO
});
// SystemConfigLibraryDto library_
test('to test the property `library_`', () async {
// TODO
});
// SystemConfigMachineLearningDto machineLearning
test('to test the property `machineLearning`', () async {
// TODO

View File

@ -0,0 +1,27 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for SystemConfigLibraryDto
void main() {
// final instance = SystemConfigLibraryDto();
group('test SystemConfigLibraryDto', () {
// SystemConfigLibraryScanDto scan
test('to test the property `scan`', () async {
// TODO
});
});
}

View File

@ -0,0 +1,32 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for SystemConfigLibraryScanDto
void main() {
// final instance = SystemConfigLibraryScanDto();
group('test SystemConfigLibraryScanDto', () {
// String cronExpression
test('to test the property `cronExpression`', () async {
// TODO
});
// bool enabled
test('to test the property `enabled`', () async {
// TODO
});
});
}

View File

@ -8061,6 +8061,9 @@
"job": {
"$ref": "#/components/schemas/SystemConfigJobDto"
},
"library": {
"$ref": "#/components/schemas/SystemConfigLibraryDto"
},
"machineLearning": {
"$ref": "#/components/schemas/SystemConfigMachineLearningDto"
},
@ -8104,7 +8107,8 @@
"job",
"thumbnail",
"trash",
"theme"
"theme",
"library"
],
"type": "object"
},
@ -8238,6 +8242,32 @@
],
"type": "object"
},
"SystemConfigLibraryDto": {
"properties": {
"scan": {
"$ref": "#/components/schemas/SystemConfigLibraryScanDto"
}
},
"required": [
"scan"
],
"type": "object"
},
"SystemConfigLibraryScanDto": {
"properties": {
"cronExpression": {
"type": "string"
},
"enabled": {
"type": "boolean"
}
},
"required": [
"enabled",
"cronExpression"
],
"type": "object"
},
"SystemConfigMachineLearningDto": {
"properties": {
"classification": {

110
server/package-lock.json generated
View File

@ -1683,66 +1683,6 @@
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
"integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
"integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
"cpu": [
"arm"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
"integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
"integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
]
},
"node_modules/@nestjs/bull-shared": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz",
@ -6118,15 +6058,6 @@
"exiftool-vendored.pl": "12.67.0"
}
},
"node_modules/exiftool-vendored.exe": {
"version": "12.67.0",
"resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz",
"integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==",
"optional": true,
"os": [
"win32"
]
},
"node_modules/exiftool-vendored.pl": {
"version": "12.67.0",
"resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz",
@ -14300,36 +14231,6 @@
"integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
"integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
"integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
"integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
"integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
"optional": true
},
"@nestjs/bull-shared": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz",
@ -16944,6 +16845,11 @@
"luxon": "^3.2.1"
}
},
"cron-validator": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz",
"integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A=="
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -17608,12 +17514,6 @@
}
}
},
"exiftool-vendored.exe": {
"version": "12.67.0",
"resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz",
"integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==",
"optional": true
},
"exiftool-vendored.pl": {
"version": "12.67.0",
"resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz",

View File

@ -1,6 +1,7 @@
import { applyDecorators } from '@nestjs/common';
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNotEmpty, IsOptional, IsString, IsUUID, ValidateIf, ValidationOptions } from 'class-validator';
import { CronJob } from 'cron';
import { basename, extname } from 'node:path';
import sanitize from 'sanitize-filename';
@ -18,6 +19,16 @@ export function ValidateUUID({ optional, each }: Options = { optional: false, ea
);
}
export function validateCronExpression(expression: string) {
try {
new CronJob(expression, () => {});
} catch (error) {
return false;
}
return true;
}
interface IValue {
value?: string;
}

View File

@ -61,7 +61,6 @@ describe(JobService.name, () => {
[{ name: JobName.PERSON_CLEANUP }],
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
[{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
]);
});
});

View File

@ -153,7 +153,6 @@ export class JobService {
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
}
/**

View File

@ -1,4 +1,4 @@
import { AssetType, LibraryType, UserEntity } from '@app/infra/entities';
import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import {
@ -12,6 +12,7 @@ import {
newJobRepositoryMock,
newLibraryRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
newUserRepositoryMock,
userStub,
} from '@test';
@ -23,8 +24,10 @@ import {
IJobRepository,
ILibraryRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
} from '../repositories';
import { SystemConfigCore } from '../system-config/system-config.core';
import { LibraryService } from './library.service';
describe(LibraryService.name, () => {
@ -32,6 +35,7 @@ describe(LibraryService.name, () => {
let accessMock: IAccessRepositoryMock;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let userMock: jest.Mocked<IUserRepository>;
let jobMock: jest.Mocked<IJobRepository>;
@ -40,6 +44,7 @@ describe(LibraryService.name, () => {
beforeEach(() => {
accessMock = newAccessRepositoryMock();
configMock = newSystemConfigRepositoryMock();
libraryMock = newLibraryRepositoryMock();
userMock = newUserRepositoryMock();
assetMock = newAssetRepositoryMock();
@ -55,13 +60,46 @@ describe(LibraryService.name, () => {
accessMock.library.hasOwnerAccess.mockResolvedValue(true);
sut = new LibraryService(accessMock, assetMock, cryptoMock, jobMock, libraryMock, storageMock, userMock);
sut = new LibraryService(
accessMock,
assetMock,
configMock,
cryptoMock,
jobMock,
libraryMock,
storageMock,
userMock,
);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('init', () => {
it('should init cron job and subscribe to config changes', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.LIBRARY_SCAN_ENABLED, value: true },
{ key: SystemConfigKey.LIBRARY_SCAN_CRON_EXPRESSION, value: '0 0 * * *' },
]);
await sut.init();
expect(configMock.load).toHaveBeenCalled();
expect(jobMock.addCronJob).toHaveBeenCalled();
SystemConfigCore.create(newSystemConfigRepositoryMock(false)).config$.next({
library: {
scan: {
enabled: true,
cronExpression: '0 1 * * *',
},
},
} as SystemConfig);
expect(jobMock.updateCronJob).toHaveBeenCalledWith('libraryScan', '0 1 * * *', true);
});
});
describe('handleQueueAssetRefresh', () => {
it("should not queue assets outside of user's external path", async () => {
const mockLibraryJob: ILibraryRefreshJob = {

View File

@ -7,7 +7,7 @@ import { basename, parse } from 'path';
import { AccessCore, Permission } from '../access';
import { AuthUserDto } from '../auth';
import { mimeTypes } from '../domain.constant';
import { usePagination } from '../domain.util';
import { usePagination, validateCronExpression } from '../domain.util';
import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import {
@ -17,9 +17,11 @@ import {
IJobRepository,
ILibraryRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
WithProperty,
} from '../repositories';
import { SystemConfigCore } from '../system-config';
import {
CreateLibraryDto,
LibraryResponseDto,
@ -33,10 +35,12 @@ import {
export class LibraryService {
readonly logger = new Logger(LibraryService.name);
private access: AccessCore;
private configCore: SystemConfigCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ILibraryRepository) private repository: ILibraryRepository,
@ -44,6 +48,26 @@ export class LibraryService {
@Inject(IUserRepository) private userRepository: IUserRepository,
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.configCore.addValidator((config) => {
if (!validateCronExpression(config.library.scan.cronExpression)) {
throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`);
}
});
}
async init() {
const config = await this.configCore.getConfig();
this.jobRepository.addCronJob(
'libraryScan',
config.library.scan.cronExpression,
() => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }),
config.library.scan.enabled,
);
this.configCore.config$.subscribe((config) => {
this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled);
});
}
async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {

View File

@ -111,6 +111,9 @@ export const IJobRepository = 'IJobRepository';
export interface IJobRepository {
addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void;
addCronJob(name: string, expression: string, onTick: () => void, start?: boolean): void;
updateCronJob(name: string, expression?: string, start?: boolean): void;
deleteCronJob(name: string): void;
setConcurrency(queueName: QueueName, concurrency: number): void;
queue(item: JobItem): Promise<void>;
pause(name: QueueName): Promise<void>;

View File

@ -1,4 +1,5 @@
export * from './system-config-ffmpeg.dto';
export * from './system-config-library.dto';
export * from './system-config-oauth.dto';
export * from './system-config-password-login.dto';
export * from './system-config-storage-template.dto';

View File

@ -0,0 +1,40 @@
import { validateCronExpression } from '@app/domain';
import { Type } from 'class-transformer';
import {
IsBoolean,
IsNotEmpty,
IsObject,
IsString,
Validate,
ValidateIf,
ValidateNested,
ValidatorConstraint,
ValidatorConstraintInterface,
} from 'class-validator';
const isEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;
@ValidatorConstraint({ name: 'cronValidator' })
class CronValidator implements ValidatorConstraintInterface {
validate(expression: string): boolean {
return validateCronExpression(expression);
}
}
export class SystemConfigLibraryScanDto {
@IsBoolean()
enabled!: boolean;
@ValidateIf(isEnabled)
@IsNotEmpty()
@Validate(CronValidator, { message: 'Invalid cron expression' })
@IsString()
cronExpression!: string;
}
export class SystemConfigLibraryDto {
@Type(() => SystemConfigLibraryScanDto)
@ValidateNested()
@IsObject()
scan!: SystemConfigLibraryScanDto;
}

View File

@ -3,6 +3,7 @@ import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
import { SystemConfigJobDto } from './system-config-job.dto';
import { SystemConfigLibraryDto } from './system-config-library.dto';
import { SystemConfigMachineLearningDto } from './system-config-machine-learning.dto';
import { SystemConfigMapDto } from './system-config-map.dto';
import { SystemConfigNewVersionCheckDto } from './system-config-new-version-check.dto';
@ -74,6 +75,11 @@ export class SystemConfigDto implements SystemConfig {
@ValidateNested()
@IsObject()
theme!: SystemConfigThemeDto;
@Type(() => SystemConfigLibraryDto)
@ValidateNested()
@IsObject()
library!: SystemConfigLibraryDto;
}
export function mapConfig(config: SystemConfig): SystemConfigDto {

View File

@ -13,6 +13,7 @@ import {
VideoCodec,
} from '@app/infra/entities';
import { BadRequestException, ForbiddenException, Injectable, Logger } from '@nestjs/common';
import { CronExpression } from '@nestjs/schedule';
import { plainToInstance } from 'class-transformer';
import { validate } from 'class-validator';
import * as _ from 'lodash';
@ -120,6 +121,12 @@ export const defaults = Object.freeze<SystemConfig>({
theme: {
customCss: '',
},
library: {
scan: {
enabled: true,
cronExpression: CronExpression.EVERY_DAY_AT_MIDNIGHT,
},
},
});
export enum FeatureFlag {

View File

@ -121,6 +121,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
theme: {
customCss: '',
},
library: {
scan: {
enabled: true,
cronExpression: '0 0 * * *',
},
},
});
describe(SystemConfigService.name, () => {

View File

@ -1,4 +1,4 @@
import { JobService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
import { JobService, LibraryService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression, Interval } from '@nestjs/schedule';
@ -8,6 +8,7 @@ export class AppService {
constructor(
private jobService: JobService,
private libraryService: LibraryService,
private searchService: SearchService,
private storageService: StorageService,
private serverService: ServerInfoService,
@ -28,6 +29,7 @@ export class AppService {
await this.searchService.init();
await this.serverService.handleVersionCheck();
this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
await this.libraryService.init();
}
async destroy() {

View File

@ -94,6 +94,9 @@ export enum SystemConfigKey {
TRASH_DAYS = 'trash.days',
THEME_CUSTOM_CSS = 'theme.customCss',
LIBRARY_SCAN_ENABLED = 'library.scan.enabled',
LIBRARY_SCAN_CRON_EXPRESSION = 'library.scan.cronExpression',
}
export enum TranscodePolicy {
@ -232,4 +235,10 @@ export interface SystemConfig {
theme: {
customCss: string;
};
library: {
scan: {
enabled: boolean;
cronExpression: string;
};
};
}

View File

@ -2,7 +2,9 @@ import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName,
import { getQueueToken } from '@nestjs/bullmq';
import { Injectable, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { SchedulerRegistry } from '@nestjs/schedule';
import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
import { CronJob, CronTime } from 'cron';
import { bullConfig } from '../infra.config';
@Injectable()
@ -10,7 +12,10 @@ export class JobRepository implements IJobRepository {
private workers: Partial<Record<QueueName, Worker>> = {};
private logger = new Logger(JobRepository.name);
constructor(private moduleRef: ModuleRef) {}
constructor(
private moduleRef: ModuleRef,
private schedulerReqistry: SchedulerRegistry,
) {}
addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise<void>) {
const workerHandler: Processor = async (job: Job) => handler(job as JobItem);
@ -18,6 +23,43 @@ export class JobRepository implements IJobRepository {
this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions);
}
addCronJob(name: string, expression: string, onTick: () => void, start = true): void {
const job = new CronJob(
expression,
onTick,
// function to run onComplete
undefined,
// whether it should start directly
start,
// timezone
undefined,
// context
undefined,
// runOnInit
undefined,
// utcOffset
undefined,
// prevents memory leaking by automatically stopping when the node process finishes
true,
);
this.schedulerReqistry.addCronJob(name, job);
}
updateCronJob(name: string, expression?: string, start?: boolean): void {
const job = this.schedulerReqistry.getCronJob(name);
if (expression) {
job.setTime(new CronTime(expression));
}
if (start !== undefined) {
start ? job.start() : job.stop();
}
}
deleteCronJob(name: string): void {
this.schedulerReqistry.deleteCronJob(name);
}
setConcurrency(queueName: QueueName, concurrency: number) {
const worker = this.workers[queueName];
if (!worker) {

View File

@ -3,6 +3,9 @@ import { IJobRepository } from '@app/domain';
export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
return {
addHandler: jest.fn(),
addCronJob: jest.fn(),
deleteCronJob: jest.fn(),
updateCronJob: jest.fn(),
setConcurrency: jest.fn(),
empty: jest.fn(),
pause: jest.fn(),

View File

@ -49,6 +49,10 @@ export const testApp = {
.overrideProvider(IJobRepository)
.useValue({
addHandler: (_queueName: QueueName, _concurrency: number, handler: JobItemHandler) => (_handler = handler),
addCronJob: jest.fn(),
updateCronJob: jest.fn(),
deleteCronJob: jest.fn(),
validateCronExpression: jest.fn(),
queue: (item: JobItem) => jobs && _handler(item),
resume: jest.fn(),
empty: jest.fn(),

View File

@ -3283,6 +3283,12 @@ export interface SystemConfigDto {
* @memberof SystemConfigDto
*/
'job': SystemConfigJobDto;
/**
*
* @type {SystemConfigLibraryDto}
* @memberof SystemConfigDto
*/
'library': SystemConfigLibraryDto;
/**
*
* @type {SystemConfigMachineLearningDto}
@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto {
*/
'videoConversion': JobSettingsDto;
}
/**
*
* @export
* @interface SystemConfigLibraryDto
*/
export interface SystemConfigLibraryDto {
/**
*
* @type {SystemConfigLibraryScanDto}
* @memberof SystemConfigLibraryDto
*/
'scan': SystemConfigLibraryScanDto;
}
/**
*
* @export
* @interface SystemConfigLibraryScanDto
*/
export interface SystemConfigLibraryScanDto {
/**
*
* @type {string}
* @memberof SystemConfigLibraryScanDto
*/
'cronExpression': string;
/**
*
* @type {boolean}
* @memberof SystemConfigLibraryScanDto
*/
'enabled': boolean;
}
/**
*
* @export

View File

@ -0,0 +1,145 @@
<script lang="ts">
import {
notificationController,
NotificationType,
} from '$lib/components/shared-components/notification/notification';
import { api, SystemConfigLibraryDto } from '@api';
import SettingButtonsRow from '../setting-buttons-row.svelte';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
import SettingSwitch from '../setting-switch.svelte';
import { isEqual } from 'lodash-es';
import { fade } from 'svelte/transition';
import { handleError } from '../../../../utils/handle-error';
import SettingAccordion from '../setting-accordion.svelte';
export let libraryConfig: SystemConfigLibraryDto; // this is the config that is being edited
export let disabled = false;
const cronExpressionOptions = [
{ title: 'Every night at midnight', expression: '0 0 * * *' },
{ title: 'Every night at 2am', expression: '0 2 * * *' },
{ title: 'Every day at 1pm', expression: '0 13 * * *' },
{ title: 'Every 6 hours', expression: '0 */6 * * *' },
];
let savedConfig: SystemConfigLibraryDto;
let defaultConfig: SystemConfigLibraryDto;
async function getConfigs() {
[savedConfig, defaultConfig] = await Promise.all([
api.systemConfigApi.getConfig().then((res) => res.data.library),
api.systemConfigApi.getDefaults().then((res) => res.data.library),
]);
}
async function saveSetting() {
try {
const { data: configs } = await api.systemConfigApi.getConfig();
const result = await api.systemConfigApi.updateConfig({
systemConfigDto: {
...configs,
library: libraryConfig,
},
});
libraryConfig = { ...result.data.library };
savedConfig = { ...result.data.library };
notificationController.show({
message: 'Library settings saved',
type: NotificationType.Info,
});
} catch (e) {
handleError(e, 'Unable to save settings');
}
}
async function reset() {
const { data: resetConfig } = await api.systemConfigApi.getConfig();
libraryConfig = { ...resetConfig.library };
savedConfig = { ...resetConfig.library };
notificationController.show({
message: 'Reset library settings to the recent saved settings',
type: NotificationType.Info,
});
}
async function resetToDefault() {
const { data: configs } = await api.systemConfigApi.getDefaults();
libraryConfig = { ...configs.library };
defaultConfig = { ...configs.library };
notificationController.show({
message: 'Reset library settings to default',
type: NotificationType.Info,
});
}
</script>
<div>
{#await getConfigs() then}
<div in:fade={{ duration: 500 }}>
<SettingAccordion title="Scanning" subtitle="Settings for library scanning" isOpen>
<form autocomplete="off" on:submit|preventDefault>
<div class="ml-4 mt-4 flex flex-col gap-4">
<SettingSwitch
title="ENABLED"
{disabled}
subtitle="Enable automatic library scanning"
bind:checked={libraryConfig.scan.enabled}
/>
<div class="flex flex-col my-2 dark:text-immich-dark-fg">
<label class="text-sm" for="expression-select">Cron Expression Presets</label>
<select
class="p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
disabled={disabled || !libraryConfig.scan.enabled}
name="expression"
id="expression-select"
bind:value={libraryConfig.scan.cronExpression}
>
{#each cronExpressionOptions as { title, expression }}
<option value={expression}>{title}</option>
{/each}
</select>
</div>
<SettingInputField
inputType={SettingInputFieldType.TEXT}
required={true}
disabled={disabled || !libraryConfig.scan.enabled}
label="Cron Expression"
bind:value={libraryConfig.scan.cronExpression}
isEdited={libraryConfig.scan.cronExpression !== savedConfig.scan.cronExpression}
>
<svelte:fragment slot="desc">
<p class="text-sm dark:text-immich-dark-fg">
Set the scanning interval using the cron format. For more information please refer to e.g. <a
href="https://crontab.guru"
class="underline"
target="_blank"
rel="noreferrer">Crontab Guru</a
>
</p>
</svelte:fragment>
</SettingInputField>
</div>
<div class="ml-4">
<SettingButtonsRow
on:reset={reset}
on:save={saveSetting}
on:reset-to-default={resetToDefault}
showResetToDefault={!isEqual(savedConfig, defaultConfig)}
{disabled}
/>
</div>
</form>
</SettingAccordion>
</div>
{/await}
</div>

View File

@ -20,6 +20,7 @@
import Icon from '$lib/components/elements/icon.svelte';
import type { PageData } from './$types';
import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte';
import LibrarySettings from '$lib/components/admin-page/settings/library-settings/library-settings.svelte';
import { mdiAlert, mdiContentCopy, mdiDownload } from '@mdi/js';
export let data: PageData;
@ -69,6 +70,10 @@
<JobSettings disabled={$featureFlags.configFile} jobConfig={configs.job} />
</SettingAccordion>
<SettingAccordion title="Library" subtitle="Manage library settings">
<LibrarySettings disabled={$featureFlags.configFile} libraryConfig={configs.library} />
</SettingAccordion>
<SettingAccordion title="Machine Learning Settings" subtitle="Manage machine learning features and settings">
<MachineLearningSettings disabled={$featureFlags.configFile} machineLearningConfig={configs.machineLearning} />
</SettingAccordion>