cotugno/immich
2
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-15 18:08:48 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): multi archive downloads (#956)

Browse Source
This commit is contained in:
Jason Rasmussen 2022-11-15 10:51:56 -05:00 committed by GitHub
parent b5d75e2016
commit f2f255e6e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 538 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mobile/openapi/README.md
View File

@ -80,6 +80,7 @@ Class | Method | HTTP request | Description
*AssetApi* | [**checkExistingAssets**](doc//AssetApi.md#checkexistingassets) | **POST** /asset/exist |
*AssetApi* | [**deleteAsset**](doc//AssetApi.md#deleteasset) | **DELETE** /asset |
*AssetApi* | [**downloadFile**](doc//AssetApi.md#downloadfile) | **GET** /asset/download |
*AssetApi* | [**downloadLibrary**](doc//AssetApi.md#downloadlibrary) | **GET** /asset/download-library |
*AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset |
*AssetApi* | [**getAssetById**](doc//AssetApi.md#getassetbyid) | **GET** /asset/assetById/{assetId} |
*AssetApi* | [**getAssetByTimeBucket**](doc//AssetApi.md#getassetbytimebucket) | **POST** /asset/time-bucket |

6
mobile/openapi/doc/AlbumApi.md
View File

@ -214,7 +214,7 @@ void (empty response body)
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **downloadArchive**
> Object downloadArchive(albumId)
> Object downloadArchive(albumId, skip)
@ -230,9 +230,10 @@ import 'package:openapi/api.dart';
final api_instance = AlbumApi();
final albumId = albumId_example; // String |
final skip = 8.14; // num |
try {
final result = api_instance.downloadArchive(albumId);
final result = api_instance.downloadArchive(albumId, skip);
print(result);
} catch (e) {
print('Exception when calling AlbumApi->downloadArchive: $e\n');
@ -244,6 +245,7 @@ try {
Name | Type | Description | Notes
------------- | ------------- | ------------- | -------------
**albumId** | **String**| |
**skip** | **num**| | [optional]
### Return type

48
mobile/openapi/doc/AssetApi.md
View File

@ -13,6 +13,7 @@ Method | HTTP request | Description
[**checkExistingAssets**](AssetApi.md#checkexistingassets) | **POST** /asset/exist |
[**deleteAsset**](AssetApi.md#deleteasset) | **DELETE** /asset |
[**downloadFile**](AssetApi.md#downloadfile) | **GET** /asset/download |
[**downloadLibrary**](AssetApi.md#downloadlibrary) | **GET** /asset/download-library |
[**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset |
[**getAssetById**](AssetApi.md#getassetbyid) | **GET** /asset/assetById/{assetId} |
[**getAssetByTimeBucket**](AssetApi.md#getassetbytimebucket) | **POST** /asset/time-bucket |
@ -227,6 +228,53 @@ Name | Type | Description | Notes
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **downloadLibrary**
> Object downloadLibrary(skip)
### Example
```dart
import 'package:openapi/api.dart';
// TODO Configure HTTP Bearer authorization: bearer
// Case 1. Use String Token
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
// Case 2. Use Function which generate token.
// String yourTokenGeneratorFunction() { ... }
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
final api_instance = AssetApi();
final skip = 8.14; // num |
try {
final result = api_instance.downloadLibrary(skip);
print(result);
} catch (e) {
print('Exception when calling AssetApi->downloadLibrary: $e\n');
}
```
### Parameters
Name | Type | Description | Notes
------------- | ------------- | ------------- | -------------
**skip** | **num**| | [optional]
### Return type
[**Object**](Object.md)
### Authorization
[bearer](../README.md#bearer)
### HTTP request headers
- **Content-Type**: Not defined
- **Accept**: application/json
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **getAllAssets**
> List<AssetResponseDto> getAllAssets()

7
mobile/openapi/doc/AssetCountByUserIdResponseDto.md
View File

@ -8,8 +8,11 @@ import 'package:openapi/api.dart';
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**photos** | **int** | |
**videos** | **int** | |
**audio** | **int** | | [default to 0]
**photos** | **int** | | [default to 0]
**videos** | **int** | | [default to 0]
**other** | **int** | | [default to 0]
**total** | **int** | | [default to 0]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

14
mobile/openapi/lib/api/album_api.dart
View File

@ -211,7 +211,9 @@ class AlbumApi {
/// Parameters:
///
/// * [String] albumId (required):
Future<Response> downloadArchiveWithHttpInfo(String albumId,) async {
///
/// * [num] skip:
Future<Response> downloadArchiveWithHttpInfo(String albumId, { num? skip, }) async {
// ignore: prefer_const_declarations
final path = r'/album/{albumId}/download'
.replaceAll('{albumId}', albumId);
@ -223,6 +225,10 @@ class AlbumApi {
final headerParams = <String, String>{};
final formParams = <String, String>{};
if (skip != null) {
queryParams.addAll(_queryParams('', 'skip', skip));
}
const contentTypes = <String>[];
@ -240,8 +246,10 @@ class AlbumApi {
/// Parameters:
///
/// * [String] albumId (required):
Future<Object?> downloadArchive(String albumId,) async {
final response = await downloadArchiveWithHttpInfo(albumId,);
///
/// * [num] skip:
Future<Object?> downloadArchive(String albumId, { num? skip, }) async {
final response = await downloadArchiveWithHttpInfo(albumId, skip: skip, );
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}

51
mobile/openapi/lib/api/asset_api.dart
View File

@ -246,6 +246,57 @@ class AssetApi {
return null;
}
/// Performs an HTTP 'GET /asset/download-library' operation and returns the [Response].
/// Parameters:
///
/// * [num] skip:
Future<Response> downloadLibraryWithHttpInfo({ num? skip, }) async {
// ignore: prefer_const_declarations
final path = r'/asset/download-library';
// ignore: prefer_final_locals
Object? postBody;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
if (skip != null) {
queryParams.addAll(_queryParams('', 'skip', skip));
}
const contentTypes = <String>[];
return apiClient.invokeAPI(
path,
'GET',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Parameters:
///
/// * [num] skip:
Future<Object?> downloadLibrary({ num? skip, }) async {
final response = await downloadLibraryWithHttpInfo( skip: skip, );
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'Object',) as Object;
}
return null;
}
///
///
/// Get all AssetEntity belong to the user

34
mobile/openapi/lib/model/asset_count_by_user_id_response_dto.dart
View File

@ -13,32 +13,50 @@ part of openapi.api;
class AssetCountByUserIdResponseDto {
/// Returns a new [AssetCountByUserIdResponseDto] instance.
AssetCountByUserIdResponseDto({
required this.photos,
required this.videos,
this.audio = 0,
this.photos = 0,
this.videos = 0,
this.other = 0,
this.total = 0,
});
int audio;
int photos;
int videos;
int other;
int total;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetCountByUserIdResponseDto &&
other.audio == audio &&
other.photos == photos &&
other.videos == videos;
other.videos == videos &&
other.other == other &&
other.total == total;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(audio.hashCode) +
(photos.hashCode) +
(videos.hashCode);
(videos.hashCode) +
(other.hashCode) +
(total.hashCode);
@override
String toString() => 'AssetCountByUserIdResponseDto[photos=$photos, videos=$videos]';
String toString() => 'AssetCountByUserIdResponseDto[audio=$audio, photos=$photos, videos=$videos, other=$other, total=$total]';
Map<String, dynamic> toJson() {
final _json = <String, dynamic>{};
_json[r'audio'] = audio;
_json[r'photos'] = photos;
_json[r'videos'] = videos;
_json[r'other'] = other;
_json[r'total'] = total;
return _json;
}
@ -61,8 +79,11 @@ class AssetCountByUserIdResponseDto {
}());
return AssetCountByUserIdResponseDto(
audio: mapValueOfType<int>(json, r'audio')!,
photos: mapValueOfType<int>(json, r'photos')!,
videos: mapValueOfType<int>(json, r'videos')!,
other: mapValueOfType<int>(json, r'other')!,
total: mapValueOfType<int>(json, r'total')!,
);
}
return null;
@ -112,8 +133,11 @@ class AssetCountByUserIdResponseDto {
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'audio',
'photos',
'videos',
'other',
'total',
};
}

19
server/apps/immich/src/api-v1/album/album.controller.ts
View File

@ -27,6 +27,12 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
import { Response as Res } from 'express';
import {
IMMICH_ARCHIVE_COMPLETE,
IMMICH_ARCHIVE_FILE_COUNT,
IMMICH_CONTENT_LENGTH_HINT,
} from '../../constants/download.constant';
import { DownloadDto } from '../asset/dto/download-library.dto';
// TODO might be worth creating a AlbumParamsDto that validates `albumId` instead of using the pipe.
@Authenticated()
@ -119,11 +125,18 @@ export class AlbumController {
async downloadArchive(
@GetAuthUser() authUser: AuthUserDto,
@Param('albumId', new ParseUUIDPipe({ version: '4' })) albumId: string,
@Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
@Response({ passthrough: true }) res: Res,
): Promise<any> {
const { stream, filename, filesize } = await this.albumService.downloadArchive(authUser, albumId);
res.attachment(filename);
res.setHeader('X-Immich-Content-Length-Hint', filesize);
const { stream, fileName, fileSize, fileCount, complete } = await this.albumService.downloadArchive(
authUser,
albumId,
dto,
);
res.attachment(fileName);
res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
return stream;
}
}

6
server/apps/immich/src/api-v1/album/album.module.ts
View File

@ -9,9 +9,13 @@ import { AssetAlbumEntity } from '@app/database/entities/asset-album.entity';
import { UserAlbumEntity } from '@app/database/entities/user-album.entity';
import { AlbumRepository, ALBUM_REPOSITORY } from './album-repository';
import { AssetRepository, ASSET_REPOSITORY } from '../asset/asset-repository';
import { DownloadModule } from '../../modules/download/download.module';
@Module({
imports: [TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity])],
imports: [
TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity]),
DownloadModule,
],
controllers: [AlbumController],
providers: [
AlbumService,

8
server/apps/immich/src/api-v1/album/album.service.spec.ts
View File

@ -6,11 +6,13 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
import { IAssetRepository } from '../asset/asset-repository';
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
import { IAlbumRepository } from './album-repository';
import { DownloadService } from '../../modules/download/download.service';
describe('Album service', () => {
let sut: AlbumService;
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
const authUser: AuthUserDto = Object.freeze({
id: '1111',
@ -142,7 +144,11 @@ describe('Album service', () => {
getExistingAssets: jest.fn(),
};
sut = new AlbumService(albumRepositoryMock, assetRepositoryMock);
downloadServiceMock = {
downloadArchive: jest.fn(),
};
sut = new AlbumService(albumRepositoryMock, assetRepositoryMock, downloadServiceMock as DownloadService);
});
it('creates album', async () => {

46
server/apps/immich/src/api-v1/album/album.service.ts
View File

@ -1,13 +1,4 @@
import {
BadRequestException,
Inject,
Injectable,
NotFoundException,
ForbiddenException,
Logger,
InternalServerErrorException,
StreamableFile,
} from '@nestjs/common';
import { BadRequestException, Inject, Injectable, NotFoundException, ForbiddenException } from '@nestjs/common';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { CreateAlbumDto } from './dto/create-album.dto';
import { AlbumEntity } from '@app/database/entities/album.entity';
@ -21,14 +12,15 @@ import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
import { ASSET_REPOSITORY, IAssetRepository } from '../asset/asset-repository';
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
import { AddAssetsDto } from './dto/add-assets.dto';
import archiver from 'archiver';
import { extname } from 'path';
import { DownloadService } from '../../modules/download/download.service';
import { DownloadDto } from '../asset/dto/download-library.dto';
@Injectable()
export class AlbumService {
constructor(
@Inject(ALBUM_REPOSITORY) private _albumRepository: IAlbumRepository,
@Inject(ASSET_REPOSITORY) private _assetRepository: IAssetRepository,
private downloadService: DownloadService,
) {}
private async _getAlbum({
@ -162,35 +154,11 @@ export class AlbumService {
return this._albumRepository.getCountByUserId(authUser.id);
}
async downloadArchive(authUser: AuthUserDto, albumId: string) {
async downloadArchive(authUser: AuthUserDto, albumId: string, dto: DownloadDto) {
const album = await this._getAlbum({ authUser, albumId, validateIsOwner: false });
if (!album.assets || album.assets.length === 0) {
throw new BadRequestException('Cannot download an empty album.');
}
const assets = (album.assets || []).map((asset) => asset.assetInfo).slice(dto.skip || 0);
try {
const archive = archiver('zip', { store: true });
const stream = new StreamableFile(archive);
let totalSize = 0;
for (const { assetInfo } of album.assets) {
const { originalPath } = assetInfo;
const name = `${assetInfo.exifInfo?.imageName || assetInfo.id}${extname(originalPath)}`;
archive.file(originalPath, { name });
totalSize += Number(assetInfo.exifInfo?.fileSizeInByte || 0);
}
archive.finalize();
return {
stream,
filename: `${album.albumName}.zip`,
filesize: totalSize,
};
} catch (e) {
Logger.error(`Error downloading album ${e}`, 'downloadArchive');
throw new InternalServerErrorException(`Failed to download album ${e}`, 'DownloadArchive');
}
return this.downloadService.downloadArchive(album.albumName, assets);
}
async _checkValidThumbnail(album: AlbumEntity) {

33
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@ -24,7 +24,7 @@ export interface IAssetRepository {
checksum?: Buffer,
): Promise<AssetEntity>;
update(asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
getAllByUserId(userId: string): Promise<AssetEntity[]>;
getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getById(assetId: string): Promise<AssetEntity>;
getLocationsByUserId(userId: string): Promise<CuratedLocationsResponseDto[]>;
@ -81,7 +81,7 @@ export class AssetRepository implements IAssetRepository {
async getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto> {
// Get asset count by AssetType
const res = await this.assetRepository
const items = await this.assetRepository
.createQueryBuilder('asset')
.select(`COUNT(asset.id)`, 'count')
.addSelect(`asset.type`, 'type')
@ -89,14 +89,24 @@ export class AssetRepository implements IAssetRepository {
.groupBy('asset.type')
.getRawMany();
const assetCountByUserId = new AssetCountByUserIdResponseDto(0, 0);
res.map((item) => {
if (item.type === 'IMAGE') {
assetCountByUserId.photos = item.count;
} else if (item.type === 'VIDEO') {
assetCountByUserId.videos = item.count;
}
});
const assetCountByUserId = new AssetCountByUserIdResponseDto();
// asset type to dto property mapping
const map: Record<AssetType, keyof AssetCountByUserIdResponseDto> = {
[AssetType.AUDIO]: 'audio',
[AssetType.IMAGE]: 'photos',
[AssetType.VIDEO]: 'videos',
[AssetType.OTHER]: 'other',
};
for (const item of items) {
const count = Number(item.count) || 0;
const assetType = item.type as AssetType;
const type = map[assetType];
assetCountByUserId[type] = count;
assetCountByUserId.total += count;
}
return assetCountByUserId;
}
@ -207,12 +217,13 @@ export class AssetRepository implements IAssetRepository {
* Get all assets belong to the user on the database
* @param userId
*/
async getAllByUserId(userId: string): Promise<AssetEntity[]> {
async getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]> {
const query = this.assetRepository
.createQueryBuilder('asset')
.where('asset.userId = :userId', { userId: userId })
.andWhere('asset.resizePath is not NULL')
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.skip(skip || 0)
.orderBy('asset.createdAt', 'DESC');
return await query.getMany();

20
server/apps/immich/src/api-v1/asset/asset.controller.ts
View File

@ -52,6 +52,12 @@ import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-use
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { DownloadDto } from './dto/download-library.dto';
import {
IMMICH_ARCHIVE_COMPLETE,
IMMICH_ARCHIVE_FILE_COUNT,
IMMICH_CONTENT_LENGTH_HINT,
} from '../../constants/download.constant';
@Authenticated()
@ApiBearerAuth()
@ -134,6 +140,20 @@ export class AssetController {
return this.assetService.downloadFile(query, res);
}
@Get('/download-library')
async downloadLibrary(
@GetAuthUser() authUser: AuthUserDto,
@Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
@Response({ passthrough: true }) res: Res,
): Promise<any> {
const { stream, fileName, fileSize, fileCount, complete } = await this.assetService.downloadLibrary(authUser, dto);
res.attachment(fileName);
res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
return stream;
}
@Get('/file')
async serveFile(
@Headers() headers: Record<string, string>,

2
server/apps/immich/src/api-v1/asset/asset.module.ts
View File

@ -9,11 +9,13 @@ import { BackgroundTaskService } from '../../modules/background-task/background-
import { CommunicationModule } from '../communication/communication.module';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { AssetRepository, ASSET_REPOSITORY } from './asset-repository';
import { DownloadModule } from '../../modules/download/download.module';
@Module({
imports: [
CommunicationModule,
BackgroundTaskModule,
DownloadModule,
TypeOrmModule.forFeature([AssetEntity]),
BullModule.registerQueue({
name: QueueNameEnum.ASSET_UPLOADED,

13
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@ -7,11 +7,13 @@ import { CreateAssetDto } from './dto/create-asset.dto';
import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group-response.dto';
import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { DownloadService } from '../../modules/download/download.service';
describe('AssetService', () => {
let sui: AssetService;
let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
const authUser: AuthUserDto = Object.freeze({
id: 'user_id_1',
@ -89,7 +91,10 @@ describe('AssetService', () => {
};
const _getAssetCountByUserId = (): AssetCountByUserIdResponseDto => {
const result = new AssetCountByUserIdResponseDto(2, 2);
const result = new AssetCountByUserIdResponseDto();
result.videos = 2;
result.photos = 2;
return result;
};
@ -114,7 +119,11 @@ describe('AssetService', () => {
getExistingAssets: jest.fn(),
};
sui = new AssetService(assetRepositoryMock, a);
downloadServiceMock = {
downloadArchive: jest.fn(),
};
sui = new AssetService(assetRepositoryMock, a, downloadServiceMock as DownloadService);
});
// Currently failing due to calculate checksum from a file

10
server/apps/immich/src/api-v1/asset/asset.service.ts
View File

@ -41,6 +41,8 @@ import { timeUtils } from '@app/common/utils';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { DownloadService } from '../../modules/download/download.service';
import { DownloadDto } from './dto/download-library.dto';
const fileInfo = promisify(stat);
@ -52,6 +54,8 @@ export class AssetService {
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
private downloadService: DownloadService,
) {}
public async createUserAsset(
@ -140,6 +144,12 @@ export class AssetService {
return mapAsset(updatedAsset);
}
public async downloadLibrary(user: AuthUserDto, dto: DownloadDto) {
const assets = await this._assetRepository.getAllByUserId(user.id, dto.skip);
return this.downloadService.downloadArchive(dto.name || `library`, assets);
}
public async downloadFile(query: ServeFileDto, res: Res) {
try {
let fileReadStream = null;

14
server/apps/immich/src/api-v1/asset/dto/download-library.dto.ts Normal file
View File

@ -0,0 +1,14 @@
import { Type } from 'class-transformer';
import { IsNumber, IsOptional, IsPositive, IsString } from 'class-validator';
export class DownloadDto {
@IsOptional()
@IsString()
name = '';
@IsOptional()
@IsPositive()
@IsNumber()
@Type(() => Number)
skip?: number;
}

16
server/apps/immich/src/api-v1/asset/response-dto/asset-count-by-user-id-response.dto.ts
View File

@ -2,13 +2,17 @@ import { ApiProperty } from '@nestjs/swagger';
export class AssetCountByUserIdResponseDto {
@ApiProperty({ type: 'integer' })
photos!: number;
audio = 0;
@ApiProperty({ type: 'integer' })
videos!: number;
photos = 0;
constructor(photos: number, videos: number) {
this.photos = photos;
this.videos = videos;
}
@ApiProperty({ type: 'integer' })
videos = 0;
@ApiProperty({ type: 'integer' })
other = 0;
@ApiProperty({ type: 'integer' })
total = 0;
}

38
server/apps/immich/src/api-v1/server-info/server-info.service.ts
View File

@ -9,6 +9,7 @@ import { Repository } from 'typeorm';
import { InjectRepository } from '@nestjs/typeorm';
import path from 'path';
import { readdirSync, statSync } from 'fs';
import { asHumanReadable } from '../../utils/human-readable.util';
@Injectable()
export class ServerInfoService {
@ -23,9 +24,9 @@ export class ServerInfoService {
const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2);
const serverInfo = new ServerInfoResponseDto();
serverInfo.diskAvailable = ServerInfoService.getHumanReadableString(diskInfo.available);
serverInfo.diskSize = ServerInfoService.getHumanReadableString(diskInfo.total);
serverInfo.diskUse = ServerInfoService.getHumanReadableString(diskInfo.total - diskInfo.free);
serverInfo.diskAvailable = asHumanReadable(diskInfo.available);
serverInfo.diskSize = asHumanReadable(diskInfo.total);
serverInfo.diskUse = asHumanReadable(diskInfo.total - diskInfo.free);
serverInfo.diskAvailableRaw = diskInfo.available;
serverInfo.diskSizeRaw = diskInfo.total;
serverInfo.diskUseRaw = diskInfo.total - diskInfo.free;
@ -33,33 +34,6 @@ export class ServerInfoService {
return serverInfo;
}
private static getHumanReadableString(sizeInByte: number) {
const pepibyte = 1.126 * Math.pow(10, 15);
const tebibyte = 1.1 * Math.pow(10, 12);
const gibibyte = 1.074 * Math.pow(10, 9);
const mebibyte = 1.049 * Math.pow(10, 6);
const kibibyte = 1024;
// Pebibyte
if (sizeInByte >= pepibyte) {
// Pe
return `${(sizeInByte / pepibyte).toFixed(1)}PB`;
} else if (tebibyte <= sizeInByte && sizeInByte < pepibyte) {
// Te
return `${(sizeInByte / tebibyte).toFixed(1)}TB`;
} else if (gibibyte <= sizeInByte && sizeInByte < tebibyte) {
// Gi
return `${(sizeInByte / gibibyte).toFixed(1)}GB`;
} else if (mebibyte <= sizeInByte && sizeInByte < gibibyte) {
// Mega
return `${(sizeInByte / mebibyte).toFixed(1)}MB`;
} else if (kibibyte <= sizeInByte && sizeInByte < mebibyte) {
// Kibi
return `${(sizeInByte / kibibyte).toFixed(1)}KB`;
} else {
return `${sizeInByte}B`;
}
}
async getStats(): Promise<ServerStatsResponseDto> {
const res = await this.assetRepository
.createQueryBuilder('asset')
@ -90,11 +64,11 @@ export class ServerInfoService {
const userDiskUsage = await ServerInfoService.getDirectoryStats(path.join(APP_UPLOAD_LOCATION, userId));
usage.usageRaw = userDiskUsage.size;
usage.objects = userDiskUsage.fileCount;
usage.usage = ServerInfoService.getHumanReadableString(usage.usageRaw);
usage.usage = asHumanReadable(usage.usageRaw);
serverStats.usageRaw += usage.usageRaw;
serverStats.objects += usage.objects;
}
serverStats.usage = ServerInfoService.getHumanReadableString(serverStats.usageRaw);
serverStats.usage = asHumanReadable(serverStats.usageRaw);
serverStats.usageByUser = Array.from(tmpMap.values());
return serverStats;
}

3
server/apps/immich/src/constants/download.constant.ts Normal file
View File

@ -0,0 +1,3 @@
export const IMMICH_CONTENT_LENGTH_HINT = 'X-Immich-Content-Length-Hint';
export const IMMICH_ARCHIVE_FILE_COUNT = 'X-Immich-Archive-File-Count';
export const IMMICH_ARCHIVE_COMPLETE = 'X-Immich-Archive-Complete';

8
server/apps/immich/src/modules/download/download.module.ts Normal file
View File

@ -0,0 +1,8 @@
import { Module } from '@nestjs/common';
import { DownloadService } from './download.service';
@Module({
providers: [DownloadService],
exports: [DownloadService],
})
export class DownloadModule {}

63
server/apps/immich/src/modules/download/download.service.ts Normal file
View File

@ -0,0 +1,63 @@
import { AssetEntity } from '@app/database/entities/asset.entity';
import { BadRequestException, Injectable, InternalServerErrorException, Logger, StreamableFile } from '@nestjs/common';
import archiver from 'archiver';
import { extname } from 'path';
import { asHumanReadable, HumanReadableSize } from '../../utils/human-readable.util';
export interface DownloadArchive {
stream: StreamableFile;
fileName: string;
fileSize: number;
fileCount: number;
complete: boolean;
}
@Injectable()
export class DownloadService {
private readonly logger = new Logger(DownloadService.name);
public async downloadArchive(name: string, assets: AssetEntity[]): Promise<DownloadArchive> {
if (!assets || assets.length === 0) {
throw new BadRequestException('No assets to download.');
}
try {
const archive = archiver('zip', { store: true });
const stream = new StreamableFile(archive);
let totalSize = 0;
let fileCount = 0;
let complete = true;
for (const { id, originalPath, exifInfo } of assets) {
const name = `${exifInfo?.imageName || id}${extname(originalPath)}`;
archive.file(originalPath, { name });
totalSize += Number(exifInfo?.fileSizeInByte || 0);
fileCount++;
// for easier testing, can be changed before merging.
if (totalSize > HumanReadableSize.GB * 20) {
complete = false;
this.logger.log(
`Archive size exceeded after ${fileCount} files, capping at ${totalSize} bytes (${asHumanReadable(
totalSize,
)})`,
);
break;
}
}
archive.finalize();
return {
stream,
fileName: `${name}.zip`,
fileSize: totalSize,
fileCount,
complete,
};
} catch (error) {
this.logger.error(`Error creating download archive ${error}`);
throw new InternalServerErrorException(`Failed to download ${name}: ${error}`, 'DownloadArchive');
}
}
}

31
server/apps/immich/src/utils/human-readable.util.ts Normal file
View File

@ -0,0 +1,31 @@
const KB = 1000;
const MB = KB * 1000;
const GB = MB * 1000;
const TB = GB * 1000;
const PB = TB * 1000;
export const HumanReadableSize = { KB, MB, GB, TB, PB };
export function asHumanReadable(bytes: number, precision = 1) {
if (bytes >= PB) {
return `${(bytes / PB).toFixed(precision)}PB`;
}
if (bytes >= TB) {
return `${(bytes / TB).toFixed(precision)}TB`;
}
if (bytes >= GB) {
return `${(bytes / GB).toFixed(precision)}GB`;
}
if (bytes >= MB) {
return `${(bytes / MB).toFixed(precision)}MB`;
}
if (bytes >= KB) {
return `${(bytes / KB).toFixed(precision)}KB`;
}
return `${bytes}B`;
}

2
server/immich-openapi-specs.json
View File

File diff suppressed because one or more lines are too long

108
web/src/api/open-api/api.ts
View File

@ -294,6 +294,12 @@ export interface AssetCountByTimeBucketResponseDto {
* @interface AssetCountByUserIdResponseDto
*/
export interface AssetCountByUserIdResponseDto {
/**
*
* @type {number}
* @memberof AssetCountByUserIdResponseDto
*/
'audio': number;
/**
*
* @type {number}
@ -306,6 +312,18 @@ export interface AssetCountByUserIdResponseDto {
* @memberof AssetCountByUserIdResponseDto
*/
'videos': number;
/**
*
* @type {number}
* @memberof AssetCountByUserIdResponseDto
*/
'other': number;
/**
*
* @type {number}
* @memberof AssetCountByUserIdResponseDto
*/
'total': number;
}
/**
*
@ -1898,10 +1916,11 @@ export const AlbumApiAxiosParamCreator = function (configuration?: Configuration
/**
*
* @param {string} albumId
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
downloadArchive: async (albumId: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
downloadArchive: async (albumId: string, skip?: number, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
// verify required parameter 'albumId' is not null or undefined
assertParamExists('downloadArchive', 'albumId', albumId)
const localVarPath = `/album/{albumId}/download`
@ -1921,6 +1940,10 @@ export const AlbumApiAxiosParamCreator = function (configuration?: Configuration
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
setSearchParams(localVarUrlObj, localVarQueryParameter);
@ -2227,11 +2250,12 @@ export const AlbumApiFp = function(configuration?: Configuration) {
/**
*
* @param {string} albumId
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async downloadArchive(albumId: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadArchive(albumId, options);
async downloadArchive(albumId: string, skip?: number, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadArchive(albumId, skip, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
@ -2348,11 +2372,12 @@ export const AlbumApiFactory = function (configuration?: Configuration, basePath
/**
*
* @param {string} albumId
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
downloadArchive(albumId: string, options?: any): AxiosPromise<object> {
return localVarFp.downloadArchive(albumId, options).then((request) => request(axios, basePath));
downloadArchive(albumId: string, skip?: number, options?: any): AxiosPromise<object> {
return localVarFp.downloadArchive(albumId, skip, options).then((request) => request(axios, basePath));
},
/**
*
@ -2470,12 +2495,13 @@ export class AlbumApi extends BaseAPI {
/**
*
* @param {string} albumId
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof AlbumApi
*/
public downloadArchive(albumId: string, options?: AxiosRequestConfig) {
return AlbumApiFp(this.configuration).downloadArchive(albumId, options).then((request) => request(this.axios, this.basePath));
public downloadArchive(albumId: string, skip?: number, options?: AxiosRequestConfig) {
return AlbumApiFp(this.configuration).downloadArchive(albumId, skip, options).then((request) => request(this.axios, this.basePath));
}
/**
@ -2722,6 +2748,44 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
*
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
downloadLibrary: async (skip?: number, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
const localVarPath = `/asset/download-library`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
// authentication bearer required
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
@ -3332,6 +3396,16 @@ export const AssetApiFp = function(configuration?: Configuration) {
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadFile(aid, did, isThumb, isWeb, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async downloadLibrary(skip?: number, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadLibrary(skip, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
* Get all AssetEntity belong to the user
* @summary
@ -3527,6 +3601,15 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
downloadFile(aid: string, did: string, isThumb?: boolean, isWeb?: boolean, options?: any): AxiosPromise<object> {
return localVarFp.downloadFile(aid, did, isThumb, isWeb, options).then((request) => request(axios, basePath));
},
/**
*
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
downloadLibrary(skip?: number, options?: any): AxiosPromise<object> {
return localVarFp.downloadLibrary(skip, options).then((request) => request(axios, basePath));
},
/**
* Get all AssetEntity belong to the user
* @summary
@ -3716,6 +3799,17 @@ export class AssetApi extends BaseAPI {
return AssetApiFp(this.configuration).downloadFile(aid, did, isThumb, isWeb, options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {number} [skip]
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof AssetApi
*/
public downloadLibrary(skip?: number, options?: AxiosRequestConfig) {
return AssetApiFp(this.configuration).downloadLibrary(skip, options).then((request) => request(this.axios, this.basePath));
}
/**
* Get all AssetEntity belong to the user
* @summary

88
web/src/lib/components/album-page/album-viewer.svelte
View File

@ -313,53 +313,69 @@
const downloadAlbum = async () => {
try {
const fileName = album.albumName + '.zip';
let skip = 0;
let count = 0;
let done = false;
// If assets is already download -> return;
if ($downloadAssets[fileName]) {
return;
}
while (!done) {
count++;
$downloadAssets[fileName] = 0;
const fileName = album.albumName + `${count === 1 ? '' : count}.zip`;
let total = 0;
const { data, status } = await api.albumApi.downloadArchive(album.id, {
responseType: 'blob',
onDownloadProgress: function (progressEvent) {
const request = this as XMLHttpRequest;
if (!total) {
total = Number(request.getResponseHeader('X-Immich-Content-Length-Hint')) || 0;
$downloadAssets[fileName] = 0;
let total = 0;
const { data, status, headers } = await api.albumApi.downloadArchive(
album.id,
skip || undefined,
{
responseType: 'blob',
onDownloadProgress: function (progressEvent) {
const request = this as XMLHttpRequest;
if (!total) {
total = Number(request.getResponseHeader('X-Immich-Content-Length-Hint')) || 0;
}
if (total) {
const current = progressEvent.loaded;
$downloadAssets[fileName] = Math.floor((current / total) * 100);
}
}
}
);
if (total) {
const current = progressEvent.loaded;
$downloadAssets[fileName] = Math.floor((current / total) * 100);
}
const isNotComplete = headers['x-immich-archive-complete'] === 'false';
const fileCount = Number(headers['x-immich-archive-file-count']) || 0;
if (isNotComplete && fileCount > 0) {
skip += fileCount;
} else {
done = true;
}
});
if (!(data instanceof Blob)) {
return;
}
if (!(data instanceof Blob)) {
return;
}
if (status === 200) {
const fileUrl = URL.createObjectURL(data);
const anchor = document.createElement('a');
anchor.href = fileUrl;
anchor.download = fileName;
if (status === 200) {
const fileUrl = URL.createObjectURL(data);
const anchor = document.createElement('a');
anchor.href = fileUrl;
anchor.download = fileName;
document.body.appendChild(anchor);
anchor.click();
document.body.removeChild(anchor);
document.body.appendChild(anchor);
anchor.click();
document.body.removeChild(anchor);
URL.revokeObjectURL(fileUrl);
URL.revokeObjectURL(fileUrl);
// Remove item from download list
setTimeout(() => {
const copy = $downloadAssets;
delete copy[fileName];
$downloadAssets = copy;
}, 2000);
// Remove item from download list
setTimeout(() => {
const copy = $downloadAssets;
delete copy[fileName];
$downloadAssets = copy;
}, 2000);
}
}
} catch (e) {
console.error('Error downloading file ', e);