diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts index aa45ce5470..67c38af6e3 100644 --- a/cli/src/commands/asset.ts +++ b/cli/src/commands/asset.ts @@ -56,14 +56,13 @@ class UploadFile extends File { export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => { await authenticate(baseOptions); - const files = await scan(paths, options); - if (files.length === 0) { + const scanFiles = await scan(paths, options); + if (scanFiles.length === 0) { console.log('No files found, exiting'); return; } - const { newFiles, duplicates } = await checkForDuplicates(files, options); - + const { newFiles, duplicates } = await checkForDuplicates(scanFiles, options); const newAssets = await uploadFiles(newFiles, options); await updateAlbums([...newAssets, ...duplicates], options); await deleteFiles(newFiles, options); @@ -84,7 +83,12 @@ const scan = async (pathsToCrawl: string[], options: UploadOptionsDto) => { return files; }; -const checkForDuplicates = async (files: string[], { concurrency }: UploadOptionsDto) => { +const checkForDuplicates = async (files: string[], { concurrency, skipHash }: UploadOptionsDto) => { + if (skipHash) { + console.log('Skipping hash check, assuming all files are new'); + return { newFiles: files, duplicates: [] }; + } + const progressBar = new SingleBar( { format: 'Checking files | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' }, Presets.shades_classic, @@ -147,17 +151,32 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio uploadProgress.start(totalSize, 0); uploadProgress.update({ value_formatted: 0, total_formatted: byteSize(totalSize) }); - let totalSizeUploaded = 0; + let duplicateCount = 0; + let duplicateSize = 0; + let successCount = 0; + let successSize = 0; + const newAssets: Asset[] = []; + try { for (const items of chunk(files, concurrency)) { await Promise.all( items.map(async (filepath) => { const stats = statsMap.get(filepath) as Stats; const response = await uploadFile(filepath, stats); - totalSizeUploaded += stats.size ?? 0; - uploadProgress.update(totalSizeUploaded, { value_formatted: byteSize(totalSizeUploaded) }); + newAssets.push({ id: response.id, filepath }); + + if (response.duplicate) { + duplicateCount++; + duplicateSize += stats.size ?? 0; + } else { + successCount++; + successSize += stats.size ?? 0; + } + + uploadProgress.update(successSize, { value_formatted: byteSize(successSize + duplicateSize) }); + return response; }), ); @@ -166,7 +185,10 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio uploadProgress.stop(); } - console.log(`Successfully uploaded ${newAssets.length} asset${s(newAssets.length)} (${byteSize(totalSizeUploaded)})`); + console.log(`Successfully uploaded ${successCount} new asset${s(successCount)} (${byteSize(successSize)})`); + if (duplicateCount > 0) { + console.log(`Skipped ${duplicateCount} duplicate asset${s(duplicateCount)} (${byteSize(duplicateSize)})`); + } return newAssets; }; diff --git a/e2e/src/cli/specs/upload.e2e-spec.ts b/e2e/src/cli/specs/upload.e2e-spec.ts index a74a57c711..b0b61bf51a 100644 --- a/e2e/src/cli/specs/upload.e2e-spec.ts +++ b/e2e/src/cli/specs/upload.e2e-spec.ts @@ -1,4 +1,5 @@ import { LoginResponseDto, getAllAlbums, getAllAssets } from '@immich/sdk'; +import { readFileSync } from 'node:fs'; import { mkdir, readdir, rm, symlink } from 'node:fs/promises'; import { asKeyAuth, immichCli, testAssetDir, utils } from 'src/utils'; import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; @@ -23,7 +24,7 @@ describe(`immich upload`, () => { const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(stderr).toBe(''); expect(stdout.split('\n')).toEqual( - expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]), + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), ); expect(exitCode).toBe(0); @@ -35,7 +36,7 @@ describe(`immich upload`, () => { const first = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(first.stderr).toBe(''); expect(first.stdout.split('\n')).toEqual( - expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]), + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), ); expect(first.exitCode).toBe(0); @@ -69,7 +70,7 @@ describe(`immich upload`, () => { const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']); expect(stderr).toBe(''); expect(stdout.split('\n')).toEqual( - expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]), + expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]), ); expect(exitCode).toBe(0); @@ -88,7 +89,7 @@ describe(`immich upload`, () => { ]); expect(stdout.split('\n')).toEqual( expect.arrayContaining([ - expect.stringContaining('Successfully uploaded 9 assets'), + expect.stringContaining('Successfully uploaded 9 new assets'), expect.stringContaining('Successfully created 1 new album'), expect.stringContaining('Successfully updated 9 assets'), ]), @@ -107,7 +108,7 @@ describe(`immich upload`, () => { it('should add existing assets to albums', async () => { const response1 = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']); expect(response1.stdout.split('\n')).toEqual( - expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]), + expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]), ); expect(response1.stderr).toBe(''); expect(response1.exitCode).toBe(0); @@ -147,7 +148,7 @@ describe(`immich upload`, () => { ]); expect(stdout.split('\n')).toEqual( expect.arrayContaining([ - expect.stringContaining('Successfully uploaded 9 assets'), + expect.stringContaining('Successfully uploaded 9 new assets'), expect.stringContaining('Successfully created 1 new album'), expect.stringContaining('Successfully updated 9 assets'), ]), @@ -180,7 +181,7 @@ describe(`immich upload`, () => { expect(stdout.split('\n')).toEqual( expect.arrayContaining([ - expect.stringContaining('Successfully uploaded 9 assets'), + expect.stringContaining('Successfully uploaded 9 new assets'), expect.stringContaining('Deleting assets that have been uploaded'), ]), ); @@ -192,6 +193,32 @@ describe(`immich upload`, () => { }); }); + describe('immich upload --skip-hash', () => { + it('should skip hashing', async () => { + const filename = `albums/nature/silver_fir.jpg`; + await utils.createAsset(admin.accessToken, { + assetData: { + bytes: readFileSync(`${testAssetDir}/${filename}`), + filename: 'silver_fit.jpg', + }, + }); + const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/${filename}`, '--skip-hash']); + + expect(stderr).toBe(''); + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + 'Skipping hash check, assuming all files are new', + expect.stringContaining('Successfully uploaded 0 new assets'), + expect.stringContaining('Skipped 1 duplicate asset'), + ]), + ); + expect(exitCode).toBe(0); + + const assets = await getAllAssets({}, { headers: asKeyAuth(key) }); + expect(assets.length).toBe(1); + }); + }); + describe('immich upload --concurrency ', () => { it('should work', async () => { const { stderr, stdout, exitCode } = await immichCli([ @@ -203,7 +230,10 @@ describe(`immich upload`, () => { expect(stderr).toBe(''); expect(stdout.split('\n')).toEqual( - expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]), + expect.arrayContaining([ + 'Found 9 new files and 0 duplicates', + expect.stringContaining('Successfully uploaded 9 new assets'), + ]), ); expect(exitCode).toBe(0);