mirror of
https://github.com/immich-app/immich.git
synced 2024-12-29 15:11:58 +00:00
feat(server): multi archive downloads (#956)
This commit is contained in:
parent
b5d75e2016
commit
f2f255e6e6
26 changed files with 389 additions and 139 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -27,6 +27,12 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
|
|||
import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
|
||||
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
|
||||
import { Response as Res } from 'express';
|
||||
import {
|
||||
IMMICH_ARCHIVE_COMPLETE,
|
||||
IMMICH_ARCHIVE_FILE_COUNT,
|
||||
IMMICH_CONTENT_LENGTH_HINT,
|
||||
} from '../../constants/download.constant';
|
||||
import { DownloadDto } from '../asset/dto/download-library.dto';
|
||||
|
||||
// TODO might be worth creating a AlbumParamsDto that validates `albumId` instead of using the pipe.
|
||||
@Authenticated()
|
||||
|
@ -119,11 +125,18 @@ export class AlbumController {
|
|||
async downloadArchive(
|
||||
@GetAuthUser() authUser: AuthUserDto,
|
||||
@Param('albumId', new ParseUUIDPipe({ version: '4' })) albumId: string,
|
||||
@Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
|
||||
@Response({ passthrough: true }) res: Res,
|
||||
): Promise<any> {
|
||||
const { stream, filename, filesize } = await this.albumService.downloadArchive(authUser, albumId);
|
||||
res.attachment(filename);
|
||||
res.setHeader('X-Immich-Content-Length-Hint', filesize);
|
||||
const { stream, fileName, fileSize, fileCount, complete } = await this.albumService.downloadArchive(
|
||||
authUser,
|
||||
albumId,
|
||||
dto,
|
||||
);
|
||||
res.attachment(fileName);
|
||||
res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
|
||||
res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
|
||||
res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,9 +9,13 @@ import { AssetAlbumEntity } from '@app/database/entities/asset-album.entity';
|
|||
import { UserAlbumEntity } from '@app/database/entities/user-album.entity';
|
||||
import { AlbumRepository, ALBUM_REPOSITORY } from './album-repository';
|
||||
import { AssetRepository, ASSET_REPOSITORY } from '../asset/asset-repository';
|
||||
import { DownloadModule } from '../../modules/download/download.module';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity])],
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity]),
|
||||
DownloadModule,
|
||||
],
|
||||
controllers: [AlbumController],
|
||||
providers: [
|
||||
AlbumService,
|
||||
|
|
|
@ -6,11 +6,13 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
|
|||
import { IAssetRepository } from '../asset/asset-repository';
|
||||
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
|
||||
import { IAlbumRepository } from './album-repository';
|
||||
import { DownloadService } from '../../modules/download/download.service';
|
||||
|
||||
describe('Album service', () => {
|
||||
let sut: AlbumService;
|
||||
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
|
||||
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
|
||||
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
|
||||
|
||||
const authUser: AuthUserDto = Object.freeze({
|
||||
id: '1111',
|
||||
|
@ -142,7 +144,11 @@ describe('Album service', () => {
|
|||
getExistingAssets: jest.fn(),
|
||||
};
|
||||
|
||||
sut = new AlbumService(albumRepositoryMock, assetRepositoryMock);
|
||||
downloadServiceMock = {
|
||||
downloadArchive: jest.fn(),
|
||||
};
|
||||
|
||||
sut = new AlbumService(albumRepositoryMock, assetRepositoryMock, downloadServiceMock as DownloadService);
|
||||
});
|
||||
|
||||
it('creates album', async () => {
|
||||
|
|
|
@ -1,13 +1,4 @@
|
|||
import {
|
||||
BadRequestException,
|
||||
Inject,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
Logger,
|
||||
InternalServerErrorException,
|
||||
StreamableFile,
|
||||
} from '@nestjs/common';
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, ForbiddenException } from '@nestjs/common';
|
||||
import { AuthUserDto } from '../../decorators/auth-user.decorator';
|
||||
import { CreateAlbumDto } from './dto/create-album.dto';
|
||||
import { AlbumEntity } from '@app/database/entities/album.entity';
|
||||
|
@ -21,14 +12,15 @@ import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
|
|||
import { ASSET_REPOSITORY, IAssetRepository } from '../asset/asset-repository';
|
||||
import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
|
||||
import { AddAssetsDto } from './dto/add-assets.dto';
|
||||
import archiver from 'archiver';
|
||||
import { extname } from 'path';
|
||||
import { DownloadService } from '../../modules/download/download.service';
|
||||
import { DownloadDto } from '../asset/dto/download-library.dto';
|
||||
|
||||
@Injectable()
|
||||
export class AlbumService {
|
||||
constructor(
|
||||
@Inject(ALBUM_REPOSITORY) private _albumRepository: IAlbumRepository,
|
||||
@Inject(ASSET_REPOSITORY) private _assetRepository: IAssetRepository,
|
||||
private downloadService: DownloadService,
|
||||
) {}
|
||||
|
||||
private async _getAlbum({
|
||||
|
@ -162,35 +154,11 @@ export class AlbumService {
|
|||
return this._albumRepository.getCountByUserId(authUser.id);
|
||||
}
|
||||
|
||||
async downloadArchive(authUser: AuthUserDto, albumId: string) {
|
||||
async downloadArchive(authUser: AuthUserDto, albumId: string, dto: DownloadDto) {
|
||||
const album = await this._getAlbum({ authUser, albumId, validateIsOwner: false });
|
||||
if (!album.assets || album.assets.length === 0) {
|
||||
throw new BadRequestException('Cannot download an empty album.');
|
||||
}
|
||||
const assets = (album.assets || []).map((asset) => asset.assetInfo).slice(dto.skip || 0);
|
||||
|
||||
try {
|
||||
const archive = archiver('zip', { store: true });
|
||||
const stream = new StreamableFile(archive);
|
||||
let totalSize = 0;
|
||||
|
||||
for (const { assetInfo } of album.assets) {
|
||||
const { originalPath } = assetInfo;
|
||||
const name = `${assetInfo.exifInfo?.imageName || assetInfo.id}${extname(originalPath)}`;
|
||||
archive.file(originalPath, { name });
|
||||
totalSize += Number(assetInfo.exifInfo?.fileSizeInByte || 0);
|
||||
}
|
||||
|
||||
archive.finalize();
|
||||
|
||||
return {
|
||||
stream,
|
||||
filename: `${album.albumName}.zip`,
|
||||
filesize: totalSize,
|
||||
};
|
||||
} catch (e) {
|
||||
Logger.error(`Error downloading album ${e}`, 'downloadArchive');
|
||||
throw new InternalServerErrorException(`Failed to download album ${e}`, 'DownloadArchive');
|
||||
}
|
||||
return this.downloadService.downloadArchive(album.albumName, assets);
|
||||
}
|
||||
|
||||
async _checkValidThumbnail(album: AlbumEntity) {
|
||||
|
|
|
@ -24,7 +24,7 @@ export interface IAssetRepository {
|
|||
checksum?: Buffer,
|
||||
): Promise<AssetEntity>;
|
||||
update(asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
|
||||
getAllByUserId(userId: string): Promise<AssetEntity[]>;
|
||||
getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]>;
|
||||
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
|
||||
getById(assetId: string): Promise<AssetEntity>;
|
||||
getLocationsByUserId(userId: string): Promise<CuratedLocationsResponseDto[]>;
|
||||
|
@ -81,7 +81,7 @@ export class AssetRepository implements IAssetRepository {
|
|||
|
||||
async getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto> {
|
||||
// Get asset count by AssetType
|
||||
const res = await this.assetRepository
|
||||
const items = await this.assetRepository
|
||||
.createQueryBuilder('asset')
|
||||
.select(`COUNT(asset.id)`, 'count')
|
||||
.addSelect(`asset.type`, 'type')
|
||||
|
@ -89,14 +89,24 @@ export class AssetRepository implements IAssetRepository {
|
|||
.groupBy('asset.type')
|
||||
.getRawMany();
|
||||
|
||||
const assetCountByUserId = new AssetCountByUserIdResponseDto(0, 0);
|
||||
res.map((item) => {
|
||||
if (item.type === 'IMAGE') {
|
||||
assetCountByUserId.photos = item.count;
|
||||
} else if (item.type === 'VIDEO') {
|
||||
assetCountByUserId.videos = item.count;
|
||||
}
|
||||
});
|
||||
const assetCountByUserId = new AssetCountByUserIdResponseDto();
|
||||
|
||||
// asset type to dto property mapping
|
||||
const map: Record<AssetType, keyof AssetCountByUserIdResponseDto> = {
|
||||
[AssetType.AUDIO]: 'audio',
|
||||
[AssetType.IMAGE]: 'photos',
|
||||
[AssetType.VIDEO]: 'videos',
|
||||
[AssetType.OTHER]: 'other',
|
||||
};
|
||||
|
||||
for (const item of items) {
|
||||
const count = Number(item.count) || 0;
|
||||
const assetType = item.type as AssetType;
|
||||
const type = map[assetType];
|
||||
|
||||
assetCountByUserId[type] = count;
|
||||
assetCountByUserId.total += count;
|
||||
}
|
||||
|
||||
return assetCountByUserId;
|
||||
}
|
||||
|
@ -207,12 +217,13 @@ export class AssetRepository implements IAssetRepository {
|
|||
* Get all assets belong to the user on the database
|
||||
* @param userId
|
||||
*/
|
||||
async getAllByUserId(userId: string): Promise<AssetEntity[]> {
|
||||
async getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]> {
|
||||
const query = this.assetRepository
|
||||
.createQueryBuilder('asset')
|
||||
.where('asset.userId = :userId', { userId: userId })
|
||||
.andWhere('asset.resizePath is not NULL')
|
||||
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
|
||||
.skip(skip || 0)
|
||||
.orderBy('asset.createdAt', 'DESC');
|
||||
|
||||
return await query.getMany();
|
||||
|
|
|
@ -52,6 +52,12 @@ import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-use
|
|||
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
|
||||
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
|
||||
import { UpdateAssetDto } from './dto/update-asset.dto';
|
||||
import { DownloadDto } from './dto/download-library.dto';
|
||||
import {
|
||||
IMMICH_ARCHIVE_COMPLETE,
|
||||
IMMICH_ARCHIVE_FILE_COUNT,
|
||||
IMMICH_CONTENT_LENGTH_HINT,
|
||||
} from '../../constants/download.constant';
|
||||
|
||||
@Authenticated()
|
||||
@ApiBearerAuth()
|
||||
|
@ -134,6 +140,20 @@ export class AssetController {
|
|||
return this.assetService.downloadFile(query, res);
|
||||
}
|
||||
|
||||
@Get('/download-library')
|
||||
async downloadLibrary(
|
||||
@GetAuthUser() authUser: AuthUserDto,
|
||||
@Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
|
||||
@Response({ passthrough: true }) res: Res,
|
||||
): Promise<any> {
|
||||
const { stream, fileName, fileSize, fileCount, complete } = await this.assetService.downloadLibrary(authUser, dto);
|
||||
res.attachment(fileName);
|
||||
res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
|
||||
res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
|
||||
res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
|
||||
return stream;
|
||||
}
|
||||
|
||||
@Get('/file')
|
||||
async serveFile(
|
||||
@Headers() headers: Record<string, string>,
|
||||
|
|
|
@ -9,11 +9,13 @@ import { BackgroundTaskService } from '../../modules/background-task/background-
|
|||
import { CommunicationModule } from '../communication/communication.module';
|
||||
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
|
||||
import { AssetRepository, ASSET_REPOSITORY } from './asset-repository';
|
||||
import { DownloadModule } from '../../modules/download/download.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
CommunicationModule,
|
||||
BackgroundTaskModule,
|
||||
DownloadModule,
|
||||
TypeOrmModule.forFeature([AssetEntity]),
|
||||
BullModule.registerQueue({
|
||||
name: QueueNameEnum.ASSET_UPLOADED,
|
||||
|
|
|
@ -7,11 +7,13 @@ import { CreateAssetDto } from './dto/create-asset.dto';
|
|||
import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group-response.dto';
|
||||
import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
|
||||
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
|
||||
import { DownloadService } from '../../modules/download/download.service';
|
||||
|
||||
describe('AssetService', () => {
|
||||
let sui: AssetService;
|
||||
let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
|
||||
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
|
||||
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
|
||||
|
||||
const authUser: AuthUserDto = Object.freeze({
|
||||
id: 'user_id_1',
|
||||
|
@ -89,7 +91,10 @@ describe('AssetService', () => {
|
|||
};
|
||||
|
||||
const _getAssetCountByUserId = (): AssetCountByUserIdResponseDto => {
|
||||
const result = new AssetCountByUserIdResponseDto(2, 2);
|
||||
const result = new AssetCountByUserIdResponseDto();
|
||||
|
||||
result.videos = 2;
|
||||
result.photos = 2;
|
||||
|
||||
return result;
|
||||
};
|
||||
|
@ -114,7 +119,11 @@ describe('AssetService', () => {
|
|||
getExistingAssets: jest.fn(),
|
||||
};
|
||||
|
||||
sui = new AssetService(assetRepositoryMock, a);
|
||||
downloadServiceMock = {
|
||||
downloadArchive: jest.fn(),
|
||||
};
|
||||
|
||||
sui = new AssetService(assetRepositoryMock, a, downloadServiceMock as DownloadService);
|
||||
});
|
||||
|
||||
// Currently failing due to calculate checksum from a file
|
||||
|
|
|
@ -41,6 +41,8 @@ import { timeUtils } from '@app/common/utils';
|
|||
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
|
||||
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
|
||||
import { UpdateAssetDto } from './dto/update-asset.dto';
|
||||
import { DownloadService } from '../../modules/download/download.service';
|
||||
import { DownloadDto } from './dto/download-library.dto';
|
||||
|
||||
const fileInfo = promisify(stat);
|
||||
|
||||
|
@ -52,6 +54,8 @@ export class AssetService {
|
|||
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
|
||||
private downloadService: DownloadService,
|
||||
) {}
|
||||
|
||||
public async createUserAsset(
|
||||
|
@ -140,6 +144,12 @@ export class AssetService {
|
|||
return mapAsset(updatedAsset);
|
||||
}
|
||||
|
||||
public async downloadLibrary(user: AuthUserDto, dto: DownloadDto) {
|
||||
const assets = await this._assetRepository.getAllByUserId(user.id, dto.skip);
|
||||
|
||||
return this.downloadService.downloadArchive(dto.name || `library`, assets);
|
||||
}
|
||||
|
||||
public async downloadFile(query: ServeFileDto, res: Res) {
|
||||
try {
|
||||
let fileReadStream = null;
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
import { Type } from 'class-transformer';
|
||||
import { IsNumber, IsOptional, IsPositive, IsString } from 'class-validator';
|
||||
|
||||
export class DownloadDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
name = '';
|
||||
|
||||
@IsOptional()
|
||||
@IsPositive()
|
||||
@IsNumber()
|
||||
@Type(() => Number)
|
||||
skip?: number;
|
||||
}
|
|
@ -2,13 +2,17 @@ import { ApiProperty } from '@nestjs/swagger';
|
|||
|
||||
export class AssetCountByUserIdResponseDto {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
photos!: number;
|
||||
audio = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
videos!: number;
|
||||
photos = 0;
|
||||
|
||||
constructor(photos: number, videos: number) {
|
||||
this.photos = photos;
|
||||
this.videos = videos;
|
||||
}
|
||||
@ApiProperty({ type: 'integer' })
|
||||
videos = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
other = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
total = 0;
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ import { Repository } from 'typeorm';
|
|||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import path from 'path';
|
||||
import { readdirSync, statSync } from 'fs';
|
||||
import { asHumanReadable } from '../../utils/human-readable.util';
|
||||
|
||||
@Injectable()
|
||||
export class ServerInfoService {
|
||||
|
@ -23,9 +24,9 @@ export class ServerInfoService {
|
|||
const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2);
|
||||
|
||||
const serverInfo = new ServerInfoResponseDto();
|
||||
serverInfo.diskAvailable = ServerInfoService.getHumanReadableString(diskInfo.available);
|
||||
serverInfo.diskSize = ServerInfoService.getHumanReadableString(diskInfo.total);
|
||||
serverInfo.diskUse = ServerInfoService.getHumanReadableString(diskInfo.total - diskInfo.free);
|
||||
serverInfo.diskAvailable = asHumanReadable(diskInfo.available);
|
||||
serverInfo.diskSize = asHumanReadable(diskInfo.total);
|
||||
serverInfo.diskUse = asHumanReadable(diskInfo.total - diskInfo.free);
|
||||
serverInfo.diskAvailableRaw = diskInfo.available;
|
||||
serverInfo.diskSizeRaw = diskInfo.total;
|
||||
serverInfo.diskUseRaw = diskInfo.total - diskInfo.free;
|
||||
|
@ -33,33 +34,6 @@ export class ServerInfoService {
|
|||
return serverInfo;
|
||||
}
|
||||
|
||||
private static getHumanReadableString(sizeInByte: number) {
|
||||
const pepibyte = 1.126 * Math.pow(10, 15);
|
||||
const tebibyte = 1.1 * Math.pow(10, 12);
|
||||
const gibibyte = 1.074 * Math.pow(10, 9);
|
||||
const mebibyte = 1.049 * Math.pow(10, 6);
|
||||
const kibibyte = 1024;
|
||||
// Pebibyte
|
||||
if (sizeInByte >= pepibyte) {
|
||||
// Pe
|
||||
return `${(sizeInByte / pepibyte).toFixed(1)}PB`;
|
||||
} else if (tebibyte <= sizeInByte && sizeInByte < pepibyte) {
|
||||
// Te
|
||||
return `${(sizeInByte / tebibyte).toFixed(1)}TB`;
|
||||
} else if (gibibyte <= sizeInByte && sizeInByte < tebibyte) {
|
||||
// Gi
|
||||
return `${(sizeInByte / gibibyte).toFixed(1)}GB`;
|
||||
} else if (mebibyte <= sizeInByte && sizeInByte < gibibyte) {
|
||||
// Mega
|
||||
return `${(sizeInByte / mebibyte).toFixed(1)}MB`;
|
||||
} else if (kibibyte <= sizeInByte && sizeInByte < mebibyte) {
|
||||
// Kibi
|
||||
return `${(sizeInByte / kibibyte).toFixed(1)}KB`;
|
||||
} else {
|
||||
return `${sizeInByte}B`;
|
||||
}
|
||||
}
|
||||
|
||||
async getStats(): Promise<ServerStatsResponseDto> {
|
||||
const res = await this.assetRepository
|
||||
.createQueryBuilder('asset')
|
||||
|
@ -90,11 +64,11 @@ export class ServerInfoService {
|
|||
const userDiskUsage = await ServerInfoService.getDirectoryStats(path.join(APP_UPLOAD_LOCATION, userId));
|
||||
usage.usageRaw = userDiskUsage.size;
|
||||
usage.objects = userDiskUsage.fileCount;
|
||||
usage.usage = ServerInfoService.getHumanReadableString(usage.usageRaw);
|
||||
usage.usage = asHumanReadable(usage.usageRaw);
|
||||
serverStats.usageRaw += usage.usageRaw;
|
||||
serverStats.objects += usage.objects;
|
||||
}
|
||||
serverStats.usage = ServerInfoService.getHumanReadableString(serverStats.usageRaw);
|
||||
serverStats.usage = asHumanReadable(serverStats.usageRaw);
|
||||
serverStats.usageByUser = Array.from(tmpMap.values());
|
||||
return serverStats;
|
||||
}
|
||||
|
|
3
server/apps/immich/src/constants/download.constant.ts
Normal file
3
server/apps/immich/src/constants/download.constant.ts
Normal file
|
@ -0,0 +1,3 @@
|
|||
export const IMMICH_CONTENT_LENGTH_HINT = 'X-Immich-Content-Length-Hint';
|
||||
export const IMMICH_ARCHIVE_FILE_COUNT = 'X-Immich-Archive-File-Count';
|
||||
export const IMMICH_ARCHIVE_COMPLETE = 'X-Immich-Archive-Complete';
|
|
@ -0,0 +1,8 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { DownloadService } from './download.service';
|
||||
|
||||
@Module({
|
||||
providers: [DownloadService],
|
||||
exports: [DownloadService],
|
||||
})
|
||||
export class DownloadModule {}
|
63
server/apps/immich/src/modules/download/download.service.ts
Normal file
63
server/apps/immich/src/modules/download/download.service.ts
Normal file
|
@ -0,0 +1,63 @@
|
|||
import { AssetEntity } from '@app/database/entities/asset.entity';
|
||||
import { BadRequestException, Injectable, InternalServerErrorException, Logger, StreamableFile } from '@nestjs/common';
|
||||
import archiver from 'archiver';
|
||||
import { extname } from 'path';
|
||||
import { asHumanReadable, HumanReadableSize } from '../../utils/human-readable.util';
|
||||
|
||||
export interface DownloadArchive {
|
||||
stream: StreamableFile;
|
||||
fileName: string;
|
||||
fileSize: number;
|
||||
fileCount: number;
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DownloadService {
|
||||
private readonly logger = new Logger(DownloadService.name);
|
||||
|
||||
public async downloadArchive(name: string, assets: AssetEntity[]): Promise<DownloadArchive> {
|
||||
if (!assets || assets.length === 0) {
|
||||
throw new BadRequestException('No assets to download.');
|
||||
}
|
||||
|
||||
try {
|
||||
const archive = archiver('zip', { store: true });
|
||||
const stream = new StreamableFile(archive);
|
||||
let totalSize = 0;
|
||||
let fileCount = 0;
|
||||
let complete = true;
|
||||
|
||||
for (const { id, originalPath, exifInfo } of assets) {
|
||||
const name = `${exifInfo?.imageName || id}${extname(originalPath)}`;
|
||||
archive.file(originalPath, { name });
|
||||
totalSize += Number(exifInfo?.fileSizeInByte || 0);
|
||||
fileCount++;
|
||||
|
||||
// for easier testing, can be changed before merging.
|
||||
if (totalSize > HumanReadableSize.GB * 20) {
|
||||
complete = false;
|
||||
this.logger.log(
|
||||
`Archive size exceeded after ${fileCount} files, capping at ${totalSize} bytes (${asHumanReadable(
|
||||
totalSize,
|
||||
)})`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
archive.finalize();
|
||||
|
||||
return {
|
||||
stream,
|
||||
fileName: `${name}.zip`,
|
||||
fileSize: totalSize,
|
||||
fileCount,
|
||||
complete,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`Error creating download archive ${error}`);
|
||||
throw new InternalServerErrorException(`Failed to download ${name}: ${error}`, 'DownloadArchive');
|
||||
}
|
||||
}
|
||||
}
|
31
server/apps/immich/src/utils/human-readable.util.ts
Normal file
31
server/apps/immich/src/utils/human-readable.util.ts
Normal file
|
@ -0,0 +1,31 @@
|
|||
const KB = 1000;
|
||||
const MB = KB * 1000;
|
||||
const GB = MB * 1000;
|
||||
const TB = GB * 1000;
|
||||
const PB = TB * 1000;
|
||||
|
||||
export const HumanReadableSize = { KB, MB, GB, TB, PB };
|
||||
|
||||
export function asHumanReadable(bytes: number, precision = 1) {
|
||||
if (bytes >= PB) {
|
||||
return `${(bytes / PB).toFixed(precision)}PB`;
|
||||
}
|
||||
|
||||
if (bytes >= TB) {
|
||||
return `${(bytes / TB).toFixed(precision)}TB`;
|
||||
}
|
||||
|
||||
if (bytes >= GB) {
|
||||
return `${(bytes / GB).toFixed(precision)}GB`;
|
||||
}
|
||||
|
||||
if (bytes >= MB) {
|
||||
return `${(bytes / MB).toFixed(precision)}MB`;
|
||||
}
|
||||
|
||||
if (bytes >= KB) {
|
||||
return `${(bytes / KB).toFixed(precision)}KB`;
|
||||
}
|
||||
|
||||
return `${bytes}B`;
|
||||
}
|
File diff suppressed because one or more lines are too long
|
@ -294,6 +294,12 @@ export interface AssetCountByTimeBucketResponseDto {
|
|||
* @interface AssetCountByUserIdResponseDto
|
||||
*/
|
||||
export interface AssetCountByUserIdResponseDto {
|
||||
/**
|
||||
*
|
||||
* @type {number}
|
||||
* @memberof AssetCountByUserIdResponseDto
|
||||
*/
|
||||
'audio': number;
|
||||
/**
|
||||
*
|
||||
* @type {number}
|
||||
|
@ -306,6 +312,18 @@ export interface AssetCountByUserIdResponseDto {
|
|||
* @memberof AssetCountByUserIdResponseDto
|
||||
*/
|
||||
'videos': number;
|
||||
/**
|
||||
*
|
||||
* @type {number}
|
||||
* @memberof AssetCountByUserIdResponseDto
|
||||
*/
|
||||
'other': number;
|
||||
/**
|
||||
*
|
||||
* @type {number}
|
||||
* @memberof AssetCountByUserIdResponseDto
|
||||
*/
|
||||
'total': number;
|
||||
}
|
||||
/**
|
||||
*
|
||||
|
@ -1898,10 +1916,11 @@ export const AlbumApiAxiosParamCreator = function (configuration?: Configuration
|
|||
/**
|
||||
*
|
||||
* @param {string} albumId
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
downloadArchive: async (albumId: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
|
||||
downloadArchive: async (albumId: string, skip?: number, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
|
||||
// verify required parameter 'albumId' is not null or undefined
|
||||
assertParamExists('downloadArchive', 'albumId', albumId)
|
||||
const localVarPath = `/album/{albumId}/download`
|
||||
|
@ -1921,6 +1940,10 @@ export const AlbumApiAxiosParamCreator = function (configuration?: Configuration
|
|||
// http bearer authentication required
|
||||
await setBearerAuthToObject(localVarHeaderParameter, configuration)
|
||||
|
||||
if (skip !== undefined) {
|
||||
localVarQueryParameter['skip'] = skip;
|
||||
}
|
||||
|
||||
|
||||
|
||||
setSearchParams(localVarUrlObj, localVarQueryParameter);
|
||||
|
@ -2227,11 +2250,12 @@ export const AlbumApiFp = function(configuration?: Configuration) {
|
|||
/**
|
||||
*
|
||||
* @param {string} albumId
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
async downloadArchive(albumId: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
|
||||
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadArchive(albumId, options);
|
||||
async downloadArchive(albumId: string, skip?: number, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
|
||||
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadArchive(albumId, skip, options);
|
||||
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
|
||||
},
|
||||
/**
|
||||
|
@ -2348,11 +2372,12 @@ export const AlbumApiFactory = function (configuration?: Configuration, basePath
|
|||
/**
|
||||
*
|
||||
* @param {string} albumId
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
downloadArchive(albumId: string, options?: any): AxiosPromise<object> {
|
||||
return localVarFp.downloadArchive(albumId, options).then((request) => request(axios, basePath));
|
||||
downloadArchive(albumId: string, skip?: number, options?: any): AxiosPromise<object> {
|
||||
return localVarFp.downloadArchive(albumId, skip, options).then((request) => request(axios, basePath));
|
||||
},
|
||||
/**
|
||||
*
|
||||
|
@ -2470,12 +2495,13 @@ export class AlbumApi extends BaseAPI {
|
|||
/**
|
||||
*
|
||||
* @param {string} albumId
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
* @memberof AlbumApi
|
||||
*/
|
||||
public downloadArchive(albumId: string, options?: AxiosRequestConfig) {
|
||||
return AlbumApiFp(this.configuration).downloadArchive(albumId, options).then((request) => request(this.axios, this.basePath));
|
||||
public downloadArchive(albumId: string, skip?: number, options?: AxiosRequestConfig) {
|
||||
return AlbumApiFp(this.configuration).downloadArchive(albumId, skip, options).then((request) => request(this.axios, this.basePath));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -2722,6 +2748,44 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
|
|||
|
||||
|
||||
|
||||
setSearchParams(localVarUrlObj, localVarQueryParameter);
|
||||
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
|
||||
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
|
||||
|
||||
return {
|
||||
url: toPathString(localVarUrlObj),
|
||||
options: localVarRequestOptions,
|
||||
};
|
||||
},
|
||||
/**
|
||||
*
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
downloadLibrary: async (skip?: number, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
|
||||
const localVarPath = `/asset/download-library`;
|
||||
// use dummy base URL string because the URL constructor only accepts absolute URLs.
|
||||
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
|
||||
let baseOptions;
|
||||
if (configuration) {
|
||||
baseOptions = configuration.baseOptions;
|
||||
}
|
||||
|
||||
const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
|
||||
const localVarHeaderParameter = {} as any;
|
||||
const localVarQueryParameter = {} as any;
|
||||
|
||||
// authentication bearer required
|
||||
// http bearer authentication required
|
||||
await setBearerAuthToObject(localVarHeaderParameter, configuration)
|
||||
|
||||
if (skip !== undefined) {
|
||||
localVarQueryParameter['skip'] = skip;
|
||||
}
|
||||
|
||||
|
||||
|
||||
setSearchParams(localVarUrlObj, localVarQueryParameter);
|
||||
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
|
||||
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
|
||||
|
@ -3332,6 +3396,16 @@ export const AssetApiFp = function(configuration?: Configuration) {
|
|||
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadFile(aid, did, isThumb, isWeb, options);
|
||||
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
|
||||
},
|
||||
/**
|
||||
*
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
async downloadLibrary(skip?: number, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<object>> {
|
||||
const localVarAxiosArgs = await localVarAxiosParamCreator.downloadLibrary(skip, options);
|
||||
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
|
||||
},
|
||||
/**
|
||||
* Get all AssetEntity belong to the user
|
||||
* @summary
|
||||
|
@ -3527,6 +3601,15 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
|
|||
downloadFile(aid: string, did: string, isThumb?: boolean, isWeb?: boolean, options?: any): AxiosPromise<object> {
|
||||
return localVarFp.downloadFile(aid, did, isThumb, isWeb, options).then((request) => request(axios, basePath));
|
||||
},
|
||||
/**
|
||||
*
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
downloadLibrary(skip?: number, options?: any): AxiosPromise<object> {
|
||||
return localVarFp.downloadLibrary(skip, options).then((request) => request(axios, basePath));
|
||||
},
|
||||
/**
|
||||
* Get all AssetEntity belong to the user
|
||||
* @summary
|
||||
|
@ -3716,6 +3799,17 @@ export class AssetApi extends BaseAPI {
|
|||
return AssetApiFp(this.configuration).downloadFile(aid, did, isThumb, isWeb, options).then((request) => request(this.axios, this.basePath));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} [skip]
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
* @memberof AssetApi
|
||||
*/
|
||||
public downloadLibrary(skip?: number, options?: AxiosRequestConfig) {
|
||||
return AssetApiFp(this.configuration).downloadLibrary(skip, options).then((request) => request(this.axios, this.basePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all AssetEntity belong to the user
|
||||
* @summary
|
||||
|
|
|
@ -313,53 +313,69 @@
|
|||
|
||||
const downloadAlbum = async () => {
|
||||
try {
|
||||
const fileName = album.albumName + '.zip';
|
||||
let skip = 0;
|
||||
let count = 0;
|
||||
let done = false;
|
||||
|
||||
// If assets is already download -> return;
|
||||
if ($downloadAssets[fileName]) {
|
||||
return;
|
||||
}
|
||||
while (!done) {
|
||||
count++;
|
||||
|
||||
$downloadAssets[fileName] = 0;
|
||||
const fileName = album.albumName + `${count === 1 ? '' : count}.zip`;
|
||||
|
||||
let total = 0;
|
||||
const { data, status } = await api.albumApi.downloadArchive(album.id, {
|
||||
responseType: 'blob',
|
||||
onDownloadProgress: function (progressEvent) {
|
||||
const request = this as XMLHttpRequest;
|
||||
if (!total) {
|
||||
total = Number(request.getResponseHeader('X-Immich-Content-Length-Hint')) || 0;
|
||||
$downloadAssets[fileName] = 0;
|
||||
|
||||
let total = 0;
|
||||
|
||||
const { data, status, headers } = await api.albumApi.downloadArchive(
|
||||
album.id,
|
||||
skip || undefined,
|
||||
{
|
||||
responseType: 'blob',
|
||||
onDownloadProgress: function (progressEvent) {
|
||||
const request = this as XMLHttpRequest;
|
||||
if (!total) {
|
||||
total = Number(request.getResponseHeader('X-Immich-Content-Length-Hint')) || 0;
|
||||
}
|
||||
|
||||
if (total) {
|
||||
const current = progressEvent.loaded;
|
||||
$downloadAssets[fileName] = Math.floor((current / total) * 100);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (total) {
|
||||
const current = progressEvent.loaded;
|
||||
$downloadAssets[fileName] = Math.floor((current / total) * 100);
|
||||
}
|
||||
const isNotComplete = headers['x-immich-archive-complete'] === 'false';
|
||||
const fileCount = Number(headers['x-immich-archive-file-count']) || 0;
|
||||
if (isNotComplete && fileCount > 0) {
|
||||
skip += fileCount;
|
||||
} else {
|
||||
done = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (!(data instanceof Blob)) {
|
||||
return;
|
||||
}
|
||||
if (!(data instanceof Blob)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (status === 200) {
|
||||
const fileUrl = URL.createObjectURL(data);
|
||||
const anchor = document.createElement('a');
|
||||
anchor.href = fileUrl;
|
||||
anchor.download = fileName;
|
||||
if (status === 200) {
|
||||
const fileUrl = URL.createObjectURL(data);
|
||||
const anchor = document.createElement('a');
|
||||
anchor.href = fileUrl;
|
||||
anchor.download = fileName;
|
||||
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
document.body.removeChild(anchor);
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
document.body.removeChild(anchor);
|
||||
|
||||
URL.revokeObjectURL(fileUrl);
|
||||
URL.revokeObjectURL(fileUrl);
|
||||
|
||||
// Remove item from download list
|
||||
setTimeout(() => {
|
||||
const copy = $downloadAssets;
|
||||
delete copy[fileName];
|
||||
$downloadAssets = copy;
|
||||
}, 2000);
|
||||
// Remove item from download list
|
||||
setTimeout(() => {
|
||||
const copy = $downloadAssets;
|
||||
delete copy[fileName];
|
||||
$downloadAssets = copy;
|
||||
}, 2000);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error downloading file ', e);
|
||||
|
|
Loading…
Reference in a new issue