c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-29 15:11:58 +00:00
Code Releases Activity

refactor(server): jobs and processors (#1787)

Browse source 
* refactor: jobs and processors

* refactor: storage migration processor

* fix: tests

* fix: code warning

* chore: ignore coverage from infra

* fix: sync move asset logic between job core and asset core

* refactor: move error handling inside of catch

* refactor(server): job core into dedicated service calls

* refactor: smart info

* fix: tests

* chore: smart info tests

* refactor: use asset repository

* refactor: thumbnail processor

* chore: coverage reqs
This commit is contained in:
Jason Rasmussen 2023-02-25 09:12:03 -05:00 committed by GitHub
parent 71d8567f18
commit 6c7679714b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
108 changed files with 1645 additions and 1072 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
server/apps/immich/src/api-v1/asset/asset-repository.ts
View file

@ -23,6 +23,7 @@ export interface IAssetRepository {
asset: Omit<AssetEntity, 'id' | 'createdAt' | 'updatedAt' | 'ownerId' | 'livePhotoVideoId'>,
): Promise<AssetEntity>;
remove(asset: AssetEntity): Promise<void>;
save(asset: Partial<AssetEntity>): Promise<AssetEntity>;
update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
getAll(): Promise<AssetEntity[]>;
@ -292,6 +293,11 @@ export class AssetRepository implements IAssetRepository {
await this.assetRepository.remove(asset);
}
async save(asset: Partial<AssetEntity>): Promise<AssetEntity> {
const { id } = await this.assetRepository.save(asset);
return this.assetRepository.findOneOrFail({ where: { id } });
}
/**
* Update asset
*/

49
server/apps/immich/src/api-v1/asset/asset.core.ts
View file

@ -1,15 +1,29 @@
import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
import { AssetEntity, UserEntity } from '@app/infra/db/entities';
import { StorageService } from '@app/storage';
import {
AuthUserDto,
IJobRepository,
IStorageRepository,
ISystemConfigRepository,
JobName,
StorageTemplateCore,
} from '@app/domain';
import { AssetEntity, SystemConfig, UserEntity } from '@app/infra/db/entities';
import { Logger } from '@nestjs/common';
import { IAssetRepository } from './asset-repository';
import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
export class AssetCore {
private templateCore: StorageTemplateCore;
private logger = new Logger(AssetCore.name);
constructor(
private repository: IAssetRepository,
private jobRepository: IJobRepository,
private storageService: StorageService,
) {}
configRepository: ISystemConfigRepository,
config: SystemConfig,
private storageRepository: IStorageRepository,
) {
this.templateCore = new StorageTemplateCore(configRepository, config, storageRepository);
}
async create(
authUser: AuthUserDto,
@ -42,10 +56,31 @@ export class AssetCore {
sharedLinks: [],
});
asset = await this.storageService.moveAsset(asset, file.originalName);
asset = await this.moveAsset(asset, file.originalName);
await this.jobRepository.add({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
return asset;
}
async moveAsset(asset: AssetEntity, originalName: string) {
const destination = await this.templateCore.getTemplatePath(asset, originalName);
if (asset.originalPath !== destination) {
const source = asset.originalPath;
try {
await this.storageRepository.moveFile(asset.originalPath, destination);
try {
await this.repository.save({ id: asset.id, originalPath: destination });
asset.originalPath = destination;
} catch (error: any) {
this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
await this.storageRepository.moveFile(destination, source);
}
} catch (error: any) {
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
}
}
return asset;
}
}

5
server/apps/immich/src/api-v1/asset/asset.module.ts
View file

@ -3,12 +3,10 @@ import { AssetService } from './asset.service';
import { AssetController } from './asset.controller';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AssetEntity } from '@app/infra';
import { CommunicationModule } from '../communication/communication.module';
import { AssetRepository, IAssetRepository } from './asset-repository';
import { DownloadModule } from '../../modules/download/download.module';
import { TagModule } from '../tag/tag.module';
import { AlbumModule } from '../album/album.module';
import { StorageModule } from '@app/storage';
const ASSET_REPOSITORY_PROVIDER = {
provide: IAssetRepository,
@ -17,11 +15,10 @@ const ASSET_REPOSITORY_PROVIDER = {
@Module({
imports: [
//
TypeOrmModule.forFeature([AssetEntity]),
CommunicationModule,
DownloadModule,
TagModule,
StorageModule,
AlbumModule,
],
controllers: [AssetController],

171
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View file

@ -8,19 +8,30 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { DownloadService } from '../../modules/download/download.service';
import { AlbumRepository, IAlbumRepository } from '../album/album-repository';
import { StorageService } from '@app/storage';
import { ICryptoRepository, IJobRepository, ISharedLinkRepository, IStorageRepository, JobName } from '@app/domain';
import {
ICryptoRepository,
IJobRepository,
ISharedLinkRepository,
IStorageRepository,
ISystemConfigRepository,
JobName,
} from '@app/domain';
import {
assetEntityStub,
authStub,
fileStub,
newCryptoRepositoryMock,
newJobRepositoryMock,
newSharedLinkRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
sharedLinkResponseStub,
sharedLinkStub,
systemConfigStub,
} from '@app/domain/../test';
import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
import { BadRequestException, ForbiddenException } from '@nestjs/common';
import { when } from 'jest-when';
const _getCreateAssetDto = (): CreateAssetDto => {
const createAssetDto = new CreateAssetDto();
@ -109,8 +120,8 @@ describe('AssetService', () => {
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
let storageServiceMock: jest.Mocked<StorageService>;
let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
@ -120,6 +131,7 @@ describe('AssetService', () => {
get: jest.fn(),
create: jest.fn(),
remove: jest.fn(),
save: jest.fn(),
update: jest.fn(),
getAll: jest.fn(),
@ -150,13 +162,9 @@ describe('AssetService', () => {
downloadArchive: jest.fn(),
};
storageServiceMock = {
moveAsset: jest.fn(),
removeEmptyDirectories: jest.fn(),
} as unknown as jest.Mocked<StorageService>;
sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
jobMock = newJobRepositoryMock();
configMock = newSystemConfigRepositoryMock();
cryptoMock = newCryptoRepositoryMock();
storageMock = newStorageRepositoryMock();
@ -165,12 +173,20 @@ describe('AssetService', () => {
albumRepositoryMock,
a,
downloadServiceMock as DownloadService,
storageServiceMock,
sharedLinkRepositoryMock,
jobMock,
configMock,
systemConfigStub.defaults,
cryptoMock,
storageMock,
);
when(assetRepositoryMock.get)
.calledWith(assetEntityStub.livePhotoStillAsset.id)
.mockResolvedValue(assetEntityStub.livePhotoStillAsset);
when(assetRepositoryMock.get)
.calledWith(assetEntityStub.livePhotoMotionAsset.id)
.mockResolvedValue(assetEntityStub.livePhotoMotionAsset);
});
describe('createAssetsSharedLink', () => {
@ -255,10 +271,16 @@ describe('AssetService', () => {
};
const dto = _getCreateAssetDto();
assetRepositoryMock.create.mockImplementation(() => Promise.resolve(assetEntity));
storageServiceMock.moveAsset.mockResolvedValue({ ...assetEntity, originalPath: 'fake_new_path/asset_123.jpeg' });
assetRepositoryMock.create.mockResolvedValue(assetEntity);
assetRepositoryMock.save.mockResolvedValue(assetEntity);
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' });
expect(assetRepositoryMock.create).toHaveBeenCalled();
expect(assetRepositoryMock.save).toHaveBeenCalledWith({
id: 'id_1',
originalPath: 'upload/user_id_1/2022/2022-06-19/asset_1.jpeg',
});
});
it('should handle a duplicate', async () => {
@ -277,59 +299,43 @@ describe('AssetService', () => {
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });
expect(jobMock.add).toHaveBeenCalledWith({
name: JobName.DELETE_FILE_ON_DISK,
data: { assets: [{ originalPath: 'fake_path/asset_1.jpeg', resizePath: null }] },
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.DELETE_FILES,
data: { files: ['fake_path/asset_1.jpeg', undefined] },
});
expect(storageServiceMock.moveAsset).not.toHaveBeenCalled();
expect(storageMock.moveFile).not.toHaveBeenCalled();
});
it('should handle a live photo', async () => {
const file = {
originalPath: 'fake_path/asset_1.jpeg',
mimeType: 'image/jpeg',
checksum: Buffer.from('file hash', 'utf8'),
originalName: 'asset_1.jpeg',
};
const asset = {
id: 'live-photo-asset',
originalPath: file.originalPath,
ownerId: authStub.user1.id,
type: AssetType.IMAGE,
isVisible: true,
} as AssetEntity;
const livePhotoFile = {
originalPath: 'fake_path/asset_1.mp4',
mimeType: 'image/jpeg',
checksum: Buffer.from('live photo file hash', 'utf8'),
originalName: 'asset_1.jpeg',
};
const livePhotoAsset = {
id: 'live-photo-motion',
originalPath: livePhotoFile.originalPath,
ownerId: authStub.user1.id,
type: AssetType.VIDEO,
isVisible: false,
} as AssetEntity;
const dto = _getCreateAssetDto();
const error = new QueryFailedError('', [], '');
(error as any).constraint = 'UQ_userid_checksum';
assetRepositoryMock.create.mockResolvedValueOnce(livePhotoAsset);
assetRepositoryMock.create.mockResolvedValueOnce(asset);
storageServiceMock.moveAsset.mockImplementation((asset) => Promise.resolve(asset));
assetRepositoryMock.create.mockResolvedValueOnce(assetEntityStub.livePhotoMotionAsset);
assetRepositoryMock.save.mockResolvedValueOnce(assetEntityStub.livePhotoMotionAsset);
assetRepositoryMock.create.mockResolvedValueOnce(assetEntityStub.livePhotoStillAsset);
assetRepositoryMock.save.mockResolvedValueOnce(assetEntityStub.livePhotoStillAsset);
await expect(sut.uploadFile(authStub.user1, dto, file, livePhotoFile)).resolves.toEqual({
await expect(
sut.uploadFile(authStub.user1, dto, fileStub.livePhotoStill, fileStub.livePhotoMotion),
).resolves.toEqual({
duplicate: false,
id: 'live-photo-asset',
id: 'live-photo-still-asset',
});
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.ASSET_UPLOADED, data: { asset: livePhotoAsset, fileName: file.originalName } }],
[{ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } }],
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.ASSET_UPLOADED,
data: { asset: assetEntityStub.livePhotoMotionAsset, fileName: 'asset_1.mp4' },
},
],
[
{
name: JobName.ASSET_UPLOADED,
data: { asset: assetEntityStub.livePhotoStillAsset, fileName: 'asset_1.jpeg' },
},
],
]);
});
});
@ -383,7 +389,7 @@ describe('AssetService', () => {
{ id: 'asset1', status: 'FAILED' },
]);
expect(jobMock.add).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should return failed status a delete fails', async () => {
@ -394,35 +400,66 @@ describe('AssetService', () => {
{ id: 'asset1', status: 'FAILED' },
]);
expect(jobMock.add).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should delete a live photo', async () => {
assetRepositoryMock.get.mockResolvedValueOnce({ id: 'asset1', livePhotoVideoId: 'live-photo' } as AssetEntity);
assetRepositoryMock.get.mockResolvedValueOnce({ id: 'live-photo' } as AssetEntity);
await expect(sut.deleteAll(authStub.user1, { ids: ['asset1'] })).resolves.toEqual([
{ id: 'asset1', status: 'SUCCESS' },
{ id: 'live-photo', status: 'SUCCESS' },
await expect(sut.deleteAll(authStub.user1, { ids: [assetEntityStub.livePhotoStillAsset.id] })).resolves.toEqual([
{ id: assetEntityStub.livePhotoStillAsset.id, status: 'SUCCESS' },
{ id: assetEntityStub.livePhotoMotionAsset.id, status: 'SUCCESS' },
]);
expect(jobMock.add).toHaveBeenCalledWith({
name: JobName.DELETE_FILE_ON_DISK,
data: { assets: [{ id: 'asset1', livePhotoVideoId: 'live-photo' }, { id: 'live-photo' }] },
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.DELETE_FILES,
data: {
files: ['fake_path/asset_1.jpeg', undefined, undefined, 'fake_path/asset_1.mp4', undefined, undefined],
},
});
});
it('should delete a batch of assets', async () => {
assetRepositoryMock.get.mockImplementation((id) => Promise.resolve({ id } as AssetEntity));
assetRepositoryMock.remove.mockImplementation(() => Promise.resolve());
const asset1 = {
id: 'asset1',
originalPath: 'original-path-1',
resizePath: 'resize-path-1',
webpPath: 'web-path-1',
};
const asset2 = {
id: 'asset2',
originalPath: 'original-path-2',
resizePath: 'resize-path-2',
webpPath: 'web-path-2',
};
when(assetRepositoryMock.get)
.calledWith(asset1.id)
.mockResolvedValue(asset1 as AssetEntity);
when(assetRepositoryMock.get)
.calledWith(asset2.id)
.mockResolvedValue(asset2 as AssetEntity);
await expect(sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'] })).resolves.toEqual([
{ id: 'asset1', status: 'SUCCESS' },
{ id: 'asset2', status: 'SUCCESS' },
]);
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.DELETE_FILE_ON_DISK, data: { assets: [{ id: 'asset1' }, { id: 'asset2' }] } }],
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.DELETE_FILES,
data: {
files: [
'original-path-1',
'web-path-1',
'resize-path-1',
'original-path-2',
'web-path-2',
'resize-path-2',
],
},
},
],
]);
});
});

40
server/apps/immich/src/api-v1/asset/asset.service.ts
View file

@ -12,7 +12,7 @@ import {
import { InjectRepository } from '@nestjs/typeorm';
import { QueryFailedError, Repository } from 'typeorm';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { AssetEntity, AssetType, SharedLinkType } from '@app/infra';
import { AssetEntity, AssetType, SharedLinkType, SystemConfig } from '@app/infra';
import { constants, createReadStream, ReadStream, stat } from 'fs';
import { ServeFileDto } from './dto/serve-file.dto';
import { Response as Res } from 'express';
@ -25,7 +25,9 @@ import { CuratedObjectsResponseDto } from './response-dto/curated-objects-respon
import {
AssetResponseDto,
ImmichReadStream,
INITIAL_SYSTEM_CONFIG,
IStorageRepository,
ISystemConfigRepository,
JobName,
mapAsset,
mapAssetWithoutExif,
@ -52,7 +54,6 @@ import { ICryptoRepository, IJobRepository } from '@app/domain';
import { DownloadService } from '../../modules/download/download.service';
import { DownloadDto } from './dto/download-library.dto';
import { IAlbumRepository } from '../album/album-repository';
import { StorageService } from '@app/storage';
import { ShareCore } from '@app/domain';
import { ISharedLinkRepository } from '@app/domain';
import { DownloadFilesDto } from './dto/download-files.dto';
@ -61,6 +62,8 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
import { AssetSearchDto } from './dto/asset-search.dto';
import { AddAssetsDto } from '../album/dto/add-assets.dto';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import path from 'path';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
const fileInfo = promisify(stat);
@ -76,13 +79,14 @@ export class AssetService {
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
private downloadService: DownloadService,
storageService: StorageService,
@Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(IStorageRepository) private storage: IStorageRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.assetCore = new AssetCore(_assetRepository, jobRepository, storageService);
this.assetCore = new AssetCore(_assetRepository, jobRepository, configRepository, config, storageRepository);
this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
}
@ -93,7 +97,10 @@ export class AssetService {
livePhotoFile?: UploadFile,
): Promise<AssetFileUploadResponseDto> {
if (livePhotoFile) {
livePhotoFile.originalName = file.originalName;
livePhotoFile = {
...livePhotoFile,
originalName: getFileNameWithoutExtension(file.originalName) + path.extname(livePhotoFile.originalName),
};
}
let livePhotoAsset: AssetEntity | null = null;
@ -109,16 +116,9 @@ export class AssetService {
return { id: asset.id, duplicate: false };
} catch (error: any) {
// clean up files
await this.jobRepository.add({
name: JobName.DELETE_FILE_ON_DISK,
data: {
assets: [
{
originalPath: file.originalPath,
resizePath: livePhotoFile?.originalPath || null,
} as AssetEntity,
],
},
await this.jobRepository.queue({
name: JobName.DELETE_FILES,
data: { files: [file.originalPath, livePhotoFile?.originalPath] },
});
// handle duplicates with a success response
@ -204,7 +204,7 @@ export class AssetService {
try {
const asset = await this._assetRepository.get(assetId);
if (asset && asset.originalPath && asset.mimeType) {
return this.storage.createReadStream(asset.originalPath, asset.mimeType);
return this.storageRepository.createReadStream(asset.originalPath, asset.mimeType);
}
} catch (e) {
Logger.error(`Error download asset ${e}`, 'downloadFile');
@ -412,7 +412,7 @@ export class AssetService {
}
public async deleteAll(authUser: AuthUserDto, dto: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
const deleteQueue: AssetEntity[] = [];
const deleteQueue: Array<string | null> = [];
const result: DeleteAssetResponseDto[] = [];
const ids = dto.ids.slice();
@ -427,7 +427,7 @@ export class AssetService {
await this._assetRepository.remove(asset);
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
deleteQueue.push(asset as any);
deleteQueue.push(asset.originalPath, asset.webpPath, asset.resizePath);
// TODO refactor this to use cascades
if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
@ -439,7 +439,7 @@ export class AssetService {
}
if (deleteQueue.length > 0) {
await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets: deleteQueue } });
await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: deleteQueue } });
}
return result;

8
server/apps/immich/src/api-v1/communication/communication.module.ts
View file

@ -1,8 +0,0 @@
import { Module } from '@nestjs/common';
import { CommunicationGateway } from './communication.gateway';
@Module({
providers: [CommunicationGateway],
exports: [CommunicationGateway],
})
export class CommunicationModule {}

18
server/apps/immich/src/api-v1/job/job.service.ts
View file

@ -48,14 +48,14 @@ export class JobService {
? await this._assetRepository.getAllVideos()
: await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
return assets.length;
}
case QueueName.CONFIG:
await this.jobRepository.add({ name: JobName.TEMPLATE_MIGRATION });
case QueueName.STORAGE_TEMPLATE_MIGRATION:
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
return 1;
case QueueName.MACHINE_LEARNING: {
@ -68,8 +68,8 @@ export class JobService {
: await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
}
return assets.length;
}
@ -81,7 +81,7 @@ export class JobService {
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({
await this.jobRepository.queue({
name: JobName.EXTRACT_VIDEO_METADATA,
data: {
asset,
@ -89,7 +89,7 @@ export class JobService {
},
});
} else {
await this.jobRepository.add({
await this.jobRepository.queue({
name: JobName.EXIF_EXTRACTION,
data: {
asset,
@ -107,7 +107,7 @@ export class JobService {
: await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}
return assets.length;
}
@ -129,7 +129,7 @@ export class JobService {
return QueueName.VIDEO_CONVERSION;
case JobId.STORAGE_TEMPLATE_MIGRATION:
return QueueName.CONFIG;
return QueueName.STORAGE_TEMPLATE_MIGRATION;
case JobId.MACHINE_LEARNING:
return QueueName.MACHINE_LEARNING;

3
server/apps/immich/src/app.module.ts
View file

@ -3,7 +3,6 @@ import { Module } from '@nestjs/common';
import { AssetModule } from './api-v1/asset/asset.module';
import { ConfigModule } from '@nestjs/config';
import { ServerInfoModule } from './api-v1/server-info/server-info.module';
import { CommunicationModule } from './api-v1/communication/communication.module';
import { AlbumModule } from './api-v1/album/album.module';
import { AppController } from './app.controller';
import { ScheduleModule } from '@nestjs/schedule';
@ -36,8 +35,6 @@ import { AuthGuard } from './middlewares/auth.guard';
ServerInfoModule,
CommunicationModule,
AlbumModule,
ScheduleModule.forRoot(),

23
server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts
View file

@ -1,26 +1,13 @@
import { Inject, Injectable } from '@nestjs/common';
import { UserService } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Not, Repository } from 'typeorm';
import { UserEntity } from '@app/infra';
import { userUtils } from '@app/common';
import { IJobRepository, JobName } from '@app/domain';
@Injectable()
export class ScheduleTasksService {
constructor(
@InjectRepository(UserEntity)
private userRepository: Repository<UserEntity>,
constructor(private userService: UserService) {}
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {}
@Cron(CronExpression.EVERY_DAY_AT_11PM)
async deleteUserAndRelatedAssets() {
const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });
for (const user of usersToDelete) {
if (userUtils.isReadyForDeletion(user)) {
await this.jobRepository.add({ name: JobName.USER_DELETION, data: { user } });
}
}
async onUserDeleteCheck() {
await this.userService.handleUserDeleteCheck();
}
}

1
server/apps/immich/test/jest-e2e.json
View file

@ -9,7 +9,6 @@
},
"moduleNameMapper": {
"^@app/common": "<rootDir>../../../libs/common/src",
"^@app/storage(|/.*)$": "<rootDir>../../../libs/storage/src/$1",
"^@app/infra(|/.*)$": "<rootDir>../../../libs/infra/src/$1",
"^@app/domain(|/.*)$": "<rootDir>../../../libs/domain/src/$1"
}

48
server/apps/microservices/src/microservices.module.ts
View file

@ -1,56 +1,30 @@
import { immichAppConfig } from '@app/common/config';
import {
AssetEntity,
ExifEntity,
SmartInfoEntity,
UserEntity,
APIKeyEntity,
InfraModule,
UserTokenEntity,
AlbumEntity,
} from '@app/infra';
import { StorageModule } from '@app/storage';
import { DomainModule } from '@app/domain';
import { ExifEntity, InfraModule } from '@app/infra';
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module';
import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
import { MachineLearningProcessor } from './processors/machine-learning.processor';
import {
BackgroundTaskProcessor,
MachineLearningProcessor,
StorageTemplateMigrationProcessor,
ThumbnailGeneratorProcessor,
} from './processors';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
import { StorageMigrationProcessor } from './processors/storage-migration.processor';
import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
import { UserDeletionProcessor } from './processors/user-deletion.processor';
import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
import { BackgroundTaskProcessor } from './processors/background-task.processor';
import { DomainModule } from '@app/domain';
@Module({
imports: [
ConfigModule.forRoot(immichAppConfig),
DomainModule.register({
imports: [InfraModule],
}),
TypeOrmModule.forFeature([
UserEntity,
ExifEntity,
AssetEntity,
SmartInfoEntity,
APIKeyEntity,
UserTokenEntity,
AlbumEntity,
]),
StorageModule,
CommunicationModule,
DomainModule.register({ imports: [InfraModule] }),
TypeOrmModule.forFeature([ExifEntity]),
],
controllers: [],
providers: [
AssetUploadedProcessor,
ThumbnailGeneratorProcessor,
MetadataExtractionProcessor,
VideoTranscodeProcessor,
MachineLearningProcessor,
UserDeletionProcessor,
StorageMigrationProcessor,
StorageTemplateMigrationProcessor,
BackgroundTaskProcessor,
],
})

87
server/apps/microservices/src/processors.ts Normal file
View file

@ -0,0 +1,87 @@
import {
AssetService,
IAssetJob,
IAssetUploadedJob,
IDeleteFilesJob,
IUserDeletionJob,
JobName,
MediaService,
QueueName,
SmartInfoService,
StorageService,
StorageTemplateService,
SystemConfigService,
UserService,
} from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
@Processor(QueueName.BACKGROUND_TASK)
export class BackgroundTaskProcessor {
constructor(
private assetService: AssetService,
private storageService: StorageService,
private systemConfigService: SystemConfigService,
private userService: UserService,
) {}
@Process(JobName.ASSET_UPLOADED)
async onAssetUpload(job: Job<IAssetUploadedJob>) {
await this.assetService.handleAssetUpload(job.data);
}
@Process(JobName.DELETE_FILES)
async onDeleteFile(job: Job<IDeleteFilesJob>) {
await this.storageService.handleDeleteFiles(job.data);
}
@Process(JobName.SYSTEM_CONFIG_CHANGE)
async onSystemConfigChange() {
await this.systemConfigService.refreshConfig();
}
@Process(JobName.USER_DELETION)
async onUserDelete(job: Job<IUserDeletionJob>) {
await this.userService.handleUserDelete(job.data);
}
}
@Processor(QueueName.MACHINE_LEARNING)
export class MachineLearningProcessor {
constructor(private smartInfoService: SmartInfoService) {}
@Process({ name: JobName.IMAGE_TAGGING, concurrency: 2 })
async onTagImage(job: Job<IAssetJob>) {
await this.smartInfoService.handleTagImage(job.data);
}
@Process({ name: JobName.OBJECT_DETECTION, concurrency: 2 })
async onDetectObject(job: Job<IAssetJob>) {
await this.smartInfoService.handleDetectObjects(job.data);
}
}
@Processor(QueueName.STORAGE_TEMPLATE_MIGRATION)
export class StorageTemplateMigrationProcessor {
constructor(private storageTemplateService: StorageTemplateService) {}
@Process({ name: JobName.STORAGE_TEMPLATE_MIGRATION })
async onTemplateMigration() {
await this.storageTemplateService.handleTemplateMigration();
}
}
@Processor(QueueName.THUMBNAIL_GENERATION)
export class ThumbnailGeneratorProcessor {
constructor(private mediaService: MediaService) {}
@Process({ name: JobName.GENERATE_JPEG_THUMBNAIL, concurrency: 3 })
async handleGenerateJpegThumbnail(job: Job<IAssetJob>) {
await this.mediaService.handleGenerateJpegThumbnail(job.data);
}
@Process({ name: JobName.GENERATE_WEBP_THUMBNAIL, concurrency: 3 })
async handleGenerateWepbThumbnail(job: Job<IAssetJob>) {
await this.mediaService.handleGenerateWepbThumbnail(job.data);
}
}

13
server/apps/microservices/src/processors/asset-uploaded.processor.ts
View file

@ -1,13 +0,0 @@
import { IAssetUploadedJob, JobName, JobService, QueueName } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
@Processor(QueueName.ASSET_UPLOADED)
export class AssetUploadedProcessor {
constructor(private jobService: JobService) {}
@Process(JobName.ASSET_UPLOADED)
async processUploadedVideo(job: Job<IAssetUploadedJob>) {
await this.jobService.handleUploadedAsset(job);
}
}

17
server/apps/microservices/src/processors/background-task.processor.ts
View file

@ -1,17 +0,0 @@
import { assetUtils } from '@app/common/utils';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
import { JobName, QueueName } from '@app/domain';
import { AssetEntity } from '@app/infra/db/entities';
@Processor(QueueName.BACKGROUND_TASK)
export class BackgroundTaskProcessor {
@Process(JobName.DELETE_FILE_ON_DISK)
async deleteFileOnDisk(job: Job<{ assets: AssetEntity[] }>) {
const { assets } = job.data;
for (const asset of assets) {
assetUtils.deleteFiles(asset);
}
}
}

68
server/apps/microservices/src/processors/machine-learning.processor.ts
View file

@ -1,68 +0,0 @@
import { AssetEntity } from '@app/infra';
import { SmartInfoEntity } from '@app/infra';
import { QueueName, JobName } from '@app/domain';
import { IMachineLearningJob } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import axios from 'axios';
import { Job } from 'bull';
import { Repository } from 'typeorm';
import { MACHINE_LEARNING_ENABLED, MACHINE_LEARNING_URL } from '@app/common';
@Processor(QueueName.MACHINE_LEARNING)
export class MachineLearningProcessor {
constructor(
@InjectRepository(SmartInfoEntity)
private smartInfoRepository: Repository<SmartInfoEntity>,
) {}
@Process({ name: JobName.IMAGE_TAGGING, concurrency: 2 })
async tagImage(job: Job<IMachineLearningJob>) {
if (!MACHINE_LEARNING_ENABLED) {
return;
}
const { asset } = job.data;
const res = await axios.post(MACHINE_LEARNING_URL + '/image-classifier/tag-image', {
thumbnailPath: asset.resizePath,
});
if (res.status == 201 && res.data.length > 0) {
const smartInfo = new SmartInfoEntity();
smartInfo.assetId = asset.id;
smartInfo.tags = [...res.data];
await this.smartInfoRepository.upsert(smartInfo, {
conflictPaths: ['assetId'],
});
}
}
@Process({ name: JobName.OBJECT_DETECTION, concurrency: 2 })
async detectObject(job: Job<IMachineLearningJob>) {
if (!MACHINE_LEARNING_ENABLED) {
return;
}
try {
const { asset }: { asset: AssetEntity } = job.data;
const res = await axios.post(MACHINE_LEARNING_URL + '/object-detection/detect-object', {
thumbnailPath: asset.resizePath,
});
if (res.status == 201 && res.data.length > 0) {
const smartInfo = new SmartInfoEntity();
smartInfo.assetId = asset.id;
smartInfo.objects = [...res.data];
await this.smartInfoRepository.upsert(smartInfo, {
conflictPaths: ['assetId'],
});
}
} catch (error) {
Logger.error(`Failed to trigger object detection pipe line ${String(error)}`);
}
}
}

65
server/apps/microservices/src/processors/metadata-extraction.processor.ts
View file

@ -1,13 +1,7 @@
import { AssetEntity, AssetType, ExifEntity } from '@app/infra';
import {
IExifExtractionProcessor,
IReverseGeocodingProcessor,
IVideoLengthExtractionProcessor,
QueueName,
JobName,
} from '@app/domain';
import { IReverseGeocodingJob, IAssetUploadedJob, QueueName, JobName, IAssetRepository } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { Inject, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Job } from 'bull';
@ -19,7 +13,6 @@ import geocoder, { InitOptions } from 'local-reverse-geocoder';
import { getName } from 'i18n-iso-countries';
import fs from 'node:fs';
import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
import { IsNull, Not } from 'typeorm';
interface ImmichTags extends Tags {
ContentIdentifier?: string;
@ -79,9 +72,7 @@ export class MetadataExtractionProcessor {
private logger = new Logger(MetadataExtractionProcessor.name);
private isGeocodeInitialized = false;
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@InjectRepository(ExifEntity)
private exifRepository: Repository<ExifEntity>,
@ -141,7 +132,7 @@ export class MetadataExtractionProcessor {
}
@Process(JobName.EXIF_EXTRACTION)
async extractExifInfo(job: Job<IExifExtractionProcessor>) {
async extractExifInfo(job: Job<IAssetUploadedJob>) {
try {
const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
@ -190,22 +181,14 @@ export class MetadataExtractionProcessor {
});
if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
const motionAsset = await this.assetRepository.findOne({
where: {
id: Not(asset.id),
type: AssetType.VIDEO,
exifInfo: {
livePhotoCID: newExif.livePhotoCID,
},
},
relations: {
exifInfo: true,
},
});
const motionAsset = await this.assetRepository.findLivePhotoMatch(
newExif.livePhotoCID,
AssetType.VIDEO,
asset.id,
);
if (motionAsset) {
await this.assetRepository.update(asset.id, { livePhotoVideoId: motionAsset.id });
await this.assetRepository.update(motionAsset.id, { isVisible: false });
await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
}
}
@ -249,7 +232,7 @@ export class MetadataExtractionProcessor {
}
@Process({ name: JobName.REVERSE_GEOCODING })
async reverseGeocoding(job: Job<IReverseGeocodingProcessor>) {
async reverseGeocoding(job: Job<IReverseGeocodingJob>) {
if (this.isGeocodeInitialized) {
const { latitude, longitude } = job.data;
const { country, state, city } = await this.reverseGeocodeExif(latitude, longitude);
@ -258,7 +241,7 @@ export class MetadataExtractionProcessor {
}
@Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 })
async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
async extractVideoMetadata(job: Job<IAssetUploadedJob>) {
const { asset, fileName } = job.data;
if (!asset.isVisible) {
@ -309,20 +292,14 @@ export class MetadataExtractionProcessor {
newExif.livePhotoCID = exifData?.ContentIdentifier || null;
if (newExif.livePhotoCID) {
const photoAsset = await this.assetRepository.findOne({
where: {
id: Not(asset.id),
type: AssetType.IMAGE,
livePhotoVideoId: IsNull(),
exifInfo: {
livePhotoCID: newExif.livePhotoCID,
},
},
});
const photoAsset = await this.assetRepository.findLivePhotoMatch(
newExif.livePhotoCID,
AssetType.IMAGE,
asset.id,
);
if (photoAsset) {
await this.assetRepository.update(photoAsset.id, { livePhotoVideoId: asset.id });
await this.assetRepository.update(asset.id, { isVisible: false });
await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
await this.assetRepository.save({ id: asset.id, isVisible: false });
}
}
@ -378,7 +355,7 @@ export class MetadataExtractionProcessor {
}
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetRepository.update({ id: asset.id }, { duration: durationString, fileCreatedAt });
await this.assetRepository.save({ id: asset.id, duration: durationString, fileCreatedAt });
} catch (err) {
``;
// do nothing

61
server/apps/microservices/src/processors/storage-migration.processor.ts
View file

@ -1,61 +0,0 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity } from '@app/infra';
import { SystemConfigService } from '@app/domain';
import { QueueName, JobName } from '@app/domain';
import { StorageService } from '@app/storage';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
@Processor(QueueName.CONFIG)
export class StorageMigrationProcessor {
readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
constructor(
private storageService: StorageService,
private systemConfigService: SystemConfigService,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) {}
/**
* Migration process when a new user set a new storage template.
* @param job
*/
@Process({ name: JobName.TEMPLATE_MIGRATION, concurrency: 100 })
async templateMigration() {
console.time('migrating-time');
const assets = await this.assetRepository.find({
relations: ['exifInfo'],
});
const livePhotoMap: Record<string, AssetEntity> = {};
for (const asset of assets) {
if (asset.livePhotoVideoId) {
livePhotoMap[asset.livePhotoVideoId] = asset;
}
}
for (const asset of assets) {
const livePhotoParentAsset = livePhotoMap[asset.id];
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
await this.storageService.moveAsset(asset, filename);
}
await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
console.timeEnd('migrating-time');
}
/**
* Update config when a new storage template is set.
* This is to ensure the synchronization between processes.
* @param job
*/
@Process({ name: JobName.CONFIG_CHANGE, concurrency: 1 })
async updateTemplate() {
await this.systemConfigService.refreshConfig();
}
}

130
server/apps/microservices/src/processors/thumbnail.processor.ts
View file

@ -1,130 +0,0 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity, AssetType } from '@app/infra';
import { WebpGeneratorProcessor, JpegGeneratorProcessor, QueueName, JobName } from '@app/domain';
import { InjectQueue, Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { mapAsset } from '@app/domain';
import { Job, Queue } from 'bull';
import ffmpeg from 'fluent-ffmpeg';
import { existsSync, mkdirSync } from 'node:fs';
import sanitize from 'sanitize-filename';
import sharp from 'sharp';
import { Repository } from 'typeorm/repository/Repository';
import { join } from 'path';
import { CommunicationGateway } from 'apps/immich/src/api-v1/communication/communication.gateway';
import { IMachineLearningJob } from '@app/domain';
import { exiftool } from 'exiftool-vendored';
@Processor(QueueName.THUMBNAIL_GENERATION)
export class ThumbnailGeneratorProcessor {
readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectQueue(QueueName.THUMBNAIL_GENERATION)
private thumbnailGeneratorQueue: Queue,
private wsCommunicationGateway: CommunicationGateway,
@InjectQueue(QueueName.MACHINE_LEARNING)
private machineLearningQueue: Queue<IMachineLearningJob>,
) {}
@Process({ name: JobName.GENERATE_JPEG_THUMBNAIL, concurrency: 3 })
async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
const basePath = APP_UPLOAD_LOCATION;
const { asset } = job.data;
const sanitizedDeviceId = sanitize(String(asset.deviceId));
const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
if (!existsSync(resizePath)) {
mkdirSync(resizePath, { recursive: true });
}
const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
if (asset.type == AssetType.IMAGE) {
try {
await sharp(asset.originalPath, { failOnError: false })
.resize(1440, 1440, { fit: 'outside', withoutEnlargement: true })
.jpeg()
.rotate()
.toFile(jpegThumbnailPath)
.catch(() => {
this.logger.warn(
'Failed to generate jpeg thumbnail for asset: ' +
asset.id +
' using sharp, failing over to exiftool-vendored',
);
return exiftool.extractThumbnail(asset.originalPath, jpegThumbnailPath);
});
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
} catch (error: any) {
this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
}
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset });
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
this.wsCommunicationGateway.server.to(asset.ownerId).emit('on_upload_success', JSON.stringify(mapAsset(asset)));
}
if (asset.type == AssetType.VIDEO) {
await new Promise((resolve, reject) => {
ffmpeg(asset.originalPath)
.outputOptions(['-ss 00:00:00.000', '-frames:v 1'])
.output(jpegThumbnailPath)
.on('start', () => {
Logger.log('Start Generating Video Thumbnail', 'generateJPEGThumbnail');
})
.on('error', (error) => {
Logger.error(`Cannot Generate Video Thumbnail ${error}`, 'generateJPEGThumbnail');
reject(error);
})
.on('end', async () => {
Logger.log(`Generating Video Thumbnail Success ${asset.id}`, 'generateJPEGThumbnail');
resolve(asset);
})
.run();
});
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset });
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
this.wsCommunicationGateway.server.to(asset.ownerId).emit('on_upload_success', JSON.stringify(mapAsset(asset)));
}
}
@Process({ name: JobName.GENERATE_WEBP_THUMBNAIL, concurrency: 3 })
async generateWepbThumbnail(job: Job<WebpGeneratorProcessor>) {
const { asset } = job.data;
if (!asset.resizePath) {
return;
}
const webpPath = asset.resizePath.replace('jpeg', 'webp');
try {
await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
} catch (error: any) {
this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
}
}
}

72
server/apps/microservices/src/processors/user-deletion.processor.ts
View file

@ -1,72 +0,0 @@
import { APP_UPLOAD_LOCATION, userUtils } from '@app/common';
import { AlbumEntity, APIKeyEntity, AssetEntity, UserEntity, UserTokenEntity } from '@app/infra';
import { QueueName, JobName } from '@app/domain';
import { IUserDeletionJob } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Job } from 'bull';
import { join } from 'path';
import fs from 'fs';
import { Repository } from 'typeorm';
@Processor(QueueName.USER_DELETION)
export class UserDeletionProcessor {
private logger = new Logger(UserDeletionProcessor.name);
constructor(
@InjectRepository(UserEntity)
private userRepository: Repository<UserEntity>,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectRepository(APIKeyEntity)
private apiKeyRepository: Repository<APIKeyEntity>,
@InjectRepository(UserTokenEntity)
private userTokenRepository: Repository<UserTokenEntity>,
@InjectRepository(AlbumEntity)
private albumRepository: Repository<AlbumEntity>,
) {}
@Process(JobName.USER_DELETION)
async processUserDeletion(job: Job<IUserDeletionJob>) {
const { user } = job.data;
// just for extra protection here
if (!userUtils.isReadyForDeletion(user)) {
this.logger.warn(`Skipped user that was not ready for deletion: id=${user.id}`);
return;
}
this.logger.log(`Deleting user: ${user.id}`);
try {
const basePath = APP_UPLOAD_LOCATION;
const userAssetDir = join(basePath, user.id);
this.logger.warn(`Removing user from filesystem: ${userAssetDir}`);
fs.rmSync(userAssetDir, { recursive: true, force: true });
this.logger.warn(`Removing user from database: ${user.id}`);
const userTokens = await this.userTokenRepository.find({
where: { user: { id: user.id } },
relations: { user: true },
withDeleted: true,
});
await this.userTokenRepository.remove(userTokens);
const albums = await this.albumRepository.find({ where: { ownerId: user.id } });
await this.albumRepository.remove(albums);
await this.apiKeyRepository.delete({ userId: user.id });
await this.assetRepository.delete({ ownerId: user.id });
await this.userRepository.remove(user);
} catch (error: any) {
this.logger.error(`Failed to remove user`);
this.logger.error(error, error?.stack);
throw error;
}
}
}

13
server/apps/microservices/src/processors/video-transcode.processor.ts
View file

@ -1,25 +1,22 @@
import { APP_UPLOAD_LOCATION } from '@app/common/constants';
import { AssetEntity } from '@app/infra';
import { IVideoConversionProcessor, JobName, QueueName, SystemConfigService } from '@app/domain';
import { IAssetJob, IAssetRepository, JobName, QueueName, SystemConfigService } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Inject, Logger } from '@nestjs/common';
import { Job } from 'bull';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import { existsSync, mkdirSync } from 'fs';
import { Repository } from 'typeorm';
@Processor(QueueName.VIDEO_CONVERSION)
export class VideoTranscodeProcessor {
readonly logger = new Logger(VideoTranscodeProcessor.name);
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
private systemConfigService: SystemConfigService,
) {}
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
async videoConversion(job: Job<IVideoConversionProcessor>) {
async videoConversion(job: Job<IAssetJob>) {
const { asset } = job.data;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.ownerId}/encoded-video`;
@ -93,7 +90,7 @@ export class VideoTranscodeProcessor {
})
.on('end', async () => {
this.logger.log(`Converting Success ${asset.id}`);
await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath });
await this.assetRepository.save({ id: asset.id, encodedVideoPath: savedEncodedPath });
resolve();
})
.run();

2
server/libs/common/src/utils/asset-utils.ts
View file

@ -1,4 +1,4 @@
import { AssetEntity } from '@app/infra';
import { AssetEntity } from '@app/infra/db/entities';
import { AssetResponseDto } from '@app/domain';
import fs from 'fs';

1
server/libs/common/src/utils/index.ts
View file

@ -2,7 +2,6 @@ import { LogLevel } from '@nestjs/common';
export * from './time-utils';
export * from './asset-utils';
export * from './user-utils';
export function getLogLevels() {
const LOG_LEVELS: LogLevel[] = ['verbose', 'debug', 'log', 'warn', 'error'];

19
server/libs/common/src/utils/user-utils.spec.ts
View file

@ -1,19 +0,0 @@
// create unit test for user utils
import { UserEntity } from '@app/infra';
import { userUtils } from './user-utils';
describe('User Utilities', () => {
describe('checkIsReadyForDeletion', () => {
it('check that user is not ready to be deleted', () => {
const result = userUtils.isReadyForDeletion({ deletedAt: new Date() } as UserEntity);
expect(result).toBeFalsy();
});
it('check that user is ready to be deleted', () => {
const aWeekAgo = new Date(new Date().getTime() - 8 * 86400000);
const result = userUtils.isReadyForDeletion({ deletedAt: aWeekAgo } as UserEntity);
expect(result).toBeTruthy();
});
});
});

16
server/libs/common/src/utils/user-utils.ts
View file

@ -1,16 +0,0 @@
import { UserEntity } from '@app/infra';
function createUserUtils() {
const isReadyForDeletion = (user: UserEntity): boolean => {
if (user.deletedAt == null) return false;
const millisecondsInDay = 86400000;
// get this number (7 days) from some configuration perhaps ?
const millisecondsDeleteWait = millisecondsInDay * 7;
const millisecondsSinceDelete = new Date().getTime() - (Date.parse(user.deletedAt.toString()) ?? 0);
return millisecondsSinceDelete >= millisecondsDeleteWait;
};
return { isReadyForDeletion };
}
export const userUtils = createUserUtils();

5
server/libs/domain/src/album/album.repository.ts Normal file
View file

@ -0,0 +1,5 @@
export const IAlbumRepository = 'IAlbumRepository';
export interface IAlbumRepository {
deleteAll(userId: string): Promise<void>;
}

1
server/libs/domain/src/album/index.ts
View file

@ -1 +1,2 @@
export * from './album.repository';
export * from './response-dto';

1
server/libs/domain/src/api-key/api-key.repository.ts
View file

@ -6,6 +6,7 @@ export interface IKeyRepository {
create(dto: Partial<APIKeyEntity>): Promise<APIKeyEntity>;
update(userId: string, id: string, dto: Partial<APIKeyEntity>): Promise<APIKeyEntity>;
delete(userId: string, id: string): Promise<void>;
deleteAll(userId: string): Promise<void>;
/**
* Includes the hashed `key` for verification
* @param id

10
server/libs/domain/src/asset/asset.repository.ts Normal file
View file

@ -0,0 +1,10 @@
import { AssetEntity, AssetType } from '@app/infra/db/entities';
export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
deleteAll(ownerId: string): Promise<void>;
getAll(): Promise<AssetEntity[]>;
save(asset: Partial<AssetEntity>): Promise<AssetEntity>;
findLivePhotoMatch(livePhotoCID: string, type: AssetType, otherAssetId: string): Promise<AssetEntity | null>;
}

45
server/libs/domain/src/asset/asset.service.spec.ts Normal file
View file

@ -0,0 +1,45 @@
import { AssetEntity, AssetType } from '@app/infra/db/entities';
import { newJobRepositoryMock } from '../../test';
import { AssetService } from '../asset';
import { IJobRepository, JobName } from '../job';
describe(AssetService.name, () => {
let sut: AssetService;
let jobMock: jest.Mocked<IJobRepository>;
it('should work', () => {
expect(sut).toBeDefined();
});
beforeEach(async () => {
jobMock = newJobRepositoryMock();
sut = new AssetService(jobMock);
});
describe(`handle asset upload`, () => {
it('should process an uploaded video', async () => {
const data = { asset: { type: AssetType.VIDEO } as AssetEntity, fileName: 'video.mp4' };
await expect(sut.handleAssetUpload(data)).resolves.toBeUndefined();
expect(jobMock.queue).toHaveBeenCalledTimes(3);
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data }],
[{ name: JobName.VIDEO_CONVERSION, data }],
[{ name: JobName.EXTRACT_VIDEO_METADATA, data }],
]);
});
it('should process an uploaded image', async () => {
const data = { asset: { type: AssetType.IMAGE } as AssetEntity, fileName: 'image.jpg' };
await sut.handleAssetUpload(data);
expect(jobMock.queue).toHaveBeenCalledTimes(2);
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data }],
[{ name: JobName.EXIF_EXTRACTION, data }],
]);
});
});
});

18
server/libs/domain/src/asset/asset.service.ts Normal file
View file

@ -0,0 +1,18 @@
import { AssetType } from '@app/infra/db/entities';
import { Inject } from '@nestjs/common';
import { IAssetUploadedJob, IJobRepository, JobName } from '../job';
export class AssetService {
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
async handleAssetUpload(data: IAssetUploadedJob) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data });
if (data.asset.type == AssetType.VIDEO) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data });
await this.jobRepository.queue({ name: JobName.EXTRACT_VIDEO_METADATA, data });
} else {
await this.jobRepository.queue({ name: JobName.EXIF_EXTRACTION, data });
}
}
}

2
server/libs/domain/src/asset/index.ts
View file

@ -1 +1,3 @@
export * from './asset.repository';
export * from './asset.service';
export * from './response-dto';

23
server/libs/domain/src/auth/auth.service.spec.ts
View file

@ -42,18 +42,6 @@ const fixtures = {
const CLIENT_IP = '127.0.0.1';
jest.mock('@nestjs/common', () => ({
...jest.requireActual('@nestjs/common'),
Logger: jest.fn().mockReturnValue({
verbose: jest.fn(),
debug: jest.fn(),
log: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
}),
}));
describe('AuthService', () => {
let sut: AuthService;
let cryptoMock: jest.Mocked<ICryptoRepository>;
@ -208,6 +196,17 @@ describe('AuthService', () => {
redirectUri: '/auth/login?autoLaunch=0',
});
});
it('should delete the access token', async () => {
const authUser = { id: '123', accessTokenId: 'token123' } as AuthUserDto;
await expect(sut.logout(authUser, AuthType.PASSWORD)).resolves.toEqual({
successful: true,
redirectUri: '/auth/login?autoLaunch=0',
});
expect(userTokenMock.delete).toHaveBeenCalledWith('token123');
});
});
describe('adminSignUp', () => {

9
server/libs/domain/src/communication/communication.repository.ts Normal file
View file

@ -0,0 +1,9 @@
export const ICommunicationRepository = 'ICommunicationRepository';
export enum CommunicationEvent {
UPLOAD_SUCCESS = 'on_upload_success',
}
export interface ICommunicationRepository {
send(event: CommunicationEvent, userId: string, data: any): void;
}

1
server/libs/domain/src/communication/index.ts Normal file
View file

@ -0,0 +1 @@
export * from './communication.repository';

2
server/libs/domain/src/device-info/device-info.service.spec.ts
View file

@ -1,4 +1,4 @@
import { DeviceInfoEntity, DeviceType } from '@app/infra';
import { DeviceInfoEntity, DeviceType } from '@app/infra/db/entities';
import { authStub, newDeviceInfoRepositoryMock } from '../../test';
import { IDeviceInfoRepository } from './device-info.repository';
import { DeviceInfoService } from './device-info.service';

12
server/libs/domain/src/domain.module.ts
View file

@ -1,19 +1,27 @@
import { DynamicModule, Global, Module, ModuleMetadata, Provider } from '@nestjs/common';
import { APIKeyService } from './api-key';
import { AssetService } from './asset';
import { AuthService } from './auth';
import { DeviceInfoService } from './device-info';
import { JobService } from './job';
import { MediaService } from './media';
import { OAuthService } from './oauth';
import { ShareService } from './share';
import { SmartInfoService } from './smart-info';
import { StorageService } from './storage';
import { StorageTemplateService } from './storage-template';
import { INITIAL_SYSTEM_CONFIG, SystemConfigService } from './system-config';
import { UserService } from './user';
const providers: Provider[] = [
AssetService,
APIKeyService,
AuthService,
DeviceInfoService,
JobService,
MediaService,
OAuthService,
SmartInfoService,
StorageService,
StorageTemplateService,
SystemConfigService,
UserService,
ShareService,

4
server/libs/domain/src/index.ts
View file

@ -2,13 +2,17 @@ export * from './album';
export * from './api-key';
export * from './asset';
export * from './auth';
export * from './communication';
export * from './crypto';
export * from './device-info';
export * from './domain.module';
export * from './job';
export * from './media';
export * from './oauth';
export * from './share';
export * from './smart-info';
export * from './storage';
export * from './storage-template';
export * from './system-config';
export * from './tag';
export * from './user';

3
server/libs/domain/src/job/index.ts
View file

@ -1,4 +1,3 @@
export * from './interfaces';
export * from './job.constants';
export * from './job.interface';
export * from './job.repository';
export * from './job.service';

13
server/libs/domain/src/job/interfaces/asset-uploaded.interface.ts
View file

@ -1,13 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IAssetUploadedJob {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}

5
server/libs/domain/src/job/interfaces/background-task.interface.ts
View file

@ -1,5 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IDeleteFileOnDiskJob {
assets: AssetEntity[];
}

7
server/libs/domain/src/job/interfaces/index.ts
View file

@ -1,7 +0,0 @@
export * from './asset-uploaded.interface';
export * from './background-task.interface';
export * from './machine-learning.interface';
export * from './metadata-extraction.interface';
export * from './thumbnail-generation.interface';
export * from './user-deletion.interface';
export * from './video-transcode.interface';

8
server/libs/domain/src/job/interfaces/machine-learning.interface.ts
View file

@ -1,8 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IMachineLearningJob {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}

36
server/libs/domain/src/job/interfaces/metadata-extraction.interface.ts
View file

@ -1,36 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IExifExtractionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}
export interface IVideoLengthExtractionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}
export interface IReverseGeocodingProcessor {
assetId: string;
latitude: number;
longitude: number;
}
export type IMetadataExtractionJob =
| IExifExtractionProcessor
| IVideoLengthExtractionProcessor
| IReverseGeocodingProcessor;

17
server/libs/domain/src/job/interfaces/thumbnail-generation.interface.ts
View file

@ -1,17 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface JpegGeneratorProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export interface WebpGeneratorProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export type IThumbnailGenerationJob = JpegGeneratorProcessor | WebpGeneratorProcessor;

8
server/libs/domain/src/job/interfaces/user-deletion.interface.ts
View file

@ -1,8 +0,0 @@
import { UserEntity } from '@app/infra/db/entities';
export interface IUserDeletionJob {
/**
* The user entity that was saved in the database
*/
user: UserEntity;
}

10
server/libs/domain/src/job/interfaces/video-transcode.interface.ts
View file

@ -1,10 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IVideoConversionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export type IVideoTranscodeJob = IVideoConversionProcessor;

11
server/libs/domain/src/job/job.constants.ts
View file

@ -2,11 +2,9 @@ export enum QueueName {
THUMBNAIL_GENERATION = 'thumbnail-generation-queue',
METADATA_EXTRACTION = 'metadata-extraction-queue',
VIDEO_CONVERSION = 'video-conversion-queue',
ASSET_UPLOADED = 'asset-uploaded-queue',
MACHINE_LEARNING = 'machine-learning-queue',
USER_DELETION = 'user-deletion-queue',
CONFIG = 'config-queue',
BACKGROUND_TASK = 'background-task',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue',
}
export enum JobName {
@ -18,9 +16,10 @@ export enum JobName {
EXTRACT_VIDEO_METADATA = 'extract-video-metadata',
REVERSE_GEOCODING = 'reverse-geocoding',
USER_DELETION = 'user-deletion',
TEMPLATE_MIGRATION = 'template-migration',
CONFIG_CHANGE = 'config-change',
USER_DELETE_CHECK = 'user-delete-check',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
SYSTEM_CONFIG_CHANGE = 'system-config-change',
OBJECT_DETECTION = 'detect-object',
IMAGE_TAGGING = 'tag-image',
DELETE_FILE_ON_DISK = 'delete-file-on-disk',
DELETE_FILES = 'delete-files',
}

26
server/libs/domain/src/job/job.interface.ts Normal file
View file

@ -0,0 +1,26 @@
import { AssetEntity, UserEntity } from '@app/infra/db/entities';
export interface IAssetJob {
asset: AssetEntity;
}
export interface IAssetUploadedJob {
asset: AssetEntity;
fileName: string;
}
export interface IDeleteFilesJob {
files: Array<string | null | undefined>;
}
export interface IUserDeletionJob {
user: UserEntity;
}
export interface IReverseGeocodingJob {
assetId: string;
latitude: number;
longitude: number;
}
export type IMetadataExtractionJob = IAssetUploadedJob | IReverseGeocodingJob;

42
server/libs/domain/src/job/job.repository.ts
View file

@ -1,16 +1,5 @@
import {
IAssetUploadedJob,
IDeleteFileOnDiskJob,
IExifExtractionProcessor,
IMachineLearningJob,
IVideoConversionProcessor,
IReverseGeocodingProcessor,
IUserDeletionJob,
IVideoLengthExtractionProcessor,
JpegGeneratorProcessor,
WebpGeneratorProcessor,
} from './interfaces';
import { JobName, QueueName } from './job.constants';
import { IAssetJob, IAssetUploadedJob, IDeleteFilesJob, IReverseGeocodingJob, IUserDeletionJob } from './job.interface';
export interface JobCounts {
active: number;
@ -20,30 +9,27 @@ export interface JobCounts {
waiting: number;
}
export interface Job<T> {
data: T;
}
export type JobItem =
| { name: JobName.ASSET_UPLOADED; data: IAssetUploadedJob }
| { name: JobName.VIDEO_CONVERSION; data: IVideoConversionProcessor }
| { name: JobName.GENERATE_JPEG_THUMBNAIL; data: JpegGeneratorProcessor }
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: WebpGeneratorProcessor }
| { name: JobName.EXIF_EXTRACTION; data: IExifExtractionProcessor }
| { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingProcessor }
| { name: JobName.VIDEO_CONVERSION; data: IAssetJob }
| { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IAssetJob }
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IAssetJob }
| { name: JobName.EXIF_EXTRACTION; data: IAssetUploadedJob }
| { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingJob }
| { name: JobName.USER_DELETE_CHECK }
| { name: JobName.USER_DELETION; data: IUserDeletionJob }
| { name: JobName.TEMPLATE_MIGRATION }
| { name: JobName.CONFIG_CHANGE }
| { name: JobName.EXTRACT_VIDEO_METADATA; data: IVideoLengthExtractionProcessor }
| { name: JobName.OBJECT_DETECTION; data: IMachineLearningJob }
| { name: JobName.IMAGE_TAGGING; data: IMachineLearningJob }
| { name: JobName.DELETE_FILE_ON_DISK; data: IDeleteFileOnDiskJob };
| { name: JobName.STORAGE_TEMPLATE_MIGRATION }
| { name: JobName.SYSTEM_CONFIG_CHANGE }
| { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetUploadedJob }
| { name: JobName.OBJECT_DETECTION; data: IAssetJob }
| { name: JobName.IMAGE_TAGGING; data: IAssetJob }
| { name: JobName.DELETE_FILES; data: IDeleteFilesJob };
export const IJobRepository = 'IJobRepository';
export interface IJobRepository {
queue(item: JobItem): Promise<void>;
empty(name: QueueName): Promise<void>;
add(item: JobItem): Promise<void>;
isActive(name: QueueName): Promise<boolean>;
getJobCounts(name: QueueName): Promise<JobCounts>;
}

54
server/libs/domain/src/job/job.service.spec.ts
View file

@ -1,54 +0,0 @@
import { AssetEntity, AssetType } from '@app/infra/db/entities';
import { newJobRepositoryMock } from '../../test';
import { IAssetUploadedJob } from './interfaces';
import { JobName } from './job.constants';
import { IJobRepository, Job } from './job.repository';
import { JobService } from './job.service';
const jobStub = {
upload: {
video: Object.freeze<Job<IAssetUploadedJob>>({
data: { asset: { type: AssetType.VIDEO } as AssetEntity, fileName: 'video.mp4' },
}),
image: Object.freeze<Job<IAssetUploadedJob>>({
data: { asset: { type: AssetType.IMAGE } as AssetEntity, fileName: 'image.jpg' },
}),
},
};
describe(JobService.name, () => {
let sut: JobService;
let jobMock: jest.Mocked<IJobRepository>;
it('should work', () => {
expect(sut).toBeDefined();
});
beforeEach(async () => {
jobMock = newJobRepositoryMock();
sut = new JobService(jobMock);
});
describe('handleUploadedAsset', () => {
it('should process a video', async () => {
await expect(sut.handleUploadedAsset(jobStub.upload.video)).resolves.toBeUndefined();
expect(jobMock.add).toHaveBeenCalledTimes(3);
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset: { type: AssetType.VIDEO } } }],
[{ name: JobName.VIDEO_CONVERSION, data: { asset: { type: AssetType.VIDEO } } }],
[{ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset: { type: AssetType.VIDEO }, fileName: 'video.mp4' } }],
]);
});
it('should process an image', async () => {
await sut.handleUploadedAsset(jobStub.upload.image);
expect(jobMock.add).toHaveBeenCalledTimes(2);
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset: { type: AssetType.IMAGE } } }],
[{ name: JobName.EXIF_EXTRACTION, data: { asset: { type: AssetType.IMAGE }, fileName: 'image.jpg' } }],
]);
});
});
});

17
server/libs/domain/src/job/job.service.ts
View file

@ -1,17 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { IAssetUploadedJob } from './interfaces';
import { JobUploadCore } from './job.upload.core';
import { IJobRepository, Job } from './job.repository';
@Injectable()
export class JobService {
private uploadCore: JobUploadCore;
constructor(@Inject(IJobRepository) repository: IJobRepository) {
this.uploadCore = new JobUploadCore(repository);
}
async handleUploadedAsset(job: Job<IAssetUploadedJob>) {
await this.uploadCore.handleAsset(job);
}
}

32
server/libs/domain/src/job/job.upload.core.ts
View file

@ -1,32 +0,0 @@
import { AssetType } from '@app/infra/db/entities';
import { IAssetUploadedJob } from './interfaces';
import { JobName } from './job.constants';
import { IJobRepository, Job } from './job.repository';
export class JobUploadCore {
constructor(private repository: IJobRepository) {}
/**
* Post processing uploaded asset to perform the following function
* 1. Generate JPEG Thumbnail
* 2. Generate Webp Thumbnail
* 3. EXIF extractor
* 4. Reverse Geocoding
*
* @param job asset-uploaded
*/
async handleAsset(job: Job<IAssetUploadedJob>) {
const { asset, fileName } = job.data;
await this.repository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.repository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
await this.repository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName } });
} else {
// Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet
await this.repository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName } });
}
}
}

2
server/libs/domain/src/media/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './media.repository';
export * from './media.service';

12
server/libs/domain/src/media/media.repository.ts Normal file
View file

@ -0,0 +1,12 @@
export const IMediaRepository = 'IMediaRepository';
export interface ResizeOptions {
size: number;
format: 'webp' | 'jpeg';
}
export interface IMediaRepository {
resize(input: string, output: string, options: ResizeOptions): Promise<void>;
extractVideoThumbnail(input: string, output: string): Promise<void>;
extractThumbnailFromExif(input: string, output: string): Promise<void>;
}

99
server/libs/domain/src/media/media.service.ts Normal file
View file

@ -0,0 +1,99 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetType } from '@app/infra/db/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { join } from 'path';
import sanitize from 'sanitize-filename';
import { IAssetRepository, mapAsset } from '../asset';
import { CommunicationEvent, ICommunicationRepository } from '../communication';
import { IAssetJob, IJobRepository, JobName } from '../job';
import { IStorageRepository } from '../storage';
import { IMediaRepository } from './media.repository';
@Injectable()
export class MediaService {
private logger = new Logger(MediaService.name);
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {}
async handleGenerateJpegThumbnail(data: IAssetJob): Promise<void> {
const { asset } = data;
const basePath = APP_UPLOAD_LOCATION;
const sanitizedDeviceId = sanitize(String(asset.deviceId));
const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
this.storageRepository.mkdirSync(resizePath);
if (asset.type == AssetType.IMAGE) {
try {
await this.mediaRepository
.resize(asset.originalPath, jpegThumbnailPath, { size: 1440, format: 'jpeg' })
.catch(() => {
this.logger.warn(
'Failed to generate jpeg thumbnail for asset: ' +
asset.id +
' using sharp, failing over to exiftool-vendored',
);
return this.mediaRepository.extractThumbnailFromExif(asset.originalPath, jpegThumbnailPath);
});
await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
} catch (error: any) {
this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
}
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
}
if (asset.type == AssetType.VIDEO) {
try {
this.logger.log('Start Generating Video Thumbnail');
await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath);
this.logger.log(`Generating Video Thumbnail Success ${asset.id}`);
await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
} catch (error: any) {
this.logger.error(`Cannot Generate Video Thumbnail: ${asset.id}`, error?.stack);
}
}
}
async handleGenerateWepbThumbnail(data: IAssetJob): Promise<void> {
const { asset } = data;
if (!asset.resizePath) {
return;
}
const webpPath = asset.resizePath.replace('jpeg', 'webp');
try {
await this.mediaRepository.resize(asset.resizePath, webpPath, { size: 250, format: 'webp' });
await this.assetRepository.save({ id: asset.id, webpPath: webpPath });
} catch (error: any) {
this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
}
}
}

12
server/libs/domain/src/oauth/oauth.service.spec.ts
View file

@ -21,18 +21,6 @@ import { newUserTokenRepositoryMock } from '../../test/user-token.repository.moc
const email = 'user@immich.com';
const sub = 'my-auth-user-sub';
jest.mock('@nestjs/common', () => ({
...jest.requireActual('@nestjs/common'),
Logger: jest.fn().mockReturnValue({
verbose: jest.fn(),
debug: jest.fn(),
log: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
}),
}));
describe('OAuthService', () => {
let sut: OAuthService;
let userMock: jest.Mocked<IUserRepository>;

3
server/libs/domain/src/smart-info/index.ts Normal file
View file

@ -0,0 +1,3 @@
export * from './machine-learning.interface';
export * from './smart-info.repository';
export * from './smart-info.service';

10
server/libs/domain/src/smart-info/machine-learning.interface.ts Normal file
View file

@ -0,0 +1,10 @@
export const IMachineLearningRepository = 'IMachineLearningRepository';
export interface MachineLearningInput {
thumbnailPath: string;
}
export interface IMachineLearningRepository {
tagImage(input: MachineLearningInput): Promise<string[]>;
detectObjects(input: MachineLearningInput): Promise<string[]>;
}

7
server/libs/domain/src/smart-info/smart-info.repository.ts Normal file
View file

@ -0,0 +1,7 @@
import { SmartInfoEntity } from '@app/infra/db/entities';
export const ISmartInfoRepository = 'ISmartInfoRepository';
export interface ISmartInfoRepository {
upsert(info: Partial<SmartInfoEntity>): Promise<void>;
}

102
server/libs/domain/src/smart-info/smart-info.service.spec.ts Normal file
View file

@ -0,0 +1,102 @@
import { AssetEntity } from '@app/infra/db/entities';
import { newMachineLearningRepositoryMock, newSmartInfoRepositoryMock } from '../../test';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
import { SmartInfoService } from './smart-info.service';
const asset = {
id: 'asset-1',
resizePath: 'path/to/resize.ext',
} as AssetEntity;
describe(SmartInfoService.name, () => {
let sut: SmartInfoService;
let smartMock: jest.Mocked<ISmartInfoRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
beforeEach(async () => {
smartMock = newSmartInfoRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
sut = new SmartInfoService(smartMock, machineMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('handleTagImage', () => {
it('should skip assets without a resize path', async () => {
await sut.handleTagImage({ asset: { resizePath: '' } as AssetEntity });
expect(smartMock.upsert).not.toHaveBeenCalled();
expect(machineMock.tagImage).not.toHaveBeenCalled();
});
it('should save the returned tags', async () => {
machineMock.tagImage.mockResolvedValue(['tag1', 'tag2', 'tag3']);
await sut.handleTagImage({ asset });
expect(machineMock.tagImage).toHaveBeenCalledWith({ thumbnailPath: 'path/to/resize.ext' });
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
tags: ['tag1', 'tag2', 'tag3'],
});
});
it('should handle an error with the machine learning pipeline', async () => {
machineMock.tagImage.mockRejectedValue(new Error('Unable to read thumbnail'));
await sut.handleTagImage({ asset });
expect(smartMock.upsert).not.toHaveBeenCalled();
});
it('should no update the smart info if no tags were returned', async () => {
machineMock.tagImage.mockResolvedValue([]);
await sut.handleTagImage({ asset });
expect(machineMock.tagImage).toHaveBeenCalled();
expect(smartMock.upsert).not.toHaveBeenCalled();
});
});
describe('handleDetectObjects', () => {
it('should skip assets without a resize path', async () => {
await sut.handleDetectObjects({ asset: { resizePath: '' } as AssetEntity });
expect(smartMock.upsert).not.toHaveBeenCalled();
expect(machineMock.detectObjects).not.toHaveBeenCalled();
});
it('should save the returned objects', async () => {
machineMock.detectObjects.mockResolvedValue(['obj1', 'obj2', 'obj3']);
await sut.handleDetectObjects({ asset });
expect(machineMock.detectObjects).toHaveBeenCalledWith({ thumbnailPath: 'path/to/resize.ext' });
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
objects: ['obj1', 'obj2', 'obj3'],
});
});
it('should handle an error with the machine learning pipeline', async () => {
machineMock.detectObjects.mockRejectedValue(new Error('Unable to read thumbnail'));
await sut.handleDetectObjects({ asset });
expect(smartMock.upsert).not.toHaveBeenCalled();
});
it('should no update the smart info if no objects were returned', async () => {
machineMock.detectObjects.mockResolvedValue([]);
await sut.handleDetectObjects({ asset });
expect(machineMock.detectObjects).toHaveBeenCalled();
expect(smartMock.upsert).not.toHaveBeenCalled();
});
});
});

49
server/libs/domain/src/smart-info/smart-info.service.ts Normal file
View file

@ -0,0 +1,49 @@
import { MACHINE_LEARNING_ENABLED } from '@app/common';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IAssetJob } from '../job';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
@Injectable()
export class SmartInfoService {
private logger = new Logger(SmartInfoService.name);
constructor(
@Inject(ISmartInfoRepository) private repository: ISmartInfoRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
) {}
async handleTagImage(data: IAssetJob) {
const { asset } = data;
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
return;
}
try {
const tags = await this.machineLearning.tagImage({ thumbnailPath: asset.resizePath });
if (tags.length > 0) {
await this.repository.upsert({ assetId: asset.id, tags });
}
} catch (error: any) {
this.logger.error(`Unable to run image tagging pipeline: ${asset.id}`, error?.stack);
}
}
async handleDetectObjects(data: IAssetJob) {
const { asset } = data;
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
return;
}
try {
const objects = await this.machineLearning.detectObjects({ thumbnailPath: asset.resizePath });
if (objects.length > 0) {
await this.repository.upsert({ assetId: asset.id, objects });
}
} catch (error: any) {
this.logger.error(`Unable run object detection pipeline: ${asset.id}`, error?.stack);
}
}
}

2
server/libs/domain/src/storage-template/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './storage-template.core';
export * from './storage-template.service';

108
server/libs/storage/src/storage.service.ts → server/libs/domain/src/storage-template/storage-template.core.ts
View file

@ -1,18 +1,7 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity, AssetType, SystemConfig } from '@app/infra';
import { SystemConfigService, INITIAL_SYSTEM_CONFIG } from '@app/domain';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import fsPromise from 'fs/promises';
import handlebar from 'handlebars';
import * as luxon from 'luxon';
import mv from 'mv';
import { constants } from 'node:fs';
import path from 'node:path';
import { promisify } from 'node:util';
import sanitize from 'sanitize-filename';
import { Repository } from 'typeorm';
import {
IStorageRepository,
ISystemConfigRepository,
supportedDayTokens,
supportedHourTokens,
supportedMinuteTokens,
@ -20,32 +9,31 @@ import {
supportedSecondTokens,
supportedYearTokens,
} from '@app/domain';
import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities';
import { Logger } from '@nestjs/common';
import handlebar from 'handlebars';
import * as luxon from 'luxon';
import path from 'node:path';
import sanitize from 'sanitize-filename';
import { SystemConfigCore } from '../system-config/system-config.core';
const moveFile = promisify<string, string, mv.Options>(mv);
@Injectable()
export class StorageService {
private readonly logger = new Logger(StorageService.name);
export class StorageTemplateCore {
private logger = new Logger(StorageTemplateCore.name);
private configCore: SystemConfigCore;
private storageTemplate: HandlebarsTemplateDelegate<any>;
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
private systemConfigService: SystemConfigService,
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
configRepository: ISystemConfigRepository,
config: SystemConfig,
private storageRepository: IStorageRepository,
) {
this.storageTemplate = this.compile(config.storageTemplate.template);
this.systemConfigService.addValidator((config) => this.validateConfig(config));
this.systemConfigService.config$.subscribe((config) => {
this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
this.storageTemplate = this.compile(config.storageTemplate.template);
});
this.configCore = new SystemConfigCore(configRepository);
this.configCore.addValidator((config) => this.validateConfig(config));
this.configCore.config$.subscribe((config) => this.onConfig(config));
}
public async moveAsset(asset: AssetEntity, filename: string): Promise<AssetEntity> {
public async getTemplatePath(asset: AssetEntity, filename: string): Promise<string> {
try {
const source = asset.originalPath;
const ext = path.extname(source).split('.').pop() as string;
@ -57,11 +45,11 @@ export class StorageService {
if (!fullPath.startsWith(rootPath)) {
this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
return asset;
return source;
}
if (source === destination) {
return asset;
return source;
}
/**
@ -82,14 +70,14 @@ export class StorageService {
const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
if (hasDuplicationAnnotation) {
return asset;
return source;
}
}
let duplicateCount = 0;
while (true) {
const exists = await this.checkFileExist(destination);
const exists = await this.storageRepository.checkFileExists(destination);
if (!exists) {
break;
}
@ -98,26 +86,10 @@ export class StorageService {
destination = `${fullPath}+${duplicateCount}.${ext}`;
}
await this.safeMove(source, destination);
asset.originalPath = destination;
return await this.assetRepository.save(asset);
return destination;
} catch (error: any) {
this.logger.error(error);
return asset;
}
}
private safeMove(source: string, destination: string): Promise<void> {
return moveFile(source, destination, { mkdirp: true, clobber: false });
}
private async checkFileExist(path: string): Promise<boolean> {
try {
await fsPromise.access(path, constants.F_OK);
return true;
} catch (_) {
return false;
this.logger.error(`Unable to get template path for ${filename}`, error);
return asset.originalPath;
}
}
@ -145,6 +117,11 @@ export class StorageService {
}
}
private onConfig(config: SystemConfig) {
this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
this.storageTemplate = this.compile(config.storageTemplate.template);
}
private compile(template: string) {
return handlebar.compile(template, {
knownHelpers: undefined,
@ -182,27 +159,4 @@ export class StorageService {
return template(substitutions);
}
public async removeEmptyDirectories(directory: string) {
// lstat does not follow symlinks (in contrast to stat)
const fileStats = await fsPromise.lstat(directory);
if (!fileStats.isDirectory()) {
return;
}
let fileNames = await fsPromise.readdir(directory);
if (fileNames.length > 0) {
const recursiveRemovalPromises = fileNames.map((fileName) =>
this.removeEmptyDirectories(path.join(directory, fileName)),
);
await Promise.all(recursiveRemovalPromises);
// re-evaluate fileNames; after deleting subdirectory
// we may have parent directory empty now
fileNames = await fsPromise.readdir(directory);
}
if (fileNames.length === 0) {
await fsPromise.rmdir(directory);
}
}
}

149
server/libs/domain/src/storage-template/storage-template.service.spec.ts Normal file
View file

@ -0,0 +1,149 @@
import { when } from 'jest-when';
import {
assetEntityStub,
newAssetRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
systemConfigStub,
} from '../../test';
import { IAssetRepository } from '../asset';
import { StorageTemplateService } from '../storage-template';
import { IStorageRepository } from '../storage/storage.repository';
import { ISystemConfigRepository } from '../system-config';
describe(StorageTemplateService.name, () => {
let sut: StorageTemplateService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
it('should work', () => {
expect(sut).toBeDefined();
});
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
storageMock = newStorageRepositoryMock();
sut = new StorageTemplateService(assetMock, configMock, systemConfigStub.defaults, storageMock);
});
describe('handle template migration', () => {
it('should handle no assets', async () => {
assetMock.getAll.mockResolvedValue([]);
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
});
it('should handle an asset with a duplicate destination', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.save.mockResolvedValue(assetEntityStub.image);
when(storageMock.checkFileExists)
.calledWith('upload/user-id/2023/2023-02-23/asset-id.ext')
.mockResolvedValue(true);
when(storageMock.checkFileExists)
.calledWith('upload/user-id/2023/2023-02-23/asset-id+1.ext')
.mockResolvedValue(false);
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
expect(assetMock.save).toHaveBeenCalledWith({
id: assetEntityStub.image.id,
originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
});
});
it('should skip when an asset already matches the template', async () => {
assetMock.getAll.mockResolvedValue([
{
...assetEntityStub.image,
originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
},
]);
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(storageMock.moveFile).not.toHaveBeenCalled();
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
expect(assetMock.save).not.toHaveBeenCalled();
});
it('should skip when an asset is probably a duplicate', async () => {
assetMock.getAll.mockResolvedValue([
{
...assetEntityStub.image,
originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
},
]);
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(storageMock.moveFile).not.toHaveBeenCalled();
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
expect(assetMock.save).not.toHaveBeenCalled();
});
it('should move an asset', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.save.mockResolvedValue(assetEntityStub.image);
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(storageMock.moveFile).toHaveBeenCalledWith(
'/original/path.ext',
'upload/user-id/2023/2023-02-23/asset-id.ext',
);
expect(assetMock.save).toHaveBeenCalledWith({
id: assetEntityStub.image.id,
originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
});
});
it('should not update the database if the move fails', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(storageMock.moveFile).toHaveBeenCalledWith(
'/original/path.ext',
'upload/user-id/2023/2023-02-23/asset-id.ext',
);
expect(assetMock.save).not.toHaveBeenCalled();
});
it('should move the asset back if the database fails', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.save.mockRejectedValue('Connection Error!');
await sut.handleTemplateMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(assetMock.save).toHaveBeenCalledWith({
id: assetEntityStub.image.id,
originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
});
expect(storageMock.moveFile.mock.calls).toEqual([
['/original/path.ext', 'upload/user-id/2023/2023-02-23/asset-id.ext'],
['upload/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
]);
});
});
it('should handle an error', async () => {
assetMock.getAll.mockResolvedValue([]);
storageMock.removeEmptyDirs.mockRejectedValue(new Error('Read only filesystem'));
await sut.handleTemplateMigration();
});
});

73
server/libs/domain/src/storage-template/storage-template.service.ts Normal file
View file

@ -0,0 +1,73 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity, SystemConfig } from '@app/infra/db/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IAssetRepository } from '../asset/asset.repository';
import { IStorageRepository } from '../storage/storage.repository';
import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
import { StorageTemplateCore } from './storage-template.core';
@Injectable()
export class StorageTemplateService {
private logger = new Logger(StorageTemplateService.name);
private core: StorageTemplateCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.core = new StorageTemplateCore(configRepository, config, storageRepository);
}
async handleTemplateMigration() {
try {
console.time('migrating-time');
const assets = await this.assetRepository.getAll();
const livePhotoMap: Record<string, AssetEntity> = {};
for (const asset of assets) {
if (asset.livePhotoVideoId) {
livePhotoMap[asset.livePhotoVideoId] = asset;
}
}
for (const asset of assets) {
const livePhotoParentAsset = livePhotoMap[asset.id];
// TODO: remove livePhoto specific stuff once upload is fixed
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
await this.moveAsset(asset, filename);
}
this.logger.debug('Cleaning up empty directories...');
await this.storageRepository.removeEmptyDirs(APP_UPLOAD_LOCATION);
} catch (error: any) {
this.logger.error('Error running template migration', error);
} finally {
console.timeEnd('migrating-time');
}
}
// TODO: use asset core (once in domain)
async moveAsset(asset: AssetEntity, originalName: string) {
const destination = await this.core.getTemplatePath(asset, originalName);
if (asset.originalPath !== destination) {
const source = asset.originalPath;
try {
await this.storageRepository.moveFile(asset.originalPath, destination);
try {
await this.assetRepository.save({ id: asset.id, originalPath: destination });
asset.originalPath = destination;
} catch (error: any) {
this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
await this.storageRepository.moveFile(destination, source);
}
} catch (error: any) {
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
}
}
return asset;
}
}

1
server/libs/domain/src/storage/index.ts
View file

@ -1 +1,2 @@
export * from './storage.repository';
export * from './storage.service';

6
server/libs/domain/src/storage/storage.repository.ts
View file

@ -10,4 +10,10 @@ export const IStorageRepository = 'IStorageRepository';
export interface IStorageRepository {
createReadStream(filepath: string, mimeType: string): Promise<ImmichReadStream>;
unlink(filepath: string): Promise<void>;
unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
removeEmptyDirs(folder: string): Promise<void>;
moveFile(source: string, target: string): Promise<void>;
checkFileExists(filepath: string): Promise<boolean>;
mkdirSync(filepath: string): void;
}

39
server/libs/domain/src/storage/storage.service.spec.ts Normal file
View file

@ -0,0 +1,39 @@
import { newStorageRepositoryMock } from '../../test';
import { IStorageRepository } from '../storage';
import { StorageService } from './storage.service';
describe(StorageService.name, () => {
let sut: StorageService;
let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
storageMock = newStorageRepositoryMock();
sut = new StorageService(storageMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('handleDeleteFiles', () => {
it('should handle null values', async () => {
await sut.handleDeleteFiles({ files: [undefined, null] });
expect(storageMock.unlink).not.toHaveBeenCalled();
});
it('should handle an error removing a file', async () => {
storageMock.unlink.mockRejectedValue(new Error('something-went-wrong'));
await sut.handleDeleteFiles({ files: ['path/to/something'] });
expect(storageMock.unlink).toHaveBeenCalledWith('path/to/something');
});
it('should remove the file', async () => {
await sut.handleDeleteFiles({ files: ['path/to/something'] });
expect(storageMock.unlink).toHaveBeenCalledWith('path/to/something');
});
});
});

26
server/libs/domain/src/storage/storage.service.ts Normal file
View file

@ -0,0 +1,26 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IDeleteFilesJob } from '../job';
import { IStorageRepository } from './storage.repository';
@Injectable()
export class StorageService {
private logger = new Logger(StorageService.name);
constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {}
async handleDeleteFiles(job: IDeleteFilesJob) {
const { files } = job;
for (const file of files) {
if (!file) {
continue;
}
try {
await this.storageRepository.unlink(file);
} catch (error: any) {
this.logger.warn('Unable to remove file from disk', error);
}
}
}
}

2
server/libs/domain/src/system-config/system-config.service.spec.ts
View file

@ -127,7 +127,7 @@ describe(SystemConfigService.name, () => {
await expect(sut.updateConfig(updatedConfig)).resolves.toEqual(updatedConfig);
expect(configMock.saveAll).toHaveBeenCalledWith(updates);
expect(jobMock.add).toHaveBeenCalledWith({ name: JobName.CONFIG_CHANGE });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.SYSTEM_CONFIG_CHANGE });
});
it('should throw an error if the config is not valid', async () => {

4
server/libs/domain/src/system-config/system-config.service.ts
View file

@ -19,7 +19,7 @@ export class SystemConfigService {
private core: SystemConfigCore;
constructor(
@Inject(ISystemConfigRepository) repository: ISystemConfigRepository,
@Inject(IJobRepository) private queue: IJobRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
this.core = new SystemConfigCore(repository);
}
@ -40,7 +40,7 @@ export class SystemConfigService {
async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
const config = await this.core.updateConfig(dto);
await this.queue.add({ name: JobName.CONFIG_CHANGE });
await this.jobRepository.queue({ name: JobName.SYSTEM_CONFIG_CHANGE });
return mapConfig(config);
}

1
server/libs/domain/src/user-token/user-token.repository.ts
View file

@ -5,5 +5,6 @@ export const IUserTokenRepository = 'IUserTokenRepository';
export interface IUserTokenRepository {
create(dto: Partial<UserTokenEntity>): Promise<UserTokenEntity>;
delete(userToken: string): Promise<void>;
deleteAll(userId: string): Promise<void>;
get(userToken: string): Promise<UserTokenEntity | null>;
}

3
server/libs/domain/src/user/user.repository.ts
View file

@ -11,9 +11,10 @@ export interface IUserRepository {
getAdmin(): Promise<UserEntity | null>;
getByEmail(email: string, withPassword?: boolean): Promise<UserEntity | null>;
getByOAuthId(oauthId: string): Promise<UserEntity | null>;
getDeletedUsers(): Promise<UserEntity[]>;
getList(filter?: UserListFilter): Promise<UserEntity[]>;
create(user: Partial<UserEntity>): Promise<UserEntity>;
update(id: string, user: Partial<UserEntity>): Promise<UserEntity>;
delete(user: UserEntity): Promise<UserEntity>;
delete(user: UserEntity, hard?: boolean): Promise<UserEntity>;
restore(user: UserEntity): Promise<UserEntity>;
}

112
server/libs/domain/src/user/user.service.spec.ts
View file

@ -1,13 +1,34 @@
import { IUserRepository } from './user.repository';
import { UserEntity } from '@app/infra/db/entities';
import { BadRequestException, ForbiddenException, NotFoundException } from '@nestjs/common';
import { when } from 'jest-when';
import { newCryptoRepositoryMock, newUserRepositoryMock } from '../../test';
import {
newAlbumRepositoryMock,
newAssetRepositoryMock,
newCryptoRepositoryMock,
newJobRepositoryMock,
newKeyRepositoryMock,
newStorageRepositoryMock,
newUserRepositoryMock,
newUserTokenRepositoryMock,
} from '../../test';
import { IAlbumRepository } from '../album';
import { IKeyRepository } from '../api-key';
import { IAssetRepository } from '../asset';
import { AuthUserDto } from '../auth';
import { ICryptoRepository } from '../crypto';
import { IJobRepository, JobName } from '../job';
import { IStorageRepository } from '../storage';
import { IUserTokenRepository } from '../user-token';
import { UpdateUserDto } from './dto/update-user.dto';
import { IUserRepository } from './user.repository';
import { UserService } from './user.service';
const makeDeletedAt = (daysAgo: number) => {
const deletedAt = new Date();
deletedAt.setDate(deletedAt.getDate() - daysAgo);
return deletedAt;
};
const adminUserAuth: AuthUserDto = Object.freeze({
id: 'admin_id',
email: 'admin@test.com',
@ -83,10 +104,35 @@ describe(UserService.name, () => {
let userRepositoryMock: jest.Mocked<IUserRepository>;
let cryptoRepositoryMock: jest.Mocked<ICryptoRepository>;
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let keyMock: jest.Mocked<IKeyRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let tokenMock: jest.Mocked<IUserTokenRepository>;
beforeEach(async () => {
userRepositoryMock = newUserRepositoryMock();
cryptoRepositoryMock = newCryptoRepositoryMock();
sut = new UserService(userRepositoryMock, cryptoRepositoryMock);
albumMock = newAlbumRepositoryMock();
assetMock = newAssetRepositoryMock();
jobMock = newJobRepositoryMock();
keyMock = newKeyRepositoryMock();
storageMock = newStorageRepositoryMock();
tokenMock = newUserTokenRepositoryMock();
userRepositoryMock = newUserRepositoryMock();
sut = new UserService(
userRepositoryMock,
cryptoRepositoryMock,
albumMock,
assetMock,
jobMock,
keyMock,
storageMock,
tokenMock,
);
when(userRepositoryMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
when(userRepositoryMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);
@ -374,4 +420,64 @@ describe(UserService.name, () => {
expect(update.password).toBeDefined();
});
});
describe('handleUserDeleteCheck', () => {
it('should skip users not ready for deletion', async () => {
userRepositoryMock.getDeletedUsers.mockResolvedValue([
{},
{ deletedAt: undefined },
{ deletedAt: null },
{ deletedAt: makeDeletedAt(5) },
] as UserEntity[]);
await sut.handleUserDeleteCheck();
expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should queue user ready for deletion', async () => {
const user = { deletedAt: makeDeletedAt(10) };
userRepositoryMock.getDeletedUsers.mockResolvedValue([user] as UserEntity[]);
await sut.handleUserDeleteCheck();
expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.USER_DELETION, data: { user } });
});
});
describe('handleUserDelete', () => {
it('should skip users not ready for deletion', async () => {
const user = { deletedAt: makeDeletedAt(5) } as UserEntity;
await sut.handleUserDelete({ user });
expect(storageMock.unlinkDir).not.toHaveBeenCalled();
expect(userRepositoryMock.delete).not.toHaveBeenCalled();
});
it('should delete the user and associated assets', async () => {
const user = { id: 'deleted-user', deletedAt: makeDeletedAt(10) } as UserEntity;
await sut.handleUserDelete({ user });
expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/deleted-user', { force: true, recursive: true });
expect(tokenMock.deleteAll).toHaveBeenCalledWith(user.id);
expect(keyMock.deleteAll).toHaveBeenCalledWith(user.id);
expect(albumMock.deleteAll).toHaveBeenCalledWith(user.id);
expect(assetMock.deleteAll).toHaveBeenCalledWith(user.id);
expect(userRepositoryMock.delete).toHaveBeenCalledWith(user, true);
});
it('should handle an error', async () => {
const user = { id: 'deleted-user', deletedAt: makeDeletedAt(10) } as UserEntity;
storageMock.unlinkDir.mockRejectedValue(new Error('Read only filesystem'));
await sut.handleUserDelete({ user });
expect(userRepositoryMock.delete).not.toHaveBeenCalled();
});
});
});

86
server/libs/domain/src/user/user.service.ts
View file

@ -1,26 +1,43 @@
import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
import { UserEntity } from '@app/infra/db/entities';
import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { randomBytes } from 'crypto';
import { ReadStream } from 'fs';
import { join } from 'path';
import { APP_UPLOAD_LOCATION } from '@app/common';
import { IAlbumRepository } from '../album/album.repository';
import { IKeyRepository } from '../api-key/api-key.repository';
import { IAssetRepository } from '../asset/asset.repository';
import { AuthUserDto } from '../auth';
import { ICryptoRepository } from '../crypto';
import { IUserRepository } from '../user';
import { CreateUserDto } from './dto/create-user.dto';
import { UpdateUserDto } from './dto/update-user.dto';
import { UserCountDto } from './dto/user-count.dto';
import { ICryptoRepository } from '../crypto/crypto.repository';
import { IJobRepository, IUserDeletionJob, JobName } from '../job';
import { IStorageRepository } from '../storage/storage.repository';
import { IUserTokenRepository } from '../user-token/user-token.repository';
import { IUserRepository } from '../user/user.repository';
import { CreateUserDto, UpdateUserDto, UserCountDto } from './dto';
import {
CreateProfileImageResponseDto,
mapCreateProfileImageResponse,
} from './response-dto/create-profile-image-response.dto';
import { mapUserCountResponse, UserCountResponseDto } from './response-dto/user-count-response.dto';
import { mapUser, UserResponseDto } from './response-dto/user-response.dto';
mapUser,
mapUserCountResponse,
UserCountResponseDto,
UserResponseDto,
} from './response-dto';
import { UserCore } from './user.core';
@Injectable()
export class UserService {
private logger = new Logger(UserService.name);
private userCore: UserCore;
constructor(
@Inject(IUserRepository) userRepository: IUserRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IKeyRepository) private keyRepository: IKeyRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IUserTokenRepository) private tokenRepository: IUserTokenRepository,
) {
this.userCore = new UserCore(userRepository, cryptoRepository);
}
@ -123,4 +140,53 @@ export class UserService {
return { admin, password, provided: !!providedPassword };
}
async handleUserDeleteCheck() {
const users = await this.userRepository.getDeletedUsers();
for (const user of users) {
if (this.isReadyForDeletion(user)) {
await this.jobRepository.queue({ name: JobName.USER_DELETION, data: { user } });
}
}
}
async handleUserDelete(data: IUserDeletionJob) {
const { user } = data;
// just for extra protection here
if (!this.isReadyForDeletion(user)) {
this.logger.warn(`Skipped user that was not ready for deletion: id=${user.id}`);
return;
}
this.logger.log(`Deleting user: ${user.id}`);
try {
const userAssetDir = join(APP_UPLOAD_LOCATION, user.id);
this.logger.warn(`Removing user from filesystem: ${userAssetDir}`);
await this.storageRepository.unlinkDir(userAssetDir, { recursive: true, force: true });
this.logger.warn(`Removing user from database: ${user.id}`);
await this.tokenRepository.deleteAll(user.id);
await this.keyRepository.deleteAll(user.id);
await this.albumRepository.deleteAll(user.id);
await this.assetRepository.deleteAll(user.id);
await this.userRepository.delete(user, true);
} catch (error: any) {
this.logger.error(`Failed to remove user`, error, { id: user.id });
}
}
private isReadyForDeletion(user: UserEntity): boolean {
if (!user.deletedAt) {
return false;
}
const msInDay = 86400000;
const msDeleteWait = msInDay * 7;
const msSinceDelete = new Date().getTime() - (Date.parse(user.deletedAt.toString()) || 0);
return msSinceDelete >= msDeleteWait;
}
}

7
server/libs/domain/test/album.repository.mock.ts Normal file
View file

@ -0,0 +1,7 @@
import { IAlbumRepository } from '../src';
export const newAlbumRepositoryMock = (): jest.Mocked<IAlbumRepository> => {
return {
deleteAll: jest.fn(),
};
};

1
server/libs/domain/test/api-key.repository.mock.ts
View file

@ -5,6 +5,7 @@ export const newKeyRepositoryMock = (): jest.Mocked<IKeyRepository> => {
create: jest.fn(),
update: jest.fn(),
delete: jest.fn(),
deleteAll: jest.fn(),
getKey: jest.fn(),
getById: jest.fn(),
getByUserId: jest.fn(),

10
server/libs/domain/test/asset.repository.mock.ts Normal file
View file

@ -0,0 +1,10 @@
import { IAssetRepository } from '../src';
export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
return {
getAll: jest.fn(),
deleteAll: jest.fn(),
save: jest.fn(),
findLivePhotoMatch: jest.fn(),
};
};

45
server/libs/domain/test/fixtures.ts
View file

@ -91,22 +91,37 @@ export const userEntityStub = {
}),
};
export const fileStub = {
livePhotoStill: Object.freeze({
originalPath: 'fake_path/asset_1.jpeg',
mimeType: 'image/jpg',
checksum: Buffer.from('file hash', 'utf8'),
originalName: 'asset_1.jpeg',
}),
livePhotoMotion: Object.freeze({
originalPath: 'fake_path/asset_1.mp4',
mimeType: 'image/jpeg',
checksum: Buffer.from('live photo file hash', 'utf8'),
originalName: 'asset_1.mp4',
}),
};
export const assetEntityStub = {
image: Object.freeze<AssetEntity>({
id: 'asset-id',
deviceAssetId: 'device-asset-id',
fileModifiedAt: today.toISOString(),
fileCreatedAt: today.toISOString(),
fileModifiedAt: '2023-02-23T05:06:29.716Z',
fileCreatedAt: '2023-02-23T05:06:29.716Z',
owner: userEntityStub.user1,
ownerId: 'user-id',
deviceId: 'device-id',
originalPath: '/original/path',
originalPath: '/original/path.ext',
resizePath: null,
type: AssetType.IMAGE,
webpPath: null,
encodedVideoPath: null,
createdAt: today.toISOString(),
updatedAt: today.toISOString(),
createdAt: '2023-02-23T05:06:29.716Z',
updatedAt: '2023-02-23T05:06:29.716Z',
mimeType: null,
isFavorite: true,
duration: null,
@ -116,6 +131,26 @@ export const assetEntityStub = {
tags: [],
sharedLinks: [],
}),
livePhotoMotionAsset: Object.freeze({
id: 'live-photo-motion-asset',
originalPath: fileStub.livePhotoMotion.originalPath,
ownerId: authStub.user1.id,
type: AssetType.VIDEO,
isVisible: false,
fileModifiedAt: '2022-06-19T23:41:36.910Z',
fileCreatedAt: '2022-06-19T23:41:36.910Z',
} as AssetEntity),
livePhotoStillAsset: Object.freeze({
id: 'live-photo-still-asset',
originalPath: fileStub.livePhotoStill.originalPath,
ownerId: authStub.user1.id,
type: AssetType.IMAGE,
livePhotoVideoId: 'live-photo-motion-asset',
isVisible: true,
fileModifiedAt: '2022-06-19T23:41:36.910Z',
fileCreatedAt: '2022-06-19T23:41:36.910Z',
} as AssetEntity),
};
const assetInfo: ExifResponseDto = {

4
server/libs/domain/test/index.ts
View file

@ -1,9 +1,13 @@
export * from './album.repository.mock';
export * from './api-key.repository.mock';
export * from './asset.repository.mock';
export * from './crypto.repository.mock';
export * from './device-info.repository.mock';
export * from './fixtures';
export * from './job.repository.mock';
export * from './machine-learning.repository.mock';
export * from './shared-link.repository.mock';
export * from './smart-info.repository.mock';
export * from './storage.repository.mock';
export * from './system-config.repository.mock';
export * from './user-token.repository.mock';

2
server/libs/domain/test/job.repository.mock.ts
View file

@ -3,7 +3,7 @@ import { IJobRepository } from '../src';
export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
return {
empty: jest.fn(),
add: jest.fn().mockImplementation(() => Promise.resolve()),
queue: jest.fn().mockImplementation(() => Promise.resolve()),
isActive: jest.fn(),
getJobCounts: jest.fn(),
};

8
server/libs/domain/test/machine-learning.repository.mock.ts Normal file
View file

@ -0,0 +1,8 @@
import { IMachineLearningRepository } from '../src';
export const newMachineLearningRepositoryMock = (): jest.Mocked<IMachineLearningRepository> => {
return {
tagImage: jest.fn(),
detectObjects: jest.fn(),
};
};

11
server/libs/domain/test/setup.ts Normal file
View file

@ -0,0 +1,11 @@
jest.mock('@nestjs/common', () => ({
...jest.requireActual('@nestjs/common'),
Logger: jest.fn().mockReturnValue({
verbose: jest.fn(),
debug: jest.fn(),
log: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
}),
}));

7
server/libs/domain/test/smart-info.repository.mock.ts Normal file
View file

@ -0,0 +1,7 @@
import { ISmartInfoRepository } from '../src';
export const newSmartInfoRepositoryMock = (): jest.Mocked<ISmartInfoRepository> => {
return {
upsert: jest.fn(),
};
};

6
server/libs/domain/test/storage.repository.mock.ts
View file

@ -3,5 +3,11 @@ import { IStorageRepository } from '../src';
export const newStorageRepositoryMock = (): jest.Mocked<IStorageRepository> => {
return {
createReadStream: jest.fn(),
unlink: jest.fn(),
unlinkDir: jest.fn(),
removeEmptyDirs: jest.fn(),
moveFile: jest.fn(),
checkFileExists: jest.fn(),
mkdirSync: jest.fn(),
};
};

1
server/libs/domain/test/user-token.repository.mock.ts
View file

@ -4,6 +4,7 @@ export const newUserTokenRepositoryMock = (): jest.Mocked<IUserTokenRepository>
return {
create: jest.fn(),
delete: jest.fn(),
deleteAll: jest.fn(),
get: jest.fn(),
};
};

1
server/libs/domain/test/user.repository.mock.ts
View file

@ -10,6 +10,7 @@ export const newUserRepositoryMock = (): jest.Mocked<IUserRepository> => {
create: jest.fn(),
update: jest.fn(),
delete: jest.fn(),
getDeletedUsers: jest.fn(),
restore: jest.fn(),
};
};

0
server/apps/immich/src/api-v1/communication/communication.gateway.ts → server/libs/infra/src/communication/communication.gateway.ts
View file

12
server/libs/infra/src/communication/communication.repository.ts Normal file
View file

@ -0,0 +1,12 @@
import { CommunicationEvent } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { CommunicationGateway } from './communication.gateway';
@Injectable()
export class CommunicationRepository {
constructor(private ws: CommunicationGateway) {}
send(event: CommunicationEvent, userId: string, data: any) {
this.ws.server.to(userId).emit(event, JSON.stringify(data));
}
}

2
server/libs/infra/src/communication/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './communication.gateway';
export * from './communication.repository';

14
server/libs/infra/src/db/repository/album.repository.ts Normal file
View file

@ -0,0 +1,14 @@
import { IAlbumRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { AlbumEntity } from '../entities';
@Injectable()
export class AlbumRepository implements IAlbumRepository {
constructor(@InjectRepository(AlbumEntity) private repository: Repository<AlbumEntity>) {}
async deleteAll(userId: string): Promise<void> {
await this.repository.delete({ ownerId: userId });
}
}

4
server/libs/infra/src/db/repository/api-key.repository.ts
View file

@ -21,6 +21,10 @@ export class APIKeyRepository implements IKeyRepository {
await this.repository.delete({ userId, id });
}
async deleteAll(userId: string): Promise<void> {
await this.repository.delete({ userId });
}
getKey(hashedToken: string): Promise<APIKeyEntity | null> {
return this.repository.findOne({
select: {

38
server/libs/infra/src/db/repository/asset.repository.ts Normal file
View file

@ -0,0 +1,38 @@
import { IAssetRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Not, Repository } from 'typeorm';
import { AssetEntity, AssetType } from '../entities';
@Injectable()
export class AssetRepository implements IAssetRepository {
constructor(@InjectRepository(AssetEntity) private repository: Repository<AssetEntity>) {}
async deleteAll(ownerId: string): Promise<void> {
await this.repository.delete({ ownerId });
}
async getAll(): Promise<AssetEntity[]> {
return this.repository.find({ relations: { exifInfo: true } });
}
async save(asset: Partial<AssetEntity>): Promise<AssetEntity> {
const { id } = await this.repository.save(asset);
return this.repository.findOneOrFail({ where: { id } });
}
findLivePhotoMatch(livePhotoCID: string, otherAssetId: string, type: AssetType): Promise<AssetEntity | null> {
return this.repository.findOne({
where: {
id: Not(otherAssetId),
type,
exifInfo: {
livePhotoCID,
},
},
relations: {
exifInfo: true,
},
});
}
}

3
server/libs/infra/src/db/repository/index.ts
View file

@ -1,6 +1,9 @@
export * from './album.repository';
export * from './api-key.repository';
export * from './asset.repository';
export * from './device-info.repository';
export * from './shared-link.repository';
export * from './smart-info.repository';
export * from './system-config.repository';
export * from './user-token.repository';
export * from './user.repository';

14
server/libs/infra/src/db/repository/smart-info.repository.ts Normal file
View file

@ -0,0 +1,14 @@
import { ISmartInfoRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { SmartInfoEntity } from '../entities';
@Injectable()
export class SmartInfoRepository implements ISmartInfoRepository {
constructor(@InjectRepository(SmartInfoEntity) private repository: Repository<SmartInfoEntity>) {}
async upsert(info: Partial<SmartInfoEntity>): Promise<void> {
await this.repository.upsert(info, { conflictPaths: ['assetId'] });
}
}

6
server/libs/infra/src/db/repository/user-token.repository.ts
View file

@ -1,7 +1,7 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { UserTokenEntity } from '@app/infra/db/entities/user-token.entity';
import { UserTokenEntity } from '../entities/user-token.entity';
import { IUserTokenRepository } from '@app/domain/user-token';
@Injectable()
@ -22,4 +22,8 @@ export class UserTokenRepository implements IUserTokenRepository {
async delete(id: string): Promise<void> {
await this.userTokenRepository.delete(id);
}
async deleteAll(userId: string): Promise<void> {
await this.userTokenRepository.delete({ user: { id: userId } });
}
}

14
server/libs/infra/src/db/repository/user.repository.ts
View file

@ -2,7 +2,7 @@ import { UserEntity } from '../entities';
import { IUserRepository, UserListFilter } from '@app/domain';
import { Injectable, InternalServerErrorException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Not, Repository } from 'typeorm';
import { IsNull, Not, Repository } from 'typeorm';
@Injectable()
export class UserRepository implements IUserRepository {
@ -33,6 +33,10 @@ export class UserRepository implements IUserRepository {
return this.userRepository.findOne({ where: { oauthId } });
}
async getDeletedUsers(): Promise<UserEntity[]> {
return this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });
}
async getList({ excludeId }: UserListFilter = {}): Promise<UserEntity[]> {
if (!excludeId) {
return this.userRepository.find(); // TODO: this should also be ordered the same as below
@ -61,8 +65,12 @@ export class UserRepository implements IUserRepository {
return updatedUser;
}
async delete(user: UserEntity): Promise<UserEntity> {
return this.userRepository.softRemove(user);
async delete(user: UserEntity, hard?: boolean): Promise<UserEntity> {
if (hard) {
return this.userRepository.remove(user);
} else {
return this.userRepository.softRemove(user);
}
}
async restore(user: UserEntity): Promise<UserEntity> {

42
server/libs/infra/src/infra.module.ts
View file

@ -1,43 +1,65 @@
import {
IAlbumRepository,
IAssetRepository,
ICommunicationRepository,
ICryptoRepository,
IDeviceInfoRepository,
IJobRepository,
IKeyRepository,
IMachineLearningRepository,
IMediaRepository,
ISharedLinkRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
IUserTokenRepository,
QueueName,
} from '@app/domain';
import { IUserTokenRepository } from '@app/domain/user-token';
import { UserTokenRepository } from '@app/infra/db/repository/user-token.repository';
import { BullModule } from '@nestjs/bull';
import { Global, Module, Provider } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CryptoRepository } from './auth/crypto.repository';
import { CommunicationGateway, CommunicationRepository } from './communication';
import {
AlbumEntity,
AlbumRepository,
APIKeyEntity,
APIKeyRepository,
AssetEntity,
AssetRepository,
databaseConfig,
DeviceInfoEntity,
DeviceInfoRepository,
SharedLinkEntity,
SharedLinkRepository,
SmartInfoEntity,
SmartInfoRepository,
SystemConfigEntity,
SystemConfigRepository,
UserEntity,
UserRepository,
UserTokenEntity,
UserTokenRepository,
} from './db';
import { JobRepository } from './job';
import { MachineLearningRepository } from './machine-learning';
import { MediaRepository } from './media';
import { FilesystemProvider } from './storage';
const providers: Provider[] = [
{ provide: IAlbumRepository, useClass: AlbumRepository },
{ provide: IAssetRepository, useClass: AssetRepository },
{ provide: ICommunicationRepository, useClass: CommunicationRepository },
{ provide: ICryptoRepository, useClass: CryptoRepository },
{ provide: ICryptoRepository, useClass: CryptoRepository },
{ provide: IDeviceInfoRepository, useClass: DeviceInfoRepository },
{ provide: IKeyRepository, useClass: APIKeyRepository },
{ provide: IJobRepository, useClass: JobRepository },
{ provide: IMachineLearningRepository, useClass: MachineLearningRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: ISharedLinkRepository, useClass: SharedLinkRepository },
{ provide: ISmartInfoRepository, useClass: SmartInfoRepository },
{ provide: IStorageRepository, useClass: FilesystemProvider },
{ provide: ISystemConfigRepository, useClass: SystemConfigRepository },
{ provide: IUserRepository, useClass: UserRepository },
@ -49,10 +71,13 @@ const providers: Provider[] = [
imports: [
TypeOrmModule.forRoot(databaseConfig),
TypeOrmModule.forFeature([
AssetEntity,
AlbumEntity,
APIKeyEntity,
DeviceInfoEntity,
UserEntity,
SharedLinkEntity,
SmartInfoEntity,
SystemConfigEntity,
UserTokenEntity,
]),
@ -73,18 +98,9 @@ const providers: Provider[] = [
},
}),
}),
BullModule.registerQueue(
{ name: QueueName.USER_DELETION },
{ name: QueueName.THUMBNAIL_GENERATION },
{ name: QueueName.ASSET_UPLOADED },
{ name: QueueName.METADATA_EXTRACTION },
{ name: QueueName.VIDEO_CONVERSION },
{ name: QueueName.MACHINE_LEARNING },
{ name: QueueName.CONFIG },
{ name: QueueName.BACKGROUND_TASK },
),
BullModule.registerQueue(...Object.values(QueueName).map((name) => ({ name }))),
],
providers: [...providers],
providers: [...providers, CommunicationGateway],
exports: [...providers, BullModule],
})
export class InfraModule {}

46
server/libs/infra/src/job/job.repository.ts
View file

@ -1,15 +1,4 @@
import {
IAssetUploadedJob,
IJobRepository,
IMachineLearningJob,
IMetadataExtractionJob,
IUserDeletionJob,
IVideoTranscodeJob,
JobCounts,
JobItem,
JobName,
QueueName,
} from '@app/domain';
import { IAssetJob, IJobRepository, IMetadataExtractionJob, JobCounts, JobItem, JobName, QueueName } from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { BadRequestException, Logger } from '@nestjs/common';
import { Queue } from 'bull';
@ -18,14 +7,12 @@ export class JobRepository implements IJobRepository {
private logger = new Logger(JobRepository.name);
constructor(
@InjectQueue(QueueName.ASSET_UPLOADED) private assetUploaded: Queue<IAssetUploadedJob>,
@InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue,
@InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue<IMachineLearningJob>,
@InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue<IAssetJob>,
@InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IMetadataExtractionJob>,
@InjectQueue(QueueName.CONFIG) private storageMigration: Queue,
@InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue,
@InjectQueue(QueueName.THUMBNAIL_GENERATION) private thumbnail: Queue,
@InjectQueue(QueueName.USER_DELETION) private userDeletion: Queue<IUserDeletionJob>,
@InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IVideoTranscodeJob>,
@InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IAssetJob>,
) {}
async isActive(name: QueueName): Promise<boolean> {
@ -41,13 +28,13 @@ export class JobRepository implements IJobRepository {
return this.getQueue(name).getJobCounts();
}
async add(item: JobItem): Promise<void> {
async queue(item: JobItem): Promise<void> {
switch (item.name) {
case JobName.ASSET_UPLOADED:
await this.assetUploaded.add(item.name, item.data, { jobId: item.data.asset.id });
await this.backgroundTask.add(item.name, item.data, { jobId: item.data.asset.id });
break;
case JobName.DELETE_FILE_ON_DISK:
case JobName.DELETE_FILES:
await this.backgroundTask.add(item.name, item.data);
break;
@ -62,18 +49,21 @@ export class JobRepository implements IJobRepository {
await this.metadataExtraction.add(item.name, item.data);
break;
case JobName.TEMPLATE_MIGRATION:
case JobName.CONFIG_CHANGE:
await this.storageMigration.add(item.name, {});
break;
case JobName.GENERATE_JPEG_THUMBNAIL:
case JobName.GENERATE_WEBP_THUMBNAIL:
await this.thumbnail.add(item.name, item.data);
break;
case JobName.USER_DELETION:
await this.userDeletion.add(item.name, item.data);
await this.backgroundTask.add(item.name, item.data);
break;
case JobName.STORAGE_TEMPLATE_MIGRATION:
await this.storageTemplateMigration.add(item.name);
break;
case JobName.SYSTEM_CONFIG_CHANGE:
await this.backgroundTask.add(item.name, {});
break;
case JobName.VIDEO_CONVERSION:
@ -88,14 +78,14 @@ export class JobRepository implements IJobRepository {
private getQueue(name: QueueName) {
switch (name) {
case QueueName.STORAGE_TEMPLATE_MIGRATION:
return this.storageTemplateMigration;
case QueueName.THUMBNAIL_GENERATION:
return this.thumbnail;
case QueueName.METADATA_EXTRACTION:
return this.metadataExtraction;
case QueueName.VIDEO_CONVERSION:
return this.videoTranscode;
case QueueName.CONFIG:
return this.storageMigration;
case QueueName.MACHINE_LEARNING:
return this.machineLearning;
default:

1
server/libs/infra/src/machine-learning/index.ts Normal file
View file

@ -0,0 +1 @@
export * from './machine-learning.repository';

Some files were not shown because too many files have changed in this diff Show more