2023-12-14 17:55:40 +01:00
|
|
|
import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
|
2024-05-14 20:43:49 +02:00
|
|
|
import { ImageFormat } from 'src/config';
|
2024-03-21 04:15:09 +01:00
|
|
|
import { FACE_THUMBNAIL_SIZE } from 'src/constants';
|
2024-03-20 21:20:38 +01:00
|
|
|
import { AccessCore, Permission } from 'src/cores/access.core';
|
|
|
|
import { StorageCore } from 'src/cores/storage.core';
|
|
|
|
import { SystemConfigCore } from 'src/cores/system-config.core';
|
2024-03-20 23:53:07 +01:00
|
|
|
import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
|
|
|
|
import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
|
|
|
|
import { AuthDto } from 'src/dtos/auth.dto';
|
2023-07-18 20:09:43 +02:00
|
|
|
import {
|
2023-12-05 16:43:15 +01:00
|
|
|
AssetFaceResponseDto,
|
|
|
|
AssetFaceUpdateDto,
|
|
|
|
FaceDto,
|
2023-07-18 20:09:43 +02:00
|
|
|
MergePersonDto,
|
|
|
|
PeopleResponseDto,
|
2023-07-23 05:00:43 +02:00
|
|
|
PeopleUpdateDto,
|
2024-03-07 21:34:57 +01:00
|
|
|
PersonCreateDto,
|
2023-07-18 20:09:43 +02:00
|
|
|
PersonResponseDto,
|
|
|
|
PersonSearchDto,
|
2023-10-24 17:53:49 +02:00
|
|
|
PersonStatisticsResponseDto,
|
2023-07-18 20:09:43 +02:00
|
|
|
PersonUpdateDto,
|
2023-12-05 16:43:15 +01:00
|
|
|
mapFaces,
|
2023-09-04 21:45:59 +02:00
|
|
|
mapPerson,
|
2024-03-20 23:53:07 +01:00
|
|
|
} from 'src/dtos/person.dto';
|
2024-06-13 04:16:26 +02:00
|
|
|
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
|
2024-03-20 22:02:51 +01:00
|
|
|
import { PersonPathType } from 'src/entities/move.entity';
|
|
|
|
import { PersonEntity } from 'src/entities/person.entity';
|
2024-03-21 12:59:49 +01:00
|
|
|
import { IAccessRepository } from 'src/interfaces/access.interface';
|
|
|
|
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
|
|
|
|
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
|
2024-03-21 04:15:09 +01:00
|
|
|
import {
|
|
|
|
IBaseJob,
|
|
|
|
IDeferrableJob,
|
|
|
|
IEntityJob,
|
|
|
|
IJobRepository,
|
|
|
|
JOBS_ASSET_PAGINATION_SIZE,
|
|
|
|
JobItem,
|
|
|
|
JobName,
|
|
|
|
JobStatus,
|
|
|
|
QueueName,
|
2024-03-21 12:59:49 +01:00
|
|
|
} from 'src/interfaces/job.interface';
|
2024-04-16 23:30:31 +02:00
|
|
|
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
2024-06-13 04:16:26 +02:00
|
|
|
import { BoundingBox, IMachineLearningRepository } from 'src/interfaces/machine-learning.interface';
|
2024-05-08 15:09:34 +02:00
|
|
|
import { CropOptions, IMediaRepository, ImageDimensions } from 'src/interfaces/media.interface';
|
2024-03-21 12:59:49 +01:00
|
|
|
import { IMoveRepository } from 'src/interfaces/move.interface';
|
|
|
|
import { IPersonRepository, UpdateFacesData } from 'src/interfaces/person.interface';
|
|
|
|
import { ISearchRepository } from 'src/interfaces/search.interface';
|
|
|
|
import { IStorageRepository } from 'src/interfaces/storage.interface';
|
2024-05-16 00:58:23 +02:00
|
|
|
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
|
2024-05-08 15:09:34 +02:00
|
|
|
import { Orientation } from 'src/services/metadata.service';
|
2024-03-21 04:15:09 +01:00
|
|
|
import { CacheControl, ImmichFileResponse } from 'src/utils/file';
|
|
|
|
import { mimeTypes } from 'src/utils/mime-types';
|
2024-05-14 21:31:36 +02:00
|
|
|
import { isFacialRecognitionEnabled } from 'src/utils/misc';
|
2024-03-21 04:15:09 +01:00
|
|
|
import { usePagination } from 'src/utils/pagination';
|
2024-03-20 19:32:04 +01:00
|
|
|
import { IsNull } from 'typeorm';
|
2023-05-17 19:07:17 +02:00
|
|
|
|
|
|
|
@Injectable()
|
|
|
|
export class PersonService {
|
2023-09-18 23:22:44 +02:00
|
|
|
private access: AccessCore;
|
2023-09-18 06:05:35 +02:00
|
|
|
private configCore: SystemConfigCore;
|
2023-09-27 22:46:46 +02:00
|
|
|
private storageCore: StorageCore;
|
2023-05-17 19:07:17 +02:00
|
|
|
|
|
|
|
constructor(
|
2023-09-27 22:46:46 +02:00
|
|
|
@Inject(IAccessRepository) accessRepository: IAccessRepository,
|
|
|
|
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
|
|
|
@Inject(IMachineLearningRepository) private machineLearningRepository: IMachineLearningRepository,
|
2023-10-11 04:14:44 +02:00
|
|
|
@Inject(IMoveRepository) moveRepository: IMoveRepository,
|
2023-09-27 22:46:46 +02:00
|
|
|
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
|
2023-05-17 19:07:17 +02:00
|
|
|
@Inject(IPersonRepository) private repository: IPersonRepository,
|
2024-05-16 00:58:23 +02:00
|
|
|
@Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository,
|
2023-05-17 19:07:17 +02:00
|
|
|
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
|
|
|
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
2024-02-13 02:50:47 +01:00
|
|
|
@Inject(ISearchRepository) private smartInfoRepository: ISearchRepository,
|
2024-03-20 19:32:04 +01:00
|
|
|
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
|
2024-04-16 23:30:31 +02:00
|
|
|
@Inject(ILoggerRepository) private logger: ILoggerRepository,
|
2023-09-18 06:05:35 +02:00
|
|
|
) {
|
2023-10-23 14:37:51 +02:00
|
|
|
this.access = AccessCore.create(accessRepository);
|
2024-04-16 23:30:31 +02:00
|
|
|
this.logger.setContext(PersonService.name);
|
2024-05-16 00:58:23 +02:00
|
|
|
this.configCore = SystemConfigCore.create(systemMetadataRepository, this.logger);
|
2023-12-29 19:41:33 +01:00
|
|
|
this.storageCore = StorageCore.create(
|
|
|
|
assetRepository,
|
2024-04-16 23:30:31 +02:00
|
|
|
cryptoRepository,
|
2023-12-29 19:41:33 +01:00
|
|
|
moveRepository,
|
|
|
|
repository,
|
|
|
|
storageRepository,
|
2024-05-16 00:58:23 +02:00
|
|
|
systemMetadataRepository,
|
2024-04-16 23:30:31 +02:00
|
|
|
this.logger,
|
2023-12-29 19:41:33 +01:00
|
|
|
);
|
2023-09-18 06:05:35 +02:00
|
|
|
}
|
2023-05-17 19:07:17 +02:00
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async getAll(auth: AuthDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning } = await this.configCore.getConfig({ withCache: false });
|
2023-12-10 05:34:12 +01:00
|
|
|
const people = await this.repository.getAllForUser(auth.user.id, {
|
2023-09-18 06:05:35 +02:00
|
|
|
minimumFaceCount: machineLearning.facialRecognition.minFaces,
|
2023-08-16 02:06:49 +02:00
|
|
|
withHidden: dto.withHidden || false,
|
|
|
|
});
|
2024-02-21 23:03:45 +01:00
|
|
|
const { total, hidden } = await this.repository.getNumberOfPeople(auth.user.id);
|
2023-07-18 20:09:43 +02:00
|
|
|
|
|
|
|
return {
|
2024-02-21 23:03:45 +01:00
|
|
|
people: people.map((person) => mapPerson(person)),
|
2024-01-28 01:54:31 +01:00
|
|
|
total,
|
2024-02-21 23:03:45 +01:00
|
|
|
hidden,
|
2023-07-18 20:09:43 +02:00
|
|
|
};
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async reassignFaces(auth: AuthDto, personId: string, dto: AssetFaceUpdateDto): Promise<PersonResponseDto[]> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_WRITE, personId);
|
2023-12-05 16:43:15 +01:00
|
|
|
const person = await this.findOrFail(personId);
|
|
|
|
const result: PersonResponseDto[] = [];
|
|
|
|
const changeFeaturePhoto: string[] = [];
|
|
|
|
for (const data of dto.data) {
|
|
|
|
const faces = await this.repository.getFacesByIds([{ personId: data.personId, assetId: data.assetId }]);
|
|
|
|
|
|
|
|
for (const face of faces) {
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_CREATE, face.id);
|
2023-12-05 16:43:15 +01:00
|
|
|
if (person.faceAssetId === null) {
|
|
|
|
changeFeaturePhoto.push(person.id);
|
|
|
|
}
|
|
|
|
if (face.person && face.person.faceAssetId === face.id) {
|
|
|
|
changeFeaturePhoto.push(face.person.id);
|
|
|
|
}
|
|
|
|
|
|
|
|
await this.repository.reassignFace(face.id, personId);
|
|
|
|
}
|
|
|
|
|
|
|
|
result.push(person);
|
|
|
|
}
|
|
|
|
if (changeFeaturePhoto.length > 0) {
|
|
|
|
// Remove duplicates
|
2024-02-02 04:18:00 +01:00
|
|
|
await this.createNewFeaturePhoto([...new Set(changeFeaturePhoto)]);
|
2023-12-05 16:43:15 +01:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async reassignFacesById(auth: AuthDto, personId: string, dto: FaceDto): Promise<PersonResponseDto> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_WRITE, personId);
|
2023-12-05 16:43:15 +01:00
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_CREATE, dto.id);
|
2023-12-05 16:43:15 +01:00
|
|
|
const face = await this.repository.getFaceById(dto.id);
|
|
|
|
const person = await this.findOrFail(personId);
|
|
|
|
|
|
|
|
await this.repository.reassignFace(face.id, personId);
|
|
|
|
if (person.faceAssetId === null) {
|
|
|
|
await this.createNewFeaturePhoto([person.id]);
|
|
|
|
}
|
|
|
|
if (face.person && face.person.faceAssetId === face.id) {
|
|
|
|
await this.createNewFeaturePhoto([face.person.id]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return await this.findOrFail(personId).then(mapPerson);
|
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async getFacesById(auth: AuthDto, dto: FaceDto): Promise<AssetFaceResponseDto[]> {
|
|
|
|
await this.access.requirePermission(auth, Permission.ASSET_READ, dto.id);
|
2023-12-05 16:43:15 +01:00
|
|
|
const faces = await this.repository.getFaces(dto.id);
|
2023-12-10 05:34:12 +01:00
|
|
|
return faces.map((asset) => mapFaces(asset, auth));
|
2023-12-05 16:43:15 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
async createNewFeaturePhoto(changeFeaturePhoto: string[]) {
|
|
|
|
this.logger.debug(
|
|
|
|
`Changing feature photos for ${changeFeaturePhoto.length} ${changeFeaturePhoto.length > 1 ? 'people' : 'person'}`,
|
|
|
|
);
|
2024-01-01 21:45:42 +01:00
|
|
|
|
|
|
|
const jobs: JobItem[] = [];
|
2023-12-05 16:43:15 +01:00
|
|
|
for (const personId of changeFeaturePhoto) {
|
|
|
|
const assetFace = await this.repository.getRandomFace(personId);
|
|
|
|
|
|
|
|
if (assetFace !== null) {
|
|
|
|
await this.repository.update({
|
|
|
|
id: personId,
|
|
|
|
faceAssetId: assetFace.id,
|
|
|
|
});
|
2024-01-01 21:45:42 +01:00
|
|
|
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: personId } });
|
2023-12-05 16:43:15 +01:00
|
|
|
}
|
|
|
|
}
|
2024-01-01 21:45:42 +01:00
|
|
|
|
|
|
|
await this.jobRepository.queueAll(jobs);
|
2023-12-05 16:43:15 +01:00
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async getById(auth: AuthDto, id: string): Promise<PersonResponseDto> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_READ, id);
|
2023-09-18 23:22:44 +02:00
|
|
|
return this.findOrFail(id).then(mapPerson);
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async getStatistics(auth: AuthDto, id: string): Promise<PersonStatisticsResponseDto> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_READ, id);
|
2023-10-24 17:53:49 +02:00
|
|
|
return this.repository.getStatistics(id);
|
|
|
|
}
|
|
|
|
|
2023-12-12 15:58:25 +01:00
|
|
|
async getThumbnail(auth: AuthDto, id: string): Promise<ImmichFileResponse> {
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_READ, id);
|
2023-09-18 23:22:44 +02:00
|
|
|
const person = await this.repository.getById(id);
|
2023-05-17 19:07:17 +02:00
|
|
|
if (!person || !person.thumbnailPath) {
|
|
|
|
throw new NotFoundException();
|
|
|
|
}
|
|
|
|
|
2023-12-12 15:58:25 +01:00
|
|
|
return new ImmichFileResponse({
|
|
|
|
path: person.thumbnailPath,
|
|
|
|
contentType: mimeTypes.lookup(person.thumbnailPath),
|
2023-12-18 17:33:46 +01:00
|
|
|
cacheControl: CacheControl.PRIVATE_WITHOUT_CACHE,
|
2023-12-12 15:58:25 +01:00
|
|
|
});
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async getAssets(auth: AuthDto, id: string): Promise<AssetResponseDto[]> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_READ, id);
|
2023-09-18 23:22:44 +02:00
|
|
|
const assets = await this.repository.getAssets(id);
|
2023-10-14 03:46:30 +02:00
|
|
|
return assets.map((asset) => mapAsset(asset));
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
|
|
|
|
2024-03-07 21:34:57 +01:00
|
|
|
create(auth: AuthDto, dto: PersonCreateDto): Promise<PersonResponseDto> {
|
|
|
|
return this.repository.create({
|
|
|
|
ownerId: auth.user.id,
|
|
|
|
name: dto.name,
|
|
|
|
birthDate: dto.birthDate,
|
|
|
|
isHidden: dto.isHidden,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async update(auth: AuthDto, id: string, dto: PersonUpdateDto): Promise<PersonResponseDto> {
|
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_WRITE, id);
|
2023-05-17 19:07:17 +02:00
|
|
|
|
2023-09-26 09:03:22 +02:00
|
|
|
const { name, birthDate, isHidden, featureFaceAssetId: assetId } = dto;
|
2024-03-07 21:34:57 +01:00
|
|
|
// TODO: set by faceId directly
|
|
|
|
let faceId: string | undefined = undefined;
|
2023-09-26 09:03:22 +02:00
|
|
|
if (assetId) {
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.access.requirePermission(auth, Permission.ASSET_READ, assetId);
|
2023-09-27 22:46:46 +02:00
|
|
|
const [face] = await this.repository.getFacesByIds([{ personId: id, assetId }]);
|
2023-07-11 23:52:41 +02:00
|
|
|
if (!face) {
|
|
|
|
throw new BadRequestException('Invalid assetId for feature face');
|
|
|
|
}
|
2023-07-03 00:46:20 +02:00
|
|
|
|
2024-03-07 21:34:57 +01:00
|
|
|
faceId = face.id;
|
|
|
|
}
|
|
|
|
|
|
|
|
const person = await this.repository.update({ id, faceAssetId: faceId, name, birthDate, isHidden });
|
|
|
|
|
|
|
|
if (assetId) {
|
2023-09-26 09:03:22 +02:00
|
|
|
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id } });
|
2023-07-03 00:46:20 +02:00
|
|
|
}
|
|
|
|
|
2023-07-11 23:52:41 +02:00
|
|
|
return mapPerson(person);
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
|
|
|
|
2024-03-07 21:34:57 +01:00
|
|
|
async updateAll(auth: AuthDto, dto: PeopleUpdateDto): Promise<BulkIdResponseDto[]> {
|
2023-07-23 05:00:43 +02:00
|
|
|
const results: BulkIdResponseDto[] = [];
|
|
|
|
for (const person of dto.people) {
|
|
|
|
try {
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.update(auth, person.id, {
|
2023-07-23 05:00:43 +02:00
|
|
|
isHidden: person.isHidden,
|
|
|
|
name: person.name,
|
2023-08-18 22:10:29 +02:00
|
|
|
birthDate: person.birthDate,
|
2023-07-23 05:00:43 +02:00
|
|
|
featureFaceAssetId: person.featureFaceAssetId,
|
|
|
|
}),
|
|
|
|
results.push({ id: person.id, success: true });
|
|
|
|
} catch (error: Error | any) {
|
|
|
|
this.logger.error(`Unable to update ${person.id} : ${error}`, error?.stack);
|
|
|
|
results.push({ id: person.id, success: false, error: BulkIdErrorReason.UNKNOWN });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
}
|
|
|
|
|
2024-01-18 06:08:48 +01:00
|
|
|
private async delete(people: PersonEntity[]) {
|
|
|
|
await Promise.all(people.map((person) => this.storageRepository.unlink(person.thumbnailPath)));
|
|
|
|
await this.repository.delete(people);
|
|
|
|
this.logger.debug(`Deleted ${people.length} people`);
|
|
|
|
}
|
2023-10-03 03:15:11 +02:00
|
|
|
|
2024-01-18 06:08:48 +01:00
|
|
|
private async deleteAllPeople() {
|
|
|
|
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
2024-01-28 23:02:01 +01:00
|
|
|
this.repository.getAll({ ...pagination, skip: 0 }),
|
2024-01-18 06:08:48 +01:00
|
|
|
);
|
2023-10-03 03:15:11 +02:00
|
|
|
|
2024-01-18 06:08:48 +01:00
|
|
|
for await (const people of personPagination) {
|
|
|
|
await this.delete(people); // deletes thumbnails too
|
|
|
|
}
|
2023-10-03 03:15:11 +02:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handlePersonCleanup(): Promise<JobStatus> {
|
2023-05-17 19:07:17 +02:00
|
|
|
const people = await this.repository.getAllWithoutFaces();
|
2024-01-18 06:08:48 +01:00
|
|
|
await this.delete(people);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|
2023-07-11 23:52:41 +02:00
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handleQueueDetectFaces({ force }: IBaseJob): Promise<JobStatus> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning } = await this.configCore.getConfig({ withCache: false });
|
2024-05-14 21:31:36 +02:00
|
|
|
if (!isFacialRecognitionEnabled(machineLearning)) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-01-18 06:08:48 +01:00
|
|
|
if (force) {
|
|
|
|
await this.deleteAllPeople();
|
|
|
|
await this.repository.deleteAllFaces();
|
|
|
|
}
|
|
|
|
|
2023-09-27 22:46:46 +02:00
|
|
|
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
|
|
|
|
return force
|
2024-04-19 03:37:55 +02:00
|
|
|
? this.assetRepository.getAll(pagination, {
|
|
|
|
orderDirection: 'DESC',
|
|
|
|
withFaces: true,
|
|
|
|
withArchived: true,
|
|
|
|
isVisible: true,
|
|
|
|
})
|
2023-09-27 22:46:46 +02:00
|
|
|
: this.assetRepository.getWithout(pagination, WithoutProperty.FACES);
|
|
|
|
});
|
|
|
|
|
|
|
|
for await (const assets of assetPagination) {
|
2024-01-01 21:45:42 +01:00
|
|
|
await this.jobRepository.queueAll(
|
2024-01-18 06:08:48 +01:00
|
|
|
assets.map((asset) => ({ name: JobName.FACE_DETECTION, data: { id: asset.id } })),
|
2024-01-01 21:45:42 +01:00
|
|
|
);
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handleDetectFaces({ id }: IEntityJob): Promise<JobStatus> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning } = await this.configCore.getConfig({ withCache: true });
|
2024-05-14 21:31:36 +02:00
|
|
|
if (!isFacialRecognitionEnabled(machineLearning)) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2023-11-06 03:15:12 +01:00
|
|
|
const relations = {
|
|
|
|
exifInfo: true,
|
|
|
|
faces: {
|
2024-01-18 06:08:48 +01:00
|
|
|
person: false,
|
2023-11-06 03:15:12 +01:00
|
|
|
},
|
|
|
|
};
|
|
|
|
const [asset] = await this.assetRepository.getByIds([id], relations);
|
2024-04-02 06:56:56 +02:00
|
|
|
if (!asset || !asset.previewPath || asset.faces?.length > 0) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-04-19 03:37:55 +02:00
|
|
|
if (!asset.isVisible) {
|
|
|
|
return JobStatus.SKIPPED;
|
|
|
|
}
|
|
|
|
|
2024-06-07 05:09:47 +02:00
|
|
|
if (!asset.isVisible) {
|
|
|
|
return JobStatus.SKIPPED;
|
|
|
|
}
|
|
|
|
|
|
|
|
const { imageHeight, imageWidth, faces } = await this.machineLearningRepository.detectFaces(
|
2023-09-27 22:46:46 +02:00
|
|
|
machineLearning.url,
|
2024-06-07 05:09:47 +02:00
|
|
|
asset.previewPath,
|
2023-09-27 22:46:46 +02:00
|
|
|
machineLearning.facialRecognition,
|
|
|
|
);
|
|
|
|
|
2024-04-02 06:56:56 +02:00
|
|
|
this.logger.debug(`${faces.length} faces detected in ${asset.previewPath}`);
|
2023-09-27 22:46:46 +02:00
|
|
|
|
2024-02-02 04:18:00 +01:00
|
|
|
if (faces.length > 0) {
|
2024-01-25 07:27:39 +01:00
|
|
|
await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } });
|
|
|
|
const mappedFaces = faces.map((face) => ({
|
2023-12-05 16:43:15 +01:00
|
|
|
assetId: asset.id,
|
2024-01-18 06:08:48 +01:00
|
|
|
embedding: face.embedding,
|
2024-06-07 05:09:47 +02:00
|
|
|
imageHeight,
|
|
|
|
imageWidth,
|
2024-01-18 06:08:48 +01:00
|
|
|
boundingBoxX1: face.boundingBox.x1,
|
|
|
|
boundingBoxY1: face.boundingBox.y1,
|
2024-06-07 05:09:47 +02:00
|
|
|
boundingBoxX2: face.boundingBox.x2,
|
2024-01-18 06:08:48 +01:00
|
|
|
boundingBoxY2: face.boundingBox.y2,
|
2024-01-25 07:27:39 +01:00
|
|
|
}));
|
2024-01-18 06:08:48 +01:00
|
|
|
|
2024-01-25 07:27:39 +01:00
|
|
|
const faceIds = await this.repository.createFaces(mappedFaces);
|
|
|
|
await this.jobRepository.queueAll(faceIds.map((id) => ({ name: JobName.FACIAL_RECOGNITION, data: { id } })));
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2023-11-10 02:55:00 +01:00
|
|
|
await this.assetRepository.upsertJobStatus({
|
|
|
|
assetId: asset.id,
|
|
|
|
facesRecognizedAt: new Date(),
|
|
|
|
});
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handleQueueRecognizeFaces({ force }: IBaseJob): Promise<JobStatus> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning } = await this.configCore.getConfig({ withCache: false });
|
2024-05-14 21:31:36 +02:00
|
|
|
if (!isFacialRecognitionEnabled(machineLearning)) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
|
2024-01-25 07:27:39 +01:00
|
|
|
const { waiting } = await this.jobRepository.getJobCounts(QueueName.FACIAL_RECOGNITION);
|
2024-01-18 06:08:48 +01:00
|
|
|
|
|
|
|
if (force) {
|
|
|
|
await this.deleteAllPeople();
|
2024-01-25 07:27:39 +01:00
|
|
|
} else if (waiting) {
|
|
|
|
this.logger.debug(
|
|
|
|
`Skipping facial recognition queueing because ${waiting} job${waiting > 1 ? 's are' : ' is'} already queued`,
|
|
|
|
);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
|
|
|
this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }),
|
|
|
|
);
|
|
|
|
|
|
|
|
for await (const page of facePagination) {
|
|
|
|
await this.jobRepository.queueAll(
|
|
|
|
page.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id, deferred: false } })),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handleRecognizeFaces({ id, deferred }: IDeferrableJob): Promise<JobStatus> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning } = await this.configCore.getConfig({ withCache: true });
|
2024-05-14 21:31:36 +02:00
|
|
|
if (!isFacialRecognitionEnabled(machineLearning)) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const face = await this.repository.getFaceByIdWithAssets(
|
|
|
|
id,
|
|
|
|
{ person: true, asset: true },
|
|
|
|
{ id: true, personId: true, embedding: true },
|
|
|
|
);
|
2024-01-29 02:17:54 +01:00
|
|
|
if (!face || !face.asset) {
|
2024-01-18 06:08:48 +01:00
|
|
|
this.logger.warn(`Face ${id} not found`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (face.personId) {
|
|
|
|
this.logger.debug(`Face ${id} already has a person assigned`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const matches = await this.smartInfoRepository.searchFaces({
|
|
|
|
userIds: [face.asset.ownerId],
|
|
|
|
embedding: face.embedding,
|
|
|
|
maxDistance: machineLearning.facialRecognition.maxDistance,
|
|
|
|
numResults: machineLearning.facialRecognition.minFaces,
|
|
|
|
});
|
|
|
|
|
2024-02-07 16:56:39 +01:00
|
|
|
// `matches` also includes the face itself
|
2024-02-17 04:32:11 +01:00
|
|
|
if (machineLearning.facialRecognition.minFaces > 1 && matches.length <= 1) {
|
|
|
|
this.logger.debug(`Face ${id} only matched the face itself, skipping`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-02-07 16:56:39 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
this.logger.debug(`Face ${id} has ${matches.length} matches`);
|
2024-01-18 06:08:48 +01:00
|
|
|
|
2024-04-18 05:47:24 +02:00
|
|
|
const isCore = matches.length >= machineLearning.facialRecognition.minFaces && !face.asset.isArchived;
|
2024-01-18 06:08:48 +01:00
|
|
|
if (!isCore && !deferred) {
|
|
|
|
this.logger.debug(`Deferring non-core face ${id} for later processing`);
|
|
|
|
await this.jobRepository.queue({ name: JobName.FACIAL_RECOGNITION, data: { id, deferred: true } });
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
2024-02-07 16:56:39 +01:00
|
|
|
let personId = matches.find((match) => match.face.personId)?.face.personId;
|
2024-01-18 06:08:48 +01:00
|
|
|
if (!personId) {
|
|
|
|
const matchWithPerson = await this.smartInfoRepository.searchFaces({
|
|
|
|
userIds: [face.asset.ownerId],
|
|
|
|
embedding: face.embedding,
|
|
|
|
maxDistance: machineLearning.facialRecognition.maxDistance,
|
|
|
|
numResults: 1,
|
|
|
|
hasPerson: true,
|
|
|
|
});
|
|
|
|
|
|
|
|
if (matchWithPerson.length > 0) {
|
|
|
|
personId = matchWithPerson[0].face.personId;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isCore && !personId) {
|
|
|
|
this.logger.log(`Creating new person for face ${id}`);
|
|
|
|
const newPerson = await this.repository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id });
|
|
|
|
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
|
|
|
|
personId = newPerson.id;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (personId) {
|
|
|
|
this.logger.debug(`Assigning face ${id} to person ${personId}`);
|
|
|
|
await this.repository.reassignFaces({ faceIds: [id], newPersonId: personId });
|
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2024-01-18 06:08:48 +01:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handlePersonMigration({ id }: IEntityJob): Promise<JobStatus> {
|
2023-09-27 22:46:46 +02:00
|
|
|
const person = await this.repository.getById(id);
|
|
|
|
if (!person) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2023-10-11 04:14:44 +02:00
|
|
|
await this.storageCore.movePersonFile(person, PersonPathType.FACE);
|
2023-09-27 22:46:46 +02:00
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
async handleGeneratePersonThumbnail(data: IEntityJob): Promise<JobStatus> {
|
2024-06-12 13:07:35 +02:00
|
|
|
const { machineLearning, image } = await this.configCore.getConfig({ withCache: true });
|
2024-05-14 21:31:36 +02:00
|
|
|
if (!isFacialRecognitionEnabled(machineLearning)) {
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SKIPPED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
const person = await this.repository.getById(data.id);
|
|
|
|
if (!person?.faceAssetId) {
|
2024-05-08 15:09:34 +02:00
|
|
|
this.logger.error(`Could not generate person thumbnail: person ${person?.id} has no face asset`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2023-12-05 16:43:15 +01:00
|
|
|
const face = await this.repository.getFaceByIdWithAssets(person.faceAssetId);
|
|
|
|
if (face === null) {
|
2024-05-08 15:09:34 +02:00
|
|
|
this.logger.error(`Could not generate person thumbnail: face ${person.faceAssetId} not found`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
const {
|
|
|
|
assetId,
|
|
|
|
boundingBoxX1: x1,
|
|
|
|
boundingBoxX2: x2,
|
|
|
|
boundingBoxY1: y1,
|
|
|
|
boundingBoxY2: y2,
|
2024-06-13 04:16:26 +02:00
|
|
|
imageWidth: oldWidth,
|
|
|
|
imageHeight: oldHeight,
|
2023-09-27 22:46:46 +02:00
|
|
|
} = face;
|
|
|
|
|
2024-05-08 15:09:34 +02:00
|
|
|
const asset = await this.assetRepository.getById(assetId, { exifInfo: true });
|
2024-06-13 04:16:26 +02:00
|
|
|
if (!asset) {
|
|
|
|
this.logger.error(`Could not generate person thumbnail: asset ${assetId} does not exist`);
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.FAILED;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
2024-05-08 15:09:34 +02:00
|
|
|
|
2024-06-13 04:16:26 +02:00
|
|
|
const { width, height, inputPath } = await this.getInputDimensions(asset);
|
|
|
|
|
2023-10-23 17:52:21 +02:00
|
|
|
const thumbnailPath = StorageCore.getPersonThumbnailPath(person);
|
2023-10-11 04:14:44 +02:00
|
|
|
this.storageCore.ensureFolders(thumbnailPath);
|
2023-09-27 22:46:46 +02:00
|
|
|
|
|
|
|
const thumbnailOptions = {
|
2024-04-02 06:56:56 +02:00
|
|
|
format: ImageFormat.JPEG,
|
2023-09-27 22:46:46 +02:00
|
|
|
size: FACE_THUMBNAIL_SIZE,
|
2024-04-02 06:56:56 +02:00
|
|
|
colorspace: image.colorspace,
|
|
|
|
quality: image.quality,
|
2024-06-13 04:16:26 +02:00
|
|
|
crop: this.getCrop({ old: { width: oldWidth, height: oldHeight }, new: { width, height } }, { x1, y1, x2, y2 }),
|
2023-09-27 22:46:46 +02:00
|
|
|
} as const;
|
|
|
|
|
2024-06-13 04:16:26 +02:00
|
|
|
await this.mediaRepository.generateThumbnail(inputPath, thumbnailPath, thumbnailOptions);
|
2023-10-24 15:12:42 +02:00
|
|
|
await this.repository.update({ id: person.id, thumbnailPath });
|
2023-09-27 22:46:46 +02:00
|
|
|
|
2024-03-15 14:16:54 +01:00
|
|
|
return JobStatus.SUCCESS;
|
2023-09-27 22:46:46 +02:00
|
|
|
}
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
async mergePerson(auth: AuthDto, id: string, dto: MergePersonDto): Promise<BulkIdResponseDto[]> {
|
2023-07-11 23:52:41 +02:00
|
|
|
const mergeIds = dto.ids;
|
2023-12-10 05:34:12 +01:00
|
|
|
await this.access.requirePermission(auth, Permission.PERSON_WRITE, id);
|
2024-01-19 18:52:26 +01:00
|
|
|
let primaryPerson = await this.findOrFail(id);
|
2023-07-11 23:52:41 +02:00
|
|
|
const primaryName = primaryPerson.name || primaryPerson.id;
|
|
|
|
|
|
|
|
const results: BulkIdResponseDto[] = [];
|
|
|
|
|
2023-12-10 05:34:12 +01:00
|
|
|
const allowedIds = await this.access.checkAccess(auth, Permission.PERSON_MERGE, mergeIds);
|
2023-09-18 23:22:44 +02:00
|
|
|
|
2023-11-23 05:04:52 +01:00
|
|
|
for (const mergeId of mergeIds) {
|
|
|
|
const hasAccess = allowedIds.has(mergeId);
|
|
|
|
if (!hasAccess) {
|
2023-09-18 23:22:44 +02:00
|
|
|
results.push({ id: mergeId, success: false, error: BulkIdErrorReason.NO_PERMISSION });
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2023-07-11 23:52:41 +02:00
|
|
|
try {
|
2023-09-18 23:22:44 +02:00
|
|
|
const mergePerson = await this.repository.getById(mergeId);
|
2023-07-11 23:52:41 +02:00
|
|
|
if (!mergePerson) {
|
|
|
|
results.push({ id: mergeId, success: false, error: BulkIdErrorReason.NOT_FOUND });
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2024-01-19 18:52:26 +01:00
|
|
|
const update: Partial<PersonEntity> = {};
|
|
|
|
if (!primaryPerson.name && mergePerson.name) {
|
|
|
|
update.name = mergePerson.name;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!primaryPerson.birthDate && mergePerson.birthDate) {
|
|
|
|
update.birthDate = mergePerson.birthDate;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (Object.keys(update).length > 0) {
|
|
|
|
primaryPerson = await this.repository.update({ id: primaryPerson.id, ...update });
|
|
|
|
}
|
|
|
|
|
2023-07-11 23:52:41 +02:00
|
|
|
const mergeName = mergePerson.name || mergePerson.id;
|
|
|
|
const mergeData: UpdateFacesData = { oldPersonId: mergeId, newPersonId: id };
|
|
|
|
this.logger.log(`Merging ${mergeName} into ${primaryName}`);
|
|
|
|
|
|
|
|
await this.repository.reassignFaces(mergeData);
|
2024-01-18 06:08:48 +01:00
|
|
|
await this.delete([mergePerson]);
|
2023-07-11 23:52:41 +02:00
|
|
|
|
|
|
|
this.logger.log(`Merged ${mergeName} into ${primaryName}`);
|
|
|
|
results.push({ id: mergeId, success: true });
|
|
|
|
} catch (error: Error | any) {
|
|
|
|
this.logger.error(`Unable to merge ${mergeId} into ${id}: ${error}`, error?.stack);
|
|
|
|
results.push({ id: mergeId, success: false, error: BulkIdErrorReason.UNKNOWN });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
}
|
|
|
|
|
2023-09-18 23:22:44 +02:00
|
|
|
private async findOrFail(id: string) {
|
|
|
|
const person = await this.repository.getById(id);
|
2023-07-11 23:52:41 +02:00
|
|
|
if (!person) {
|
|
|
|
throw new BadRequestException('Person not found');
|
|
|
|
}
|
|
|
|
return person;
|
|
|
|
}
|
2024-05-08 15:09:34 +02:00
|
|
|
|
2024-06-13 04:16:26 +02:00
|
|
|
private async getInputDimensions(asset: AssetEntity): Promise<ImageDimensions & { inputPath: string }> {
|
|
|
|
if (!asset.exifInfo?.exifImageHeight || !asset.exifInfo.exifImageWidth) {
|
|
|
|
throw new Error(`Asset ${asset.id} dimensions are unknown`);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!asset.previewPath) {
|
|
|
|
throw new Error(`Asset ${asset.id} has no preview path`);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (asset.type === AssetType.IMAGE) {
|
|
|
|
const { width, height } = this.withOrientation(asset.exifInfo.orientation as Orientation, {
|
|
|
|
width: asset.exifInfo.exifImageWidth,
|
|
|
|
height: asset.exifInfo.exifImageHeight,
|
|
|
|
});
|
|
|
|
return { width, height, inputPath: asset.originalPath };
|
|
|
|
}
|
|
|
|
|
|
|
|
const { width, height } = await this.mediaRepository.getImageDimensions(asset.previewPath);
|
|
|
|
return { width, height, inputPath: asset.previewPath };
|
|
|
|
}
|
|
|
|
|
2024-05-08 15:09:34 +02:00
|
|
|
private withOrientation(orientation: Orientation, { width, height }: ImageDimensions): ImageDimensions {
|
|
|
|
switch (orientation) {
|
|
|
|
case Orientation.MirrorHorizontalRotate270CW:
|
|
|
|
case Orientation.Rotate90CW:
|
|
|
|
case Orientation.MirrorHorizontalRotate90CW:
|
|
|
|
case Orientation.Rotate270CW: {
|
|
|
|
return { width: height, height: width };
|
|
|
|
}
|
|
|
|
default: {
|
|
|
|
return { width, height };
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2024-06-13 04:16:26 +02:00
|
|
|
|
|
|
|
private getCrop(dims: { old: ImageDimensions; new: ImageDimensions }, { x1, y1, x2, y2 }: BoundingBox): CropOptions {
|
|
|
|
const widthScale = dims.new.width / dims.old.width;
|
|
|
|
const heightScale = dims.new.height / dims.old.height;
|
|
|
|
|
|
|
|
const halfWidth = (widthScale * (x2 - x1)) / 2;
|
|
|
|
const halfHeight = (heightScale * (y2 - y1)) / 2;
|
|
|
|
|
|
|
|
const middleX = Math.round(widthScale * x1 + halfWidth);
|
|
|
|
const middleY = Math.round(heightScale * y1 + halfHeight);
|
|
|
|
|
|
|
|
// zoom out 10%
|
|
|
|
const targetHalfSize = Math.floor(Math.max(halfWidth, halfHeight) * 1.1);
|
|
|
|
|
|
|
|
// get the longest distance from the center of the image without overflowing
|
|
|
|
const newHalfSize = Math.min(
|
|
|
|
middleX - Math.max(0, middleX - targetHalfSize),
|
|
|
|
middleY - Math.max(0, middleY - targetHalfSize),
|
|
|
|
Math.min(dims.new.width - 1, middleX + targetHalfSize) - middleX,
|
|
|
|
Math.min(dims.new.height - 1, middleY + targetHalfSize) - middleY,
|
|
|
|
);
|
|
|
|
|
|
|
|
return {
|
|
|
|
left: middleX - newHalfSize,
|
|
|
|
top: middleY - newHalfSize,
|
|
|
|
width: newHalfSize * 2,
|
|
|
|
height: newHalfSize * 2,
|
|
|
|
};
|
|
|
|
}
|
2023-05-17 19:07:17 +02:00
|
|
|
}
|