From 03dffc8414bab81939a2af17855c29867e1c013f Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:43:48 -0500 Subject: [PATCH] optimize metadata extraction --- server/src/services/job.service.ts | 1 - server/src/services/metadata.service.ts | 196 ++++++++++++++---------- 2 files changed, 115 insertions(+), 82 deletions(-) diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 2faed0a516..49c6871959 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -240,7 +240,6 @@ export class JobService extends BaseService { this.eventRepository.clientSend('on_asset_update', asset.ownerId, mapAsset(asset)); } } - await this.jobRepository.queue({ name: JobName.LINK_LIVE_PHOTOS, data: item.data }); break; } diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 79a7d519d6..b6f1a3beaf 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -5,7 +5,6 @@ import _ from 'lodash'; import { Duration } from 'luxon'; import { constants } from 'node:fs/promises'; import path from 'node:path'; -import { SystemConfig } from 'src/config'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; @@ -66,6 +65,8 @@ const validateRange = (value: number | undefined, min: number, max: number): Non return val; }; +type ImmichTagsWithFaces = ImmichTags & { RegionInfo: NonNullable }; + @Injectable() export class MetadataService extends BaseService { @OnEvent({ name: 'app.bootstrap', workers: [ImmichWorker.MICROSERVICES] }) @@ -101,13 +102,17 @@ export class MetadataService extends BaseService { return JobStatus.FAILED; } - if (!asset.exifInfo.livePhotoCID) { + return this.linkLivePhotos(asset, asset.exifInfo); + } + + private async linkLivePhotos(asset: AssetEntity, exifInfo: ExifEntity): Promise { + if (!exifInfo.livePhotoCID) { return JobStatus.SKIPPED; } const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO; const match = await this.assetRepository.findLivePhotoMatch({ - livePhotoCID: asset.exifInfo.livePhotoCID, + livePhotoCID: exifInfo.livePhotoCID, ownerId: asset.ownerId, libraryId: asset.libraryId, otherAssetId: asset.id, @@ -119,10 +124,11 @@ export class MetadataService extends BaseService { } const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset]; - - await this.assetRepository.update({ id: photoAsset.id, livePhotoVideoId: motionAsset.id }); - await this.assetRepository.update({ id: motionAsset.id, isVisible: false }); - await this.albumRepository.removeAsset(motionAsset.id); + await Promise.all([ + this.assetRepository.update({ id: photoAsset.id, livePhotoVideoId: motionAsset.id }), + this.assetRepository.update({ id: motionAsset.id, isVisible: false }), + this.albumRepository.removeAsset(motionAsset.id), + ]); await this.eventRepository.emit('asset.hide', { assetId: motionAsset.id, userId: motionAsset.ownerId }); @@ -148,25 +154,37 @@ export class MetadataService extends BaseService { } @OnJob({ name: JobName.METADATA_EXTRACTION, queue: QueueName.METADATA_EXTRACTION }) - async handleMetadataExtraction({ id }: JobOf): Promise { - const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true }); - const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } }); + async handleMetadataExtraction(data: JobOf): Promise { + const [{ metadata, reverseGeocoding }, [asset]] = await Promise.all([ + this.getConfig({ withCache: true }), + this.assetRepository.getByIds([data.id], { faces: { person: false } }), + ]); + if (!asset) { return JobStatus.FAILED; } - const stats = await this.storageRepository.stat(asset.originalPath); - - const exifTags = await this.getExifTags(asset); + const [stats, exifTags] = await Promise.all([ + this.storageRepository.stat(asset.originalPath), + this.getExifTags(asset), + ]); this.logger.verbose('Exif Tags', exifTags); const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags); - const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding); - const { width, height } = this.getImageDimensions(exifTags); - const exifData: Partial = { + let geo: ReverseGeocodeResult = { country: null, state: null, city: null }; + let latitude: number | null = null; + let longitude: number | null = null; + if (reverseGeocoding.enabled && this.hasGeo(exifTags)) { + latitude = exifTags.GPSLatitude; + longitude = exifTags.GPSLongitude; + geo = await this.mapRepository.reverseGeocode({ latitude, longitude }); + } + + const livePhotoCID = (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null; + const exifData: ExifEntity = { assetId: asset.id, // dates @@ -175,11 +193,11 @@ export class MetadataService extends BaseService { timeZone, // gps - latitude, - longitude, - country, - state, - city, + latitude: exifTags.GPSLatitude ?? null, + longitude: exifTags.GPSLongitude ?? null, + country: geo.country, + state: geo.state, + city: geo.city, // image/file fileSizeInByte: stats.size, @@ -206,31 +224,42 @@ export class MetadataService extends BaseService { rating: validateRange(exifTags.Rating, 0, 5), // grouping - livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null, + livePhotoCID, autoStackId: this.getAutoStackId(exifTags), }; - await this.applyTagList(asset, exifTags); - await this.applyMotionPhotos(asset, exifTags); + const promises: Promise[] = [ + this.assetRepository.upsertExif(exifData), + this.assetRepository.update({ + id: asset.id, + duration: exifTags.Duration?.toString() ?? null, + localDateTime, + fileCreatedAt: exifData.dateTimeOriginal ?? undefined, + }), + ]; - await this.assetRepository.upsertExif(exifData); - - await this.assetRepository.update({ - id: asset.id, - duration: exifTags.Duration?.toString() ?? null, - localDateTime, - fileCreatedAt: exifData.dateTimeOriginal ?? undefined, - }); - - await this.assetRepository.upsertJobStatus({ - assetId: asset.id, - metadataExtractedAt: new Date(), - }); - - if (isFaceImportEnabled(metadata)) { - await this.applyTaggedFaces(asset, exifTags); + if (this.hasTagList(exifTags)) { + promises.push(this.applyTagList(asset, exifTags)); } + if (asset.type === AssetType.IMAGE) { + promises.push(this.applyMotionPhotos(asset, exifTags)); + } + + if (isFaceImportEnabled(metadata) && this.hasTaggedFaces(exifTags)) { + promises.push(this.applyTaggedFaces(asset, exifTags)); + } + + if (livePhotoCID) { + promises.push(this.linkLivePhotos(asset, exifData)); + } + + await Promise.all(promises); + await Promise.all([ + this.assetRepository.upsertJobStatus({ assetId: asset.id, metadataExtractedAt: new Date() }), + this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data }), + ]); + return JobStatus.SUCCESS; } @@ -326,45 +355,60 @@ export class MetadataService extends BaseService { } private async getExifTags(asset: AssetEntity): Promise { - const mediaTags = await this.metadataRepository.readTags(asset.originalPath); - const sidecarTags = asset.sidecarPath ? await this.metadataRepository.readTags(asset.sidecarPath) : {}; - const videoTags = asset.type === AssetType.VIDEO ? await this.getVideoTags(asset.originalPath) : {}; + const promises = [this.metadataRepository.readTags(asset.originalPath)]; + if (asset.sidecarPath) { + promises.push(this.metadataRepository.readTags(asset.sidecarPath)); + } + if (asset.type === AssetType.VIDEO) { + promises.push(this.getVideoTags(asset.originalPath)); + } + + const [mediaTags, sidecarTags, videoTags] = await Promise.all(promises); + if (!sidecarTags && !videoTags) { + return mediaTags; + } // prefer dates from sidecar tags - const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS); - if (sidecarDate) { - for (const tag of EXIF_DATE_TAGS) { - delete mediaTags[tag]; + if (sidecarTags) { + const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS); + if (sidecarDate) { + for (const tag of EXIF_DATE_TAGS) { + delete mediaTags[tag]; + } } } // prefer duration from video tags delete mediaTags.Duration; - delete sidecarTags.Duration; + delete sidecarTags?.Duration; return { ...mediaTags, ...videoTags, ...sidecarTags }; } + private hasTagList(tags: ImmichTags): tags is ImmichTags & { TagsList: string[] } { + return tags.TagsList !== undefined || tags.HierarchicalSubject !== undefined || tags.Keywords !== undefined; + } + private async applyTagList(asset: AssetEntity, exifTags: ImmichTags) { - const tags: string[] = []; + let tags: string[] = []; if (exifTags.TagsList) { - tags.push(...exifTags.TagsList.map(String)); + tags = exifTags.TagsList.map(String); } else if (exifTags.HierarchicalSubject) { - tags.push( - ...exifTags.HierarchicalSubject.map((tag) => - String(tag) - // convert | to / - .replaceAll('/', '') - .replaceAll('|', '/') - .replaceAll('', '|'), - ), + tags = exifTags.HierarchicalSubject.map((tag) => + typeof tag === 'number' + ? String(tag) + : tag + // convert | to / + .replaceAll('/', '') + .replaceAll('|', '/') + .replaceAll('', '|'), ); } else if (exifTags.Keywords) { let keywords = exifTags.Keywords; if (!Array.isArray(keywords)) { keywords = [keywords]; } - tags.push(...keywords.map(String)); + tags = keywords.map(String); } const results = await upsertTags(this.tagRepository, { userId: asset.ownerId, tags }); @@ -503,11 +547,13 @@ export class MetadataService extends BaseService { } } - private async applyTaggedFaces(asset: AssetEntity, tags: ImmichTags) { - if (!tags.RegionInfo?.AppliedToDimensions || tags.RegionInfo.RegionList.length === 0) { - return; - } + private hasTaggedFaces(tags: ImmichTags): tags is ImmichTagsWithFaces { + return ( + tags.RegionInfo !== undefined && tags.RegionInfo.AppliedToDimensions && tags.RegionInfo.RegionList.length > 0 + ); + } + private async applyTaggedFaces(asset: AssetEntity, tags: ImmichTagsWithFaces) { const facesToAdd: Partial[] = []; const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true }); const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id])); @@ -609,24 +655,12 @@ export class MetadataService extends BaseService { }; } - private async getGeo(tags: ImmichTags, reverseGeocoding: SystemConfig['reverseGeocoding']) { - let latitude = validate(tags.GPSLatitude); - let longitude = validate(tags.GPSLongitude); - - // TODO take ref into account - - if (latitude === 0 && longitude === 0) { - this.logger.warn('Latitude and longitude of 0, setting to null'); - latitude = null; - longitude = null; - } - - let result: ReverseGeocodeResult = { country: null, state: null, city: null }; - if (reverseGeocoding.enabled && longitude && latitude) { - result = await this.mapRepository.reverseGeocode({ latitude, longitude }); - } - - return { ...result, latitude, longitude }; + private hasGeo(tags: ImmichTags): tags is ImmichTags & { GPSLatitude: number; GPSLongitude: number } { + return ( + tags.GPSLatitude !== undefined && + tags.GPSLongitude !== undefined && + (tags.GPSLatitude !== 0 || tags.GPSLatitude !== 0) + ); } private getAutoStackId(tags: ImmichTags | null): string | null {