From 9c30d58b106775cc4098f9e0826e4646ec32b555 Mon Sep 17 00:00:00 2001
From: Thanh Pham <pano.tjoan@yahoo.com>
Date: Sun, 21 Aug 2022 06:31:37 +0700
Subject: [PATCH] feat(server): preserve caption fields and extract mediainfo
for video (#505)
* feat(server): preserve caption fields and extract mediainfo for video
* Fixed Geocoding missing info leads to fail EXIF extraction for the whole file
Co-authored-by: Alex <alex.tran1502@gmail.com>
---
.../metadata-extraction.processor.ts | 173 +++++++++++++++---
.../libs/database/src/entities/exif.entity.ts | 57 +++---
.../src/entities/smart-info.entity.ts | 2 +-
.../migrations/1661011331242-AddCaption.ts | 18 ++
4 files changed, 197 insertions(+), 53 deletions(-)
create mode 100644 server/libs/database/src/migrations/1661011331242-AddCaption.ts
diff --git a/server/apps/microservices/src/processors/metadata-extraction.processor.ts b/server/apps/microservices/src/processors/metadata-extraction.processor.ts
index 1701d41358..f956e17aa4 100644
--- a/server/apps/microservices/src/processors/metadata-extraction.processor.ts
+++ b/server/apps/microservices/src/processors/metadata-extraction.processor.ts
@@ -85,9 +85,21 @@ export class MetadataExtractionProcessor {
const res: [] = geoCodeInfo.body['features'];
- const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
- const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
- const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+ let city = '';
+ let state = '';
+ let country = '';
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+ city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+ state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+ country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+ }
newExif.city = city || null;
newExif.state = state || null;
@@ -114,9 +126,21 @@ export class MetadataExtractionProcessor {
const res: [] = geoCodeInfo.body['features'];
- const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
- const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
- const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+ let city = '';
+ let state = '';
+ let country = '';
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+ city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+ state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+ country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+ }
await this.exifRepository.update({ id: exif.id }, { city, state, country });
}
@@ -168,31 +192,126 @@ export class MetadataExtractionProcessor {
async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
const { asset } = job.data;
- ffmpeg.ffprobe(asset.originalPath, async (err, data) => {
- if (!err) {
- let durationString = asset.duration;
- let createdAt = asset.createdAt;
+ try {
+ const data = await new Promise<ffmpeg.FfprobeData>((resolve, reject) =>
+ ffmpeg.ffprobe(asset.originalPath, (err, data) => {
+ if (err) return reject(err);
+ return resolve(data);
+ }),
+ );
+ let durationString = asset.duration;
+ let createdAt = asset.createdAt;
- if (data.format.duration) {
- durationString = this.extractDuration(data.format.duration);
- }
+ if (data.format.duration) {
+ durationString = this.extractDuration(data.format.duration);
+ }
- const videoTags = data.format.tags;
- if (videoTags) {
- if (videoTags['com.apple.quicktime.creationdate']) {
- createdAt = String(videoTags['com.apple.quicktime.creationdate']);
- } else if (videoTags['creation_time']) {
- createdAt = String(videoTags['creation_time']);
- } else {
- createdAt = asset.createdAt;
- }
+ const videoTags = data.format.tags;
+ if (videoTags) {
+ if (videoTags['com.apple.quicktime.creationdate']) {
+ createdAt = String(videoTags['com.apple.quicktime.creationdate']);
+ } else if (videoTags['creation_time']) {
+ createdAt = String(videoTags['creation_time']);
} else {
createdAt = asset.createdAt;
}
-
- await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
+ } else {
+ createdAt = asset.createdAt;
}
- });
+
+ const newExif = new ExifEntity();
+ newExif.assetId = asset.id;
+ newExif.description = '';
+ newExif.fileSizeInByte = data.format.size || null;
+ newExif.dateTimeOriginal = createdAt ? new Date(createdAt) : null;
+ newExif.modifyDate = null;
+ newExif.latitude = null;
+ newExif.longitude = null;
+ newExif.city = null;
+ newExif.state = null;
+ newExif.country = null;
+ newExif.fps = null;
+
+ if (videoTags && videoTags['location']) {
+ const location = videoTags['location'] as string;
+ const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
+ const match = location.match(locationRegex);
+
+ if (match?.length === 3) {
+ newExif.latitude = parseFloat(match[0]);
+ newExif.longitude = parseFloat(match[1]);
+ }
+ } else if (videoTags && videoTags['com.apple.quicktime.location.ISO6709']) {
+ const location = videoTags['com.apple.quicktime.location.ISO6709'] as string;
+ const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
+ const match = location.match(locationRegex);
+ if (match?.length === 4) {
+ newExif.latitude = parseFloat(match[1]);
+ newExif.longitude = parseFloat(match[2]);
+ }
+ }
+
+ // Reverse GeoCoding
+ if (this.geocodingClient && newExif.longitude && newExif.latitude) {
+ const geoCodeInfo: MapiResponse = await this.geocodingClient
+ .reverseGeocode({
+ query: [newExif.longitude, newExif.latitude],
+ types: ['country', 'region', 'place'],
+ })
+ .send();
+
+ const res: [] = geoCodeInfo.body['features'];
+
+ let city = '';
+ let state = '';
+ let country = '';
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+ city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+ state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+ }
+
+ if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+ country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+ }
+
+ newExif.city = city || null;
+ newExif.state = state || null;
+ newExif.country = country || null;
+ }
+
+ for (const stream of data.streams) {
+ if (stream.codec_type === 'video') {
+ newExif.exifImageWidth = stream.width || null;
+ newExif.exifImageHeight = stream.height || null;
+
+ if (typeof stream.rotation === 'string') {
+ newExif.orientation = stream.rotation;
+ } else if (typeof stream.rotation === 'number') {
+ newExif.orientation = `${stream.rotation}`;
+ } else {
+ newExif.orientation = null;
+ }
+
+ if (stream.r_frame_rate) {
+ let fpsParts = stream.r_frame_rate.split('/');
+
+ if (fpsParts.length === 2) {
+ newExif.fps = Math.round(parseInt(fpsParts[0]) / parseInt(fpsParts[1]));
+ }
+ }
+ }
+ }
+
+ await this.exifRepository.save(newExif);
+ await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
+ } catch (err) {
+ // do nothing
+ console.log('Error in video metadata extraction', err);
+ }
}
private extractDuration(duration: number) {
@@ -202,8 +321,6 @@ export class MetadataExtractionProcessor {
const minutes = Math.floor((videoDurationInSecond - hours * 3600) / 60);
const seconds = videoDurationInSecond - hours * 3600 - minutes * 60;
- return `${hours}:${minutes < 10 ? '0' + minutes.toString() : minutes}:${
- seconds < 10 ? '0' + seconds.toString() : seconds
- }.000000`;
+ return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.000000`;
}
}
diff --git a/server/libs/database/src/entities/exif.entity.ts b/server/libs/database/src/entities/exif.entity.ts
index 5cca536894..0bd3b06e2a 100644
--- a/server/libs/database/src/entities/exif.entity.ts
+++ b/server/libs/database/src/entities/exif.entity.ts
@@ -13,14 +13,9 @@ export class ExifEntity {
@Column({ type: 'uuid' })
assetId!: string;
- @Column({ type: 'varchar', nullable: true })
- make!: string | null;
-
- @Column({ type: 'varchar', nullable: true })
- model!: string | null;
-
- @Column({ type: 'varchar', nullable: true })
- imageName!: string | null;
+ /* General info */
+ @Column({ type: 'text', nullable: true, default: '' })
+ description!: string; // or caption
@Column({ type: 'integer', nullable: true })
exifImageWidth!: number | null;
@@ -40,21 +35,6 @@ export class ExifEntity {
@Column({ type: 'timestamptz', nullable: true })
modifyDate!: Date | null;
- @Column({ type: 'varchar', nullable: true })
- lensModel!: string | null;
-
- @Column({ type: 'float8', nullable: true })
- fNumber!: number | null;
-
- @Column({ type: 'float8', nullable: true })
- focalLength!: number | null;
-
- @Column({ type: 'integer', nullable: true })
- iso!: number | null;
-
- @Column({ type: 'float', nullable: true })
- exposureTime!: number | null;
-
@Column({ type: 'float', nullable: true })
latitude!: number | null;
@@ -70,9 +50,38 @@ export class ExifEntity {
@Column({ type: 'varchar', nullable: true })
country!: string | null;
+ /* Image info */
+ @Column({ type: 'varchar', nullable: true })
+ make!: string | null;
+
+ @Column({ type: 'varchar', nullable: true })
+ model!: string | null;
+
+ @Column({ type: 'varchar', nullable: true })
+ imageName!: string | null;
+
+ @Column({ type: 'varchar', nullable: true })
+ lensModel!: string | null;
+
+ @Column({ type: 'float8', nullable: true })
+ fNumber!: number | null;
+
+ @Column({ type: 'float8', nullable: true })
+ focalLength!: number | null;
+
+ @Column({ type: 'integer', nullable: true })
+ iso!: number | null;
+
+ @Column({ type: 'float', nullable: true })
+ exposureTime!: number | null;
+
+ /* Video info */
+ @Column({ type: 'float8', nullable: true })
+ fps?: number | null;
+
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
- asset?: ExifEntity;
+ asset?: AssetEntity;
@Index('exif_text_searchable', { synchronize: false })
@Column({
diff --git a/server/libs/database/src/entities/smart-info.entity.ts b/server/libs/database/src/entities/smart-info.entity.ts
index f1b46e5cee..f40990f59f 100644
--- a/server/libs/database/src/entities/smart-info.entity.ts
+++ b/server/libs/database/src/entities/smart-info.entity.ts
@@ -18,5 +18,5 @@ export class SmartInfoEntity {
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
- asset?: SmartInfoEntity;
+ asset?: AssetEntity;
}
diff --git a/server/libs/database/src/migrations/1661011331242-AddCaption.ts b/server/libs/database/src/migrations/1661011331242-AddCaption.ts
new file mode 100644
index 0000000000..b1d0566e64
--- /dev/null
+++ b/server/libs/database/src/migrations/1661011331242-AddCaption.ts
@@ -0,0 +1,18 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddCaption1661011331242 implements MigrationInterface {
+ name = 'AddCaption1661011331242'
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`ALTER TABLE "exif" ADD "description" text DEFAULT ''`);
+ await queryRunner.query(`ALTER TABLE "exif" ADD "fps" double precision`);
+ // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" SET NOT NULL`);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" DROP NOT NULL`);
+ await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "fps"`);
+ await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "description"`);
+ }
+
+}