From bc65bbfcc47987217f4d4991ef7fffd7af8bced7 Mon Sep 17 00:00:00 2001
From: martin <74269598+martabal@users.noreply.github.com>
Date: Fri, 8 Dec 2023 05:18:33 +0100
Subject: [PATCH] fix(web): create face from video (#5544)

* fix: create face from video

* fix: remove comment

* fix: inaccurate bounding boxes
---
 .../asset-viewer/detail-panel.svelte          |  1 +
 .../asset-viewer/photo-viewer.svelte          |  2 +-
 .../faces-page/assign-face-side-panel.svelte  | 31 ++++++++++++++-----
 .../faces-page/person-side-panel.svelte       |  5 ++-
 4 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/web/src/lib/components/asset-viewer/detail-panel.svelte b/web/src/lib/components/asset-viewer/detail-panel.svelte
index 8b55dbd965..7f73fb2ed5 100644
--- a/web/src/lib/components/asset-viewer/detail-panel.svelte
+++ b/web/src/lib/components/asset-viewer/detail-panel.svelte
@@ -635,6 +635,7 @@
 {#if showEditFaces}
   <PersonSidePanel
     assetId={asset.id}
+    assetType={asset.type}
     on:close={() => {
       showEditFaces = false;
     }}
diff --git a/web/src/lib/components/asset-viewer/photo-viewer.svelte b/web/src/lib/components/asset-viewer/photo-viewer.svelte
index 68be9c12b2..958ac520a0 100644
--- a/web/src/lib/components/asset-viewer/photo-viewer.svelte
+++ b/web/src/lib/components/asset-viewer/photo-viewer.svelte
@@ -143,7 +143,7 @@
       />
       {#each getBoundingBox($boundingBoxesArray, $photoZoomState, $photoViewer) as boundingbox}
         <div
-          class="absolute border-solid border-white border-[3px] rounded-lg p-3"
+          class="absolute border-solid border-white border-[3px] rounded-lg"
           style="top: {boundingbox.top}px; left: {boundingbox.left}px; height: {boundingbox.height}px; width: {boundingbox.width}px;"
         />
       {/each}
diff --git a/web/src/lib/components/faces-page/assign-face-side-panel.svelte b/web/src/lib/components/faces-page/assign-face-side-panel.svelte
index a4d1319e4c..acae0971e5 100644
--- a/web/src/lib/components/faces-page/assign-face-side-panel.svelte
+++ b/web/src/lib/components/faces-page/assign-face-side-panel.svelte
@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { api, type AssetFaceResponseDto, type PersonResponseDto } from '@api';
+  import { api, AssetTypeEnum, type AssetFaceResponseDto, type PersonResponseDto, ThumbnailFormat } from '@api';
   import { createEventDispatcher } from 'svelte';
   import { linear } from 'svelte/easing';
   import { fly } from 'svelte/transition';
@@ -14,6 +14,8 @@
   export let peopleWithFaces: AssetFaceResponseDto[];
   export let allPeople: PersonResponseDto[];
   export let editedPersonIndex: number;
+  export let assetType: AssetTypeEnum;
+  export let assetId: string;
 
   // loading spinners
   let isShowLoadingNewPerson = false;
@@ -31,23 +33,38 @@
     dispatch('close');
   };
   const zoomImageToBase64 = async (face: AssetFaceResponseDto): Promise<string | null> => {
-    if ($photoViewer === null) {
+    let image: HTMLImageElement | null = null;
+    if (assetType === AssetTypeEnum.Image) {
+      image = $photoViewer;
+    } else if (assetType === AssetTypeEnum.Video) {
+      const data = await api.getAssetThumbnailUrl(assetId, ThumbnailFormat.Webp);
+      const img: HTMLImageElement = new Image();
+      img.src = data;
+
+      await new Promise<void>((resolve) => {
+        img.onload = () => resolve();
+        img.onerror = () => resolve();
+      });
+
+      image = img;
+    }
+    if (image === null) {
       return null;
     }
     const { boundingBoxX1: x1, boundingBoxX2: x2, boundingBoxY1: y1, boundingBoxY2: y2 } = face;
 
     const coordinates = {
-      x1: ($photoViewer.naturalWidth / face.imageWidth) * x1,
-      x2: ($photoViewer.naturalWidth / face.imageWidth) * x2,
-      y1: ($photoViewer.naturalHeight / face.imageHeight) * y1,
-      y2: ($photoViewer.naturalHeight / face.imageHeight) * y2,
+      x1: (image.naturalWidth / face.imageWidth) * x1,
+      x2: (image.naturalWidth / face.imageWidth) * x2,
+      y1: (image.naturalHeight / face.imageHeight) * y1,
+      y2: (image.naturalHeight / face.imageHeight) * y2,
     };
 
     const faceWidth = coordinates.x2 - coordinates.x1;
     const faceHeight = coordinates.y2 - coordinates.y1;
 
     const faceImage = new Image();
-    faceImage.src = $photoViewer.src;
+    faceImage.src = image.src;
 
     await new Promise((resolve) => {
       faceImage.onload = resolve;
diff --git a/web/src/lib/components/faces-page/person-side-panel.svelte b/web/src/lib/components/faces-page/person-side-panel.svelte
index 14b60d7eea..8cb3651f88 100644
--- a/web/src/lib/components/faces-page/person-side-panel.svelte
+++ b/web/src/lib/components/faces-page/person-side-panel.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
   import { fly } from 'svelte/transition';
   import { linear } from 'svelte/easing';
-  import { api, type PersonResponseDto, AssetFaceResponseDto } from '@api';
+  import { api, type PersonResponseDto, AssetFaceResponseDto, AssetTypeEnum } from '@api';
   import ImageThumbnail from '../assets/thumbnail/image-thumbnail.svelte';
   import { handleError } from '$lib/utils/handle-error';
   import { createEventDispatcher, onMount } from 'svelte';
@@ -15,6 +15,7 @@
   import { getPersonNameWithHiddenValue } from '$lib/utils/person';
 
   export let assetId: string;
+  export let assetType: AssetTypeEnum;
 
   // keep track of the changes
   let numberOfPersonToCreate: string[] = [];
@@ -271,6 +272,8 @@
     {peopleWithFaces}
     {allPeople}
     {editedPersonIndex}
+    {assetType}
+    {assetId}
     on:close={() => (showSeletecFaces = false)}
     on:createPerson={(event) => handleCreatePerson(event.detail)}
     on:reassign={(event) => handleReassignFace(event.detail)}