feat(server): Merge Faces sorted by Similarity (#14635)

* Merge Faces sorted by Similarity

* Adds face sorting to the side panel face merger

* run make open-api

* Make it one query

* Only have the single order by when sorting by closest face

open-api/immich-openapi-specs.json

@@ -3846,6 +3846,24 @@
       "get": {
         "operationId": "getAllPeople",
         "parameters": [
+          {
+            "name": "closestAssetId",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "format": "uuid",
+              "type": "string"
+            }
+          },
+          {
+            "name": "closestPersonId",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "format": "uuid",
+              "type": "string"
+            }
+          },
           {
             "name": "page",
             "required": false,

open-api/typescript-sdk/src/fetch-client.ts

@@ -2362,7 +2362,9 @@ export function updatePartner({ id, updatePartnerDto }: {
         body: updatePartnerDto
     })));
 }
-export function getAllPeople({ page, size, withHidden }: {
+export function getAllPeople({ closestAssetId, closestPersonId, page, size, withHidden }: {
+    closestAssetId?: string;
+    closestPersonId?: string;
     page?: number;
     size?: number;
     withHidden?: boolean;
@@ -2371,6 +2373,8 @@ export function getAllPeople({ page, size, withHidden }: {
         status: 200;
         data: PeopleResponseDto;
     }>(`/people${QS.query(QS.explode({
+        closestAssetId,
+        closestPersonId,
         page,
         size,
         withHidden

server/src/controllers/person.controller.ts

@@ -31,8 +31,8 @@ export class PersonController {
 
   @Get()
   @Authenticated({ permission: Permission.PERSON_READ })
-  getAllPeople(@Auth() auth: AuthDto, @Query() withHidden: PersonSearchDto): Promise<PeopleResponseDto> {
-    return this.service.getAll(auth, withHidden);
+  getAllPeople(@Auth() auth: AuthDto, @Query() options: PersonSearchDto): Promise<PeopleResponseDto> {
+    return this.service.getAll(auth, options);
   }
 
   @Post()

server/src/dtos/person.dto.ts

@@ -67,6 +67,10 @@ export class MergePersonDto {
 export class PersonSearchDto {
   @ValidateBoolean({ optional: true })
   withHidden?: boolean;
+  @ValidateUUID({ optional: true })
+  closestPersonId?: string;
+  @ValidateUUID({ optional: true })
+  closestAssetId?: string;
 
   /** Page number for pagination */
   @ApiPropertyOptional()

server/src/interfaces/person.interface.ts

@@ -10,6 +10,7 @@ export const IPersonRepository = 'IPersonRepository';
 export interface PersonSearchOptions {
   minimumFaceCount: number;
   withHidden: boolean;
+  closestFaceAssetId?: string;
 }
 
 export interface PersonNameSearchOptions {

server/src/repositories/person.repository.ts

@@ -83,7 +83,11 @@ export class PersonRepository implements IPersonRepository {
   }
 
   @GenerateSql({ params: [{ take: 10, skip: 10 }, DummyValue.UUID] })
-  getAllForUser(pagination: PaginationOptions, userId: string, options?: PersonSearchOptions): Paginated<PersonEntity> {
+  async getAllForUser(
+    pagination: PaginationOptions,
+    userId: string,
+    options?: PersonSearchOptions,
+  ): Paginated<PersonEntity> {
     const queryBuilder = this.personRepository
       .createQueryBuilder('person')
       .innerJoin('person.faces', 'face')
@@ -97,10 +101,22 @@ export class PersonRepository implements IPersonRepository {
       .addOrderBy('person.createdAt')
       .having("person.name != '' OR COUNT(face.assetId) >= :faces", { faces: options?.minimumFaceCount || 1 })
       .groupBy('person.id');
+    if (options?.closestFaceAssetId) {
+      const innerQueryBuilder = this.faceSearchRepository
+        .createQueryBuilder('face_search')
+        .select('embedding', 'embedding')
+        .where('"face_search"."faceId" = "person"."faceAssetId"');
+      const faceSelectQueryBuilder = this.faceSearchRepository
+        .createQueryBuilder('face_search')
+        .select('embedding', 'embedding')
+        .where('"face_search"."faceId" = :faceId', { faceId: options.closestFaceAssetId });
+      queryBuilder
+        .orderBy('(' + innerQueryBuilder.getQuery() + ') <=> (' + faceSelectQueryBuilder.getQuery() + ')')
+        .setParameters(faceSelectQueryBuilder.getParameters());
+    }
     if (!options?.withHidden) {
       queryBuilder.andWhere('person.isHidden = false');
     }
-
     return paginatedBuilder(queryBuilder, {
       mode: PaginationMode.LIMIT_OFFSET,
       ...pagination,

server/src/services/person.service.ts

@@ -55,16 +55,25 @@ import { IsNull } from 'typeorm';
 @Injectable()
 export class PersonService extends BaseService {
   async getAll(auth: AuthDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
-    const { withHidden = false, page, size } = dto;
+    const { withHidden = false, closestAssetId, closestPersonId, page, size } = dto;
+    let closestFaceAssetId = closestAssetId;
     const pagination = {
       take: size,
       skip: (page - 1) * size,
     };
 
+    if (closestPersonId) {
+      const person = await this.personRepository.getById(closestPersonId);
+      if (!person?.faceAssetId) {
+        throw new NotFoundException('Person not found');
+      }
+      closestFaceAssetId = person.faceAssetId;
+    }
     const { machineLearning } = await this.getConfig({ withCache: false });
     const { items, hasNextPage } = await this.personRepository.getAllForUser(pagination, auth.user.id, {
       minimumFaceCount: machineLearning.facialRecognition.minFaces,
       withHidden,
+      closestFaceAssetId,
     });
     const { total, hidden } = await this.personRepository.getNumberOfPeople(auth.user.id);
 

web/src/lib/components/faces-page/assign-face-side-panel.svelte

@@ -1,8 +1,8 @@
 <script lang="ts">
   import { timeBeforeShowLoadingSpinner } from '$lib/constants';
   import { getPersonNameWithHiddenValue } from '$lib/utils/person';
-  import { getPeopleThumbnailUrl } from '$lib/utils';
-  import { AssetTypeEnum, type AssetFaceResponseDto, type PersonResponseDto } from '@immich/sdk';
+  import { getPeopleThumbnailUrl, handlePromiseError } from '$lib/utils';
+  import { AssetTypeEnum, type AssetFaceResponseDto, type PersonResponseDto, getAllPeople } from '@immich/sdk';
   import { mdiArrowLeftThin, mdiClose, mdiMagnify, mdiPlus } from '@mdi/js';
   import { linear } from 'svelte/easing';
   import { fly } from 'svelte/transition';
@@ -13,9 +13,10 @@
   import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
   import { zoomImageToBase64 } from '$lib/utils/people-utils';
   import { t } from 'svelte-i18n';
+  import { handleError } from '$lib/utils/handle-error';
+  import { onMount } from 'svelte';
 
   interface Props {
-    allPeople: PersonResponseDto[];
     editedFace: AssetFaceResponseDto;
     assetId: string;
     assetType: AssetTypeEnum;
@@ -24,7 +25,24 @@
     onReassign: (person: PersonResponseDto) => void;
   }
 
-  let { allPeople, editedFace, assetId, assetType, onClose, onCreatePerson, onReassign }: Props = $props();
+  let { editedFace, assetId, assetType, onClose, onCreatePerson, onReassign }: Props = $props();
+
+  let allPeople: PersonResponseDto[] = $state([]);
+
+  let isShowLoadingPeople = $state(false);
+
+  async function loadPeople() {
+    const timeout = setTimeout(() => (isShowLoadingPeople = true), timeBeforeShowLoadingSpinner);
+    try {
+      const { people } = await getAllPeople({ withHidden: true, closestAssetId: editedFace.id });
+      allPeople = people;
+    } catch (error) {
+      handleError(error, $t('errors.cant_get_faces'));
+    } finally {
+      clearTimeout(timeout);
+    }
+    isShowLoadingPeople = false;
+  }
 
   // loading spinners
   let isShowLoadingNewPerson = $state(false);
@@ -37,6 +55,10 @@
 
   let showPeople = $derived(searchName ? searchedPeople : allPeople.filter((person) => !person.isHidden));
 
+  onMount(() => {
+    handlePromiseError(loadPeople());
+  });
+
   const handleCreatePerson = async () => {
     const timeout = setTimeout(() => (isShowLoadingNewPerson = true), timeBeforeShowLoadingSpinner);
 
@@ -96,31 +118,40 @@
   </div>
   <div class="px-4 py-4 text-sm">
     <h2 class="mb-8 mt-4 uppercase">{$t('all_people')}</h2>
-    <div class="immich-scrollbar mt-4 flex flex-wrap gap-2 overflow-y-auto">
-      {#each showPeople as person (person.id)}
-        {#if !editedFace.person || person.id !== editedFace.person.id}
-          <div class="w-fit">
-            <button type="button" class="w-[90px]" onclick={() => onReassign(person)}>
-              <div class="relative">
-                <ImageThumbnail
-                  curve
-                  shadow
-                  url={getPeopleThumbnailUrl(person)}
-                  altText={$getPersonNameWithHiddenValue(person.name, person.isHidden)}
-                  title={$getPersonNameWithHiddenValue(person.name, person.isHidden)}
-                  widthStyle="90px"
-                  heightStyle="90px"
-                  hidden={person.isHidden}
-                />
-              </div>
+    {#if isShowLoadingPeople}
+      <div class="flex w-full justify-center">
+        <LoadingSpinner />
+      </div>
+    {:else}
+      <div class="immich-scrollbar mt-4 flex flex-wrap gap-2 overflow-y-auto">
+        {#each showPeople as person (person.id)}
+          {#if !editedFace.person || person.id !== editedFace.person.id}
+            <div class="w-fit">
+              <button type="button" class="w-[90px]" onclick={() => onReassign(person)}>
+                <div class="relative">
+                  <ImageThumbnail
+                    curve
+                    shadow
+                    url={getPeopleThumbnailUrl(person)}
+                    altText={$getPersonNameWithHiddenValue(person.name, person.isHidden)}
+                    title={$getPersonNameWithHiddenValue(person.name, person.isHidden)}
+                    widthStyle="90px"
+                    heightStyle="90px"
+                    hidden={person.isHidden}
+                  />
+                </div>
 
-              <p class="mt-1 truncate font-medium" title={$getPersonNameWithHiddenValue(person.name, person.isHidden)}>
-                {person.name}
-              </p>
-            </button>
-          </div>
-        {/if}
-      {/each}
-    </div>
+                <p
+                  class="mt-1 truncate font-medium"
+                  title={$getPersonNameWithHiddenValue(person.name, person.isHidden)}
+                >
+                  {person.name}
+                </p>
+              </button>
+            </div>
+          {/if}
+        {/each}
+      </div>
+    {/if}
   </div>
 </section>

web/src/lib/components/faces-page/merge-face-selector.svelte

@@ -35,7 +35,7 @@
   let peopleToNotShow = $derived([...selectedPeople, person]);
 
   onMount(async () => {
-    const data = await getAllPeople({ withHidden: false });
+    const data = await getAllPeople({ withHidden: false, closestPersonId: person.id });
     people = data.people;
   });
 

web/src/lib/components/faces-page/person-side-panel.svelte

@@ -8,7 +8,6 @@
   import { getPersonNameWithHiddenValue } from '$lib/utils/person';
   import {
     createPerson,
-    getAllPeople,
     getFaces,
     reassignFacesById,
     AssetTypeEnum,
@@ -53,7 +52,6 @@
 
   // search people
   let showSelectedFaces = $state(false);
-  let allPeople: PersonResponseDto[] = $state([]);
 
   // timers
   let loaderLoadingDoneTimeout: ReturnType<typeof setTimeout>;
@@ -64,8 +62,6 @@
   async function loadPeople() {
     const timeout = setTimeout(() => (isShowLoadingPeople = true), timeBeforeShowLoadingSpinner);
     try {
-      const { people } = await getAllPeople({ withHidden: true });
-      allPeople = people;
       peopleWithFaces = await getFaces({ id: assetId });
     } catch (error) {
       handleError(error, $t('errors.cant_get_faces'));
@@ -322,7 +318,6 @@
 
 {#if showSelectedFaces && editedFace}
   <AssignFaceSidePanel
-    {allPeople}
     {editedFace}
     {assetId}
     {assetType}