c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 06:31:58 +00:00
Code Releases Activity

fix(server): /search/random failing with certain options (#13040)

Browse source 
* fix relation handling, remove pagination

* update api, sql

* update mock
This commit is contained in:
Mert 2024-09-30 00:29:35 -04:00 committed by GitHub
parent 2f13db51df
commit 7adb35e59e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 243 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
mobile/openapi/lib/api/search_api.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/model/random_search_dto.dart generated
View file

Binary file not shown.

9
open-api/immich-openapi-specs.json
View file

@ -4615,7 +4615,10 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SearchResponseDto"
"items": {
"$ref": "#/components/schemas/AssetResponseDto"
},
"type": "array"
}
}
},
@ -10463,10 +10466,6 @@
"nullable": true,
"type": "string"
},
"page": {
"minimum": 1,
"type": "number"
},
"personIds": {
"items": {
"format": "uuid",

3
open-api/typescript-sdk/src/fetch-client.ts
View file

@ -852,7 +852,6 @@ export type RandomSearchDto = {
libraryId?: string | null;
make?: string;
model?: string | null;
page?: number;
personIds?: string[];
size?: number;
state?: string | null;
@ -2523,7 +2522,7 @@ export function searchRandom({ randomSearchDto }: {
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: SearchResponseDto;
data: AssetResponseDto[];
}>("/search/random", oazapfts.json({
...opts,
method: "POST",

2
server/src/controllers/search.controller.ts
View file

@ -32,7 +32,7 @@ export class SearchController {
@Post('random')
@HttpCode(HttpStatus.OK)
@Authenticated()
searchRandom(@Auth() auth: AuthDto, @Body() dto: RandomSearchDto): Promise<SearchResponseDto> {
searchRandom(@Auth() auth: AuthDto, @Body() dto: RandomSearchDto): Promise<AssetResponseDto[]> {
return this.service.searchRandom(auth, dto);
}

18
server/src/dtos/search.dto.ts
View file

@ -99,12 +99,6 @@ class BaseSearchDto {
@Optional({ nullable: true, emptyToNull: true })
lensModel?: string | null;
@IsInt()
@Min(1)
@Type(() => Number)
@Optional()
page?: number;
@IsInt()
@Min(1)
@Max(1000)
@ -170,12 +164,24 @@ export class MetadataSearchDto extends RandomSearchDto {
@Optional()
@ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder })
order?: AssetOrder;
@IsInt()
@Min(1)
@Type(() => Number)
@Optional()
page?: number;
}
export class SmartSearchDto extends BaseSearchDto {
@IsString()
@IsNotEmpty()
query!: string;
@IsInt()
@Min(1)
@Type(() => Number)
@Optional()
page?: number;
}
export class SearchPlacesDto {

2
server/src/interfaces/search.interface.ts
View file

@ -116,7 +116,6 @@ export interface SearchPeopleOptions {
export interface SearchOrderOptions {
orderDirection?: 'ASC' | 'DESC';
random?: boolean;
}
export interface SearchPaginationOptions {
@ -177,6 +176,7 @@ export interface ISearchRepository {
searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity>;
searchDuplicates(options: AssetDuplicateSearch): Promise<AssetDuplicateResult[]>;
searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]>;
upsert(assetId: string, embedding: number[]): Promise<void>;
searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;
getAssetsByCity(userIds: string[]): Promise<AssetEntity[]>;

189
server/src/queries/search.repository.sql
View file

@ -77,10 +77,11 @@ FROM
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $3
AND "asset"."isArchived" = $4
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
)
AND ("asset"."deletedAt" IS NULL)
@ -91,6 +92,190 @@ ORDER BY
LIMIT
101
-- SearchRepository.searchRandom
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" > $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" < $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
-- SearchRepository.searchSmart
START TRANSACTION
SET

41
server/src/repositories/search.repository.ts
View file

@ -1,5 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { randomUUID } from 'node:crypto';
import { getVectorExtension } from 'src/database.config';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
@ -63,22 +64,15 @@ export class SearchRepository implements ISearchRepository {
{
takenAfter: DummyValue.DATE,
lensModel: DummyValue.STRING,
ownerId: DummyValue.UUID,
withStacked: true,
isFavorite: true,
ownerIds: [DummyValue.UUID],
userIds: [DummyValue.UUID],
},
],
})
async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> {
let builder = this.assetRepository.createQueryBuilder('asset');
builder = searchAssetBuilder(builder, options);
builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
if (options.random) {
// TODO replace with complicated SQL magic after kysely migration
builder.addSelect('RANDOM() as r').orderBy('r');
}
builder = searchAssetBuilder(builder, options).orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE,
@ -87,6 +81,35 @@ export class SearchRepository implements ISearchRepository {
});
}
@GenerateSql({
params: [
100,
{
takenAfter: DummyValue.DATE,
lensModel: DummyValue.STRING,
withStacked: true,
isFavorite: true,
userIds: [DummyValue.UUID],
},
],
})
async searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]> {
const builder1 = searchAssetBuilder(this.assetRepository.createQueryBuilder('asset'), options);
const builder2 = builder1.clone();
const uuid = randomUUID();
builder1.andWhere('asset.id > :uuid', { uuid }).orderBy('asset.id').take(size);
builder2.andWhere('asset.id < :uuid', { uuid }).orderBy('asset.id').take(size);
const [assets1, assets2] = await Promise.all([builder1.getMany(), builder2.getMany()]);
const missingCount = size - assets1.length;
for (let i = 0; i < missingCount && i < assets2.length; i++) {
assets1.push(assets2[i]);
}
return assets1;
}
private createPersonFilter(builder: SelectQueryBuilder<AssetFaceEntity>, personIds: string[]) {
return builder
.select(`${builder.alias}."assetId"`)

16
server/src/services/search.service.ts
View file

@ -94,20 +94,10 @@ export class SearchService {
return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null, { auth });
}
async searchRandom(auth: AuthDto, dto: RandomSearchDto): Promise<SearchResponseDto> {
async searchRandom(auth: AuthDto, dto: RandomSearchDto): Promise<AssetResponseDto[]> {
const userIds = await this.getUserIdsToSearch(auth);
const page = dto.page ?? 1;
const size = dto.size || 250;
const { hasNextPage, items } = await this.searchRepository.searchMetadata(
{ page, size },
{
...dto,
userIds,
random: true,
},
);
return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null, { auth });
const items = await this.searchRepository.searchRandom(dto.size || 250, { ...dto, userIds });
return items.map((item) => mapAsset(item, { auth }));
}
async searchSmart(auth: AuthDto, dto: SmartSearchDto): Promise<SearchResponseDto> {

2
server/src/utils/database.ts
View file

@ -120,7 +120,7 @@ export function searchAssetBuilder(
}
if (withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
builder.leftJoinAndSelect('faces.person', 'person');
}
if (withSmartInfo) {

1
server/test/repositories/search.repository.mock.ts
View file

@ -7,6 +7,7 @@ export const newSearchRepositoryMock = (): Mocked<ISearchRepository> => {
searchSmart: vitest.fn(),
searchDuplicates: vitest.fn(),
searchFaces: vitest.fn(),
searchRandom: vitest.fn(),
upsert: vitest.fn(),
searchPlaces: vitest.fn(),
getAssetsByCity: vitest.fn(),