From 030cab8d499376909c7fb4a1b07db1ea30f9b539 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:27:31 -0500 Subject: [PATCH] wip --- e2e/src/api/specs/asset.e2e-spec.ts | 45 +- e2e/src/api/specs/timeline.e2e-spec.ts | 6 +- open-api/immich-openapi-specs.json | 1 + server/package-lock.json | 337 ++++ server/package.json | 4 + server/src/app.module.ts | 4 +- server/src/bin/database.ts | 2 +- server/src/bin/sync-sql.ts | 15 +- server/src/constants.ts | 4 + server/src/db.d.ts | 439 ++++++ server/src/dtos/asset-response.dto.ts | 15 +- server/src/dtos/duplicate.dto.ts | 14 - server/src/dtos/search.dto.ts | 2 +- server/src/entities/asset.entity.ts | 255 ++- server/src/entities/face-search.entity.ts | 3 +- server/src/interfaces/asset.interface.ts | 106 +- server/src/interfaces/config.interface.ts | 3 +- server/src/interfaces/search.interface.ts | 10 +- .../1734574016301-AddTimeBucketIndices.ts | 25 + server/src/queries/asset.repository.sql | 1373 ++++------------- server/src/queries/search.repository.sql | 784 ++-------- server/src/queries/view.repository.sql | 102 +- server/src/repositories/asset.repository.ts | 1124 +++++++------- .../repositories/config.repository.spec.ts | 23 +- server/src/repositories/config.repository.ts | 64 +- .../src/repositories/database.repository.ts | 2 +- server/src/repositories/search.repository.ts | 520 +++---- server/src/repositories/view-repository.ts | 65 +- .../src/services/asset-media.service.spec.ts | 18 +- server/src/services/asset-media.service.ts | 5 +- server/src/services/asset.service.spec.ts | 24 +- server/src/services/asset.service.ts | 46 +- server/src/services/backup.service.ts | 6 +- server/src/services/database.service.spec.ts | 61 +- server/src/services/duplicate.service.spec.ts | 13 +- server/src/services/duplicate.service.ts | 26 +- server/src/services/library.service.spec.ts | 8 - server/src/services/metadata.service.spec.ts | 11 +- server/src/services/metadata.service.ts | 5 +- server/src/services/person.service.spec.ts | 26 +- server/src/services/person.service.ts | 14 +- server/src/services/search.service.spec.ts | 6 +- server/src/services/search.service.ts | 17 +- server/src/services/timeline.service.spec.ts | 28 +- server/src/utils/database.ts | 168 +- server/src/utils/pagination.ts | 5 +- .../repositories/config.repository.mock.ts | 30 +- server/tsconfig.json | 3 +- 48 files changed, 2702 insertions(+), 3165 deletions(-) create mode 100644 server/src/db.d.ts create mode 100644 server/src/migrations/1734574016301-AddTimeBucketIndices.ts diff --git a/e2e/src/api/specs/asset.e2e-spec.ts b/e2e/src/api/specs/asset.e2e-spec.ts index a0c429a82e..21aebf1775 100644 --- a/e2e/src/api/specs/asset.e2e-spec.ts +++ b/e2e/src/api/specs/asset.e2e-spec.ts @@ -538,7 +538,7 @@ describe('/asset', () => { expect(body).toMatchObject({ id: user1Assets[0].id, exifInfo: expect.objectContaining({ - dateTimeOriginal: '2023-11-20T01:11:00.000Z', + dateTimeOriginal: '2023-11-20T01:11:00+00:00', }), }); expect(status).toEqual(200); @@ -608,7 +608,7 @@ describe('/asset', () => { await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); const assetInfo = await utils.getAssetInfo(user1.accessToken, id); - expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52.000Z'); + expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52+00:00'); const { status, body } = await request(app) .put(`/assets/${id}`) @@ -618,7 +618,7 @@ describe('/asset', () => { expect(body).toMatchObject({ id, exifInfo: expect.objectContaining({ - dateTimeOriginal: '2023-11-20T01:11:00.000Z', + dateTimeOriginal: '2023-11-20T01:11:00+00:00', }), }); expect(status).toEqual(200); @@ -953,8 +953,6 @@ describe('/asset', () => { exifImageHeight: 1080, exifImageWidth: 1617, fileSizeInByte: 862_424, - latitude: null, - longitude: null, }, }, }, @@ -964,11 +962,9 @@ describe('/asset', () => { type: AssetTypeEnum.Image, originalFileName: 'el_torcal_rocks.jpg', exifInfo: { - dateTimeOriginal: '2012-08-05T11:39:59.000Z', + dateTimeOriginal: '2012-08-05T11:39:59+00:00', exifImageWidth: 512, exifImageHeight: 341, - latitude: null, - longitude: null, focalLength: 75, iso: 200, fNumber: 11, @@ -976,7 +972,6 @@ describe('/asset', () => { fileSizeInByte: 53_493, make: 'SONY', model: 'DSLR-A550', - orientation: null, description: 'SONY DSC', }, }, @@ -991,8 +986,6 @@ describe('/asset', () => { exifImageHeight: 1080, exifImageWidth: 1440, fileSizeInByte: 1_780_777, - latitude: null, - longitude: null, }, }, }, @@ -1003,7 +996,7 @@ describe('/asset', () => { originalFileName: 'IMG_2682.heic', fileCreatedAt: '2019-03-21T16:04:22.348Z', exifInfo: { - dateTimeOriginal: '2019-03-21T16:04:22.348Z', + dateTimeOriginal: '2019-03-21T16:04:22.348+00:00', exifImageWidth: 4032, exifImageHeight: 3024, latitude: 41.2203, @@ -1028,8 +1021,6 @@ describe('/asset', () => { exifInfo: { exifImageWidth: 800, exifImageHeight: 800, - latitude: null, - longitude: null, fileSizeInByte: 25_408, }, }, @@ -1048,9 +1039,7 @@ describe('/asset', () => { focalLength: 18, iso: 100, fileSizeInByte: 9_057_784, - dateTimeOriginal: '2010-07-20T17:27:12.000Z', - latitude: null, - longitude: null, + dateTimeOriginal: '2010-07-20T17:27:12+00:00', orientation: '1', }, }, @@ -1069,9 +1058,7 @@ describe('/asset', () => { focalLength: 85, iso: 200, fileSizeInByte: 15_856_335, - dateTimeOriginal: '2016-09-22T21:10:29.060Z', - latitude: null, - longitude: null, + dateTimeOriginal: '2016-09-22T21:10:29.06+00:00', orientation: '1', timeZone: 'UTC-4', }, @@ -1093,9 +1080,7 @@ describe('/asset', () => { focalLength: 35, iso: 400, fileSizeInByte: 19_587_072, - dateTimeOriginal: '2018-05-10T08:42:37.842Z', - latitude: null, - longitude: null, + dateTimeOriginal: '2018-05-10T08:42:37.842+00:00', orientation: '1', }, }, @@ -1117,9 +1102,7 @@ describe('/asset', () => { iso: 100, lensModel: 'E PZ 18-105mm F4 G OSS', fileSizeInByte: 25_001_984, - dateTimeOriginal: '2016-09-27T10:51:44.000Z', - latitude: null, - longitude: null, + dateTimeOriginal: '2016-09-27T10:51:44+00:00', orientation: '1', }, }, @@ -1141,9 +1124,7 @@ describe('/asset', () => { iso: 100, lensModel: 'E 25mm F2', fileSizeInByte: 49_512_448, - dateTimeOriginal: '2016-01-08T14:08:01.000Z', - latitude: null, - longitude: null, + dateTimeOriginal: '2016-01-08T14:08:01+00:00', orientation: '1', }, }, @@ -1165,7 +1146,7 @@ describe('/asset', () => { iso: 80, lensModel: null, fileSizeInByte: 11_113_617, - dateTimeOriginal: '2015-12-27T09:55:40.000Z', + dateTimeOriginal: '2015-12-27T09:55:40+00:00', latitude: null, longitude: null, orientation: '1', @@ -1189,7 +1170,7 @@ describe('/asset', () => { iso: 160, lensModel: null, fileSizeInByte: 13_551_312, - dateTimeOriginal: '2024-10-12T21:01:01.000Z', + dateTimeOriginal: '2024-10-12T21:01:01+00:00', latitude: null, longitude: null, orientation: '6', @@ -1203,7 +1184,7 @@ describe('/asset', () => { originalFileName: 'Ricoh_GR3-450.DNG', fileCreatedAt: '2024-06-08T13:48:39.000Z', exifInfo: { - dateTimeOriginal: '2024-06-08T13:48:39.000Z', + dateTimeOriginal: '2024-06-08T13:48:39+00:00', exifImageHeight: 4064, exifImageWidth: 6112, exposureTime: '1/400', diff --git a/e2e/src/api/specs/timeline.e2e-spec.ts b/e2e/src/api/specs/timeline.e2e-spec.ts index 2a1f891583..bf330e994a 100644 --- a/e2e/src/api/specs/timeline.e2e-spec.ts +++ b/e2e/src/api/specs/timeline.e2e-spec.ts @@ -151,7 +151,7 @@ describe('/timeline', () => { it('should require authentication', async () => { const { status, body } = await request(app).get('/timeline/bucket').query({ size: TimeBucketSize.Month, - timeBucket: '1900-01-01T00:00:00.000Z', + timeBucket: '1900-01-01', }); expect(status).toBe(401); @@ -161,7 +161,7 @@ describe('/timeline', () => { it('should handle 5 digit years', async () => { const { status, body } = await request(app) .get('/timeline/bucket') - .query({ size: TimeBucketSize.Month, timeBucket: '+012345-01-01T00:00:00.000Z' }) + .query({ size: TimeBucketSize.Month, timeBucket: '012345-01-01' }) .set('Authorization', `Bearer ${timeBucketUser.accessToken}`); expect(status).toBe(200); @@ -183,7 +183,7 @@ describe('/timeline', () => { const { status, body } = await request(app) .get('/timeline/bucket') .set('Authorization', `Bearer ${timeBucketUser.accessToken}`) - .query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10T00:00:00.000Z' }); + .query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10' }); expect(status).toBe(200); expect(body).toEqual([]); diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 7c8aba3b5e..ef5055ff3c 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -8377,6 +8377,7 @@ "type": "string" }, "AssetOrder": { + "default": "desc", "enum": [ "asc", "desc" diff --git a/server/package-lock.json b/server/package-lock.json index 347757a90b..d447151cfe 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -42,10 +42,13 @@ "ioredis": "^5.3.2", "joi": "^17.10.0", "js-yaml": "^4.1.0", + "kysely": "^0.27.3", + "kysely-postgres-js": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.4.2", "nest-commander": "^3.11.1", "nestjs-cls": "^4.3.0", + "nestjs-kysely": "^1.0.0", "nestjs-otel": "^6.0.0", "nodemailer": "^6.9.13", "openid-client": "^5.4.3", @@ -99,6 +102,7 @@ "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-unicorn": "^56.0.1", "globals": "^15.9.0", + "kysely-codegen": "^0.16.3", "mock-fs": "^5.2.0", "pngjs": "^7.0.0", "prettier": "^3.0.2", @@ -9169,6 +9173,102 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/git-diff": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/git-diff/-/git-diff-2.0.6.tgz", + "integrity": "sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==", + "dev": true, + "dependencies": { + "chalk": "^2.3.2", + "diff": "^3.5.0", + "loglevel": "^1.6.1", + "shelljs": "^0.8.1", + "shelljs.exec": "^1.1.7" + }, + "engines": { + "node": ">= 4.8.0" + } + }, + "node_modules/git-diff/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-diff/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-diff/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/git-diff/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/git-diff/node_modules/diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/git-diff/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/git-diff/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-diff/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -9610,6 +9710,15 @@ "node": ">=12.0.0" } }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ioredis": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", @@ -10041,6 +10150,100 @@ "json-buffer": "3.0.1" } }, + "node_modules/kysely": { + "version": "0.27.4", + "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.27.4.tgz", + "integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/kysely-codegen": { + "version": "0.16.3", + "resolved": "https://registry.npmjs.org/kysely-codegen/-/kysely-codegen-0.16.3.tgz", + "integrity": "sha512-SOOF3AhrsjREJuRewXmKl0nb6CkEzpP7VavHXzWdfIdIdfoJnlWlozuZhgMsYoIFmzL8aG4skvKGXF/dF3mbwg==", + "dev": true, + "dependencies": { + "chalk": "4.1.2", + "dotenv": "^16.4.5", + "dotenv-expand": "^11.0.6", + "git-diff": "^2.0.6", + "micromatch": "^4.0.8", + "minimist": "^1.2.8", + "pluralize": "^8.0.0" + }, + "bin": { + "kysely-codegen": "dist/cli/bin.js" + }, + "peerDependencies": { + "@libsql/kysely-libsql": "^0.3.0", + "@tediousjs/connection-string": "^0.5.0", + "better-sqlite3": ">=7.6.2", + "kysely": "^0.27.0", + "kysely-bun-sqlite": "^0.3.2", + "kysely-bun-worker": "^0.5.3", + "mysql2": "^2.3.3 || ^3.0.0", + "pg": "^8.8.0", + "tarn": "^3.0.0", + "tedious": "^18.0.0" + }, + "peerDependenciesMeta": { + "@libsql/kysely-libsql": { + "optional": true + }, + "@tediousjs/connection-string": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "kysely": { + "optional": false + }, + "kysely-bun-sqlite": { + "optional": true + }, + "kysely-bun-worker": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "tarn": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, + "node_modules/kysely-codegen/node_modules/dotenv-expand": { + "version": "11.0.6", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.6.tgz", + "integrity": "sha512-8NHi73otpWsZGBSZwwknTXS5pqMOrk9+Ssrna8xCaxkzEpU9OTf9R5ArQGVw03//Zmk9MOwLPng9WwndvpAJ5g==", + "dev": true, + "dependencies": { + "dotenv": "^16.4.4" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/kysely-postgres-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kysely-postgres-js/-/kysely-postgres-js-2.0.0.tgz", + "integrity": "sha512-R1tWx6/x3tSatWvsmbHJxpBZYhNNxcnMw52QzZaHKg7ZOWtHib4iZyEaw4gb2hNKVctWQ3jfMxZT/ZaEMK6kBQ==", + "peerDependencies": { + "kysely": ">= 0.24.0 < 1", + "postgres": ">= 3.4.0 < 4" + } + }, "node_modules/lazystream": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", @@ -10214,6 +10417,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/loglevel": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz", + "integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, "node_modules/long": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", @@ -10818,6 +11034,17 @@ "rxjs": ">= 7" } }, + "node_modules/nestjs-kysely": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/nestjs-kysely/-/nestjs-kysely-1.0.0.tgz", + "integrity": "sha512-0XwXm3H/sGQ8DhOY1UOMuosdgii+qePJpbdlWzRPP7ueP/Z/JGnxnu2jUHurFGS+bMGPipGnk9ynIkJuM05HUw==", + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "kysely": "0.x", + "reflect-metadata": "^0.1.13 || ^0.2.2" + } + }, "node_modules/nestjs-otel": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/nestjs-otel/-/nestjs-otel-6.1.1.tgz", @@ -11729,6 +11956,19 @@ "license": "MIT", "peer": true }, + "node_modules/postgres": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz", + "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, "node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -12483,6 +12723,18 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -13166,6 +13418,53 @@ "node": ">=8" } }, + "node_modules/shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/shelljs.exec": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/shelljs.exec/-/shelljs.exec-1.1.8.tgz", + "integrity": "sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/shelljs/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", @@ -15896,6 +16195,44 @@ "engines": { "node": ">= 14" } + }, + "node_modules/zip-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } } } } diff --git a/server/package.json b/server/package.json index dcb166bb06..0c1327c2c6 100644 --- a/server/package.json +++ b/server/package.json @@ -67,10 +67,13 @@ "ioredis": "^5.3.2", "joi": "^17.10.0", "js-yaml": "^4.1.0", + "kysely": "^0.27.3", + "kysely-postgres-js": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.4.2", "nest-commander": "^3.11.1", "nestjs-cls": "^4.3.0", + "nestjs-kysely": "^1.0.0", "nestjs-otel": "^6.0.0", "nodemailer": "^6.9.13", "openid-client": "^5.4.3", @@ -124,6 +127,7 @@ "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-unicorn": "^56.0.1", "globals": "^15.9.0", + "kysely-codegen": "^0.16.3", "mock-fs": "^5.2.0", "pngjs": "^7.0.0", "prettier": "^3.0.2", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index da8fa55606..9d96a0499b 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -4,6 +4,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE, ModuleRef } from '@ne import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule'; import { TypeOrmModule } from '@nestjs/typeorm'; import { ClsModule } from 'nestjs-cls'; +import { KyselyModule } from 'nestjs-kysely'; import { OpenTelemetryModule } from 'nestjs-otel'; import { commands } from 'src/commands'; import { IWorker } from 'src/constants'; @@ -48,7 +49,7 @@ const imports = [ inject: [ModuleRef], useFactory: (moduleRef: ModuleRef) => { return { - ...database.config, + ...database.config.typeorm, poolErrorHandler: (error) => { moduleRef.get(DatabaseService, { strict: false }).handleConnectionError(error); }, @@ -56,6 +57,7 @@ const imports = [ }, }), TypeOrmModule.forFeature(entities), + KyselyModule.forRoot(database.config.kysely), ]; class BaseModule implements OnModuleInit, OnModuleDestroy { diff --git a/server/src/bin/database.ts b/server/src/bin/database.ts index c861902b4e..7ea56e0fc0 100644 --- a/server/src/bin/database.ts +++ b/server/src/bin/database.ts @@ -8,4 +8,4 @@ const { database } = new ConfigRepository().getEnv(); * * this export is ONLY to be used for TypeORM commands in package.json#scripts */ -export const dataSource = new DataSource({ ...database.config, host: 'localhost' }); +export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' }); diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.ts index 98f26d879a..2de4fb4127 100644 --- a/server/src/bin/sync-sql.ts +++ b/server/src/bin/sync-sql.ts @@ -4,6 +4,7 @@ import { Reflector } from '@nestjs/core'; import { SchedulerRegistry } from '@nestjs/schedule'; import { Test } from '@nestjs/testing'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { KyselyModule } from 'nestjs-kysely'; import { OpenTelemetryModule } from 'nestjs-otel'; import { mkdir, rm, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; @@ -73,13 +74,23 @@ class SqlGenerator { await rm(this.options.targetDir, { force: true, recursive: true }); await mkdir(this.options.targetDir); + process.env.DB_HOSTNAME = 'localhost'; const { database, otel } = new ConfigRepository().getEnv(); const moduleFixture = await Test.createTestingModule({ imports: [ + KyselyModule.forRoot({ + ...database.config.kysely, + log: (event) => { + if (event.level === 'query') { + this.sqlLogger.logQuery(event.query.sql); + } else if (event.level === 'error') { + this.sqlLogger.logQueryError(event.error as Error, event.query.sql); + } + }, + }), TypeOrmModule.forRoot({ - ...database.config, - host: 'localhost', + ...database.config.typeorm, entities, logging: ['query'], logger: this.sqlLogger, diff --git a/server/src/constants.ts b/server/src/constants.ts index fc2442130e..050a7d06fa 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -7,6 +7,10 @@ export const POSTGRES_VERSION_RANGE = '>=14.0.0'; export const VECTORS_VERSION_RANGE = '>=0.2 <0.4'; export const VECTOR_VERSION_RANGE = '>=0.5 <1'; +export const ASSET_FILE_CONFLICT_KEYS = ['assetId', 'type'] as const; +export const EXIF_CONFLICT_KEYS = ['assetId'] as const; +export const JOB_STATUS_CONFLICT_KEYS = ['assetId'] as const; + export const NEXT_RELEASE = 'NEXT_RELEASE'; export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle'; export const DEPRECATED_IN_PREFIX = 'This property was deprecated in '; diff --git a/server/src/db.d.ts b/server/src/db.d.ts new file mode 100644 index 0000000000..454c5176de --- /dev/null +++ b/server/src/db.d.ts @@ -0,0 +1,439 @@ +/** + * This file was generated by kysely-codegen. + * Please do not edit it manually. + */ + +import type { ColumnType } from 'kysely'; + +export type ArrayType = ArrayTypeImpl extends (infer U)[] ? U[] : ArrayTypeImpl; + +export type ArrayTypeImpl = T extends ColumnType ? ColumnType : T[]; + +export type AssetsStatusEnum = 'active' | 'deleted' | 'trashed'; + +export type Generated = + T extends ColumnType ? ColumnType : ColumnType; + +export type Int8 = ColumnType; + +export type Json = JsonValue; + +export type JsonArray = JsonValue[]; + +export type JsonObject = { + [x: string]: JsonValue | undefined; +}; + +export type JsonPrimitive = boolean | number | string | null; + +export type JsonValue = JsonArray | JsonObject | JsonPrimitive; + +export type Sourcetype = 'exif' | 'machine-learning'; + +export type Timestamp = ColumnType; + +export interface Activity { + albumId: string; + assetId: string | null; + comment: string | null; + createdAt: Generated; + id: Generated; + isLiked: Generated; + updatedAt: Generated; + userId: string; +} + +export interface Albums { + albumName: Generated; + /** + * Asset ID to be used as thumbnail + */ + albumThumbnailAssetId: string | null; + createdAt: Generated; + deletedAt: Timestamp | null; + description: Generated; + id: Generated; + isActivityEnabled: Generated; + order: Generated; + ownerId: string; + updatedAt: Generated; +} + +export interface AlbumsAssetsAssets { + albumsId: string; + assetsId: string; +} + +export interface AlbumsSharedUsersUsers { + albumsId: string; + role: Generated; + usersId: string; +} + +export interface ApiKeys { + createdAt: Generated; + id: Generated; + key: string; + name: string; + permissions: string[]; + updatedAt: Generated; + userId: string; +} + +export interface AssetFaces { + assetId: string; + boundingBoxX1: Generated; + boundingBoxX2: Generated; + boundingBoxY1: Generated; + boundingBoxY2: Generated; + id: Generated; + imageHeight: Generated; + imageWidth: Generated; + personId: string | null; + sourceType: Generated; +} + +export interface AssetFiles { + assetId: string; + createdAt: Generated; + id: Generated; + path: string; + type: string; + updatedAt: Generated; +} + +export interface AssetJobStatus { + assetId: string; + duplicatesDetectedAt: Timestamp | null; + facesRecognizedAt: Timestamp | null; + metadataExtractedAt: Timestamp | null; + previewAt: Timestamp | null; + thumbnailAt: Timestamp | null; +} + +export interface Assets { + checksum: Buffer; + createdAt: Generated; + deletedAt: Timestamp | null; + deviceAssetId: string; + deviceId: string; + duplicateId: string | null; + duration: string | null; + encodedVideoPath: Generated; + fileCreatedAt: Timestamp; + fileModifiedAt: Timestamp; + id: Generated; + isArchived: Generated; + isExternal: Generated; + isFavorite: Generated; + isOffline: Generated; + isVisible: Generated; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Timestamp; + originalFileName: string; + originalPath: string; + ownerId: string; + sidecarPath: string | null; + stackId: string | null; + status: Generated; + thumbhash: Buffer | null; + type: string; + updatedAt: Generated; +} + +export interface AssetStack { + id: Generated; + ownerId: string; + primaryAssetId: string; +} + +export interface Audit { + action: string; + createdAt: Generated; + entityId: string; + entityType: string; + id: Generated; + ownerId: string; +} + +export interface Exif { + assetId: string; + autoStackId: string | null; + bitsPerSample: number | null; + city: string | null; + colorspace: string | null; + country: string | null; + dateTimeOriginal: Timestamp | null; + description: Generated; + exifImageHeight: number | null; + exifImageWidth: number | null; + exposureTime: string | null; + fileSizeInByte: Int8 | null; + fNumber: number | null; + focalLength: number | null; + fps: number | null; + iso: number | null; + latitude: number | null; + lensModel: string | null; + livePhotoCID: string | null; + longitude: number | null; + make: string | null; + model: string | null; + modifyDate: Timestamp | null; + orientation: string | null; + profileDescription: string | null; + projectionType: string | null; + rating: number | null; + state: string | null; + timeZone: string | null; +} + +export interface FaceSearch { + embedding: string; + faceId: string; +} + +export interface GeodataPlaces { + admin1Code: string | null; + admin1Name: string | null; + admin2Code: string | null; + admin2Name: string | null; + alternateNames: string | null; + countryCode: string; + earthCoord: Generated; + id: number; + latitude: number; + longitude: number; + modificationDate: Timestamp; + name: string; +} + +export interface Libraries { + createdAt: Generated; + deletedAt: Timestamp | null; + exclusionPatterns: string[]; + id: Generated; + importPaths: string[]; + name: string; + ownerId: string; + refreshedAt: Timestamp | null; + updatedAt: Generated; +} + +export interface Memories { + createdAt: Generated; + data: Json; + deletedAt: Timestamp | null; + id: Generated; + isSaved: Generated; + memoryAt: Timestamp; + ownerId: string; + seenAt: Timestamp | null; + type: string; + updatedAt: Generated; +} + +export interface MemoriesAssetsAssets { + assetsId: string; + memoriesId: string; +} + +export interface Migrations { + id: Generated; + name: string; + timestamp: Int8; +} + +export interface MoveHistory { + entityId: string; + id: Generated; + newPath: string; + oldPath: string; + pathType: string; +} + +export interface NaturalearthCountries { + admin: string; + admin_a3: string; + coordinates: string; + id: Generated; + type: string; +} + +export interface Partners { + createdAt: Generated; + inTimeline: Generated; + sharedById: string; + sharedWithId: string; + updatedAt: Generated; +} + +export interface Person { + birthDate: Timestamp | null; + createdAt: Generated; + faceAssetId: string | null; + id: Generated; + isHidden: Generated; + name: Generated; + ownerId: string; + thumbnailPath: Generated; + updatedAt: Generated; +} + +export interface Sessions { + createdAt: Generated; + deviceOS: Generated; + deviceType: Generated; + id: Generated; + token: string; + updatedAt: Generated; + userId: string; +} + +export interface SharedLinkAsset { + assetsId: string; + sharedLinksId: string; +} + +export interface SharedLinks { + albumId: string | null; + allowDownload: Generated; + allowUpload: Generated; + createdAt: Generated; + description: string | null; + expiresAt: Timestamp | null; + id: Generated; + key: Buffer; + password: string | null; + showExif: Generated; + type: string; + userId: string; +} + +export interface SmartInfo { + assetId: string; + objects: string[] | null; + smartInfoTextSearchableColumn: Generated; + tags: string[] | null; +} + +export interface SmartSearch { + assetId: string; + embedding: string; +} + +export interface SocketIoAttachments { + created_at: Generated; + id: Generated; + payload: Buffer | null; +} + +export interface SystemConfig { + key: string; + value: string | null; +} + +export interface SystemMetadata { + key: string; + value: Json; +} + +export interface TagAsset { + assetsId: string; + tagsId: string; +} + +export interface Tags { + color: string | null; + createdAt: Generated; + id: Generated; + parentId: string | null; + updatedAt: Generated; + userId: string; + value: string; +} + +export interface TagsClosure { + id_ancestor: string; + id_descendant: string; +} + +export interface UserMetadata { + key: string; + userId: string; + value: Json; +} + +export interface Users { + createdAt: Generated; + deletedAt: Timestamp | null; + email: string; + id: Generated; + isAdmin: Generated; + name: Generated; + oauthId: Generated; + password: Generated; + profileChangedAt: Generated; + profileImagePath: Generated; + quotaSizeInBytes: Int8 | null; + quotaUsageInBytes: Generated; + shouldChangePassword: Generated; + status: Generated; + storageLabel: string | null; + updatedAt: Generated; +} + +export interface VectorsPgVectorIndexStat { + idx_growing: ArrayType | null; + idx_indexing: boolean | null; + idx_options: string | null; + idx_sealed: ArrayType | null; + idx_size: Int8 | null; + idx_status: string | null; + idx_tuples: Int8 | null; + idx_write: Int8 | null; + indexname: string | null; + indexrelid: number | null; + tablename: string | null; + tablerelid: number | null; +} + +export interface DB { + activity: Activity; + albums: Albums; + albums_assets_assets: AlbumsAssetsAssets; + albums_shared_users_users: AlbumsSharedUsersUsers; + api_keys: ApiKeys; + asset_faces: AssetFaces; + asset_files: AssetFiles; + asset_job_status: AssetJobStatus; + asset_stack: AssetStack; + assets: Assets; + audit: Audit; + exif: Exif; + face_search: FaceSearch; + geodata_places: GeodataPlaces; + libraries: Libraries; + memories: Memories; + memories_assets_assets: MemoriesAssetsAssets; + migrations: Migrations; + move_history: MoveHistory; + naturalearth_countries: NaturalearthCountries; + partners: Partners; + person: Person; + sessions: Sessions; + shared_link__asset: SharedLinkAsset; + shared_links: SharedLinks; + smart_info: SmartInfo; + smart_search: SmartSearch; + socket_io_attachments: SocketIoAttachments; + system_config: SystemConfig; + system_metadata: SystemMetadata; + tag_asset: TagAsset; + tags: Tags; + tags_closure: TagsClosure; + user_metadata: UserMetadata; + users: Users; + 'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat; +} diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index a255ac103b..0658567912 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -97,10 +97,19 @@ const mapStack = (entity: AssetEntity) => { return { id: entity.stack.id, primaryAssetId: entity.stack.primaryAssetId, - assetCount: entity.stack.assetCount ?? entity.stack.assets.length, + assetCount: entity.stack.assetCount ?? entity.stack.assets.length + 1, }; }; +// if an asset is jsonified in the DB before being returned, its buffer fields will be hex-encoded strings +const hexOrBufferToBase64 = (encoded: string | Buffer) => { + if (typeof encoded === 'string') { + return Buffer.from(encoded.slice(2), 'hex').toString('base64'); + } + + return encoded.toString('base64'); +}; + export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto { const { stripMetadata = false, withStack = false } = options; @@ -129,7 +138,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As originalPath: entity.originalPath, originalFileName: entity.originalFileName, originalMimeType: mimeTypes.lookup(entity.originalFileName), - thumbhash: entity.thumbhash?.toString('base64') ?? null, + thumbhash: entity.thumbhash ? hexOrBufferToBase64(entity.thumbhash) : null, fileCreatedAt: entity.fileCreatedAt, fileModifiedAt: entity.fileModifiedAt, localDateTime: entity.localDateTime, @@ -143,7 +152,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As tags: entity.tags?.map((tag) => mapTag(tag)), people: peopleWithFaces(entity.faces), unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)), - checksum: entity.checksum.toString('base64'), + checksum: hexOrBufferToBase64(entity.checksum), stack: withStack ? mapStack(entity) : undefined, isOffline: entity.isOffline, hasMetadata: true, diff --git a/server/src/dtos/duplicate.dto.ts b/server/src/dtos/duplicate.dto.ts index 09976b3213..b12580ef18 100644 --- a/server/src/dtos/duplicate.dto.ts +++ b/server/src/dtos/duplicate.dto.ts @@ -1,5 +1,4 @@ import { IsNotEmpty } from 'class-validator'; -import { groupBy, sortBy } from 'lodash'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { ValidateUUID } from 'src/validation'; @@ -13,16 +12,3 @@ export class ResolveDuplicatesDto { @ValidateUUID({ each: true }) assetIds!: string[]; } - -export function mapDuplicateResponse(assets: AssetResponseDto[]): DuplicateResponseDto[] { - const result = []; - - const grouped = groupBy(assets, (a) => a.duplicateId); - - for (const [duplicateId, unsortedAssets] of Object.entries(grouped)) { - const assets = sortBy(unsortedAssets, (asset) => asset.localDateTime); - result.push({ duplicateId, assets }); - } - - return result; -} diff --git a/server/src/dtos/search.dto.ts b/server/src/dtos/search.dto.ts index 5c5dce1a11..f3f45af44d 100644 --- a/server/src/dtos/search.dto.ts +++ b/server/src/dtos/search.dto.ts @@ -162,7 +162,7 @@ export class MetadataSearchDto extends RandomSearchDto { @IsEnum(AssetOrder) @Optional() - @ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder }) + @ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder, default: AssetOrder.DESC }) order?: AssetOrder; @IsInt() diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts index f9e5c5e981..6890b8f5a9 100644 --- a/server/src/entities/asset.entity.ts +++ b/server/src/entities/asset.entity.ts @@ -1,3 +1,6 @@ +import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, Selectable, SelectQueryBuilder, sql } from 'kysely'; +import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; +import { Assets, DB } from 'src/db'; import { AlbumEntity } from 'src/entities/album.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFileEntity } from 'src/entities/asset-files.entity'; @@ -9,7 +12,9 @@ import { SmartSearchEntity } from 'src/entities/smart-search.entity'; import { StackEntity } from 'src/entities/stack.entity'; import { TagEntity } from 'src/entities/tag.entity'; import { UserEntity } from 'src/entities/user.entity'; -import { AssetStatus, AssetType } from 'src/enum'; +import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; +import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface'; +import { anyUuid, asUuid } from 'src/utils/database'; import { Column, CreateDateColumn, @@ -38,8 +43,8 @@ export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; unique: true, where: '"libraryId" IS NOT NULL', }) -@Index('IDX_day_of_month', { synchronize: false }) -@Index('IDX_month', { synchronize: false }) +@Index('idx_local_date_time', { synchronize: false }) +@Index('idx_local_date_time_month', { synchronize: false }) @Index('IDX_originalPath_libraryId', ['originalPath', 'libraryId']) @Index('IDX_asset_id_stackId', ['id', 'stackId']) @Index('idx_originalFileName_trigram', { synchronize: false }) @@ -173,3 +178,247 @@ export class AssetEntity { @Column({ type: 'uuid', nullable: true }) duplicateId!: string | null; } + +export const withExif = (qb: SelectQueryBuilder) => { + return qb + .leftJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')); +}; + +export const withExifInner = (qb: SelectQueryBuilder) => { + return qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')); +}; + +export const withSmartSearch = (qb: SelectQueryBuilder, options?: { inner: boolean }) => { + const join = options?.inner + ? qb.innerJoin('smart_search', 'assets.id', 'smart_search.assetId') + : qb.leftJoin('smart_search', 'assets.id', 'smart_search.assetId'); + return join.select(sql`smart_search.embedding`.as('embedding')); +}; + +export const withFaces = (eb: ExpressionBuilder) => + jsonArrayFrom(eb.selectFrom('asset_faces').selectAll().whereRef('asset_faces.assetId', '=', 'assets.id')).as('faces'); + +export const withFiles = (eb: ExpressionBuilder, type?: AssetFileType) => + jsonArrayFrom( + eb + .selectFrom('asset_files') + .selectAll() + .whereRef('asset_files.assetId', '=', 'assets.id') + .$if(!!type, (qb) => qb.where('type', '=', type!)), + ).as('files'); + +export const withFacesAndPeople = (eb: ExpressionBuilder) => + eb + .selectFrom('asset_faces') + .leftJoin('person', 'person.id', 'asset_faces.personId') + .whereRef('asset_faces.assetId', '=', 'assets.id') + .select((eb) => + eb + .fn('jsonb_agg', [ + eb + .case() + .when('person.id', 'is not', null) + .then( + eb.fn('jsonb_insert', [ + eb.fn('to_jsonb', [eb.table('asset_faces')]), + sql`'{person}'::text[]`, + eb.fn('to_jsonb', [eb.table('person')]), + ]), + ) + .else(eb.fn('to_jsonb', [eb.table('asset_faces')])) + .end(), + ]) + .as('faces'), + ) + .as('faces'); + +/** Adds a `has_people` CTE that can be inner joined on to filter out assets */ +export const hasPeopleCte = (db: Kysely, personIds: string[]) => + db.with('has_people', (qb) => + qb + .selectFrom('asset_faces') + .select('assetId') + .where('personId', '=', anyUuid(personIds!)) + .groupBy('assetId') + .having((eb) => eb.fn.count('personId'), '>=', personIds.length), + ); + +export const hasPeople = (db: Kysely, personIds?: string[]) => + personIds && personIds.length > 0 + ? hasPeopleCte(db, personIds).selectFrom('assets').innerJoin('has_people', 'has_people.assetId', 'assets.id') + : db.selectFrom('assets'); + +export const withOwner = (eb: ExpressionBuilder) => + jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner'); + +export const withLibrary = (eb: ExpressionBuilder) => + jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as( + 'library', + ); + +type Stacked = SelectQueryBuilder< + DB & { stacked: Selectable }, + 'assets' | 'asset_stack' | 'stacked', + { assets: Selectable[] } +>; + +type StackExpression = (eb: Stacked) => Stacked; + +export const withStack = ( + qb: SelectQueryBuilder, + { assets }: { assets?: boolean | StackExpression }, +) => + qb + .leftJoinLateral( + (eb) => + eb + .selectFrom('asset_stack') + .selectAll('asset_stack') + .whereRef('assets.stackId', '=', 'asset_stack.id') + .$if(!!assets, (qb) => + qb + .innerJoinLateral( + (eb: ExpressionBuilder) => + eb + .selectFrom('assets as stacked') + .select((eb) => eb.fn[]>('array_agg', [eb.table('stacked')]).as('assets')) + .whereRef('asset_stack.id', '=', 'stacked.stackId') + .whereRef('asset_stack.primaryAssetId', '!=', 'stacked.id') + .$if(typeof assets === 'function', assets as StackExpression) + .as('s'), + (join) => + join.on((eb) => + eb.or([ + eb('asset_stack.primaryAssetId', '=', eb.ref('assets.id')), + eb('assets.stackId', 'is', null), + ]), + ), + ) + .select('s.assets'), + ) + .as('stacked_assets'), + (join) => join.onTrue(), + ) + .select((eb) => eb.fn('to_jsonb', [eb.table('stacked_assets')]).as('stack')); + +export const withAlbums = (qb: SelectQueryBuilder, { albumId }: { albumId?: string }) => { + return qb + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('albums') + .selectAll() + .innerJoin('albums_assets_assets', (join) => + join + .onRef('albums.id', '=', 'albums_assets_assets.albumsId') + .onRef('assets.id', '=', 'albums_assets_assets.assetsId'), + ) + .whereRef('albums.id', '=', 'albums_assets_assets.albumsId') + .$if(!!albumId, (qb) => qb.where('albums.id', '=', asUuid(albumId!))), + ).as('albums'), + ) + .$if(!!albumId, (qb) => + qb.where((eb) => + eb.exists((eb) => + eb + .selectFrom('albums_assets_assets') + .whereRef('albums_assets_assets.assetsId', '=', 'assets.id') + .where('albums_assets_assets.albumsId', '=', asUuid(albumId!)), + ), + ), + ); +}; + +export const withTags = (eb: ExpressionBuilder) => + jsonArrayFrom( + eb + .selectFrom('tags') + .selectAll('tags') + .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') + .whereRef('assets.id', '=', 'tag_asset.assetsId'), + ).as('tags'); + +const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); + +/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ +export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { + options.isArchived ??= options.withArchived ? undefined : false; + options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore); + return hasPeople(kysely.withPlugin(joinDeduplicationPlugin), options.personIds) + .selectAll('assets') + .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) + .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) + .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) + .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) + .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) + .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) + .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) + .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) + .$if(options.city !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.city', options.city === null ? 'is' : '=', options.city!), + ) + .$if(options.state !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.state', options.state === null ? 'is' : '=', options.state!), + ) + .$if(options.country !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.country', options.country === null ? 'is' : '=', options.country!), + ) + .$if(options.make !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.make', options.make === null ? 'is' : '=', options.make!), + ) + .$if(options.model !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.model', options.model === null ? 'is' : '=', options.model!), + ) + .$if(options.lensModel !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), + ) + .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) + .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) + .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) + .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) + .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) + .$if(!!options.originalPath, (qb) => qb.where('assets.originalPath', '=', options.originalPath!)) + .$if(!!options.originalFileName, (qb) => + qb.where( + sql`f_unaccent(assets."originalFileName")`, + 'ilike', + sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, + ), + ) + .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) + .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isEncoded !== undefined, (qb) => + qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), + ) + .$if(options.isMotion !== undefined, (qb) => + qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), + ) + .$if(!!options.isNotInAlbum, (qb) => + qb.where((eb) => + eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), + ), + ) + .$if(!!options.withExif, withExifInner) + .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) + .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); +} diff --git a/server/src/entities/face-search.entity.ts b/server/src/entities/face-search.entity.ts index 3fd3c65f28..e353260fe0 100644 --- a/server/src/entities/face-search.entity.ts +++ b/server/src/entities/face-search.entity.ts @@ -1,5 +1,4 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity'; -import { asVector } from 'src/utils/database'; import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; @Entity('face_search', { synchronize: false }) @@ -15,7 +14,7 @@ export class FaceSearchEntity { @Column({ type: 'float4', array: true, - transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v) }, + transformer: { from: (v) => JSON.parse(v), to: (v) => `[${v}]` }, }) embedding!: number[]; } diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index b25e42ba0e..5abaf9af26 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -1,10 +1,9 @@ -import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; +import { Insertable, Updateable } from 'kysely'; +import { AssetFiles, AssetJobStatus, Assets, Exif } from 'src/db'; import { AssetEntity } from 'src/entities/asset.entity'; -import { ExifEntity } from 'src/entities/exif.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { Paginated, PaginationOptions } from 'src/utils/pagination'; -import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect } from 'typeorm'; export type AssetStats = Record; @@ -66,43 +65,6 @@ export interface TimeBucketItem { count: number; } -export type AssetCreate = Pick< - AssetEntity, - | 'deviceAssetId' - | 'ownerId' - | 'libraryId' - | 'deviceId' - | 'type' - | 'originalPath' - | 'fileCreatedAt' - | 'localDateTime' - | 'fileModifiedAt' - | 'checksum' - | 'originalFileName' -> & - Partial; - -export type AssetWithoutRelations = Omit< - AssetEntity, - | 'livePhotoVideo' - | 'stack' - | 'albums' - | 'faces' - | 'owner' - | 'library' - | 'exifInfo' - | 'sharedLinks' - | 'smartSearch' - | 'tags' ->; - -type AssetUpdateWithoutRelations = Pick & Partial; -type AssetUpdateWithLivePhotoRelation = Pick & Pick; - -export type AssetUpdateOptions = AssetUpdateWithoutRelations | AssetUpdateWithLivePhotoRelation; - -export type AssetUpdateAllOptions = Omit, 'id'>; - export interface MonthDay { day: number; month: number; @@ -113,12 +75,6 @@ export interface AssetExploreFieldOptions { minAssetsPerField: number; } -export interface AssetExploreOptions extends AssetExploreFieldOptions { - relation: keyof AssetEntity; - relatedField: string; - unnest?: boolean; -} - export interface AssetFullSyncOptions { ownerId: string; lastId?: string; @@ -144,8 +100,30 @@ export interface UpsertFileOptions { path: string; } +export interface AssetGetByChecksumOptions { + ownerId: string; + checksum: Buffer; + libraryId?: string; +} + export type AssetPathEntity = Pick; +export interface GetByIdsRelations { + exifInfo?: boolean; + faces?: { person?: boolean }; + files?: boolean; + library?: boolean; + owner?: boolean; + smartSearch?: boolean; + stack?: { assets?: boolean }; + tags?: boolean; +} + +export interface DuplicateGroup { + duplicateId: string; + assets: AssetEntity[]; +} + export interface DayOfYearAssets { yearsAgo: number; assets: AssetEntity[]; @@ -154,47 +132,39 @@ export interface DayOfYearAssets { export const IAssetRepository = 'IAssetRepository'; export interface IAssetRepository { - create(asset: AssetCreate): Promise; - getByIds( - ids: string[], - relations?: FindOptionsRelations, - select?: FindOptionsSelect, - ): Promise; + create(asset: Insertable): Promise; + getByIds(ids: string[], relations?: GetByIdsRelations): Promise; getByIdsWithAllRelations(ids: string[]): Promise; getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise; - getByChecksum(options: { ownerId: string; checksum: Buffer; libraryId?: string }): Promise; + getByChecksum(options: AssetGetByChecksumOptions): Promise; getByChecksums(userId: string, checksums: Buffer[]): Promise; getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise; getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated; getByDeviceIds(ownerId: string, deviceId: string, deviceAssetIds: string[]): Promise; getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated; - getById( - id: string, - relations?: FindOptionsRelations, - order?: FindOptionsOrder, - ): Promise; + getById(id: string, relations?: GetByIdsRelations): Promise; getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated; getRandom(userIds: string[], count: number): Promise; - getLastUpdatedAssetForAlbumId(albumId: string): Promise; - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise; + getLastUpdatedAssetForAlbumId(albumId: string): Promise; + getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise; deleteAll(ownerId: string): Promise; getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated; getAllByDeviceId(userId: string, deviceId: string): Promise; getLivePhotoCount(motionId: string): Promise; - updateAll(ids: string[], options: Partial): Promise; + updateAll(ids: string[], options: Updateable): Promise; updateDuplicates(options: AssetUpdateDuplicateOptions): Promise; - update(asset: AssetUpdateOptions): Promise; + update(asset: Updateable & { id: string }): Promise; remove(asset: AssetEntity): Promise; - findLivePhotoMatch(options: LivePhotoSearchOptions): Promise; + findLivePhotoMatch(options: LivePhotoSearchOptions): Promise; getStatistics(ownerId: string, options: AssetStatsOptions): Promise; getTimeBuckets(options: TimeBucketOptions): Promise; getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise; - upsertExif(exif: Partial): Promise; - upsertJobStatus(...jobStatus: Partial[]): Promise; + upsertExif(exif: Insertable): Promise; + upsertJobStatus(...jobStatus: Insertable[]): Promise; getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise>; - getDuplicates(options: AssetBuilderOptions): Promise; + getDuplicates(userId: string): Promise; getAllForUserFullSync(options: AssetFullSyncOptions): Promise; getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; - upsertFile(file: UpsertFileOptions): Promise; - upsertFiles(files: UpsertFileOptions[]): Promise; + upsertFile(options: Insertable): Promise; + upsertFiles(options: Insertable[]): Promise; } diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index 300b55f27b..8b45078039 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -1,6 +1,7 @@ import { RegisterQueueOptions } from '@nestjs/bullmq'; import { QueueOptions } from 'bullmq'; import { RedisOptions } from 'ioredis'; +import { KyselyConfig } from 'kysely'; import { ClsModuleOptions } from 'nestjs-cls'; import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum'; @@ -42,7 +43,7 @@ export interface EnvData { }; database: { - config: PostgresConnectionOptions & DatabaseConnectionParams; + config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: KyselyConfig }; skipMigrations: boolean; vectorExtension: VectorExtension; }; diff --git a/server/src/interfaces/search.interface.ts b/server/src/interfaces/search.interface.ts index d59291c883..0de8ef07d5 100644 --- a/server/src/interfaces/search.interface.ts +++ b/server/src/interfaces/search.interface.ts @@ -1,4 +1,3 @@ -import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { AssetStatus, AssetType } from 'src/enum'; @@ -114,7 +113,7 @@ export interface SearchPeopleOptions { } export interface SearchOrderOptions { - orderDirection?: 'ASC' | 'DESC'; + orderDirection?: 'asc' | 'desc'; } export interface SearchPaginationOptions { @@ -148,20 +147,21 @@ export type SmartSearchOptions = SearchDateOptions & export interface FaceEmbeddingSearch extends SearchEmbeddingOptions { hasPerson?: boolean; numResults: number; - maxDistance?: number; + maxDistance: number; } export interface AssetDuplicateSearch { assetId: string; embedding: number[]; - maxDistance?: number; + maxDistance: number; type: AssetType; userIds: string[]; } export interface FaceSearchResult { distance: number; - face: AssetFaceEntity; + id: string; + personId: string | null; } export interface AssetDuplicateResult { diff --git a/server/src/migrations/1734574016301-AddTimeBucketIndices.ts b/server/src/migrations/1734574016301-AddTimeBucketIndices.ts new file mode 100644 index 0000000000..9aae98be2b --- /dev/null +++ b/server/src/migrations/1734574016301-AddTimeBucketIndices.ts @@ -0,0 +1,25 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddTimeBucketIndices1734574016301 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX idx_local_date_time_month ON public.assets (date_trunc('MONTH', "localDateTime" at time zone 'UTC'))`, + ); + await queryRunner.query( + `CREATE INDEX idx_local_date_time ON public.assets ((("localDateTime" at time zone 'UTC')::date))`, + ); + await queryRunner.query(`DROP INDEX "IDX_day_of_month"`); + await queryRunner.query(`DROP INDEX "IDX_month"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX idx_local_date_time_month`); + await queryRunner.query(`DROP INDEX idx_local_date_time`); + await queryRunner.query( + `CREATE INDEX "IDX_day_of_month" ON assets (EXTRACT(DAY FROM "localDateTime" AT TIME ZONE 'UTC'))`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_month" ON assets (EXTRACT(MONTH FROM "localDateTime" AT TIME ZONE 'UTC'))`, + ); + } +} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index 4694cd20fc..439d66b82d 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -1,1134 +1,319 @@ -- NOTE: This file is auto generated by ./sql-generator -- AssetRepository.getByDayOfYear -SELECT - "entity"."id" AS "entity_id", - "entity"."deviceAssetId" AS "entity_deviceAssetId", - "entity"."ownerId" AS "entity_ownerId", - "entity"."libraryId" AS "entity_libraryId", - "entity"."deviceId" AS "entity_deviceId", - "entity"."type" AS "entity_type", - "entity"."status" AS "entity_status", - "entity"."originalPath" AS "entity_originalPath", - "entity"."thumbhash" AS "entity_thumbhash", - "entity"."encodedVideoPath" AS "entity_encodedVideoPath", - "entity"."createdAt" AS "entity_createdAt", - "entity"."updatedAt" AS "entity_updatedAt", - "entity"."deletedAt" AS "entity_deletedAt", - "entity"."fileCreatedAt" AS "entity_fileCreatedAt", - "entity"."localDateTime" AS "entity_localDateTime", - "entity"."fileModifiedAt" AS "entity_fileModifiedAt", - "entity"."isFavorite" AS "entity_isFavorite", - "entity"."isArchived" AS "entity_isArchived", - "entity"."isExternal" AS "entity_isExternal", - "entity"."isOffline" AS "entity_isOffline", - "entity"."checksum" AS "entity_checksum", - "entity"."duration" AS "entity_duration", - "entity"."isVisible" AS "entity_isVisible", - "entity"."livePhotoVideoId" AS "entity_livePhotoVideoId", - "entity"."originalFileName" AS "entity_originalFileName", - "entity"."sidecarPath" AS "entity_sidecarPath", - "entity"."stackId" AS "entity_stackId", - "entity"."duplicateId" AS "entity_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "files"."id" AS "files_id", - "files"."assetId" AS "files_assetId", - "files"."createdAt" AS "files_createdAt", - "files"."updatedAt" AS "files_updatedAt", - "files"."type" AS "files_type", - "files"."path" AS "files_path" -FROM - "assets" "entity" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "entity"."id" - INNER JOIN "asset_files" "files" ON "files"."assetId" = "entity"."id" -WHERE - ( - "entity"."ownerId" IN ($1) - AND "entity"."isVisible" = true - AND "entity"."isArchived" = false - AND EXTRACT( - DAY - FROM - "entity"."localDateTime" AT TIME ZONE 'UTC' - ) = $2 - AND EXTRACT( - MONTH - FROM - "entity"."localDateTime" AT TIME ZONE 'UTC' - ) = $3 - AND "files"."type" = $4 - AND EXTRACT( - YEAR - FROM - CURRENT_DATE AT TIME ZONE 'UTC' - ) - EXTRACT( - YEAR - FROM - "entity"."localDateTime" AT TIME ZONE 'UTC' - ) > 0 +with + "res" as ( + with + "today" as ( + select + make_date(year::int, $1::int, $2::int) as "date" + from + generate_series( + $3, + extract( + year + from + current_date + ) - 1 + ) as "year" + ) + select + "a".*, + to_jsonb("exif") as "exifInfo" + from + "today" + inner join lateral ( + select + "assets".* + from + "assets" + inner join "asset_job_status" on "assets"."id" = "asset_job_status"."assetId" + where + "asset_job_status"."previewAt" is not null + and (assets."localDateTime" at time zone 'UTC')::date = today.date + and "assets"."ownerId" = any ($4::uuid []) + and "assets"."isVisible" = $5 + and "assets"."isArchived" = $6 + and exists ( + select + from + "asset_files" + where + "assetId" = "assets"."id" + and "asset_files"."type" = $7 + ) + and "assets"."deletedAt" is null + limit + $8 + ) as "a" on true + inner join "exif" on "a"."id" = "exif"."assetId" ) - AND ("entity"."deletedAt" IS NULL) -ORDER BY - "entity"."fileCreatedAt" ASC +select + ( + (now() at time zone 'UTC')::date - ("localDateTime" at time zone 'UTC')::date + ) / 365 as "yearsAgo", + jsonb_agg("res") as "assets" +from + "res" +group by + ("localDateTime" at time zone 'UTC')::date +order by + ("localDateTime" at time zone 'UTC')::date desc +limit + $9 -- AssetRepository.getByIds -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId" -FROM - "assets" "AssetEntity" -WHERE - (("AssetEntity"."id" IN ($1))) +select + "assets".* +from + "assets" +where + "assets"."id" = any ($1::uuid []) -- AssetRepository.getByIdsWithAllRelations -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId", - "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId", - "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description", - "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth", - "AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight", - "AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte", - "AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation", - "AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal", - "AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate", - "AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone", - "AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude", - "AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude", - "AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType", - "AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city", - "AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID", - "AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId", - "AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state", - "AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country", - "AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make", - "AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model", - "AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel", - "AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber", - "AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength", - "AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso", - "AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime", - "AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription", - "AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace", - "AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample", - "AssetEntity__AssetEntity_exifInfo"."rating" AS "AssetEntity__AssetEntity_exifInfo_rating", - "AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps", - "AssetEntity__AssetEntity_tags"."id" AS "AssetEntity__AssetEntity_tags_id", - "AssetEntity__AssetEntity_tags"."value" AS "AssetEntity__AssetEntity_tags_value", - "AssetEntity__AssetEntity_tags"."createdAt" AS "AssetEntity__AssetEntity_tags_createdAt", - "AssetEntity__AssetEntity_tags"."updatedAt" AS "AssetEntity__AssetEntity_tags_updatedAt", - "AssetEntity__AssetEntity_tags"."color" AS "AssetEntity__AssetEntity_tags_color", - "AssetEntity__AssetEntity_tags"."parentId" AS "AssetEntity__AssetEntity_tags_parentId", - "AssetEntity__AssetEntity_tags"."userId" AS "AssetEntity__AssetEntity_tags_userId", - "AssetEntity__AssetEntity_faces"."id" AS "AssetEntity__AssetEntity_faces_id", - "AssetEntity__AssetEntity_faces"."assetId" AS "AssetEntity__AssetEntity_faces_assetId", - "AssetEntity__AssetEntity_faces"."personId" AS "AssetEntity__AssetEntity_faces_personId", - "AssetEntity__AssetEntity_faces"."imageWidth" AS "AssetEntity__AssetEntity_faces_imageWidth", - "AssetEntity__AssetEntity_faces"."imageHeight" AS "AssetEntity__AssetEntity_faces_imageHeight", - "AssetEntity__AssetEntity_faces"."boundingBoxX1" AS "AssetEntity__AssetEntity_faces_boundingBoxX1", - "AssetEntity__AssetEntity_faces"."boundingBoxY1" AS "AssetEntity__AssetEntity_faces_boundingBoxY1", - "AssetEntity__AssetEntity_faces"."boundingBoxX2" AS "AssetEntity__AssetEntity_faces_boundingBoxX2", - "AssetEntity__AssetEntity_faces"."boundingBoxY2" AS "AssetEntity__AssetEntity_faces_boundingBoxY2", - "AssetEntity__AssetEntity_faces"."sourceType" AS "AssetEntity__AssetEntity_faces_sourceType", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."id" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_id", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."createdAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_createdAt", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."updatedAt" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_updatedAt", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."ownerId" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_ownerId", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."name" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_name", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."birthDate" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_birthDate", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."thumbnailPath" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_thumbnailPath", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."faceAssetId" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_faceAssetId", - "8258e303a73a72cf6abb13d73fb592dde0d68280"."isHidden" AS "8258e303a73a72cf6abb13d73fb592dde0d68280_isHidden", - "AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id", - "AssetEntity__AssetEntity_stack"."ownerId" AS "AssetEntity__AssetEntity_stack_ownerId", - "AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId", - "bd93d5747511a4dad4923546c51365bf1a803774"."id" AS "bd93d5747511a4dad4923546c51365bf1a803774_id", - "bd93d5747511a4dad4923546c51365bf1a803774"."deviceAssetId" AS "bd93d5747511a4dad4923546c51365bf1a803774_deviceAssetId", - "bd93d5747511a4dad4923546c51365bf1a803774"."ownerId" AS "bd93d5747511a4dad4923546c51365bf1a803774_ownerId", - "bd93d5747511a4dad4923546c51365bf1a803774"."libraryId" AS "bd93d5747511a4dad4923546c51365bf1a803774_libraryId", - "bd93d5747511a4dad4923546c51365bf1a803774"."deviceId" AS "bd93d5747511a4dad4923546c51365bf1a803774_deviceId", - "bd93d5747511a4dad4923546c51365bf1a803774"."type" AS "bd93d5747511a4dad4923546c51365bf1a803774_type", - "bd93d5747511a4dad4923546c51365bf1a803774"."status" AS "bd93d5747511a4dad4923546c51365bf1a803774_status", - "bd93d5747511a4dad4923546c51365bf1a803774"."originalPath" AS "bd93d5747511a4dad4923546c51365bf1a803774_originalPath", - "bd93d5747511a4dad4923546c51365bf1a803774"."thumbhash" AS "bd93d5747511a4dad4923546c51365bf1a803774_thumbhash", - "bd93d5747511a4dad4923546c51365bf1a803774"."encodedVideoPath" AS "bd93d5747511a4dad4923546c51365bf1a803774_encodedVideoPath", - "bd93d5747511a4dad4923546c51365bf1a803774"."createdAt" AS "bd93d5747511a4dad4923546c51365bf1a803774_createdAt", - "bd93d5747511a4dad4923546c51365bf1a803774"."updatedAt" AS "bd93d5747511a4dad4923546c51365bf1a803774_updatedAt", - "bd93d5747511a4dad4923546c51365bf1a803774"."deletedAt" AS "bd93d5747511a4dad4923546c51365bf1a803774_deletedAt", - "bd93d5747511a4dad4923546c51365bf1a803774"."fileCreatedAt" AS "bd93d5747511a4dad4923546c51365bf1a803774_fileCreatedAt", - "bd93d5747511a4dad4923546c51365bf1a803774"."localDateTime" AS "bd93d5747511a4dad4923546c51365bf1a803774_localDateTime", - "bd93d5747511a4dad4923546c51365bf1a803774"."fileModifiedAt" AS "bd93d5747511a4dad4923546c51365bf1a803774_fileModifiedAt", - "bd93d5747511a4dad4923546c51365bf1a803774"."isFavorite" AS "bd93d5747511a4dad4923546c51365bf1a803774_isFavorite", - "bd93d5747511a4dad4923546c51365bf1a803774"."isArchived" AS "bd93d5747511a4dad4923546c51365bf1a803774_isArchived", - "bd93d5747511a4dad4923546c51365bf1a803774"."isExternal" AS "bd93d5747511a4dad4923546c51365bf1a803774_isExternal", - "bd93d5747511a4dad4923546c51365bf1a803774"."isOffline" AS "bd93d5747511a4dad4923546c51365bf1a803774_isOffline", - "bd93d5747511a4dad4923546c51365bf1a803774"."checksum" AS "bd93d5747511a4dad4923546c51365bf1a803774_checksum", - "bd93d5747511a4dad4923546c51365bf1a803774"."duration" AS "bd93d5747511a4dad4923546c51365bf1a803774_duration", - "bd93d5747511a4dad4923546c51365bf1a803774"."isVisible" AS "bd93d5747511a4dad4923546c51365bf1a803774_isVisible", - "bd93d5747511a4dad4923546c51365bf1a803774"."livePhotoVideoId" AS "bd93d5747511a4dad4923546c51365bf1a803774_livePhotoVideoId", - "bd93d5747511a4dad4923546c51365bf1a803774"."originalFileName" AS "bd93d5747511a4dad4923546c51365bf1a803774_originalFileName", - "bd93d5747511a4dad4923546c51365bf1a803774"."sidecarPath" AS "bd93d5747511a4dad4923546c51365bf1a803774_sidecarPath", - "bd93d5747511a4dad4923546c51365bf1a803774"."stackId" AS "bd93d5747511a4dad4923546c51365bf1a803774_stackId", - "bd93d5747511a4dad4923546c51365bf1a803774"."duplicateId" AS "bd93d5747511a4dad4923546c51365bf1a803774_duplicateId", - "AssetEntity__AssetEntity_files"."id" AS "AssetEntity__AssetEntity_files_id", - "AssetEntity__AssetEntity_files"."assetId" AS "AssetEntity__AssetEntity_files_assetId", - "AssetEntity__AssetEntity_files"."createdAt" AS "AssetEntity__AssetEntity_files_createdAt", - "AssetEntity__AssetEntity_files"."updatedAt" AS "AssetEntity__AssetEntity_files_updatedAt", - "AssetEntity__AssetEntity_files"."type" AS "AssetEntity__AssetEntity_files_type", - "AssetEntity__AssetEntity_files"."path" AS "AssetEntity__AssetEntity_files_path" -FROM - "assets" "AssetEntity" - LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id" - LEFT JOIN "tag_asset" "AssetEntity_AssetEntity__AssetEntity_tags" ON "AssetEntity_AssetEntity__AssetEntity_tags"."assetsId" = "AssetEntity"."id" - LEFT JOIN "tags" "AssetEntity__AssetEntity_tags" ON "AssetEntity__AssetEntity_tags"."id" = "AssetEntity_AssetEntity__AssetEntity_tags"."tagsId" - LEFT JOIN "asset_faces" "AssetEntity__AssetEntity_faces" ON "AssetEntity__AssetEntity_faces"."assetId" = "AssetEntity"."id" - LEFT JOIN "person" "8258e303a73a72cf6abb13d73fb592dde0d68280" ON "8258e303a73a72cf6abb13d73fb592dde0d68280"."id" = "AssetEntity__AssetEntity_faces"."personId" - LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId" - LEFT JOIN "assets" "bd93d5747511a4dad4923546c51365bf1a803774" ON "bd93d5747511a4dad4923546c51365bf1a803774"."stackId" = "AssetEntity__AssetEntity_stack"."id" - LEFT JOIN "asset_files" "AssetEntity__AssetEntity_files" ON "AssetEntity__AssetEntity_files"."assetId" = "AssetEntity"."id" -WHERE - (("AssetEntity"."id" IN ($1))) +select + "assets".*, + ( + select + jsonb_agg( + case + when "person"."id" is not null then jsonb_insert( + to_jsonb("asset_faces"), + '{person}'::text[], + to_jsonb("person") + ) + else to_jsonb("asset_faces") + end + ) as "faces" + from + "asset_faces" + left join "person" on "person"."id" = "asset_faces"."personId" + where + "asset_faces"."assetId" = "assets"."id" + ) as "faces", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "tags".* + from + "tags" + inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId" + where + "assets"."id" = "tag_asset"."assetsId" + ) as agg + ) as "tags", + to_jsonb("exif") as "exifInfo", + to_jsonb("stacked_assets") as "stack" +from + "assets" + left join "exif" on "assets"."id" = "exif"."assetId" + left join lateral ( + select + "asset_stack".*, + "s"."assets" + from + "asset_stack" + inner join lateral ( + select + array_agg("stacked") as "assets" + from + "assets" as "stacked" + where + "asset_stack"."id" = "stacked"."stackId" + and "asset_stack"."primaryAssetId" != "stacked"."id" + ) as "s" on ( + "asset_stack"."primaryAssetId" = "assets"."id" + or "assets"."stackId" is null + ) + where + "assets"."stackId" = "asset_stack"."id" + ) as "stacked_assets" on true +where + "assets"."id" = any ($1::uuid []) -- AssetRepository.deleteAll -DELETE FROM "assets" -WHERE +delete from "assets" +where "ownerId" = $1 -- AssetRepository.getByLibraryIdAndOriginalPath -SELECT DISTINCT - "distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id" -FROM - ( - SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId" - FROM - "assets" "AssetEntity" - LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId" - WHERE - ( - ((("AssetEntity__AssetEntity_library"."id" = $1))) - AND ("AssetEntity"."originalPath" = $2) - ) - ) "distinctAlias" -ORDER BY - "AssetEntity_id" ASC -LIMIT - 1 - --- AssetRepository.getPathsNotInLibrary -WITH - paths AS ( - SELECT - unnest($2::text[]) AS path - ) -SELECT - path -FROM - paths -WHERE - NOT EXISTS ( - SELECT - 1 - FROM - assets - WHERE - "libraryId" = $1 - AND "originalPath" = path - ); +select + "assets".* +from + "assets" +where + "libraryId" = $1::uuid + and "originalPath" = $2 +limit + $3 -- AssetRepository.getAllByDeviceId -SELECT - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."id" AS "AssetEntity_id" -FROM - "assets" "AssetEntity" -WHERE - ( - ("AssetEntity"."ownerId" = $1) - AND ("AssetEntity"."deviceId" = $2) - AND ("AssetEntity"."isVisible" = $3) - ) +select + "deviceAssetId" +from + "assets" +where + "ownerId" = $1::uuid + and "deviceId" = $2 + and "isVisible" = $3 + and "deletedAt" is null -- AssetRepository.getLivePhotoCount -SELECT - COUNT(1) AS "cnt" -FROM - "assets" "AssetEntity" -WHERE - (("AssetEntity"."livePhotoVideoId" = $1)) +select + count(*) as "count" +from + "assets" +where + "livePhotoVideoId" = $1::uuid -- AssetRepository.getById -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId" -FROM - "assets" "AssetEntity" -WHERE - (("AssetEntity"."id" = $1)) -LIMIT - 1 +select + "assets".* +from + "assets" +where + "assets"."id" = $1::uuid +limit + $2 -- AssetRepository.updateAll -UPDATE "assets" -SET - "deviceId" = $1, - "updatedAt" = CURRENT_TIMESTAMP -WHERE - "id" IN ($2) +update "assets" +set + "deviceId" = $1 +where + "id" = any ($2::uuid []) -- AssetRepository.updateDuplicates -UPDATE "assets" -SET - "duplicateId" = $1, - "updatedAt" = CURRENT_TIMESTAMP -WHERE - "duplicateId" IN ($2) - OR "id" IN ($3) +update "assets" +set + "duplicateId" = $1 +where + ( + "duplicateId" = any ($2::uuid []) + or "id" = any ($3::uuid []) + ) -- AssetRepository.getByChecksum -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId" -FROM - "assets" "AssetEntity" -WHERE - ( - ( - ("AssetEntity"."ownerId" = $1) - AND ("AssetEntity"."libraryId" = $2) - AND ("AssetEntity"."checksum" = $3) - ) - ) - AND ("AssetEntity"."deletedAt" IS NULL) -LIMIT - 1 - --- AssetRepository.getByChecksums -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."checksum" AS "AssetEntity_checksum" -FROM - "assets" "AssetEntity" -WHERE - ( - ("AssetEntity"."ownerId" = $1) - AND ( - "AssetEntity"."checksum" IN ($2, $3, $4, $5, $6, $7, $8, $9, $10) - ) - ) +select + "assets".* +from + "assets" +where + "ownerId" = $1::uuid + and "checksum" = $2 + and "libraryId" = $3::uuid +limit + $4 -- AssetRepository.getUploadAssetIdByChecksum -SELECT - "AssetEntity"."id" AS "AssetEntity_id" -FROM - "assets" "AssetEntity" -WHERE - ( - ("AssetEntity"."ownerId" = $1) - AND ("AssetEntity"."checksum" = $2) - AND ("AssetEntity"."libraryId" IS NULL) - ) -LIMIT - 1 +select + "id" +from + "assets" +where + "ownerId" = $1::uuid + and "checksum" = $2 + and "libraryId" is null +limit + $3 -- AssetRepository.getWithout (sidecar) -SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."status" AS "AssetEntity_status", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId" -FROM - "assets" "AssetEntity" -WHERE +select + "assets".* +from + "assets" +where ( - ( - ( - ( - ("AssetEntity"."sidecarPath" IS NULL) - AND ("AssetEntity"."isVisible" = $1) - ) - ) - OR ( - ( - ("AssetEntity"."sidecarPath" = $2) - AND ("AssetEntity"."isVisible" = $3) - ) - ) - ) + "assets"."sidecarPath" = $1 + or "assets"."sidecarPath" is null ) - AND ("AssetEntity"."deletedAt" IS NULL) -ORDER BY - "AssetEntity"."createdAt" ASC -LIMIT - 11 - --- AssetRepository.getRandom -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) -WHERE - ( - "asset"."isVisible" = true - AND "asset"."ownerId" IN ($1) - ) - AND ("asset"."deletedAt" IS NULL) -ORDER BY - RANDOM() ASC -LIMIT - 50 + and "assets"."isVisible" = $2 + and "deletedAt" is null +order by + "createdAt" asc +limit + $3 +offset + $4 -- AssetRepository.getTimeBuckets -SELECT - COUNT("asset"."id")::int AS "count", - ( - date_trunc( - 'month', - (asset."localDateTime" at time zone 'UTC') - ) at time zone 'UTC' - )::timestamptz AS "timeBucket" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) -WHERE - ("asset"."isVisible" = true) - AND ("asset"."deletedAt" IS NULL) -GROUP BY - ( - date_trunc( - 'month', - (asset."localDateTime" at time zone 'UTC') - ) at time zone 'UTC' - )::timestamptz -ORDER BY - ( - date_trunc( - 'month', - (asset."localDateTime" at time zone 'UTC') - ) at time zone 'UTC' - )::timestamptz DESC +with + "assets" as ( + select + date_trunc($1, "localDateTime" at time zone 'UTC') as "timeBucket" + from + "assets" + where + "assets"."deletedAt" is null + and "assets"."isVisible" = $2 + ) +select + "timeBucket", + count(*) as "count" +from + "assets" +group by + "timeBucket" +order by + "timeBucket" desc -- AssetRepository.getTimeBucket -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) -WHERE - ( - "asset"."isVisible" = true - AND ( - date_trunc( - 'month', - (asset."localDateTime" at time zone 'UTC') - ) at time zone 'UTC' - )::timestamptz = $1 - ) - AND ("asset"."deletedAt" IS NULL) -ORDER BY - ( - date_trunc( - 'month', - (asset."localDateTime" at time zone 'UTC') - ) at time zone 'UTC' - )::timestamptz DESC, - "asset"."fileCreatedAt" DESC - --- AssetRepository.getDuplicates -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) -WHERE - ( - "asset"."isVisible" = true - AND "asset"."ownerId" IN ($1, $2) - AND "asset"."duplicateId" IS NOT NULL - ) - AND ("asset"."deletedAt" IS NULL) -ORDER BY - "asset"."duplicateId" ASC +select + "assets".*, + to_jsonb("exif") as "exifInfo" +from + "assets" + left join "exif" on "assets"."id" = "exif"."assetId" +where + "assets"."deletedAt" is null + and "assets"."isVisible" = $1 + and date_trunc($2, assets."localDateTime" at time zone 'UTC') = $3 +order by + "assets"."localDateTime" desc -- AssetRepository.getAssetIdByCity -WITH - "cities" AS ( - SELECT - city - FROM - "exif" "e" - GROUP BY - city - HAVING - count(city) >= $1 +with + "cities" as ( + select + "city" + from + "exif" + where + "city" is not null + group by + "city" + having + count("assetId") >= $1 ) -SELECT DISTINCT - ON (c.city) "asset"."id" AS "data", - c.city AS "value" -FROM - "assets" "asset" - INNER JOIN "exif" "e" ON "asset"."id" = e."assetId" - INNER JOIN "cities" "c" ON c.city = "e"."city" -WHERE - ( - "asset"."isVisible" = true - AND "asset"."type" = $2 - AND "asset"."ownerId" IN ($3) - AND "asset"."isArchived" = $4 - ) - AND ("asset"."deletedAt" IS NULL) -LIMIT - 12 +select distinct + on ("exif"."city") "assetId" as "data", + "exif"."city" as "value" +from + "assets" + inner join "exif" on "assets"."id" = "exif"."assetId" + inner join "cities" on "exif"."city" = "cities"."city" +where + "ownerId" = $2::uuid + and "isVisible" = $3 + and "isArchived" = $4 + and "type" = $5 + and "deletedAt" is null +limit + $6 -- AssetRepository.getAllForUserFullSync -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" -WHERE - "asset"."isVisible" = true - AND "asset"."ownerId" IN ($1) - AND "asset"."id" > $2 - AND "asset"."updatedAt" <= $3 -ORDER BY - "asset"."id" ASC -LIMIT - 10 - --- AssetRepository.getChangedDeltaSync -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" -WHERE - "asset"."isVisible" = true - AND "asset"."ownerId" IN ($1) - AND "asset"."updatedAt" > $2 - --- AssetRepository.upsertFile -INSERT INTO - "asset_files" ( - "id", - "assetId", - "createdAt", - "updatedAt", - "type", - "path" - ) -VALUES - (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3) -ON CONFLICT ("assetId", "type") DO -UPDATE -SET - "assetId" = EXCLUDED."assetId", - "type" = EXCLUDED."type", - "path" = EXCLUDED."path", - "updatedAt" = DEFAULT -RETURNING - "id", - "createdAt", - "updatedAt" - --- AssetRepository.upsertFiles -INSERT INTO - "asset_files" ( - "id", - "assetId", - "createdAt", - "updatedAt", - "type", - "path" - ) -VALUES - (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3) -ON CONFLICT ("assetId", "type") DO -UPDATE -SET - "assetId" = EXCLUDED."assetId", - "type" = EXCLUDED."type", - "path" = EXCLUDED."path", - "updatedAt" = DEFAULT -RETURNING - "id", - "createdAt", - "updatedAt" +select +from + "assets" +where + "ownerId" = $1::uuid + and "isVisible" = $2 + and "updatedAt" <= $3 + and "id" > $4 +order by + "id" asc +limit + $5 diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index 1084375059..19322e684b 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -1,641 +1,199 @@ -- NOTE: This file is auto generated by ./sql-generator -- SearchRepository.searchMetadata -SELECT DISTINCT - "distinctAlias"."asset_id" AS "ids_asset_id", - "distinctAlias"."asset_fileCreatedAt" -FROM - ( - SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" - FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) - WHERE - ( - "asset"."fileCreatedAt" >= $1 - AND "exifInfo"."lensModel" = $2 - AND 1 = 1 - AND "asset"."ownerId" IN ($3) - AND 1 = 1 - AND ( - "asset"."isFavorite" = $4 - AND "asset"."isArchived" = $5 - ) - ) - AND ("asset"."deletedAt" IS NULL) - ) "distinctAlias" -ORDER BY - "distinctAlias"."asset_fileCreatedAt" DESC, - "asset_id" ASC -LIMIT - 101 - --- SearchRepository.searchRandom -SELECT DISTINCT - "distinctAlias"."asset_id" AS "ids_asset_id", - "distinctAlias"."asset_id" -FROM - ( - SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" - FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) - WHERE - ( - "asset"."fileCreatedAt" >= $1 - AND "exifInfo"."lensModel" = $2 - AND 1 = 1 - AND "asset"."ownerId" IN ($3) - AND 1 = 1 - AND ( - "asset"."isFavorite" = $4 - AND "asset"."isArchived" = $5 - ) - AND "asset"."id" > $6 - ) - AND ("asset"."deletedAt" IS NULL) - ) "distinctAlias" -ORDER BY - "distinctAlias"."asset_id" ASC, - "asset_id" ASC -LIMIT - 100 -SELECT DISTINCT - "distinctAlias"."asset_id" AS "ids_asset_id", - "distinctAlias"."asset_id" -FROM - ( - SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" - FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) - WHERE - ( - "asset"."fileCreatedAt" >= $1 - AND "exifInfo"."lensModel" = $2 - AND 1 = 1 - AND "asset"."ownerId" IN ($3) - AND 1 = 1 - AND ( - "asset"."isFavorite" = $4 - AND "asset"."isArchived" = $5 - ) - AND "asset"."id" < $6 - ) - AND ("asset"."deletedAt" IS NULL) - ) "distinctAlias" -ORDER BY - "distinctAlias"."asset_id" ASC, - "asset_id" ASC -LIMIT - 100 +select + "assets".* +from + "assets" + inner join "exif" on "assets"."id" = "exif"."assetId" +where + "assets"."fileCreatedAt" >= $1 + and "exif"."lensModel" = $2 + and "assets"."ownerId" = any ($3::uuid []) + and "assets"."isFavorite" = $4 + and "assets"."isArchived" = $5 + and "assets"."deletedAt" is null +order by + "assets"."fileCreatedAt" desc +limit + $6 +offset + $7 -- SearchRepository.searchSmart -START TRANSACTION -SET - LOCAL vectors.hnsw_ef_search = 200; -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "stack"."id" AS "stack_id", - "stack"."ownerId" AS "stack_ownerId", - "stack"."primaryAssetId" AS "stack_primaryAssetId", - "stackedAssets"."id" AS "stackedAssets_id", - "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", - "stackedAssets"."ownerId" AS "stackedAssets_ownerId", - "stackedAssets"."libraryId" AS "stackedAssets_libraryId", - "stackedAssets"."deviceId" AS "stackedAssets_deviceId", - "stackedAssets"."type" AS "stackedAssets_type", - "stackedAssets"."status" AS "stackedAssets_status", - "stackedAssets"."originalPath" AS "stackedAssets_originalPath", - "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", - "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", - "stackedAssets"."createdAt" AS "stackedAssets_createdAt", - "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", - "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", - "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", - "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", - "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", - "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", - "stackedAssets"."isArchived" AS "stackedAssets_isArchived", - "stackedAssets"."isExternal" AS "stackedAssets_isExternal", - "stackedAssets"."isOffline" AS "stackedAssets_isOffline", - "stackedAssets"."checksum" AS "stackedAssets_checksum", - "stackedAssets"."duration" AS "stackedAssets_duration", - "stackedAssets"."isVisible" AS "stackedAssets_isVisible", - "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", - "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", - "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", - "stackedAssets"."stackId" AS "stackedAssets_stackId", - "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" - LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" - LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" - AND ("stackedAssets"."deletedAt" IS NULL) - INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id" -WHERE - ( - "asset"."fileCreatedAt" >= $1 - AND "exifInfo"."lensModel" = $2 - AND 1 = 1 - AND 1 = 1 - AND ( - "asset"."isFavorite" = $3 - AND "asset"."isArchived" = $4 - ) - AND "asset"."ownerId" IN ($5) - ) - AND ("asset"."deletedAt" IS NULL) -ORDER BY - "search"."embedding" <= > $6 ASC -LIMIT - 201 -COMMIT - --- SearchRepository.searchDuplicates -WITH - "cte" AS ( - SELECT - "asset"."duplicateId" AS "duplicateId", - "search"."assetId" AS "assetId", - "search"."embedding" <= > $1 AS "distance" - FROM - "assets" "asset" - INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id" - WHERE - ( - "asset"."ownerId" IN ($2) - AND "asset"."id" != $3 - AND "asset"."isVisible" = $4 - AND "asset"."type" = $5 - ) - AND ("asset"."deletedAt" IS NULL) - ORDER BY - "search"."embedding" <= > $1 ASC - LIMIT - 64 - ) -SELECT - res.* -FROM - "cte" "res" -WHERE - res.distance <= $6 +select + "assets".* +from + "assets" + inner join "exif" on "assets"."id" = "exif"."assetId" + inner join "smart_search" on "assets"."id" = "smart_search"."assetId" +where + "assets"."fileCreatedAt" >= $1 + and "exif"."lensModel" = $2 + and "assets"."ownerId" = any ($3::uuid []) + and "assets"."isFavorite" = $4 + and "assets"."isArchived" = $5 + and "assets"."deletedAt" is null +order by + smart_search.embedding <= > $6::vector +limit + $7 +offset + $8 -- SearchRepository.searchFaces -START TRANSACTION -SET - LOCAL vectors.hnsw_ef_search = 100; -WITH - "cte" AS ( - SELECT - "faces"."id" AS "id", - "faces"."assetId" AS "assetId", - "faces"."personId" AS "personId", - "faces"."imageWidth" AS "imageWidth", - "faces"."imageHeight" AS "imageHeight", - "faces"."boundingBoxX1" AS "boundingBoxX1", - "faces"."boundingBoxY1" AS "boundingBoxY1", - "faces"."boundingBoxX2" AS "boundingBoxX2", - "faces"."boundingBoxY2" AS "boundingBoxY2", - "faces"."sourceType" AS "sourceType", - "search"."embedding" <= > $1 AS "distance" - FROM - "asset_faces" "faces" - INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId" - AND ("asset"."deletedAt" IS NULL) - INNER JOIN "face_search" "search" ON "search"."faceId" = "faces"."id" - WHERE - "asset"."ownerId" IN ($2) - ORDER BY - "search"."embedding" <= > $1 ASC - LIMIT - 64 +with + "cte" as ( + select + "asset_faces"."id", + "asset_faces"."personId", + face_search.embedding <= > $1::vector as "distance" + from + "asset_faces" + inner join "assets" on "assets"."id" = "asset_faces"."assetId" + inner join "face_search" on "face_search"."faceId" = "asset_faces"."id" + where + "assets"."ownerId" = any ($2::uuid []) + and "assets"."deletedAt" is null + order by + face_search.embedding <= > $3::vector + limit + $4 ) -SELECT - res.* -FROM - "cte" "res" -WHERE - res.distance <= $3 -ORDER BY - res.distance ASC -COMMIT +select + * +from + "cte" +where + "cte"."distance" <= $5 -- SearchRepository.searchPlaces -SELECT - "geoplaces"."id" AS "geoplaces_id", - "geoplaces"."name" AS "geoplaces_name", - "geoplaces"."longitude" AS "geoplaces_longitude", - "geoplaces"."latitude" AS "geoplaces_latitude", - "geoplaces"."countryCode" AS "geoplaces_countryCode", - "geoplaces"."admin1Code" AS "geoplaces_admin1Code", - "geoplaces"."admin2Code" AS "geoplaces_admin2Code", - "geoplaces"."admin1Name" AS "geoplaces_admin1Name", - "geoplaces"."admin2Name" AS "geoplaces_admin2Name", - "geoplaces"."alternateNames" AS "geoplaces_alternateNames", - "geoplaces"."modificationDate" AS "geoplaces_modificationDate" -FROM - "geodata_places" "geoplaces" -WHERE +select + * +from + "geodata_places" +where f_unaccent (name) %>> f_unaccent ($1) - OR f_unaccent ("admin2Name") %>> f_unaccent ($1) - OR f_unaccent ("admin1Name") %>> f_unaccent ($1) - OR f_unaccent ("alternateNames") %>> f_unaccent ($1) -ORDER BY - COALESCE(f_unaccent (name) <->>> f_unaccent ($1), 0.1) + COALESCE( - f_unaccent ("admin2Name") <->>> f_unaccent ($1), + or f_unaccent ("admin2Name") %>> f_unaccent ($2) + or f_unaccent ("admin1Name") %>> f_unaccent ($3) + or f_unaccent ("alternateNames") %>> f_unaccent ($4) +order by + coalesce(f_unaccent (name) <->>> f_unaccent ($5), 0.1) + coalesce( + f_unaccent ("admin2Name") <->>> f_unaccent ($6), 0.1 - ) + COALESCE( - f_unaccent ("admin1Name") <->>> f_unaccent ($1), + ) + coalesce( + f_unaccent ("admin1Name") <->>> f_unaccent ($7), 0.1 - ) + COALESCE( - f_unaccent ("alternateNames") <->>> f_unaccent ($1), + ) + coalesce( + f_unaccent ("alternateNames") <->>> f_unaccent ($8), 0.1 - ) ASC -LIMIT - 20 + ) +limit + $9 -- SearchRepository.getAssetsByCity -WITH RECURSIVE - cte AS ( +with recursive + "cte" as ( ( - SELECT - city, + select + "city", "assetId" - FROM - exif - INNER JOIN assets ON exif."assetId" = assets.id - WHERE - "ownerId" = ANY ($1::uuid []) - AND "isVisible" = $2 - AND "isArchived" = $3 - AND type = $4 - ORDER BY - city - LIMIT - 1 + from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" + where + "assets"."ownerId" = any ($1::uuid []) + and "assets"."isVisible" = $2 + and "assets"."isArchived" = $3 + and "assets"."type" = $4 + and "assets"."deletedAt" is null + order by + "city" + limit + $5 + ) + union all + ( + select + "l"."city", + "l"."assetId" + from + "cte" + inner join lateral ( + select + "city", + "assetId" + from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" + where + "assets"."ownerId" = any ($6::uuid []) + and "assets"."isVisible" = $7 + and "assets"."isArchived" = $8 + and "assets"."type" = $9 + and "assets"."deletedAt" is null + and "exif"."city" > "cte"."city" + order by + "city" + limit + $10 + ) as "l" on true ) - UNION ALL - SELECT - l.city, - l."assetId" - FROM - cte c, - LATERAL ( - SELECT - city, - "assetId" - FROM - exif - INNER JOIN assets ON exif."assetId" = assets.id - WHERE - city > c.city - AND "ownerId" = ANY ($1::uuid []) - AND "isVisible" = $2 - AND "isArchived" = $3 - AND type = $4 - ORDER BY - city - LIMIT - 1 - ) l ) -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exif"."assetId" AS "exif_assetId", - "exif"."description" AS "exif_description", - "exif"."exifImageWidth" AS "exif_exifImageWidth", - "exif"."exifImageHeight" AS "exif_exifImageHeight", - "exif"."fileSizeInByte" AS "exif_fileSizeInByte", - "exif"."orientation" AS "exif_orientation", - "exif"."dateTimeOriginal" AS "exif_dateTimeOriginal", - "exif"."modifyDate" AS "exif_modifyDate", - "exif"."timeZone" AS "exif_timeZone", - "exif"."latitude" AS "exif_latitude", - "exif"."longitude" AS "exif_longitude", - "exif"."projectionType" AS "exif_projectionType", - "exif"."city" AS "exif_city", - "exif"."livePhotoCID" AS "exif_livePhotoCID", - "exif"."autoStackId" AS "exif_autoStackId", - "exif"."state" AS "exif_state", - "exif"."country" AS "exif_country", - "exif"."make" AS "exif_make", - "exif"."model" AS "exif_model", - "exif"."lensModel" AS "exif_lensModel", - "exif"."fNumber" AS "exif_fNumber", - "exif"."focalLength" AS "exif_focalLength", - "exif"."iso" AS "exif_iso", - "exif"."exposureTime" AS "exif_exposureTime", - "exif"."profileDescription" AS "exif_profileDescription", - "exif"."colorspace" AS "exif_colorspace", - "exif"."bitsPerSample" AS "exif_bitsPerSample", - "exif"."rating" AS "exif_rating", - "exif"."fps" AS "exif_fps" -FROM - "assets" "asset" - INNER JOIN "exif" "exif" ON "exif"."assetId" = "asset"."id" - INNER JOIN cte ON asset.id = cte."assetId" -ORDER BY - exif.city - --- SearchRepository.getCountries -SELECT DISTINCT - ON ("exif"."country") "exif"."country" AS "country" -FROM - "exif" "exif" - INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" IN ($1) - AND "exif"."country" != '' - AND "exif"."country" IS NOT NULL +select + "assets".*, + to_jsonb("exif") as "exifInfo" +from + "assets" + inner join "exif" on "assets"."id" = "exif"."assetId" + inner join "cte" on "assets"."id" = "cte"."assetId" +order by + "exif"."city" -- SearchRepository.getStates -SELECT DISTINCT - ON ("exif"."state") "exif"."state" AS "state" -FROM - "exif" "exif" - INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" IN ($1) - AND "exif"."state" != '' - AND "exif"."state" IS NOT NULL +select distinct + on ("state") "state" +from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" +where + "ownerId" = any ($1::uuid []) + and "isVisible" = $2 + and "deletedAt" is null + and "state" is not null -- SearchRepository.getCities -SELECT DISTINCT - ON ("exif"."city") "exif"."city" AS "city" -FROM - "exif" "exif" - INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" IN ($1) - AND "exif"."city" != '' - AND "exif"."city" IS NOT NULL +select distinct + on ("city") "city" +from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" +where + "ownerId" = any ($1::uuid []) + and "isVisible" = $2 + and "deletedAt" is null + and "city" is not null -- SearchRepository.getCameraMakes -SELECT DISTINCT - ON ("exif"."make") "exif"."make" AS "make" -FROM - "exif" "exif" - INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" IN ($1) - AND "exif"."make" != '' - AND "exif"."make" IS NOT NULL +select distinct + on ("make") "make" +from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" +where + "ownerId" = any ($1::uuid []) + and "isVisible" = $2 + and "deletedAt" is null + and "make" is not null -- SearchRepository.getCameraModels -SELECT DISTINCT - ON ("exif"."model") "exif"."model" AS "model" -FROM - "exif" "exif" - INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" IN ($1) - AND "exif"."model" != '' - AND "exif"."model" IS NOT NULL +select distinct + on ("model") "model" +from + "exif" + inner join "assets" on "assets"."id" = "exif"."assetId" +where + "ownerId" = any ($1::uuid []) + and "isVisible" = $2 + and "deletedAt" is null + and "model" is not null diff --git a/server/src/queries/view.repository.sql b/server/src/queries/view.repository.sql index e5b88ffef9..948f60fd4d 100644 --- a/server/src/queries/view.repository.sql +++ b/server/src/queries/view.repository.sql @@ -1,79 +1,29 @@ -- NOTE: This file is auto generated by ./sql-generator +-- ViewRepository.getUniqueOriginalPaths +select distinct + substring("assets"."originalPath", $1) as "directoryPath" +from + "assets" +where + "ownerId" = $2::uuid + and "isVisible" = $3 + and "isArchived" = $4 + and "deletedAt" is null + -- ViewRepository.getAssetsByOriginalPath -SELECT - "asset"."id" AS "asset_id", - "asset"."deviceAssetId" AS "asset_deviceAssetId", - "asset"."ownerId" AS "asset_ownerId", - "asset"."libraryId" AS "asset_libraryId", - "asset"."deviceId" AS "asset_deviceId", - "asset"."type" AS "asset_type", - "asset"."status" AS "asset_status", - "asset"."originalPath" AS "asset_originalPath", - "asset"."thumbhash" AS "asset_thumbhash", - "asset"."encodedVideoPath" AS "asset_encodedVideoPath", - "asset"."createdAt" AS "asset_createdAt", - "asset"."updatedAt" AS "asset_updatedAt", - "asset"."deletedAt" AS "asset_deletedAt", - "asset"."fileCreatedAt" AS "asset_fileCreatedAt", - "asset"."localDateTime" AS "asset_localDateTime", - "asset"."fileModifiedAt" AS "asset_fileModifiedAt", - "asset"."isFavorite" AS "asset_isFavorite", - "asset"."isArchived" AS "asset_isArchived", - "asset"."isExternal" AS "asset_isExternal", - "asset"."isOffline" AS "asset_isOffline", - "asset"."checksum" AS "asset_checksum", - "asset"."duration" AS "asset_duration", - "asset"."isVisible" AS "asset_isVisible", - "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", - "asset"."originalFileName" AS "asset_originalFileName", - "asset"."sidecarPath" AS "asset_sidecarPath", - "asset"."stackId" AS "asset_stackId", - "asset"."duplicateId" AS "asset_duplicateId", - "exifInfo"."assetId" AS "exifInfo_assetId", - "exifInfo"."description" AS "exifInfo_description", - "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", - "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", - "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", - "exifInfo"."orientation" AS "exifInfo_orientation", - "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", - "exifInfo"."modifyDate" AS "exifInfo_modifyDate", - "exifInfo"."timeZone" AS "exifInfo_timeZone", - "exifInfo"."latitude" AS "exifInfo_latitude", - "exifInfo"."longitude" AS "exifInfo_longitude", - "exifInfo"."projectionType" AS "exifInfo_projectionType", - "exifInfo"."city" AS "exifInfo_city", - "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", - "exifInfo"."autoStackId" AS "exifInfo_autoStackId", - "exifInfo"."state" AS "exifInfo_state", - "exifInfo"."country" AS "exifInfo_country", - "exifInfo"."make" AS "exifInfo_make", - "exifInfo"."model" AS "exifInfo_model", - "exifInfo"."lensModel" AS "exifInfo_lensModel", - "exifInfo"."fNumber" AS "exifInfo_fNumber", - "exifInfo"."focalLength" AS "exifInfo_focalLength", - "exifInfo"."iso" AS "exifInfo_iso", - "exifInfo"."exposureTime" AS "exifInfo_exposureTime", - "exifInfo"."profileDescription" AS "exifInfo_profileDescription", - "exifInfo"."colorspace" AS "exifInfo_colorspace", - "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", - "exifInfo"."rating" AS "exifInfo_rating", - "exifInfo"."fps" AS "exifInfo_fps" -FROM - "assets" "asset" - LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" -WHERE - ( - ( - "asset"."isVisible" = $1 - AND "asset"."isArchived" = $2 - AND "asset"."ownerId" = $3 - ) - AND ( - "asset"."originalPath" LIKE $4 - AND "asset"."originalPath" NOT LIKE $5 - ) - ) - AND ("asset"."deletedAt" IS NULL) -ORDER BY - regexp_replace("asset"."originalPath", '.*/(.+)', '\1') ASC +select + "assets".*, + to_jsonb("exif") as "exifInfo" +from + "assets" + left join "exif" on "assets"."id" = "exif"."assetId" +where + "ownerId" = $1::uuid + and "isVisible" = $2 + and "isArchived" = $3 + and "deletedAt" is null + and "originalPath" like $4 + and "originalPath" not like $5 +order by + regexp_replace("assets"."originalPath", $6, $7) asc diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 33d1e2457e..a5c0e2268c 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1,23 +1,39 @@ import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { Insertable, Kysely, Updateable, sql } from 'kysely'; +import { isEmpty, isUndefined, omitBy } from 'lodash'; +import { InjectKysely } from 'nestjs-kysely'; +import { ASSET_FILE_CONFLICT_KEYS, EXIF_CONFLICT_KEYS, JOB_STATUS_CONFLICT_KEYS } from 'src/constants'; +import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; -import { AssetFileEntity } from 'src/entities/asset-files.entity'; -import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { ExifEntity } from 'src/entities/exif.entity'; -import { AssetFileType, AssetOrder, AssetStatus, AssetType, PaginationMode } from 'src/enum'; import { - AssetBuilderOptions, - AssetCreate, + AssetEntity, + hasPeople, + hasPeopleCte, + searchAssetBuilder, + withAlbums, + withExif, + withFaces, + withFacesAndPeople, + withFiles, + withLibrary, + withOwner, + withSmartSearch, + withStack, + withTags, +} from 'src/entities/asset.entity'; +import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; +import { AssetDeltaSyncOptions, AssetExploreFieldOptions, AssetFullSyncOptions, + AssetGetByChecksumOptions, AssetStats, AssetStatsOptions, - AssetUpdateAllOptions, AssetUpdateDuplicateOptions, - AssetUpdateOptions, DayOfYearAssets, + DuplicateGroup, + GetByIdsRelations, IAssetRepository, LivePhotoSearchOptions, MonthDay, @@ -27,155 +43,169 @@ import { WithProperty, WithoutProperty, } from 'src/interfaces/asset.interface'; -import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; -import { searchAssetBuilder } from 'src/utils/database'; -import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination'; -import { - Brackets, - FindOptionsOrder, - FindOptionsRelations, - FindOptionsSelect, - FindOptionsWhere, - In, - IsNull, - MoreThan, - Not, - Repository, -} from 'typeorm'; - -const truncateMap: Record = { - [TimeBucketSize.DAY]: 'day', - [TimeBucketSize.MONTH]: 'month', -}; - -const dateTrunc = (options: TimeBucketOptions) => - `(date_trunc('${ - truncateMap[options.size] - }', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`; +import { MapMarker, MapMarkerSearchOptions } from 'src/interfaces/map.interface'; +import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/interfaces/search.interface'; +import { anyUuid, asUuid, introspectTables, mapUpsertColumns } from 'src/utils/database'; +import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; +import { DataSource } from 'typeorm'; @Injectable() export class AssetRepository implements IAssetRepository { constructor( - @InjectRepository(AssetEntity) private repository: Repository, - @InjectRepository(AssetFileEntity) private fileRepository: Repository, - @InjectRepository(ExifEntity) private exifRepository: Repository, - @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, - ) {} - - async upsertExif(exif: Partial): Promise { - await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] }); + @InjectKysely() private db: Kysely, + @InjectDataSource() private dataSource: DataSource, + ) { + introspectTables(this.dataSource, 'exif', 'asset_job_status', 'asset_files'); } - async upsertJobStatus(...jobStatus: Partial[]): Promise { - await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] }); - } - - create(asset: AssetCreate): Promise { - return this.repository.save(asset); - } - - @GenerateSql({ params: [[DummyValue.UUID], { day: 1, month: 1 }] }) - async getByDayOfYear(ownerIds: string[], { day, month }: MonthDay): Promise { - const assets = await this.repository - .createQueryBuilder('entity') - .where( - `entity.ownerId IN (:...ownerIds) - AND entity.isVisible = true - AND entity.isArchived = false - AND EXTRACT(DAY FROM entity.localDateTime AT TIME ZONE 'UTC') = :day - AND EXTRACT(MONTH FROM entity.localDateTime AT TIME ZONE 'UTC') = :month`, - { - ownerIds, - day, - month, - }, + async upsertExif(exif: Insertable): Promise { + const value = { ...exif, assetId: asUuid(exif.assetId) }; + await this.db + .insertInto('exif') + .values(value) + .onConflict((oc) => + oc.columns(EXIF_CONFLICT_KEYS).doUpdateSet(() => mapUpsertColumns('exif', value, EXIF_CONFLICT_KEYS)), ) - .leftJoinAndSelect('entity.exifInfo', 'exifInfo') - .innerJoinAndSelect('entity.files', 'files') - .andWhere('files.type = :type', { type: AssetFileType.THUMBNAIL }) - .andWhere( - `EXTRACT(YEAR FROM CURRENT_DATE AT TIME ZONE 'UTC') - EXTRACT(YEAR FROM entity.localDateTime AT TIME ZONE 'UTC') > 0`, - ) - .orderBy('entity.fileCreatedAt', 'ASC') - .getMany(); + .execute(); + } - const groups: Record = {}; - const currentYear = new Date().getFullYear(); - for (const asset of assets) { - const yearsAgo = currentYear - asset.localDateTime.getFullYear(); - if (!groups[yearsAgo]) { - groups[yearsAgo] = { yearsAgo, assets: [] }; - } - groups[yearsAgo].assets.push(asset); + async upsertJobStatus(...jobStatus: Insertable[]): Promise { + if (jobStatus.length === 0) { + return; } - return Object.values(groups); + const values = jobStatus.map((row) => ({ ...row, assetId: asUuid(row.assetId) })); + await this.db + .insertInto('asset_job_status') + .values(values) + .onConflict((oc) => + oc + .columns(JOB_STATUS_CONFLICT_KEYS) + .doUpdateSet(() => mapUpsertColumns('asset_job_status', values[0], JOB_STATUS_CONFLICT_KEYS)), + ) + .execute(); + } + + create(asset: Insertable): Promise { + return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise; + } + + @GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] }) + getByDayOfYear(ownerIds: string[], { day, month }: MonthDay): Promise { + // TODO: CREATE INDEX idx_local_date_time ON public.assets ((("localDateTime" at time zone 'UTC')::date)); + // TODO: drop IDX_day_of_month and IDX_month + return this.db + .with('res', (qb) => + qb + .with('today', (qb) => + qb + .selectFrom((eb) => + eb.fn('generate_series', [eb.val(1970), sql`extract(year from current_date) - 1`]).as('year'), + ) + .select((eb) => eb.fn('make_date', [sql`year::int`, sql`${month}::int`, sql`${day}::int`]).as('date')), + ) + .selectFrom('today') + .innerJoinLateral( + (qb) => + qb + .selectFrom('assets') + .selectAll('assets') + .innerJoin('asset_job_status', 'assets.id', 'asset_job_status.assetId') + .where('asset_job_status.previewAt', 'is not', null) + .where(sql`(assets."localDateTime" at time zone 'UTC')::date`, '=', sql`today.date`) + .where('assets.ownerId', '=', anyUuid(ownerIds)) + .where('assets.isVisible', '=', true) + .where('assets.isArchived', '=', false) + .where((eb) => + eb.exists((qb) => + qb + .selectFrom('asset_files') + .whereRef('assetId', '=', 'assets.id') + .where('asset_files.type', '=', AssetFileType.PREVIEW), + ), + ) + .where('assets.deletedAt', 'is', null) + .limit(10) + .as('a'), + (join) => join.onTrue(), + ) + .innerJoin('exif', 'a.id', 'exif.assetId') + .selectAll('a') + .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')), + ) + .selectFrom('res') + .select( + sql`((now() at time zone 'UTC')::date - ("localDateTime" at time zone 'UTC')::date) / 365`.as( + 'yearsAgo', + ), + ) + .select((eb) => eb.fn('jsonb_agg', [eb.table('res')]).as('assets')) + .groupBy(sql`("localDateTime" at time zone 'UTC')::date`) + .orderBy(sql`("localDateTime" at time zone 'UTC')::date`, 'desc') + .limit(10) + .execute() as any as Promise; } @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIds( + async getByIds( ids: string[], - relations?: FindOptionsRelations, - select?: FindOptionsSelect, + { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, ): Promise { - return this.repository.find({ - where: { id: In(ids) }, - relations, - select, - withDeleted: true, - }); + const res = await this.db + .selectFrom('assets') + .selectAll('assets') + .where('assets.id', '=', anyUuid(ids)) + .$if(!!exifInfo, withExif) + .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces)) + .$if(!!files, (qb) => qb.select(withFiles)) + .$if(!!library, (qb) => qb.select(withLibrary)) + .$if(!!owner, (qb) => qb.select(withOwner)) + .$if(!!smartSearch, (qb) => withSmartSearch(qb)) + .$if(!!stack, (qb) => withStack(qb, { assets: stack!.assets })) + .$if(!!tags, (qb) => qb.select(withTags)) + .execute(); + + return res as any as AssetEntity[]; } @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() getByIdsWithAllRelations(ids: string[]): Promise { - return this.repository.find({ - where: { id: In(ids) }, - relations: { - exifInfo: true, - tags: true, - faces: { - person: true, - }, - stack: { - assets: true, - }, - files: true, - }, - withDeleted: true, - }); + return this.db + .selectFrom('assets') + .selectAll('assets') + .select(withFacesAndPeople) + .select(withTags) + .$call(withExif) + .$call((qb) => withStack(qb, { assets: true })) + .where('assets.id', '=', anyUuid(ids)) + .execute() as any as Promise; } @GenerateSql({ params: [DummyValue.UUID] }) async deleteAll(ownerId: string): Promise { - await this.repository.delete({ ownerId }); + await this.db.deleteFrom('assets').where('ownerId', '=', ownerId).execute(); } - getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated { - return paginate(this.repository, pagination, { - where: { - albums: { - id: albumId, - }, - }, - relations: { - albums: true, - exifInfo: true, - }, - }); + async getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated { + const items = await withAlbums(this.db.selectFrom('assets'), { albumId }) + .selectAll('assets') + .where('deletedAt', 'is', null) + .orderBy('fileCreatedAt', 'desc') + .execute(); + + return paginationHelper(items as any as AssetEntity[], pagination.take); } async getByDeviceIds(ownerId: string, deviceId: string, deviceAssetIds: string[]): Promise { - const assets = await this.repository.find({ - select: { deviceAssetId: true }, - where: { - deviceAssetId: In(deviceAssetIds), - deviceId, - ownerId, - }, - withDeleted: true, - }); + const assets = await this.db + .selectFrom('assets') + .select(['deviceAssetId']) + .where('deviceAssetId', 'in', deviceAssetIds) + .where('deviceId', '=', deviceId) + .where('ownerId', '=', asUuid(ownerId)) + .execute(); return assets.map((asset) => asset.deviceAssetId); } @@ -189,37 +219,27 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise { - return this.repository.findOne({ - where: { library: { id: libraryId }, originalPath }, - withDeleted: true, - }); + getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .where('libraryId', '=', asUuid(libraryId)) + .where('originalPath', '=', originalPath) + .limit(1) + .executeTakeFirst() as any as Promise; } - @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] }) - @ChunkedArray({ paramIndex: 1 }) - async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise { - const result = await this.repository.query( - ` - WITH paths AS (SELECT unnest($2::text[]) AS path) - SELECT path - FROM paths - WHERE NOT EXISTS (SELECT 1 FROM assets WHERE "libraryId" = $1 AND "originalPath" = path); - `, - [libraryId, originalPaths], - ); - return result.map((row: { path: string }) => row.path); - } - - getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated { - let builder = this.repository.createQueryBuilder('asset').leftJoinAndSelect('asset.files', 'files'); - builder = searchAssetBuilder(builder, options); - builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC'); - return paginatedBuilder(builder, { - mode: PaginationMode.SKIP_TAKE, - skip: pagination.skip, - take: pagination.take, - }); + async getAll( + pagination: PaginationOptions, + { orderDirection, ...options }: AssetSearchOptions = {}, + ): Paginated { + const builder = searchAssetBuilder(this.db, options) + .select(withFiles) + .orderBy('assets.createdAt', orderDirection ?? 'asc') + .limit(pagination.take + 1) + .offset(pagination.skip ?? 0); + const items = await builder.execute(); + return paginationHelper(items as any as AssetEntity[], pagination.take); } /** @@ -231,140 +251,139 @@ export class AssetRepository implements IAssetRepository { */ @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) async getAllByDeviceId(ownerId: string, deviceId: string): Promise { - const items = await this.repository.find({ - select: { deviceAssetId: true }, - where: { - ownerId, - deviceId, - isVisible: true, - }, - withDeleted: true, - }); + const items = await this.db + .selectFrom('assets') + .select(['deviceAssetId']) + .where('ownerId', '=', asUuid(ownerId)) + .where('deviceId', '=', deviceId) + .where('isVisible', '=', true) + .where('deletedAt', 'is', null) + .execute(); return items.map((asset) => asset.deviceAssetId); } @GenerateSql({ params: [DummyValue.UUID] }) - getLivePhotoCount(motionId: string): Promise { - return this.repository.count({ - where: { - livePhotoVideoId: motionId, - }, - withDeleted: true, - }); + async getLivePhotoCount(motionId: string): Promise { + const [{ count }] = await this.db + .selectFrom('assets') + .select((eb) => eb.fn.countAll().as('count')) + .where('livePhotoVideoId', '=', asUuid(motionId)) + .execute(); + return count as number; } @GenerateSql({ params: [DummyValue.UUID] }) getById( id: string, - relations: FindOptionsRelations, - order?: FindOptionsOrder, - ): Promise { - return this.repository.findOne({ - where: { id }, - relations, - // We are specifically asking for this asset. Return it even if it is soft deleted - withDeleted: true, - order, - }); + { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, + ): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .where('assets.id', '=', asUuid(id)) + .$if(!!exifInfo, withExif) + .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces)) + .$if(!!library, (qb) => qb.select(withLibrary)) + .$if(!!owner, (qb) => qb.select(withOwner)) + .$if(!!smartSearch, withSmartSearch) + .$if(!!stack, (qb) => withStack(qb, { assets: stack!.assets })) + .$if(!!files, (qb) => qb.select(withFiles)) + .$if(!!tags, (qb) => qb.select(withTags)) + .limit(1) + .executeTakeFirst() as any as Promise; } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) @Chunked() - async updateAll(ids: string[], options: AssetUpdateAllOptions): Promise { - await this.repository.update({ id: In(ids) }, options); + async updateAll(ids: string[], options: Updateable): Promise { + if (ids.length === 0) { + return; + } + await this.db.updateTable('assets').set(options).where('id', '=', anyUuid(ids)).execute(); } @GenerateSql({ params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }], }) async updateDuplicates(options: AssetUpdateDuplicateOptions): Promise { - await this.repository - .createQueryBuilder() - .update() + await this.db + .updateTable('assets') .set({ duplicateId: options.targetDuplicateId }) - .where({ - duplicateId: In(options.duplicateIds), - }) - .orWhere({ id: In(options.assetIds) }) + .where((eb) => + eb.or([eb('duplicateId', '=', anyUuid(options.duplicateIds)), eb('id', '=', anyUuid(options.assetIds))]), + ) .execute(); } - async update(asset: AssetUpdateOptions): Promise { - await this.repository.update(asset.id, asset); + async update(asset: Updateable & { id: string }): Promise { + const value = omitBy(asset, isUndefined); + delete value.id; + if (!isEmpty(value)) { + return this.db + .with('assets', (qb) => qb.updateTable('assets').set(asset).where('id', '=', asUuid(asset.id)).returningAll()) + .selectFrom('assets') + .selectAll('assets') + .$call(withExif) + .$call((qb) => qb.select(withFacesAndPeople)) + .executeTakeFirst() as Promise; + } + + return this.getById(asset.id, { exifInfo: true, faces: { person: true } }) as Promise; } async remove(asset: AssetEntity): Promise { - await this.repository.remove(asset); + await this.db.deleteFrom('assets').where('id', '=', asUuid(asset.id)).execute(); } @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] }) - getByChecksum({ - ownerId, - libraryId, - checksum, - }: { - ownerId: string; - checksum: Buffer; - libraryId?: string; - }): Promise { - return this.repository.findOne({ - where: { - ownerId, - libraryId: libraryId || IsNull(), - checksum, - }, - }); + getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .where('ownerId', '=', asUuid(ownerId)) + .where('checksum', '=', checksum) + .$call((qb) => (libraryId ? qb.where('libraryId', '=', asUuid(libraryId)) : qb.where('libraryId', 'is', null))) + .limit(1) + .executeTakeFirst() as Promise; } @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) - getByChecksums(ownerId: string, checksums: Buffer[]): Promise { - return this.repository.find({ - select: { - id: true, - checksum: true, - deletedAt: true, - }, - where: { - ownerId, - checksum: In(checksums), - }, - withDeleted: true, - }); + getByChecksums(userId: string, checksums: Buffer[]): Promise { + return this.db + .selectFrom('assets') + .select(['id', 'checksum', 'deletedAt']) + .where('ownerId', '=', asUuid(userId)) + .where('checksum', 'in', checksums) + .execute() as any as Promise; } @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) async getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise { - const asset = await this.repository.findOne({ - select: { id: true }, - where: { - ownerId, - checksum, - library: IsNull(), - }, - withDeleted: true, - }); + const asset = await this.db + .selectFrom('assets') + .select('id') + .where('ownerId', '=', asUuid(ownerId)) + .where('checksum', '=', checksum) + .where('libraryId', 'is', null) + .limit(1) + .executeTakeFirst(); return asset?.id; } - findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { - const { ownerId, libraryId, otherAssetId, livePhotoCID, type } = options; + findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { + const { ownerId, otherAssetId, livePhotoCID, type } = options; - return this.repository.findOne({ - where: { - id: Not(otherAssetId), - ownerId, - libraryId: libraryId || IsNull(), - type, - exifInfo: { - livePhotoCID, - }, - }, - relations: { - exifInfo: true, - }, - }); + return this.db + .selectFrom('assets') + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('id', '!=', asUuid(otherAssetId)) + .where('ownerId', '=', asUuid(ownerId)) + .where('type', '=', type) + .where('exif.livePhotoCID', '=', livePhotoCID) + .limit(1) + .executeTakeFirst() as Promise; } @GenerateSql( @@ -373,194 +392,251 @@ export class AssetRepository implements IAssetRepository { params: [DummyValue.PAGINATION, property], })), ) - getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated { - let relations: FindOptionsRelations = {}; - let where: FindOptionsWhere | FindOptionsWhere[] = {}; + async getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated { + const items = await this.db + .selectFrom('assets') + .selectAll('assets') + .$if(property === WithoutProperty.DUPLICATE, (qb) => + qb + .innerJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId') + .where('job_status.duplicatesDetectedAt', 'is', null) + .where('job_status.previewAt', 'is not', null) + .where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id')))) + .where('assets.isVisible', '=', true), + ) + .$if(property === WithoutProperty.ENCODED_VIDEO, (qb) => + qb + .where('assets.type', '=', AssetType.VIDEO) + .where((eb) => eb.or([eb('assets.encodedVideoPath', 'is', null), eb('assets.encodedVideoPath', '=', '')])), + ) + .$if(property === WithoutProperty.EXIF, (qb) => + qb + .innerJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId') + .where('job_status.metadataExtractedAt', 'is', null) + .where('assets.isVisible', '=', true), + ) + .$if(property === WithoutProperty.FACES, (qb) => + qb + .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id') + .where('job_status.previewAt', 'is not', null) + .where('job_status.facesRecognizedAt', 'is', null) + .where('assets.isVisible', '=', true), + ) + .$if(property === WithoutProperty.SIDECAR, (qb) => + qb + .where((eb) => eb.or([eb('assets.sidecarPath', '=', ''), eb('assets.sidecarPath', 'is', null)])) + .where('assets.isVisible', '=', true), + ) + .$if(property === WithoutProperty.SMART_SEARCH, (qb) => + qb + .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id') + .where('job_status.previewAt', 'is not', null) + .where('assets.isVisible', '=', true) + .where((eb) => + eb.not((eb) => eb.exists(eb.selectFrom('smart_search').whereRef('assetId', '=', 'assets.id'))), + ), + ) + .$if(property === WithoutProperty.THUMBNAIL, (qb) => + qb + .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id') + .select(withFiles) + .where('assets.isVisible', '=', true) + .where((eb) => + eb.or([ + eb('job_status.previewAt', 'is', null), + eb('job_status.thumbnailAt', 'is', null), + eb('assets.thumbhash', 'is', null), + ]), + ), + ) + .where('deletedAt', 'is', null) + .limit(pagination.take + 1) + .offset(pagination.skip ?? 0) + .orderBy('createdAt', 'asc') + .execute(); - switch (property) { - case WithoutProperty.THUMBNAIL: { - relations = { jobStatus: true, files: true }; - where = [ - { jobStatus: { previewAt: IsNull() }, isVisible: true }, - { jobStatus: { thumbnailAt: IsNull() }, isVisible: true }, - { thumbhash: IsNull(), isVisible: true }, - ]; - break; - } - - case WithoutProperty.ENCODED_VIDEO: { - where = [ - { type: AssetType.VIDEO, encodedVideoPath: IsNull() }, - { type: AssetType.VIDEO, encodedVideoPath: '' }, - ]; - break; - } - - case WithoutProperty.EXIF: { - relations = { - exifInfo: true, - jobStatus: true, - }; - where = { - isVisible: true, - jobStatus: { - metadataExtractedAt: IsNull(), - }, - }; - break; - } - - case WithoutProperty.SMART_SEARCH: { - relations = { - smartSearch: true, - }; - where = { - isVisible: true, - jobStatus: { previewAt: Not(IsNull()) }, - smartSearch: { - embedding: IsNull(), - }, - }; - break; - } - - case WithoutProperty.DUPLICATE: { - where = { - isVisible: true, - smartSearch: true, - jobStatus: { - previewAt: Not(IsNull()), - duplicatesDetectedAt: IsNull(), - }, - }; - break; - } - - case WithoutProperty.FACES: { - relations = { - faces: true, - jobStatus: true, - }; - where = { - isVisible: true, - faces: { - assetId: IsNull(), - personId: IsNull(), - }, - jobStatus: { - previewAt: Not(IsNull()), - facesRecognizedAt: IsNull(), - }, - }; - break; - } - - case WithoutProperty.SIDECAR: { - where = [ - { sidecarPath: IsNull(), isVisible: true }, - { sidecarPath: '', isVisible: true }, - ]; - break; - } - - default: { - throw new Error(`Invalid getWithout property: ${property}`); - } - } - - return paginate(this.repository, pagination, { - relations, - where, - order: { - // Ensures correct order when paginating - createdAt: 'ASC', - }, - }); + return paginationHelper(items as any as AssetEntity[], pagination.take); } - getLastUpdatedAssetForAlbumId(albumId: string): Promise { - return this.repository.findOne({ - where: { albums: { id: albumId } }, - order: { updatedAt: 'DESC' }, - }); + getLastUpdatedAssetForAlbumId(albumId: string): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId') + .where('albums_assets_assets.albumsId', '=', asUuid(albumId)) + .orderBy('updatedAt', 'desc') + .limit(1) + .executeTakeFirst() as Promise; } - async getStatistics(ownerId: string, options: AssetStatsOptions): Promise { - const builder = this.repository - .createQueryBuilder('asset') - .select(`COUNT(asset.id)`, 'count') - .addSelect(`asset.type`, 'type') - .where('"ownerId" = :ownerId', { ownerId }) - .andWhere('asset.isVisible = true') - .groupBy('asset.type'); + getMapMarkers(ownerIds: string[], options: MapMarkerSearchOptions = {}): Promise { + const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options; - const { isArchived, isFavorite, isTrashed } = options; - if (isArchived !== undefined) { - builder.andWhere(`asset.isArchived = :isArchived`, { isArchived }); - } - - if (isFavorite !== undefined) { - builder.andWhere(`asset.isFavorite = :isFavorite`, { isFavorite }); - } - - if (isTrashed !== undefined) { - builder - .withDeleted() - .andWhere(`asset.deletedAt is not null`) - .andWhere('asset.status = :status', { status: AssetStatus.TRASHED }); - } - - const items = await builder.getRawMany(); - - const result: AssetStats = { - [AssetType.AUDIO]: 0, - [AssetType.IMAGE]: 0, - [AssetType.VIDEO]: 0, - [AssetType.OTHER]: 0, - }; - - for (const item of items) { - result[item.type as AssetType] = Number(item.count) || 0; - } - - return result; + return this.db + .selectFrom('assets') + .leftJoin('exif', 'assets.id', 'exif.assetId') + .select(['id', 'latitude as lat', 'longitude as lon', 'city', 'state', 'country']) + .where('ownerId', '=', anyUuid(ownerIds)) + .where('latitude', 'is not', null) + .where('longitude', 'is not', null) + .where('isVisible', '=', true) + .where('deletedAt', 'is', null) + .$if(!!isArchived, (qb) => qb.where('isArchived', '=', isArchived!)) + .$if(!!isFavorite, (qb) => qb.where('isFavorite', '=', isFavorite!)) + .$if(!!fileCreatedAfter, (qb) => qb.where('fileCreatedAt', '>=', fileCreatedAfter!)) + .$if(!!fileCreatedBefore, (qb) => qb.where('fileCreatedAt', '<=', fileCreatedBefore!)) + .orderBy('fileCreatedAt', 'desc') + .execute() as Promise; } - @GenerateSql({ params: [[DummyValue.UUID], DummyValue.NUMBER] }) - getRandom(userIds: string[], count: number): Promise { - return this.getBuilder({ userIds, exifInfo: true }).orderBy('RANDOM()').limit(count).getMany(); + getStatistics(ownerId: string, { isArchived, isFavorite, isTrashed }: AssetStatsOptions): Promise { + return this.db + .selectFrom('assets') + .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.AUDIO).as(AssetType.AUDIO)) + .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.IMAGE).as(AssetType.IMAGE)) + .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.VIDEO).as(AssetType.VIDEO)) + .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.OTHER).as(AssetType.OTHER)) + .where('ownerId', '=', asUuid(ownerId)) + .where('isVisible', '=', true) + .$if(isArchived !== undefined, (qb) => qb.where('isArchived', '=', isArchived!)) + .$if(isFavorite !== undefined, (qb) => qb.where('isFavorite', '=', isFavorite!)) + .$if(!!isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) + .where('deletedAt', isTrashed ? 'is not' : 'is', null) + .executeTakeFirst() as Promise; + } + + getRandom(userIds: string[], take: number): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .$call(withExif) + .where('ownerId', '=', anyUuid(userIds)) + .where('isVisible', '=', true) + .where('deletedAt', 'is', null) + .orderBy((eb) => eb.fn('random')) + .limit(take) + .execute() as any as Promise; } @GenerateSql({ params: [{ size: TimeBucketSize.MONTH }] }) - getTimeBuckets(options: TimeBucketOptions): Promise { - const truncated = dateTrunc(options); - return this.getBuilder(options) - .select(`COUNT(asset.id)::int`, 'count') - .addSelect(truncated, 'timeBucket') - .groupBy(truncated) - .orderBy(truncated, options.order === AssetOrder.ASC ? 'ASC' : 'DESC') - .getRawMany(); - } - - @GenerateSql({ params: [DummyValue.TIME_BUCKET, { size: TimeBucketSize.MONTH }] }) - getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise { - const truncated = dateTrunc(options); + async getTimeBuckets(options: TimeBucketOptions): Promise { return ( - this.getBuilder(options) - .andWhere(`${truncated} = :timeBucket`, { timeBucket: timeBucket.replace(/^[+-]/, '') }) - // First sort by the day in localtime (put it in the right bucket) - .orderBy(truncated, 'DESC') - // and then sort by the actual time - .addOrderBy('asset.fileCreatedAt', options.order === AssetOrder.ASC ? 'ASC' : 'DESC') - .getMany() + ((options.personId ? hasPeopleCte(this.db, [options.personId]) : this.db) as Kysely) + .with('assets', (qb) => + qb + .selectFrom('assets') + .select((eb) => + eb + .fn('date_trunc', [eb.val(options.size), sql`"localDateTime" at time zone 'UTC'`]) + .as('timeBucket'), + ) + .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) + .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) + .where('assets.isVisible', '=', true) + .$if(!!options.albumId, (qb) => + qb + .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId') + .where('albums_assets_assets.albumsId', '=', asUuid(options.albumId!)), + ) + .$if(!!options.personId, (qb) => + qb.innerJoin(sql.table('has_people').as('has_people'), (join) => + join.onRef(sql`has_people."assetId"`, '=', 'assets.id'), + ), + ) + .$if(!!options.withStacked, (qb) => + qb + .leftJoin('asset_stack', (join) => + join + .onRef('asset_stack.id', '=', 'assets.stackId') + .onRef('asset_stack.primaryAssetId', '=', 'assets.id'), + ) + .where((eb) => eb.or([eb('assets.stackId', 'is', null), eb(eb.table('asset_stack'), 'is not', null)])), + ) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(!!options.isArchived, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(!!options.isFavorite, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) + .$if(!!options.isDuplicate, (qb) => + qb.where('assets.duplicateId', options.isDuplicate ? 'is not' : 'is', null), + ), + ) + .selectFrom('assets') + .select('timeBucket') + /* + TODO: the above line outputs in ISO format, which bloats the response. + The line below outputs in YYYY-MM-DD format, but needs a change in the web app to work. + .select(sql`"timeBucket"::date::text`.as('timeBucket')) + */ + .select((eb) => eb.fn.countAll().as('count')) + .groupBy('timeBucket') + .orderBy('timeBucket', 'desc') + .execute() as any as Promise ); } + @GenerateSql({ params: [DummyValue.TIME_BUCKET, { size: TimeBucketSize.MONTH }] }) + async getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise { + // TODO: CREATE INDEX idx_local_date_time_month ON public.assets (date_trunc('MONTH', "localDateTime" at time zone 'UTC')); + return hasPeople(this.db, options.personId ? [options.personId] : undefined) + .selectAll('assets') + .$call(withExif) + .$if(!!options.albumId, (qb) => withAlbums(qb, { albumId: options.albumId })) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(!!options.withStacked, (qb) => withStack(qb, { assets: true })) // TODO: optimize this; it's a huge performance hit + .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) + .$if(options.isDuplicate !== undefined, (qb) => + qb.where('assets.duplicateId', options.isDuplicate ? 'is not' : 'is', null), + ) + .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) + .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) + .where('assets.isVisible', '=', true) + .where( + (eb) => eb.fn('date_trunc', [eb.val(options.size), sql`assets."localDateTime" at time zone 'UTC'`]), + '=', + timeBucket.replace(/^[+-]/, ''), + ) + .orderBy('assets.localDateTime', 'desc') + .execute() as any as Promise; + } + @GenerateSql({ params: [{ userIds: [DummyValue.UUID, DummyValue.UUID] }] }) - getDuplicates(options: AssetBuilderOptions): Promise { - return this.getBuilder({ ...options, isDuplicate: true }) - .orderBy('asset.duplicateId') - .getMany(); + getDuplicates(userId: string): Promise { + return ( + this.db + .with('duplicates', (qb) => + qb + .selectFrom('assets') + .select('duplicateId') + .select((eb) => eb.fn('jsonb_agg', [eb.table('assets')]).as('assets')) + .where('ownerId', '=', asUuid(userId)) + .where('duplicateId', 'is not', null) + .where('deletedAt', 'is', null) + .where('isVisible', '=', true) + .groupBy('duplicateId'), + ) + .with('unique', (qb) => + qb + .selectFrom('duplicates') + .select('duplicateId') + .where((eb) => eb(eb.fn('jsonb_array_length', ['assets']), '=', 1)), + ) + .with('removed_unique', (qb) => + qb + .updateTable('assets') + .set({ duplicateId: null }) + .from('unique') + .whereRef('assets.duplicateId', '=', 'unique.duplicateId'), + ) + .selectFrom('duplicates') + .selectAll() + // TODO: compare with filtering by jsonb_array_length > 1 + .where(({ not, exists }) => + not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), + ) + .execute() as any as Promise + ); } @GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] }) @@ -568,111 +644,35 @@ export class AssetRepository implements IAssetRepository { ownerId: string, { minAssetsPerField, maxFields }: AssetExploreFieldOptions, ): Promise> { - const cte = this.exifRepository - .createQueryBuilder('e') - .select('city') - .groupBy('city') - .having('count(city) >= :minAssetsPerField', { minAssetsPerField }); - - const items = await this.getBuilder({ - userIds: [ownerId], - exifInfo: false, - assetType: AssetType.IMAGE, - isArchived: false, - }) - .select('c.city', 'value') - .addSelect('asset.id', 'data') - .distinctOn(['c.city']) - .innerJoin('exif', 'e', 'asset.id = e."assetId"') - .addCommonTableExpression(cte, 'cities') - .innerJoin('cities', 'c', 'c.city = e.city') + const items = await this.db + .with('cities', (qb) => + qb + .selectFrom('exif') + .select('city') + .where('city', 'is not', null) + .groupBy('city') + .having((eb) => eb.fn('count', [eb.ref('assetId')]), '>=', minAssetsPerField), + ) + .selectFrom('assets') + .innerJoin('exif', 'assets.id', 'exif.assetId') + .innerJoin('cities', 'exif.city', 'cities.city') + .distinctOn('exif.city') + .select(['assetId as data', 'exif.city as value']) + .where('ownerId', '=', asUuid(ownerId)) + .where('isVisible', '=', true) + .where('isArchived', '=', false) + .where('type', '=', AssetType.IMAGE) + .where('deletedAt', 'is', null) .limit(maxFields) - .getRawMany(); + .execute(); - return { fieldName: 'exifInfo.city', items }; - } - - private getBuilder(options: AssetBuilderOptions) { - const builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true'); - - if (options.assetType !== undefined) { - builder.andWhere('asset.type = :assetType', { assetType: options.assetType }); - } - - if (options.tagId) { - builder.innerJoin( - 'asset.tags', - 'asset_tags', - 'asset_tags.id IN (SELECT id_descendant FROM tags_closure WHERE id_ancestor = :tagId)', - { tagId: options.tagId }, - ); - } - - let stackJoined = false; - - if (options.exifInfo !== false) { - stackJoined = true; - builder - .leftJoinAndSelect('asset.exifInfo', 'exifInfo') - .leftJoinAndSelect('asset.stack', 'stack') - .leftJoinAndSelect('stack.assets', 'stackedAssets'); - } - - if (options.albumId) { - builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId: options.albumId }); - } - - if (options.userIds) { - builder.andWhere('asset.ownerId IN (:...userIds )', { userIds: options.userIds }); - } - - if (options.isArchived !== undefined) { - builder.andWhere('asset.isArchived = :isArchived', { isArchived: options.isArchived }); - } - - if (options.isFavorite !== undefined) { - builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite: options.isFavorite }); - } - - if (options.isTrashed !== undefined) { - builder.andWhere(`asset.deletedAt ${options.isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted(); - - if (options.isTrashed) { - // TODO: Temporarily inverted to support showing offline assets in the trash queries. - // Once offline assets are handled in a separate screen, this should be set back to status = TRASHED - // and the offline screens should use a separate isOffline = true parameter in the timeline query. - builder.andWhere('asset.status != :status', { status: AssetStatus.DELETED }); - } - } - - if (options.isDuplicate !== undefined) { - builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`); - } - - if (options.personId !== undefined) { - builder - .innerJoin('asset.faces', 'faces') - .innerJoin('faces.person', 'person') - .andWhere('person.id = :personId', { personId: options.personId }); - } - - if (options.withStacked) { - if (!stackJoined) { - builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets'); - } - builder.andWhere( - new Brackets((qb) => qb.where('stack.primaryAssetId = asset.id').orWhere('asset.stackId IS NULL')), - ); - } - - return builder; + return { fieldName: 'exifInfo.city', items: items as SearchExploreItemSet }; } @GenerateSql({ params: [ { ownerId: DummyValue.UUID, - lastCreationDate: DummyValue.DATE, lastId: DummyValue.UUID, updatedUntil: DummyValue.DATE, limit: 10, @@ -681,49 +681,65 @@ export class AssetRepository implements IAssetRepository { }) getAllForUserFullSync(options: AssetFullSyncOptions): Promise { const { ownerId, lastId, updatedUntil, limit } = options; - const builder = this.getBuilder({ - userIds: [ownerId], - exifInfo: false, // need to do this manually because `exifInfo: true` also loads stacked assets messing with `limit` - withStacked: false, // return all assets individually as expected by the app - }) - .leftJoinAndSelect('asset.exifInfo', 'exifInfo') - .leftJoinAndSelect('asset.stack', 'stack') - .loadRelationCountAndMap('stack.assetCount', 'stack.assets', 'stackedAssetsCount'); - - if (lastId !== undefined) { - builder.andWhere('asset.id > :lastId', { lastId }); - } - builder - .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil }) - .orderBy('asset.id', 'ASC') - .limit(limit) // cannot use `take` for performance reasons - .withDeleted(); - return builder.getMany(); + return this.db + .selectFrom('assets') + .where('ownerId', '=', asUuid(ownerId)) + .where('isVisible', '=', true) + .where('updatedAt', '<=', updatedUntil) + .$if(!!lastId, (qb) => qb.where('id', '>', lastId!)) + .orderBy('id', 'asc') + .limit(limit) + .execute() as any as Promise; } @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] }) - getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { - const builder = this.getBuilder({ - userIds: options.userIds, - exifInfo: false, // need to do this manually because `exifInfo: true` also loads stacked assets messing with `limit` - withStacked: false, // return all assets individually as expected by the app - }) - .leftJoinAndSelect('asset.exifInfo', 'exifInfo') - .leftJoinAndSelect('asset.stack', 'stack') - .loadRelationCountAndMap('stack.assetCount', 'stack.assets', 'stackedAssetsCount') - .andWhere({ updatedAt: MoreThan(options.updatedAfter) }) - .limit(options.limit) // cannot use `take` for performance reasons - .withDeleted(); - return builder.getMany(); + async getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + return this.db + .selectFrom('assets') + .selectAll('assets') + .select((eb) => + eb + .selectFrom('asset_stack') + .select((eb) => eb.fn.countAll().as('stackedAssetsCount')) + .whereRef('asset_stack.id', '=', 'assets.stackId') + .as('stackedAssetsCount'), + ) + .where('ownerId', '=', anyUuid(options.userIds)) + .where('isVisible', '=', true) + .where('updatedAt', '>', options.updatedAfter) + .limit(options.limit) + .execute() as any as Promise; } @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) - async upsertFile(file: { assetId: string; type: AssetFileType; path: string }): Promise { - await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] }); + async upsertFile(file: Pick, 'assetId' | 'path' | 'type'>): Promise { + const value = { ...file, assetId: asUuid(file.assetId) }; + await this.db + .insertInto('asset_files') + .values(value) + .onConflict((oc) => + oc + .columns(ASSET_FILE_CONFLICT_KEYS) + .doUpdateSet(() => mapUpsertColumns('asset_files', value, ASSET_FILE_CONFLICT_KEYS)), + ) + .execute(); } @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) - async upsertFiles(files: { assetId: string; type: AssetFileType; path: string }[]): Promise { - await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] }); + async upsertFiles(files: Pick, 'assetId' | 'path' | 'type'>[]): Promise { + if (files.length === 0) { + return; + } + + const values = files.map((row) => ({ ...row, assetId: asUuid(row.assetId) })); + await this.db + .insertInto('asset_files') + .values(values) + .onConflict((oc) => + oc + .columns(ASSET_FILE_CONFLICT_KEYS) + .doUpdateSet(() => mapUpsertColumns('asset_files', values[0], ASSET_FILE_CONFLICT_KEYS)), + ) + .execute(); } } diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts index 2ff5f53073..f5d0072890 100644 --- a/server/src/repositories/config.repository.spec.ts +++ b/server/src/repositories/config.repository.spec.ts @@ -1,3 +1,4 @@ +import { PostgresJSDialect } from 'kysely-postgres-js'; import { ImmichTelemetry } from 'src/enum'; import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository'; @@ -79,14 +80,20 @@ describe('getEnv', () => { it('should use defaults', () => { const { database } = getEnv(); expect(database).toEqual({ - config: expect.objectContaining({ - type: 'postgres', - host: 'database', - port: 5432, - database: 'immich', - username: 'postgres', - password: 'postgres', - }), + config: { + kysely: { + dialect: expect.any(PostgresJSDialect), + log: ['error'], + }, + typeorm: expect.objectContaining({ + type: 'postgres', + host: 'database', + port: 5432, + database: 'immich', + username: 'postgres', + password: 'postgres', + }), + }, skipMigrations: false, vectorExtension: 'vectors', }); diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index a8a1c9972b..9883d71448 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -2,8 +2,10 @@ import { Inject, Injectable, Optional } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; import { validateSync } from 'class-validator'; import { Request, Response } from 'express'; +import { PostgresJSDialect } from 'kysely-postgres-js'; import { CLS_ID } from 'nestjs-cls'; import { join, resolve } from 'node:path'; +import postgres from 'postgres'; import { citiesFile, excludePaths, IWorker } from 'src/constants'; import { Telemetry } from 'src/decorators'; import { EnvDto } from 'src/dtos/env.dto'; @@ -96,6 +98,33 @@ const getEnv = (): EnvData => { } } + const driverOptions = { + max: 10, + types: { + date: { + to: 1184, + from: [1082, 1114, 1184], + serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x), + parse: (x: string) => new Date(x), + }, + bigint: { + to: 20, + from: [20], + parse: (value: string) => Number.parseInt(value), + serialize: (value: number) => value.toString(), + }, + }, + }; + + const parts = { + connectionType: 'parts', + host: dto.DB_HOSTNAME || 'database', + port: dto.DB_PORT || 5432, + username: dto.DB_USERNAME || 'postgres', + password: dto.DB_PASSWORD || 'postgres', + database: dto.DB_DATABASE_NAME || 'immich', + } as const; + return { host: dto.IMMICH_HOST, port: dto.IMMICH_PORT || 2283, @@ -150,24 +179,23 @@ const getEnv = (): EnvData => { database: { config: { - type: 'postgres', - entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'], - migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'], - subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'], - migrationsRun: false, - synchronize: false, - connectTimeoutMS: 10_000, // 10 seconds - parseInt8: true, - ...(databaseUrl - ? { connectionType: 'url', url: databaseUrl } - : { - connectionType: 'parts', - host: dto.DB_HOSTNAME || 'database', - port: dto.DB_PORT || 5432, - username: dto.DB_USERNAME || 'postgres', - password: dto.DB_PASSWORD || 'postgres', - database: dto.DB_DATABASE_NAME || 'immich', - }), + typeorm: { + type: 'postgres', + entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'], + migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'], + subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'], + migrationsRun: false, + synchronize: false, + connectTimeoutMS: 10_000, // 10 seconds + parseInt8: true, + ...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts), + }, + kysely: { + dialect: new PostgresJSDialect({ + postgres: databaseUrl ? postgres(databaseUrl, driverOptions) : postgres({ ...parts, ...driverOptions }), + }), + log: ['error'] as const, + }, }, skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false, diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index b5e2edfdea..41c6bd4490 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -21,7 +21,7 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm'; @Injectable() export class DatabaseRepository implements IDatabaseRepository { private vectorExtension: VectorExtension; - readonly asyncLock = new AsyncLock(); + private readonly asyncLock = new AsyncLock(); constructor( @InjectDataSource() private dataSource: DataSource, diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 0a529f2f6e..0dd3a691b1 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -1,22 +1,16 @@ import { Inject, Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; +import { Kysely, OrderByDirectionExpression, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; import { randomUUID } from 'node:crypto'; +import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetFaceEntity } from 'src/entities/asset-face.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { ExifEntity } from 'src/entities/exif.entity'; +import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; -import { SmartSearchEntity } from 'src/entities/smart-search.entity'; -import { AssetType, PaginationMode } from 'src/enum'; -import { IConfigRepository } from 'src/interfaces/config.interface'; -import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { - AssetDuplicateResult, AssetDuplicateSearch, AssetSearchOptions, FaceEmbeddingSearch, - FaceSearchResult, GetCameraMakesOptions, GetCameraModelsOptions, GetCitiesOptions, @@ -25,40 +19,17 @@ import { SearchPaginationOptions, SmartSearchOptions, } from 'src/interfaces/search.interface'; -import { asVector, searchAssetBuilder } from 'src/utils/database'; -import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination'; +import { anyUuid, asUuid, asVector } from 'src/utils/database'; +import { Paginated } from 'src/utils/pagination'; import { isValidInteger } from 'src/validation'; -import { Repository } from 'typeorm'; @Injectable() export class SearchRepository implements ISearchRepository { - private vectorExtension: VectorExtension; - private faceColumns: string[]; - private assetsByCityQuery: string; - constructor( - @InjectRepository(AssetEntity) private assetRepository: Repository, - @InjectRepository(ExifEntity) private exifRepository: Repository, - @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, - @InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository, - @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository, @Inject(ILoggerRepository) private logger: ILoggerRepository, - @Inject(IConfigRepository) configRepository: IConfigRepository, + @InjectKysely() private db: Kysely, ) { - this.vectorExtension = configRepository.getEnv().database.vectorExtension; this.logger.setContext(SearchRepository.name); - this.faceColumns = this.assetFaceRepository.manager.connection - .getMetadata(AssetFaceEntity) - .ownColumns.map((column) => column.propertyName) - .filter((propertyName) => propertyName !== 'embedding'); - this.assetsByCityQuery = - assetsByCityCte + - this.assetRepository - .createQueryBuilder('asset') - .innerJoinAndSelect('asset.exifInfo', 'exif') - .withDeleted() - .getQuery() + - ' INNER JOIN cte ON asset.id = cte."assetId" ORDER BY exif.city'; } @GenerateSql({ @@ -74,14 +45,15 @@ export class SearchRepository implements ISearchRepository { ], }) async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated { - let builder = this.assetRepository.createQueryBuilder('asset'); - builder = searchAssetBuilder(builder, options).orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC'); - - return paginatedBuilder(builder, { - mode: PaginationMode.SKIP_TAKE, - skip: (pagination.page - 1) * pagination.size, - take: pagination.size, - }); + const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirectionExpression; + const items = await searchAssetBuilder(this.db, options) + .orderBy('assets.fileCreatedAt', orderDirection) + .limit(pagination.size + 1) + .offset((pagination.page - 1) * pagination.size) + .execute(); + const hasNextPage = items.length > pagination.size; + items.splice(pagination.size); + return { items: items as any as AssetEntity[], hasNextPage }; } @GenerateSql({ @@ -96,21 +68,15 @@ export class SearchRepository implements ISearchRepository { }, ], }) - async searchRandom(size: number, options: AssetSearchOptions): Promise { - const builder1 = searchAssetBuilder(this.assetRepository.createQueryBuilder('asset'), options); - const builder2 = builder1.clone(); - + searchRandom(size: number, options: AssetSearchOptions): Promise { const uuid = randomUUID(); - builder1.andWhere('asset.id > :uuid', { uuid }).orderBy('asset.id').take(size); - builder2.andWhere('asset.id < :uuid', { uuid }).orderBy('asset.id').take(size); - - const [assets1, assets2] = await Promise.all([builder1.getMany(), builder2.getMany()]); - const missingCount = size - assets1.length; - for (let i = 0; i < missingCount && i < assets2.length; i++) { - assets1.push(assets2[i]); - } - - return assets1; + const builder = searchAssetBuilder(this.db, options); + return builder + .where('assets.id', '>', uuid) + .orderBy('assets.id') + .limit(size) + .unionAll(() => builder.where('assets.id', '<', uuid).orderBy('assets.id').limit(size)) + .execute() as any as Promise; } @GenerateSql({ @@ -126,33 +92,21 @@ export class SearchRepository implements ISearchRepository { }, ], }) - async searchSmart( - pagination: SearchPaginationOptions, - { embedding, userIds, ...options }: SmartSearchOptions, - ): Paginated { - let results: PaginationResult = { items: [], hasNextPage: false }; + async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated { + if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) { + throw new Error(`Invalid value for 'size': ${pagination.size}`); + } - await this.assetRepository.manager.transaction(async (manager) => { - let builder = manager.createQueryBuilder(AssetEntity, 'asset'); - builder = searchAssetBuilder(builder, options); - builder - .innerJoin('asset.smartSearch', 'search') - .andWhere('asset.ownerId IN (:...userIds )') - .orderBy('search.embedding <=> :embedding') - .setParameters({ userIds, embedding: asVector(embedding) }); + const items = (await searchAssetBuilder(this.db, options) + .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') + .orderBy(sql`smart_search.embedding <=> ${asVector(options.embedding)}`) + .limit(pagination.size + 1) + .offset((pagination.page - 1) * pagination.size) + .execute()) as any as AssetEntity[]; - const runtimeConfig = this.getRuntimeConfig(pagination.size); - if (runtimeConfig) { - await manager.query(runtimeConfig); - } - results = await paginatedBuilder(builder, { - mode: PaginationMode.LIMIT_OFFSET, - skip: (pagination.page - 1) * pagination.size, - take: pagination.size, - }); - }); - - return results; + const hasNextPage = items.length > pagination.size; + items.splice(pagination.size); + return { items, hasNextPage }; } @GenerateSql({ @@ -164,38 +118,30 @@ export class SearchRepository implements ISearchRepository { }, ], }) - searchDuplicates({ - assetId, - embedding, - maxDistance, - type, - userIds, - }: AssetDuplicateSearch): Promise { - const cte = this.assetRepository.createQueryBuilder('asset'); - cte - .select('search.assetId', 'assetId') - .addSelect('asset.duplicateId', 'duplicateId') - .addSelect(`search.embedding <=> :embedding`, 'distance') - .innerJoin('asset.smartSearch', 'search') - .where('asset.ownerId IN (:...userIds )') - .andWhere('asset.id != :assetId') - .andWhere('asset.isVisible = :isVisible') - .andWhere('asset.type = :type') - .orderBy('search.embedding <=> :embedding') - .limit(64) - .setParameters({ assetId, embedding: asVector(embedding), isVisible: true, type, userIds }); - - const builder = this.assetRepository.manager - .createQueryBuilder() - .addCommonTableExpression(cte, 'cte') - .from('cte', 'res') - .select('res.*'); - - if (maxDistance) { - builder.where('res.distance <= :maxDistance', { maxDistance }); - } - - return builder.getRawMany() as Promise; + searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) { + const vector = asVector(embedding); + return this.db + .with('cte', (qb) => + qb + .selectFrom('assets') + .select([ + 'assets.id as assetId', + 'assets.duplicateId', + sql`smart_search.embedding <=> ${vector}`.as('distance'), + ]) + .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') + .where('assets.ownerId', '=', anyUuid(userIds)) + .where('assets.deletedAt', 'is', null) + .where('assets.isVisible', '=', true) + .where('assets.type', '=', type) + .where('assets.id', '!=', assetId) + .orderBy(sql`smart_search.embedding <=> ${vector}`) + .limit(64), + ) + .selectFrom('cte') + .selectAll() + .where('cte.distance', '<=', maxDistance as number) + .execute(); } @GenerateSql({ @@ -208,120 +154,131 @@ export class SearchRepository implements ISearchRepository { }, ], }) - async searchFaces({ - userIds, - embedding, - numResults, - maxDistance, - hasPerson, - }: FaceEmbeddingSearch): Promise { - if (!isValidInteger(numResults, { min: 1 })) { + searchFaces({ userIds, embedding, numResults, maxDistance, hasPerson }: FaceEmbeddingSearch) { + if (!isValidInteger(numResults, { min: 1, max: 1000 })) { throw new Error(`Invalid value for 'numResults': ${numResults}`); } - // setting this too low messes with prefilter recall - numResults = Math.max(numResults, 64); - - let results: Array = []; - await this.assetRepository.manager.transaction(async (manager) => { - const cte = manager - .createQueryBuilder(AssetFaceEntity, 'faces') - .select('search.embedding <=> :embedding', 'distance') - .innerJoin('faces.asset', 'asset') - .innerJoin('faces.faceSearch', 'search') - .where('asset.ownerId IN (:...userIds )') - .orderBy('search.embedding <=> :embedding') - .setParameters({ userIds, embedding: asVector(embedding) }); - - cte.limit(numResults); - - if (hasPerson) { - cte.andWhere('faces."personId" IS NOT NULL'); - } - - for (const col of this.faceColumns) { - cte.addSelect(`faces.${col}`, col); - } - - const runtimeConfig = this.getRuntimeConfig(numResults); - if (runtimeConfig) { - await manager.query(runtimeConfig); - } - results = await manager - .createQueryBuilder() - .select('res.*') - .addCommonTableExpression(cte, 'cte') - .from('cte', 'res') - .where('res.distance <= :maxDistance', { maxDistance }) - .orderBy('res.distance') - .getRawMany(); - }); - return results.map((row) => ({ - face: this.assetFaceRepository.create(row), - distance: row.distance, - })); + const vector = asVector(embedding); + return this.db + .with('cte', (qb) => + qb + .selectFrom('asset_faces') + .select([ + 'asset_faces.id', + 'asset_faces.personId', + sql`face_search.embedding <=> ${vector}`.as('distance'), + ]) + .innerJoin('assets', 'assets.id', 'asset_faces.assetId') + .innerJoin('face_search', 'face_search.faceId', 'asset_faces.id') + .where('assets.ownerId', '=', anyUuid(userIds)) + .where('assets.deletedAt', 'is', null) + .$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null)) + .orderBy(sql`face_search.embedding <=> ${vector}`) + .limit(numResults), + ) + .selectFrom('cte') + .selectAll() + .where('cte.distance', '<=', maxDistance) + .execute(); } @GenerateSql({ params: [DummyValue.STRING] }) - async searchPlaces(placeName: string): Promise { - return await this.geodataPlacesRepository - .createQueryBuilder('geoplaces') - .where(`f_unaccent(name) %>> f_unaccent(:placeName)`) - .orWhere(`f_unaccent("admin2Name") %>> f_unaccent(:placeName)`) - .orWhere(`f_unaccent("admin1Name") %>> f_unaccent(:placeName)`) - .orWhere(`f_unaccent("alternateNames") %>> f_unaccent(:placeName)`) + searchPlaces(placeName: string): Promise { + return this.db + .selectFrom('geodata_places') + .selectAll() + .where( + () => + // kysely doesn't support trigram %>> or <->>> operators + sql` + f_unaccent(name) %>> f_unaccent(${placeName}) or + f_unaccent("admin2Name") %>> f_unaccent(${placeName}) or + f_unaccent("admin1Name") %>> f_unaccent(${placeName}) or + f_unaccent("alternateNames") %>> f_unaccent(${placeName}) + `, + ) .orderBy( - ` - COALESCE(f_unaccent(name) <->>> f_unaccent(:placeName), 0.1) + - COALESCE(f_unaccent("admin2Name") <->>> f_unaccent(:placeName), 0.1) + - COALESCE(f_unaccent("admin1Name") <->>> f_unaccent(:placeName), 0.1) + - COALESCE(f_unaccent("alternateNames") <->>> f_unaccent(:placeName), 0.1) + sql` + coalesce(f_unaccent(name) <->>> f_unaccent(${placeName}), 0.1) + + coalesce(f_unaccent("admin2Name") <->>> f_unaccent(${placeName}), 0.1) + + coalesce(f_unaccent("admin1Name") <->>> f_unaccent(${placeName}), 0.1) + + coalesce(f_unaccent("alternateNames") <->>> f_unaccent(${placeName}), 0.1) `, ) - .setParameters({ placeName }) .limit(20) - .getMany(); + .execute() as Promise; } @GenerateSql({ params: [[DummyValue.UUID]] }) - async getAssetsByCity(userIds: string[]): Promise { - const parameters = [userIds, true, false, AssetType.IMAGE]; - const rawRes = await this.assetRepository.query(this.assetsByCityQuery, parameters); + getAssetsByCity(userIds: string[]): Promise { + return this.db + .withRecursive('cte', (qb) => { + const base = qb + .selectFrom('exif') + .select(['city', 'assetId']) + .innerJoin('assets', 'assets.id', 'exif.assetId') + .where('assets.ownerId', '=', anyUuid(userIds)) + .where('assets.isVisible', '=', true) + .where('assets.isArchived', '=', false) + .where('assets.type', '=', 'IMAGE') + .where('assets.deletedAt', 'is', null) + .orderBy('city') + .limit(1); - const items: AssetEntity[] = []; - for (const res of rawRes) { - const item = { exifInfo: {} as Record } as Record; - for (const [key, value] of Object.entries(res)) { - if (key.startsWith('exif_')) { - item.exifInfo[key.replace('exif_', '')] = value; - } else { - item[key.replace('asset_', '')] = value; - } - } - items.push(item as AssetEntity); - } + const recursive = qb + .selectFrom('cte') + .select(['l.city', 'l.assetId']) + .innerJoinLateral( + (qb) => + qb + .selectFrom('exif') + .select(['city', 'assetId']) + .innerJoin('assets', 'assets.id', 'exif.assetId') + .where('assets.ownerId', '=', anyUuid(userIds)) + .where('assets.isVisible', '=', true) + .where('assets.isArchived', '=', false) + .where('assets.type', '=', 'IMAGE') + .where('assets.deletedAt', 'is', null) + .whereRef('exif.city', '>', 'cte.city') + .orderBy('city') + .limit(1) + .as('l'), + (join) => join.onTrue(), + ); - return items; + return sql<{ city: string; assetId: string }>`(${base} union all ${recursive})`; + }) + .selectFrom('assets') + .innerJoin('exif', 'assets.id', 'exif.assetId') + .innerJoin('cte', 'assets.id', 'cte.assetId') + .selectAll('assets') + .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')) + .orderBy('exif.city') + .execute() as any as Promise; } async upsert(assetId: string, embedding: number[]): Promise { - await this.smartSearchRepository.upsert( - { assetId, embedding: () => asVector(embedding, true) }, - { conflictPaths: ['assetId'] }, - ); + const vector = asVector(embedding); + await this.db + .insertInto('smart_search') + .values({ assetId: asUuid(assetId), embedding: vector } as any) + .onConflict((oc) => oc.column('assetId').doUpdateSet({ embedding: vector } as any)) + .execute(); } async getDimensionSize(): Promise { - const res = await this.smartSearchRepository.manager.query(` - SELECT atttypmod as dimsize - FROM pg_attribute f - JOIN pg_class c ON c.oid = f.attrelid - WHERE c.relkind = 'r'::char - AND f.attnum > 0 - AND c.relname = 'smart_search' - AND f.attname = 'embedding'`); + const { rows } = await sql<{ dimsize: number }>` + select atttypmod as dimsize + from pg_attribute f + join pg_class c ON c.oid = f.attrelid + where c.relkind = 'r'::char + and f.attnum > 0 + and c.relname = 'smart_search' + and f.attname = 'embedding' + `.execute(this.db); - const dimSize = res[0]['dimsize']; + const dimSize = rows[0]['dimsize']; if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) { throw new Error(`Could not retrieve CLIP dimension size`); } @@ -333,146 +290,71 @@ export class SearchRepository implements ISearchRepository { throw new Error(`Invalid CLIP dimension size: ${dimSize}`); } - return this.smartSearchRepository.manager.transaction(async (manager) => { - await manager.clear(SmartSearchEntity); - await manager.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE vector(${dimSize})`); - await manager.query(`REINDEX INDEX clip_index`); + return this.db.transaction().execute(async (trx) => { + await sql`truncate ${sql.table('smart_search')}`.execute(trx); + await trx.schema + .alterTable('smart_search') + .alterColumn('embedding', (col) => col.setDataType(sql.lit(`vector(${dimSize})`))) + .execute(); + await sql`reindex index clip_index`.execute(trx); }); } async deleteAllSearchEmbeddings(): Promise { - return this.smartSearchRepository.clear(); + await sql`truncate ${sql.table('smart_search')}`.execute(this.db); } - @GenerateSql({ params: [[DummyValue.UUID]] }) async getCountries(userIds: string[]): Promise { - const query = this.exifRepository - .createQueryBuilder('exif') - .innerJoin('exif.asset', 'asset') - .where('asset.ownerId IN (:...userIds )', { userIds }) - .andWhere(`exif.country != ''`) - .andWhere('exif.country IS NOT NULL') - .select('exif.country', 'country') - .distinctOn(['exif.country']); - - const results = await query.getRawMany<{ country: string }>(); - return results.map(({ country }) => country); + const res = await this.getExifField('country', userIds).execute(); + return res.map((row) => row.country!); } @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) async getStates(userIds: string[], { country }: GetStatesOptions): Promise { - const query = this.exifRepository - .createQueryBuilder('exif') - .innerJoin('exif.asset', 'asset') - .where('asset.ownerId IN (:...userIds )', { userIds }) - .andWhere(`exif.state != ''`) - .andWhere('exif.state IS NOT NULL') - .select('exif.state', 'state') - .distinctOn(['exif.state']); + const res = await this.getExifField('state', userIds) + .$if(!!country, (qb) => qb.where('country', '=', country!)) + .execute(); - if (country) { - query.andWhere('exif.country = :country', { country }); - } - - const result = await query.getRawMany<{ state: string }>(); - return result.map(({ state }) => state); + return res.map((row) => row.state!); } @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING, DummyValue.STRING] }) async getCities(userIds: string[], { country, state }: GetCitiesOptions): Promise { - const query = this.exifRepository - .createQueryBuilder('exif') - .innerJoin('exif.asset', 'asset') - .where('asset.ownerId IN (:...userIds )', { userIds }) - .andWhere(`exif.city != ''`) - .andWhere('exif.city IS NOT NULL') - .select('exif.city', 'city') - .distinctOn(['exif.city']); + const res = await this.getExifField('city', userIds) + .$if(!!country, (qb) => qb.where('country', '=', country!)) + .$if(!!state, (qb) => qb.where('state', '=', state!)) + .execute(); - if (country) { - query.andWhere('exif.country = :country', { country }); - } - - if (state) { - query.andWhere('exif.state = :state', { state }); - } - - const results = await query.getRawMany<{ city: string }>(); - return results.map(({ city }) => city); + return res.map((row) => row.city!); } @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) async getCameraMakes(userIds: string[], { model }: GetCameraMakesOptions): Promise { - const query = this.exifRepository - .createQueryBuilder('exif') - .innerJoin('exif.asset', 'asset') - .where('asset.ownerId IN (:...userIds )', { userIds }) - .andWhere(`exif.make != ''`) - .andWhere('exif.make IS NOT NULL') - .select('exif.make', 'make') - .distinctOn(['exif.make']); + const res = await this.getExifField('make', userIds) + .$if(!!model, (qb) => qb.where('model', '=', model!)) + .execute(); - if (model) { - query.andWhere('exif.model = :model', { model }); - } - - const results = await query.getRawMany<{ make: string }>(); - return results.map(({ make }) => make); + return res.map((row) => row.make!); } @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) async getCameraModels(userIds: string[], { make }: GetCameraModelsOptions): Promise { - const query = this.exifRepository - .createQueryBuilder('exif') - .innerJoin('exif.asset', 'asset') - .where('asset.ownerId IN (:...userIds )', { userIds }) - .andWhere(`exif.model != ''`) - .andWhere('exif.model IS NOT NULL') - .select('exif.model', 'model') - .distinctOn(['exif.model']); + const res = await this.getExifField('model', userIds) + .$if(!!make, (qb) => qb.where('make', '=', make!)) + .execute(); - if (make) { - query.andWhere('exif.make = :make', { make }); - } - - const results = await query.getRawMany<{ model: string }>(); - return results.map(({ model }) => model); + return res.map((row) => row.model!); } - private getRuntimeConfig(numResults?: number): string | undefined { - if (this.vectorExtension === DatabaseExtension.VECTOR) { - return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall - } - - if (numResults && numResults !== 100) { - return `SET LOCAL vectors.hnsw_ef_search = ${Math.max(numResults, 100)};`; - } + private getExifField(field: K, userIds: string[]) { + return this.db + .selectFrom('exif') + .select(field) + .distinctOn(field) + .innerJoin('assets', 'assets.id', 'exif.assetId') + .where('ownerId', '=', anyUuid(userIds)) + .where('isVisible', '=', true) + .where('deletedAt', 'is', null) + .where(field, 'is not', null); } } - -// the performance difference between this and the normal way is too huge to ignore, e.g. 3s vs 4ms -const assetsByCityCte = ` -WITH RECURSIVE cte AS ( - ( - SELECT city, "assetId" - FROM exif - INNER JOIN assets ON exif."assetId" = assets.id - WHERE "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4 - ORDER BY city - LIMIT 1 - ) - - UNION ALL - - SELECT l.city, l."assetId" - FROM cte c - , LATERAL ( - SELECT city, "assetId" - FROM exif - INNER JOIN assets ON exif."assetId" = assets.id - WHERE city > c.city AND "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4 - ORDER BY city - LIMIT 1 - ) l -) -`; diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.ts index 3645e3638a..13a042a174 100644 --- a/server/src/repositories/view-repository.ts +++ b/server/src/repositories/view-repository.ts @@ -1,48 +1,47 @@ -import { InjectRepository } from '@nestjs/typeorm'; +import { Kysely } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { AssetEntity, withExif } from 'src/entities/asset.entity'; import { IViewRepository } from 'src/interfaces/view.interface'; -import { Brackets, Repository } from 'typeorm'; +import { asUuid } from 'src/utils/database'; export class ViewRepository implements IViewRepository { - constructor(@InjectRepository(AssetEntity) private assetRepository: Repository) {} + constructor(@InjectKysely() private db: Kysely) {} + @GenerateSql({ params: [DummyValue.UUID] }) async getUniqueOriginalPaths(userId: string): Promise { - const results = await this.assetRepository - .createQueryBuilder('asset') - .where({ - isVisible: true, - isArchived: false, - ownerId: userId, - }) - .select("DISTINCT substring(asset.originalPath FROM '^(.*/)[^/]*$')", 'directoryPath') - .getRawMany(); + const results = await this.db + .selectFrom('assets') + .select((eb) => eb.fn('substring', ['assets.originalPath', eb.val('^(.*/)[^/]*$')]).as('directoryPath')) + .distinct() + .where('ownerId', '=', asUuid(userId)) + .where('isVisible', '=', true) + .where('isArchived', '=', false) + .where('deletedAt', 'is', null) + .execute(); - return results.map((row: { directoryPath: string }) => row.directoryPath.replaceAll(/^\/|\/$/g, '')); + return results.map((row) => row.directoryPath.replaceAll(/^\/|\/$/g, '')); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) async getAssetsByOriginalPath(userId: string, partialPath: string): Promise { const normalizedPath = partialPath.replaceAll(/^\/|\/$/g, ''); - const assets = await this.assetRepository - .createQueryBuilder('asset') - .where({ - isVisible: true, - isArchived: false, - ownerId: userId, - }) - .leftJoinAndSelect('asset.exifInfo', 'exifInfo') - .andWhere( - new Brackets((qb) => { - qb.where('asset.originalPath LIKE :likePath', { likePath: `%${normalizedPath}/%` }).andWhere( - 'asset.originalPath NOT LIKE :notLikePath', - { notLikePath: `%${normalizedPath}/%/%` }, - ); - }), - ) - .orderBy(String.raw`regexp_replace(asset.originalPath, '.*/(.+)', '\1')`, 'ASC') - .getMany(); - return assets; + return this.db + .selectFrom('assets') + .selectAll('assets') + .$call(withExif) + .where('ownerId', '=', asUuid(userId)) + .where('isVisible', '=', true) + .where('isArchived', '=', false) + .where('deletedAt', 'is', null) + .where('originalPath', 'like', `%${normalizedPath}/%`) + .where('originalPath', 'not like', `%${normalizedPath}/%/%`) + .orderBy( + (eb) => eb.fn('regexp_replace', ['assets.originalPath', eb.val('.*/(.+)'), eb.val(String.raw`\1`)]), + 'asc', + ) + .execute() as any as Promise; } } diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index 1daeb99d0b..14a8d9c83b 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -23,7 +23,6 @@ import { fileStub } from 'test/fixtures/file.stub'; import { userStub } from 'test/fixtures/user.stub'; import { IAccessRepositoryMock } from 'test/repositories/access.repository.mock'; import { newTestService } from 'test/utils'; -import { QueryFailedError } from 'typeorm'; import { Mocked } from 'vitest'; const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); @@ -370,8 +369,8 @@ describe(AssetMediaService.name, () => { originalName: 'asset_1.jpeg', size: 0, }; - const error = new QueryFailedError('', [], new Error('unique key violation')); - (error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; + const error = new Error('unique key violation'); + (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT; assetMock.create.mockRejectedValue(error); assetMock.getUploadAssetIdByChecksum.mockResolvedValue(assetEntity.id); @@ -397,8 +396,8 @@ describe(AssetMediaService.name, () => { originalName: 'asset_1.jpeg', size: 0, }; - const error = new QueryFailedError('', [], new Error('unique key violation')); - (error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; + const error = new Error('unique key violation'); + (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT; assetMock.create.mockRejectedValue(error); @@ -480,7 +479,6 @@ describe(AssetMediaService.name, () => { it('should throw an error if the asset is not found', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); - assetMock.getById.mockResolvedValue(null); await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(NotFoundException); @@ -512,7 +510,6 @@ describe(AssetMediaService.name, () => { it('should throw an error if the asset does not exist', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); - assetMock.getById.mockResolvedValue(null); await expect( sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.PREVIEW }), @@ -618,7 +615,6 @@ describe(AssetMediaService.name, () => { it('should throw an error if the asset does not exist', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); - assetMock.getById.mockResolvedValue(null); await expect(sut.playbackVideo(authStub.admin, assetStub.image.id)).rejects.toBeInstanceOf(NotFoundException); }); @@ -670,8 +666,6 @@ describe(AssetMediaService.name, () => { describe('replaceAsset', () => { it('should error when update photo does not exist', async () => { - assetMock.getById.mockResolvedValueOnce(null); - await expect(sut.replaceAsset(authStub.user1, 'id', replaceDto, fileStub.photo)).rejects.toThrow( 'Not found or no asset.update access', ); @@ -785,8 +779,8 @@ describe(AssetMediaService.name, () => { it('should handle a photo with sidecar to duplicate photo ', async () => { const updatedFile = fileStub.photo; - const error = new QueryFailedError('', [], new Error('unique key violation')); - (error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; + const error = new Error('unique key violation'); + (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT; assetMock.update.mockRejectedValue(error); assetMock.getById.mockResolvedValueOnce(sidecarAsset); diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index e96d1fd0a6..fab836db94 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -30,7 +30,6 @@ import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util'; import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; import { fromChecksum } from 'src/utils/request'; -import { QueryFailedError } from 'typeorm'; export interface UploadRequest { auth: AuthDto | null; fieldName: UploadFieldName; @@ -302,7 +301,7 @@ export class AssetMediaService extends BaseService { }); // handle duplicates with a success response - if (error instanceof QueryFailedError && (error as any).constraint === ASSET_CHECKSUM_CONSTRAINT) { + if (error.constraint_name === ASSET_CHECKSUM_CONSTRAINT) { const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, file.checksum); if (!duplicateId) { this.logger.error(`Error locating duplicate for checksum constraint`); @@ -343,7 +342,7 @@ export class AssetMediaService extends BaseService { localDateTime: dto.fileCreatedAt, duration: dto.duration || null, - livePhotoVideo: null, + livePhotoVideoId: null, sidecarPath: sidecarPath || null, }); diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 5aab5032af..cc8f0a1ab0 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -51,9 +51,7 @@ describe(AssetService.name, () => { }); const mockGetById = (assets: AssetEntity[]) => { - assetMock.getById.mockImplementation((assetId) => - Promise.resolve(assets.find((asset) => asset.id === assetId) ?? null), - ); + assetMock.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId))); }; beforeEach(() => { @@ -250,27 +248,34 @@ describe(AssetService.name, () => { it('should update the asset', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); assetMock.getById.mockResolvedValue(assetStub.image); + assetMock.update.mockResolvedValue(assetStub.image); + await sut.update(authStub.admin, 'asset-1', { isFavorite: true }); + expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-1', isFavorite: true }); }); it('should update the exif description', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); assetMock.getById.mockResolvedValue(assetStub.image); + assetMock.update.mockResolvedValue(assetStub.image); + await sut.update(authStub.admin, 'asset-1', { description: 'Test description' }); + expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' }); }); it('should update the exif rating', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); - assetMock.getById.mockResolvedValue(assetStub.image); + assetMock.getById.mockResolvedValueOnce(assetStub.image); + assetMock.update.mockResolvedValueOnce(assetStub.image); + await sut.update(authStub.admin, 'asset-1', { rating: 3 }); expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 }); }); it('should fail linking a live video if the motion part could not be found', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); - assetMock.getById.mockResolvedValue(null); await expect( sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { @@ -339,6 +344,7 @@ describe(AssetService.name, () => { isVisible: true, }); assetMock.getById.mockResolvedValueOnce(assetStub.image); + assetMock.update.mockResolvedValue(assetStub.image); await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: assetStub.livePhotoMotionAsset.id, @@ -366,7 +372,7 @@ describe(AssetService.name, () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoStillAsset); assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoMotionAsset); - assetMock.getById.mockResolvedValueOnce(assetStub.image); + assetMock.update.mockResolvedValueOnce(assetStub.image); await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }); @@ -383,15 +389,15 @@ describe(AssetService.name, () => { it('should fail unlinking a live video if the asset could not be found', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); - assetMock.getById.mockResolvedValue(null); + // eslint-disable-next-line unicorn/no-useless-undefined + assetMock.getById.mockResolvedValueOnce(undefined); await expect( sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }), ).rejects.toBeInstanceOf(BadRequestException); expect(assetMock.update).not.toHaveBeenCalled(); - expect(assetMock.update).not.toHaveBeenCalledWith(); - expect(eventMock.emit).not.toHaveBeenCalledWith(); + expect(eventMock.emit).not.toHaveBeenCalled(); }); }); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 8751037119..dff6592b17 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -74,29 +74,13 @@ export class AssetService extends BaseService { async get(auth: AuthDto, id: string): Promise { await this.requireAccess({ auth, permission: Permission.ASSET_READ, ids: [id] }); - const asset = await this.assetRepository.getById( - id, - { - exifInfo: true, - sharedLinks: true, - tags: true, - owner: true, - faces: { - person: true, - }, - stack: { - assets: { - exifInfo: true, - }, - }, - files: true, - }, - { - faces: { - boundingBoxX1: 'ASC', - }, - }, - ); + const asset = await this.assetRepository.getById(id, { + exifInfo: true, + owner: true, + faces: { person: true }, + stack: { assets: true }, + tags: true, + }); if (!asset) { throw new BadRequestException('Asset not found'); @@ -137,22 +121,12 @@ export class AssetService extends BaseService { await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude, rating }); - await this.assetRepository.update({ id, ...rest }); + const asset = await this.assetRepository.update({ id, ...rest }); if (previousMotion) { await onAfterUnlink(repos, { userId: auth.user.id, livePhotoVideoId: previousMotion.id }); } - const asset = await this.assetRepository.getById(id, { - exifInfo: true, - owner: true, - tags: true, - faces: { - person: true, - }, - files: true, - }); - if (!asset) { throw new BadRequestException('Asset not found'); } @@ -202,9 +176,7 @@ export class AssetService extends BaseService { const { id, deleteOnDisk } = job; const asset = await this.assetRepository.getById(id, { - faces: { - person: true, - }, + faces: { person: true }, library: true, stack: { assets: true }, exifInfo: true, diff --git a/server/src/services/backup.service.ts b/server/src/services/backup.service.ts index b3bc1dd8d1..324bc4cc13 100644 --- a/server/src/services/backup.service.ts +++ b/server/src/services/backup.service.ts @@ -71,10 +71,8 @@ export class BackupService extends BaseService { @OnJob({ name: JobName.BACKUP_DATABASE, queue: QueueName.BACKUP_DATABASE }) async handleBackupDatabase(): Promise { this.logger.debug(`Database Backup Started`); - - const { - database: { config }, - } = this.configRepository.getEnv(); + const { database } = this.configRepository.getEnv(); + const config = database.config.typeorm; const isUrlConnection = config.connectionType === 'url'; diff --git a/server/src/services/database.service.spec.ts b/server/src/services/database.service.spec.ts index 958fb158a0..ef60415402 100644 --- a/server/src/services/database.service.spec.ts +++ b/server/src/services/database.service.spec.ts @@ -1,3 +1,4 @@ +import { PostgresJSDialect } from 'kysely-postgres-js'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension, @@ -61,13 +62,19 @@ describe(DatabaseService.name, () => { mockEnvData({ database: { config: { - connectionType: 'parts', - type: 'postgres', - host: 'database', - port: 5432, - username: 'postgres', - password: 'postgres', - database: 'immich', + kysely: { + dialect: expect.any(PostgresJSDialect), + log: ['error'], + }, + typeorm: { + connectionType: 'parts', + type: 'postgres', + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + database: 'immich', + }, }, skipMigrations: false, vectorExtension: extension, @@ -291,13 +298,19 @@ describe(DatabaseService.name, () => { mockEnvData({ database: { config: { - connectionType: 'parts', - type: 'postgres', - host: 'database', - port: 5432, - username: 'postgres', - password: 'postgres', - database: 'immich', + kysely: { + dialect: expect.any(PostgresJSDialect), + log: ['error'], + }, + typeorm: { + connectionType: 'parts', + type: 'postgres', + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + database: 'immich', + }, }, skipMigrations: true, vectorExtension: DatabaseExtension.VECTORS, @@ -315,13 +328,19 @@ describe(DatabaseService.name, () => { mockEnvData({ database: { config: { - connectionType: 'parts', - type: 'postgres', - host: 'database', - port: 5432, - username: 'postgres', - password: 'postgres', - database: 'immich', + kysely: { + dialect: expect.any(PostgresJSDialect), + log: ['error'], + }, + typeorm: { + connectionType: 'parts', + type: 'postgres', + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + database: 'immich', + }, }, skipMigrations: true, vectorExtension: DatabaseExtension.VECTOR, diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.ts index 75af1ef6f1..c954d81a74 100644 --- a/server/src/services/duplicate.service.spec.ts +++ b/server/src/services/duplicate.service.spec.ts @@ -31,7 +31,12 @@ describe(SearchService.name, () => { describe('getDuplicates', () => { it('should get duplicates', async () => { - assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe, assetStub.hasDupe]); + assetMock.getDuplicates.mockResolvedValue([ + { + duplicateId: assetStub.hasDupe.duplicateId!, + assets: [assetStub.hasDupe, assetStub.hasDupe], + }, + ]); await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([ { duplicateId: assetStub.hasDupe.duplicateId, @@ -42,12 +47,6 @@ describe(SearchService.name, () => { }, ]); }); - - it('should update assets with duplicateId', async () => { - assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe]); - await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([]); - expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.hasDupe.id], { duplicateId: null }); - }); }); describe('handleQueueSearchDuplicates', () => { diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 0d91df5790..7e8ea49991 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { OnJob } from 'src/decorators'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { DuplicateResponseDto, mapDuplicateResponse } from 'src/dtos/duplicate.dto'; +import { DuplicateResponseDto } from 'src/dtos/duplicate.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { WithoutProperty } from 'src/interfaces/asset.interface'; import { JOBS_ASSET_PAGINATION_SIZE, JobName, JobOf, JobStatus, QueueName } from 'src/interfaces/job.interface'; @@ -15,25 +15,11 @@ import { usePagination } from 'src/utils/pagination'; @Injectable() export class DuplicateService extends BaseService { async getDuplicates(auth: AuthDto): Promise { - const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] }); - const uniqueAssetIds: string[] = []; - const duplicates = mapDuplicateResponse(res.map((a) => mapAsset(a, { auth, withStack: true }))).filter( - (duplicate) => { - if (duplicate.assets.length === 1) { - uniqueAssetIds.push(duplicate.assets[0].id); - return false; - } - return true; - }, - ); - if (uniqueAssetIds.length > 0) { - try { - await this.assetRepository.updateAll(uniqueAssetIds, { duplicateId: null }); - } catch (error: any) { - this.logger.error(`Failed to remove duplicateId from assets: ${error.message}`); - } - } - return duplicates; + const duplicates = await this.assetRepository.getDuplicates(auth.user.id); + return duplicates.map(({ duplicateId, assets }) => ({ + duplicateId, + assets: assets.map((asset) => mapAsset(asset, { auth })), + })); } @OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION }) diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index 43d6662d65..8d2d804e8b 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -256,8 +256,6 @@ describe(LibraryService.name, () => { exclusionPatterns: [], }; - assetMock.getById.mockResolvedValue(null); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED); expect(assetMock.remove).not.toHaveBeenCalled(); @@ -394,7 +392,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/photo.jpg', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.image); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); @@ -440,7 +437,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/photo.jpg', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.image); storageMock.checkFileExists.mockResolvedValue(true); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); @@ -487,7 +483,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/video.mp4', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.video); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); @@ -533,7 +528,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/photo.jpg', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.image); libraryMock.get.mockResolvedValue({ ...libraryStub.externalLibrary1, deletedAt: new Date() }); @@ -599,7 +593,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/photo.jpg', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.image); await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED); @@ -618,7 +611,6 @@ describe(LibraryService.name, () => { assetPath: '/data/user1/photo.jpg', }; - assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.image); await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED); diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 390f18b777..9a5f53a5e2 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -2,6 +2,7 @@ import { BinaryField, ExifDateTime } from 'exiftool-vendored'; import { randomBytes } from 'node:crypto'; import { Stats } from 'node:fs'; import { constants } from 'node:fs/promises'; +import { AssetEntity } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum'; import { IAlbumRepository } from 'src/interfaces/album.interface'; @@ -200,7 +201,6 @@ describe(MetadataService.name, () => { exifInfo: { livePhotoCID: 'CID' } as ExifEntity, }, ]); - assetMock.findLivePhotoMatch.mockResolvedValue(null); await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe( JobStatus.SKIPPED, @@ -545,7 +545,6 @@ describe(MetadataService.name, () => { EmbeddedVideoType: 'MotionPhoto_Data', }); cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); - assetMock.getByChecksum.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); const video = randomBytes(512); @@ -590,7 +589,6 @@ describe(MetadataService.name, () => { EmbeddedVideoType: 'MotionPhoto_Data', }); cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); - assetMock.getByChecksum.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); const video = randomBytes(512); @@ -636,7 +634,6 @@ describe(MetadataService.name, () => { MicroVideoOffset: 1, }); cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); - assetMock.getByChecksum.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); const video = randomBytes(512); @@ -682,8 +679,9 @@ describe(MetadataService.name, () => { MicroVideoOffset: 1, }); cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); - assetMock.getByChecksum.mockResolvedValue(null); - assetMock.create.mockImplementation((asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset })); + assetMock.create.mockImplementation( + (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, + ); const video = randomBytes(512); storageMock.readFile.mockResolvedValue(video); @@ -755,7 +753,6 @@ describe(MetadataService.name, () => { MicroVideoOffset: 1, }); cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); - assetMock.getByChecksum.mockResolvedValue(null); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); const video = randomBytes(512); storageMock.readFile.mockResolvedValue(video); diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 79a7d519d6..c2e20feed4 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -1,16 +1,17 @@ import { Injectable } from '@nestjs/common'; import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored'; import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime'; +import { Insertable } from 'kysely'; import _ from 'lodash'; import { Duration } from 'luxon'; import { constants } from 'node:fs/promises'; import path from 'node:path'; import { SystemConfig } from 'src/config'; import { StorageCore } from 'src/cores/storage.core'; +import { Exif } from 'src/db'; import { OnEvent, OnJob } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; -import { ExifEntity } from 'src/entities/exif.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum'; import { WithoutProperty } from 'src/interfaces/asset.interface'; @@ -166,7 +167,7 @@ export class MetadataService extends BaseService { const { width, height } = this.getImageDimensions(exifTags); - const exifData: Partial = { + const exifData: Insertable = { assetId: asset.id, // dates diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index 3b749c0ab6..60cb370881 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -728,11 +728,13 @@ describe(PersonService.name, () => { assetId: assetStub.image.id, facesRecognizedAt: expect.any(Date), }); - expect(assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt?.getTime()).toBeGreaterThan(start); + const facesRecognizedAt = assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt as Date; + expect(facesRecognizedAt.getTime()).toBeGreaterThan(start); }); it('should create a face with no person and queue recognition job', async () => { machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock); + searchMock.searchFaces.mockResolvedValue([{ ...faceStub.face1, distance: 0.7 }]); assetMock.getByIds.mockResolvedValue([assetStub.image]); await sut.handleDetectFaces({ id: assetStub.image.id }); @@ -840,10 +842,10 @@ describe(PersonService.name, () => { } const faces = [ - { face: faceStub.noPerson1, distance: 0 }, - { face: faceStub.primaryFace1, distance: 0.2 }, - { face: faceStub.noPerson2, distance: 0.3 }, - { face: faceStub.face1, distance: 0.4 }, + { ...faceStub.noPerson1, distance: 0 }, + { ...faceStub.primaryFace1, distance: 0.2 }, + { ...faceStub.noPerson2, distance: 0.3 }, + { ...faceStub.face1, distance: 0.4 }, ] as FaceSearchResult[]; systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); @@ -867,8 +869,8 @@ describe(PersonService.name, () => { it('should create a new person if the face is a core point with no person', async () => { const faces = [ - { face: faceStub.noPerson1, distance: 0 }, - { face: faceStub.noPerson2, distance: 0.3 }, + { ...faceStub.noPerson1, distance: 0 }, + { ...faceStub.noPerson2, distance: 0.3 }, ] as FaceSearchResult[]; systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); @@ -889,7 +891,7 @@ describe(PersonService.name, () => { }); it('should not queue face with no matches', async () => { - const faces = [{ face: faceStub.noPerson1, distance: 0 }] as FaceSearchResult[]; + const faces = [{ ...faceStub.noPerson1, distance: 0 }] as FaceSearchResult[]; searchMock.searchFaces.mockResolvedValue(faces); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); @@ -905,8 +907,8 @@ describe(PersonService.name, () => { it('should defer non-core faces to end of queue', async () => { const faces = [ - { face: faceStub.noPerson1, distance: 0 }, - { face: faceStub.noPerson2, distance: 0.4 }, + { ...faceStub.noPerson1, distance: 0 }, + { ...faceStub.noPerson2, distance: 0.4 }, ] as FaceSearchResult[]; systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); @@ -927,8 +929,8 @@ describe(PersonService.name, () => { it('should not assign person to deferred non-core face with no matching person', async () => { const faces = [ - { face: faceStub.noPerson1, distance: 0 }, - { face: faceStub.noPerson2, distance: 0.4 }, + { ...faceStub.noPerson1, distance: 0 }, + { ...faceStub.noPerson2, distance: 0.4 }, ] as FaceSearchResult[]; systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index bdec6f88e8..cc488a7f4e 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -261,7 +261,7 @@ export class PersonService extends BaseService { return force === false ? this.assetRepository.getWithout(pagination, WithoutProperty.FACES) : this.assetRepository.getAll(pagination, { - orderDirection: 'DESC', + orderDirection: 'desc', withFaces: true, withArchived: true, isVisible: true, @@ -288,13 +288,7 @@ export class PersonService extends BaseService { return JobStatus.SKIPPED; } - const relations = { - exifInfo: true, - faces: { - person: false, - }, - files: true, - }; + const relations = { exifInfo: true, faces: { person: false }, files: true }; const [asset] = await this.assetRepository.getByIds([id], relations); const { previewFile } = getAssetFiles(asset.files); if (!asset || !previewFile) { @@ -491,7 +485,7 @@ export class PersonService extends BaseService { return JobStatus.SKIPPED; } - let personId = matches.find((match) => match.face.personId)?.face.personId; + let personId = matches.find((match) => match.personId)?.personId; if (!personId) { const matchWithPerson = await this.searchRepository.searchFaces({ userIds: [face.asset.ownerId], @@ -502,7 +496,7 @@ export class PersonService extends BaseService { }); if (matchWithPerson.length > 0) { - personId = matchWithPerson[0].face.personId; + personId = matchWithPerson[0].personId; } } diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 3933526167..5c59e24b21 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -45,11 +45,11 @@ describe(SearchService.name, () => { it('should get assets by city and tag', async () => { assetMock.getAssetIdByCity.mockResolvedValue({ fieldName: 'exifInfo.city', - items: [{ value: 'Paris', data: assetStub.image.id }], + items: [{ value: 'test-city', data: assetStub.withLocation.id }], }); - assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.image, assetStub.imageFrom2015]); + assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]); const expectedResponse = [ - { fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] }, + { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] }, ]; const result = await sut.getExploreData(authStub.user1); diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index 7fc947a8b5..b833d0184c 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -34,16 +34,10 @@ export class SearchService extends BaseService { async getExploreData(auth: AuthDto): Promise[]> { const options = { maxFields: 12, minAssetsPerField: 5 }; - const result = await this.assetRepository.getAssetIdByCity(auth.user.id, options); - const results = [result]; - const assetIds = new Set(results.flatMap((field) => field.items.map((item) => item.data))); - const assets = await this.assetRepository.getByIdsWithAllRelations([...assetIds]); - const assetMap = new Map(assets.map((asset) => [asset.id, mapAsset(asset)])); - - return results.map(({ fieldName, items }) => ({ - fieldName, - items: items.map(({ value, data }) => ({ value, data: assetMap.get(data) as AssetResponseDto })), - })); + const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options); + const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data)); + const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) })); + return [{ fieldName: cities.fieldName, items }]; } async searchMetadata(auth: AuthDto, dto: MetadataSearchDto): Promise { @@ -57,14 +51,13 @@ export class SearchService extends BaseService { const page = dto.page ?? 1; const size = dto.size || 250; - const enumToOrder = { [AssetOrder.ASC]: 'ASC', [AssetOrder.DESC]: 'DESC' } as const; const { hasNextPage, items } = await this.searchRepository.searchMetadata( { page, size }, { ...dto, checksum, userIds, - orderDirection: dto.order ? enumToOrder[dto.order] : 'DESC', + orderDirection: dto.order ?? AssetOrder.DESC, }, ); diff --git a/server/src/services/timeline.service.spec.ts b/server/src/services/timeline.service.spec.ts index db6890c27b..41f9919189 100644 --- a/server/src/services/timeline.service.spec.ts +++ b/server/src/services/timeline.service.spec.ts @@ -61,12 +61,15 @@ describe(TimelineService.name, () => { userId: authStub.admin.user.id, }), ).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })])); - expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', { - size: TimeBucketSize.DAY, - timeBucket: 'bucket', - isArchived: true, - userIds: [authStub.admin.user.id], - }); + expect(assetMock.getTimeBucket).toHaveBeenCalledWith( + 'bucket', + expect.objectContaining({ + size: TimeBucketSize.DAY, + timeBucket: 'bucket', + isArchived: true, + userIds: [authStub.admin.user.id], + }), + ); }); it('should include partner shared assets', async () => { @@ -143,11 +146,14 @@ describe(TimelineService.name, () => { userId: authStub.admin.user.id, }), ).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })])); - expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', { - size: TimeBucketSize.DAY, - timeBucket: 'bucket', - userIds: [authStub.admin.user.id], - }); + expect(assetMock.getTimeBucket).toHaveBeenCalledWith( + 'bucket', + expect.objectContaining({ + size: TimeBucketSize.DAY, + timeBucket: 'bucket', + userIds: [authStub.admin.user.id], + }), + ); }); it('should throw an error if withParners is true and isArchived true or undefined', async () => { diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index ad2198b38c..ec96466a86 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -1,8 +1,7 @@ -import _ from 'lodash'; -import { AssetFaceEntity } from 'src/entities/asset-face.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface'; -import { Between, IsNull, LessThanOrEqual, MoreThanOrEqual, Not, SelectQueryBuilder } from 'typeorm'; +import { Expression, RawBuilder, sql, ValueExpression } from 'kysely'; +import { InsertObject } from 'node_modules/kysely/dist/cjs'; +import { DB } from 'src/db'; +import { Between, DataSource, LessThanOrEqual, MoreThanOrEqual } from 'typeorm'; /** * Allows optional values unlike the regular Between and uses MoreThanOrEqual @@ -18,131 +17,54 @@ export function OptionalBetween(from?: T, to?: T) { } } -export const asVector = (embedding: number[], quote = false) => - quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`; - -export function searchAssetBuilder( - builder: SelectQueryBuilder, - options: AssetSearchBuilderOptions, -): SelectQueryBuilder { - builder.andWhere( - _.omitBy( - { - createdAt: OptionalBetween(options.createdAfter, options.createdBefore), - updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore), - deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore), - fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore), - }, - _.isUndefined, - ), - ); - - const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined); - const hasExifQuery = Object.keys(exifInfo).length > 0; - - if (options.withExif && !hasExifQuery) { - builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo'); - } - - if (hasExifQuery) { - if (options.withExif) { - builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo'); - } else { - builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo'); +export const UPSERT_COLUMNS = {} as { [T in keyof DB]: { [K in keyof DB[T]]: RawBuilder } }; +/** Any repository that upserts to a table using `mapUpsertColumns` should call this method in its constructor with that table. */ +export const introspectTables = (dataSource: DataSource, ...tables: (keyof DB)[]) => { + for (const table of tables) { + if (table in UPSERT_COLUMNS) { + continue; } - for (const [key, value] of Object.entries(exifInfo)) { - if (value === null) { - builder.andWhere(`exifInfo.${key} IS NULL`); - } else { - builder.andWhere(`exifInfo.${key} = :${key}`, { [key]: value }); - } + const metadata = dataSource.manager.connection.getMetadata(table); + UPSERT_COLUMNS[table] = Object.fromEntries( + metadata.ownColumns.map((column) => [column.propertyName, sql`excluded.${sql.ref(column.propertyName)}`]), + ) as any; + } +}; + +/** Generates the columns for an upsert statement, excluding the conflict keys. + * Assumes that all entries have the same keys. */ +export const mapUpsertColumns = ( + table: T, + entry: InsertObject, + conflictKeys: readonly (keyof DB[T])[], +) => { + const columns = UPSERT_COLUMNS[table] as { [K in keyof DB[T]]: RawBuilder }; + const upsertColumns: Partial>> = {}; + for (const entryColumn in entry) { + if (!conflictKeys.includes(entryColumn as keyof DB[T])) { + upsertColumns[entryColumn as keyof typeof entry] = columns[entryColumn as keyof DB[T]]; } } - const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId']); + return upsertColumns as Expand>>; +}; - if (id.libraryId === null) { - id.libraryId = IsNull() as unknown as string; - } +export const asUuid = (id: string | Expression) => sql`${id}::uuid`; - builder.andWhere(_.omitBy(id, _.isUndefined)); +export const anyUuid = (ids: string[]) => sql`any(${`{${ids}}`}::uuid[])`; - if (options.userIds) { - builder.andWhere(`${builder.alias}.ownerId IN (:...userIds)`, { userIds: options.userIds }); - } +export const asVector = (embedding: number[]) => sql`${`[${embedding}]`}::vector`; - const path = _.pick(options, ['encodedVideoPath', 'originalPath']); - builder.andWhere(_.omitBy(path, _.isUndefined)); +/** + * Mainly for type debugging to make VS Code display a more useful tooltip. + * Source: https://stackoverflow.com/a/69288824 + */ +export type Expand = T extends infer O ? { [K in keyof O]: O[K] } : never; - if (options.originalFileName) { - builder.andWhere(`f_unaccent(${builder.alias}.originalFileName) ILIKE f_unaccent(:originalFileName)`, { - originalFileName: `%${options.originalFileName}%`, - }); - } - - const status = _.pick(options, ['isFavorite', 'isVisible', 'type']); - const { - isArchived, - isEncoded, - isMotion, - withArchived, - isNotInAlbum, - withFaces, - withPeople, - personIds, - withStacked, - trashedAfter, - trashedBefore, - } = options; - builder.andWhere( - _.omitBy( - { - ...status, - isArchived: isArchived ?? (withArchived ? undefined : false), - encodedVideoPath: isEncoded ? Not(IsNull()) : undefined, - livePhotoVideoId: isMotion ? Not(IsNull()) : undefined, - }, - _.isUndefined, - ), - ); - - if (isNotInAlbum) { - builder - .leftJoin(`${builder.alias}.albums`, 'albums') - .andWhere('albums.id IS NULL') - .andWhere(`${builder.alias}.isVisible = true`); - } - - if (withFaces || withPeople) { - builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces'); - } - - if (withPeople) { - builder.leftJoinAndSelect('faces.person', 'person'); - } - - if (personIds && personIds.length > 0) { - const cte = builder - .createQueryBuilder() - .select('faces."assetId"') - .from(AssetFaceEntity, 'faces') - .where('faces."personId" IN (:...personIds)', { personIds }) - .groupBy(`faces."assetId"`) - .having(`COUNT(DISTINCT faces."personId") = :personCount`, { personCount: personIds.length }); - builder.addCommonTableExpression(cte, 'face_ids').innerJoin('face_ids', 'a', 'a."assetId" = asset.id'); - - builder.getQuery(); // typeorm mixes up parameters without this (੭ °ཀ°)੭ - } - - if (withStacked) { - builder.leftJoinAndSelect(`${builder.alias}.stack`, 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets'); - } - - const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined); - if (withDeleted) { - builder.withDeleted(); - } - - return builder; -} +/** Recursive version of {@link Expand} from the same source. */ +export type ExpandRecursively = T extends object + ? T extends infer O + ? { [K in keyof O]: ExpandRecursively } + : never + : T; diff --git a/server/src/utils/pagination.ts b/server/src/utils/pagination.ts index 4009f219c1..4f1bd1a7f8 100644 --- a/server/src/utils/pagination.ts +++ b/server/src/utils/pagination.ts @@ -33,7 +33,10 @@ export async function* usePagination( } } -function paginationHelper(items: Entity[], take: number): PaginationResult { +export function paginationHelper( + items: Entity[], + take: number, +): PaginationResult { const hasNextPage = items.length > take; items.splice(take); diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index df26f7f725..00cca308a7 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -1,3 +1,5 @@ +import { PostgresJSDialect } from 'kysely-postgres-js'; +import postgres from 'postgres'; import { ImmichEnvironment, ImmichWorker } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface'; @@ -21,16 +23,24 @@ const envData: EnvData = { database: { config: { - connectionType: 'parts', - database: 'immich', - type: 'postgres', - host: 'database', - port: 5432, - username: 'postgres', - password: 'postgres', - name: 'immich', - synchronize: false, - migrationsRun: true, + kysely: { + dialect: new PostgresJSDialect({ + postgres: postgres({ database: 'immich', host: 'database', port: 5432 }), + }), + log: ['error'], + }, + typeorm: { + connectionType: 'parts', + database: 'immich', + type: 'postgres', + host: 'database', + port: 5432, + username: 'postgres', + password: 'postgres', + name: 'immich', + synchronize: false, + migrationsRun: true, + }, }, skipMigrations: false, diff --git a/server/tsconfig.json b/server/tsconfig.json index 1ffc110e83..8d8d12c54e 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -19,7 +19,8 @@ "preserveWatchOutput": true, "baseUrl": "./", "jsx": "react", - "types": ["vitest/globals"] + "types": ["vitest/globals"], + "noErrorTruncation": true }, "exclude": ["dist", "node_modules", "upload"] }