1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-23 12:12:45 +01:00

refactor(server): use kysely ()

This commit is contained in:
Mert 2025-01-09 11:15:41 -05:00 committed by GitHub
parent 1489d69f81
commit 2e12c46980
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
59 changed files with 2887 additions and 3275 deletions

View file

@ -538,7 +538,7 @@ describe('/asset', () => {
expect(body).toMatchObject({
id: user1Assets[0].id,
exifInfo: expect.objectContaining({
dateTimeOriginal: '2023-11-20T01:11:00.000Z',
dateTimeOriginal: '2023-11-20T01:11:00+00:00',
}),
});
expect(status).toEqual(200);
@ -608,7 +608,7 @@ describe('/asset', () => {
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const assetInfo = await utils.getAssetInfo(user1.accessToken, id);
expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52.000Z');
expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52+00:00');
const { status, body } = await request(app)
.put(`/assets/${id}`)
@ -618,7 +618,7 @@ describe('/asset', () => {
expect(body).toMatchObject({
id,
exifInfo: expect.objectContaining({
dateTimeOriginal: '2023-11-20T01:11:00.000Z',
dateTimeOriginal: '2023-11-20T01:11:00+00:00',
}),
});
expect(status).toEqual(200);
@ -985,8 +985,6 @@ describe('/asset', () => {
exifImageHeight: 1080,
exifImageWidth: 1617,
fileSizeInByte: 862_424,
latitude: null,
longitude: null,
},
},
},
@ -996,11 +994,9 @@ describe('/asset', () => {
type: AssetTypeEnum.Image,
originalFileName: 'el_torcal_rocks.jpg',
exifInfo: {
dateTimeOriginal: '2012-08-05T11:39:59.000Z',
dateTimeOriginal: '2012-08-05T11:39:59+00:00',
exifImageWidth: 512,
exifImageHeight: 341,
latitude: null,
longitude: null,
focalLength: 75,
iso: 200,
fNumber: 11,
@ -1008,7 +1004,6 @@ describe('/asset', () => {
fileSizeInByte: 53_493,
make: 'SONY',
model: 'DSLR-A550',
orientation: null,
description: 'SONY DSC',
},
},
@ -1023,8 +1018,6 @@ describe('/asset', () => {
exifImageHeight: 1080,
exifImageWidth: 1440,
fileSizeInByte: 1_780_777,
latitude: null,
longitude: null,
},
},
},
@ -1035,7 +1028,7 @@ describe('/asset', () => {
originalFileName: 'IMG_2682.heic',
fileCreatedAt: '2019-03-21T16:04:22.348Z',
exifInfo: {
dateTimeOriginal: '2019-03-21T16:04:22.348Z',
dateTimeOriginal: '2019-03-21T16:04:22.348+00:00',
exifImageWidth: 4032,
exifImageHeight: 3024,
latitude: 41.2203,
@ -1060,8 +1053,6 @@ describe('/asset', () => {
exifInfo: {
exifImageWidth: 800,
exifImageHeight: 800,
latitude: null,
longitude: null,
fileSizeInByte: 25_408,
},
},
@ -1080,9 +1071,7 @@ describe('/asset', () => {
focalLength: 18,
iso: 100,
fileSizeInByte: 9_057_784,
dateTimeOriginal: '2010-07-20T17:27:12.000Z',
latitude: null,
longitude: null,
dateTimeOriginal: '2010-07-20T17:27:12+00:00',
orientation: '1',
},
},
@ -1101,9 +1090,7 @@ describe('/asset', () => {
focalLength: 85,
iso: 200,
fileSizeInByte: 15_856_335,
dateTimeOriginal: '2016-09-22T21:10:29.060Z',
latitude: null,
longitude: null,
dateTimeOriginal: '2016-09-22T21:10:29.06+00:00',
orientation: '1',
timeZone: 'UTC-4',
},
@ -1125,9 +1112,7 @@ describe('/asset', () => {
focalLength: 35,
iso: 400,
fileSizeInByte: 19_587_072,
dateTimeOriginal: '2018-05-10T08:42:37.842Z',
latitude: null,
longitude: null,
dateTimeOriginal: '2018-05-10T08:42:37.842+00:00',
orientation: '1',
},
},
@ -1149,9 +1134,7 @@ describe('/asset', () => {
iso: 100,
lensModel: 'E PZ 18-105mm F4 G OSS',
fileSizeInByte: 25_001_984,
dateTimeOriginal: '2016-09-27T10:51:44.000Z',
latitude: null,
longitude: null,
dateTimeOriginal: '2016-09-27T10:51:44+00:00',
orientation: '1',
},
},
@ -1173,9 +1156,7 @@ describe('/asset', () => {
iso: 100,
lensModel: 'E 25mm F2',
fileSizeInByte: 49_512_448,
dateTimeOriginal: '2016-01-08T14:08:01.000Z',
latitude: null,
longitude: null,
dateTimeOriginal: '2016-01-08T14:08:01+00:00',
orientation: '1',
},
},
@ -1197,7 +1178,7 @@ describe('/asset', () => {
iso: 80,
lensModel: null,
fileSizeInByte: 11_113_617,
dateTimeOriginal: '2015-12-27T09:55:40.000Z',
dateTimeOriginal: '2015-12-27T09:55:40+00:00',
latitude: null,
longitude: null,
orientation: '1',
@ -1221,7 +1202,7 @@ describe('/asset', () => {
iso: 160,
lensModel: null,
fileSizeInByte: 13_551_312,
dateTimeOriginal: '2024-10-12T21:01:01.000Z',
dateTimeOriginal: '2024-10-12T21:01:01+00:00',
latitude: null,
longitude: null,
orientation: '6',
@ -1235,7 +1216,7 @@ describe('/asset', () => {
originalFileName: 'Ricoh_GR3-450.DNG',
fileCreatedAt: '2024-06-08T13:48:39.000Z',
exifInfo: {
dateTimeOriginal: '2024-06-08T13:48:39.000Z',
dateTimeOriginal: '2024-06-08T13:48:39+00:00',
exifImageHeight: 4064,
exifImageWidth: 6112,
exposureTime: '1/400',

View file

@ -151,7 +151,7 @@ describe('/timeline', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).get('/timeline/bucket').query({
size: TimeBucketSize.Month,
timeBucket: '1900-01-01T00:00:00.000Z',
timeBucket: '1900-01-01',
});
expect(status).toBe(401);
@ -161,7 +161,7 @@ describe('/timeline', () => {
it('should handle 5 digit years', async () => {
const { status, body } = await request(app)
.get('/timeline/bucket')
.query({ size: TimeBucketSize.Month, timeBucket: '+012345-01-01T00:00:00.000Z' })
.query({ size: TimeBucketSize.Month, timeBucket: '012345-01-01' })
.set('Authorization', `Bearer ${timeBucketUser.accessToken}`);
expect(status).toBe(200);
@ -183,7 +183,7 @@ describe('/timeline', () => {
const { status, body } = await request(app)
.get('/timeline/bucket')
.set('Authorization', `Bearer ${timeBucketUser.accessToken}`)
.query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10T00:00:00.000Z' });
.query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10' });
expect(status).toBe(200);
expect(body).toEqual([]);

Binary file not shown.

View file

@ -10000,7 +10000,8 @@
{
"$ref": "#/components/schemas/AssetOrder"
}
]
],
"default": "desc"
},
"originalFileName": {
"type": "string"

337
server/package-lock.json generated
View file

@ -42,10 +42,13 @@
"ioredis": "^5.3.2",
"joi": "^17.10.0",
"js-yaml": "^4.1.0",
"kysely": "^0.27.3",
"kysely-postgres-js": "^2.0.0",
"lodash": "^4.17.21",
"luxon": "^3.4.2",
"nest-commander": "^3.11.1",
"nestjs-cls": "^4.3.0",
"nestjs-kysely": "^1.0.0",
"nestjs-otel": "^6.0.0",
"nodemailer": "^6.9.13",
"openid-client": "^5.4.3",
@ -99,6 +102,7 @@
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-unicorn": "^56.0.1",
"globals": "^15.9.0",
"kysely-codegen": "^0.16.3",
"mock-fs": "^5.2.0",
"pngjs": "^7.0.0",
"prettier": "^3.0.2",
@ -9176,6 +9180,102 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/git-diff": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/git-diff/-/git-diff-2.0.6.tgz",
"integrity": "sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==",
"dev": true,
"dependencies": {
"chalk": "^2.3.2",
"diff": "^3.5.0",
"loglevel": "^1.6.1",
"shelljs": "^0.8.1",
"shelljs.exec": "^1.1.7"
},
"engines": {
"node": ">= 4.8.0"
}
},
"node_modules/git-diff/node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"dependencies": {
"color-convert": "^1.9.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dev": true,
"dependencies": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"dependencies": {
"color-name": "1.1.3"
}
},
"node_modules/git-diff/node_modules/color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
"dev": true
},
"node_modules/git-diff/node_modules/diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true,
"engines": {
"node": ">=0.3.1"
}
},
"node_modules/git-diff/node_modules/escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
"dev": true,
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/git-diff/node_modules/has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
"dev": true,
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"dev": true,
"dependencies": {
"has-flag": "^3.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/glob": {
"version": "10.4.5",
"resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@ -9617,6 +9717,15 @@
"node": ">=12.0.0"
}
},
"node_modules/interpret": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
"integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
"dev": true,
"engines": {
"node": ">= 0.10"
}
},
"node_modules/ioredis": {
"version": "5.4.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz",
@ -10048,6 +10157,100 @@
"json-buffer": "3.0.1"
}
},
"node_modules/kysely": {
"version": "0.27.4",
"resolved": "https://registry.npmjs.org/kysely/-/kysely-0.27.4.tgz",
"integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/kysely-codegen": {
"version": "0.16.3",
"resolved": "https://registry.npmjs.org/kysely-codegen/-/kysely-codegen-0.16.3.tgz",
"integrity": "sha512-SOOF3AhrsjREJuRewXmKl0nb6CkEzpP7VavHXzWdfIdIdfoJnlWlozuZhgMsYoIFmzL8aG4skvKGXF/dF3mbwg==",
"dev": true,
"dependencies": {
"chalk": "4.1.2",
"dotenv": "^16.4.5",
"dotenv-expand": "^11.0.6",
"git-diff": "^2.0.6",
"micromatch": "^4.0.8",
"minimist": "^1.2.8",
"pluralize": "^8.0.0"
},
"bin": {
"kysely-codegen": "dist/cli/bin.js"
},
"peerDependencies": {
"@libsql/kysely-libsql": "^0.3.0",
"@tediousjs/connection-string": "^0.5.0",
"better-sqlite3": ">=7.6.2",
"kysely": "^0.27.0",
"kysely-bun-sqlite": "^0.3.2",
"kysely-bun-worker": "^0.5.3",
"mysql2": "^2.3.3 || ^3.0.0",
"pg": "^8.8.0",
"tarn": "^3.0.0",
"tedious": "^18.0.0"
},
"peerDependenciesMeta": {
"@libsql/kysely-libsql": {
"optional": true
},
"@tediousjs/connection-string": {
"optional": true
},
"better-sqlite3": {
"optional": true
},
"kysely": {
"optional": false
},
"kysely-bun-sqlite": {
"optional": true
},
"kysely-bun-worker": {
"optional": true
},
"mysql2": {
"optional": true
},
"pg": {
"optional": true
},
"tarn": {
"optional": true
},
"tedious": {
"optional": true
}
}
},
"node_modules/kysely-codegen/node_modules/dotenv-expand": {
"version": "11.0.6",
"resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.6.tgz",
"integrity": "sha512-8NHi73otpWsZGBSZwwknTXS5pqMOrk9+Ssrna8xCaxkzEpU9OTf9R5ArQGVw03//Zmk9MOwLPng9WwndvpAJ5g==",
"dev": true,
"dependencies": {
"dotenv": "^16.4.4"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/kysely-postgres-js": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/kysely-postgres-js/-/kysely-postgres-js-2.0.0.tgz",
"integrity": "sha512-R1tWx6/x3tSatWvsmbHJxpBZYhNNxcnMw52QzZaHKg7ZOWtHib4iZyEaw4gb2hNKVctWQ3jfMxZT/ZaEMK6kBQ==",
"peerDependencies": {
"kysely": ">= 0.24.0 < 1",
"postgres": ">= 3.4.0 < 4"
}
},
"node_modules/lazystream": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz",
@ -10221,6 +10424,19 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/loglevel": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz",
"integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==",
"dev": true,
"engines": {
"node": ">= 0.6.0"
},
"funding": {
"type": "tidelift",
"url": "https://tidelift.com/funding/github/npm/loglevel"
}
},
"node_modules/long": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
@ -10825,6 +11041,17 @@
"rxjs": ">= 7"
}
},
"node_modules/nestjs-kysely": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/nestjs-kysely/-/nestjs-kysely-1.0.0.tgz",
"integrity": "sha512-0XwXm3H/sGQ8DhOY1UOMuosdgii+qePJpbdlWzRPP7ueP/Z/JGnxnu2jUHurFGS+bMGPipGnk9ynIkJuM05HUw==",
"peerDependencies": {
"@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0",
"@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0",
"kysely": "0.x",
"reflect-metadata": "^0.1.13 || ^0.2.2"
}
},
"node_modules/nestjs-otel": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/nestjs-otel/-/nestjs-otel-6.1.1.tgz",
@ -11736,6 +11963,19 @@
"license": "MIT",
"peer": true
},
"node_modules/postgres": {
"version": "3.4.4",
"resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz",
"integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==",
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/porsager"
}
},
"node_modules/postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
@ -12490,6 +12730,18 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/rechoir": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
"integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
"dev": true,
"dependencies": {
"resolve": "^1.1.6"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
@ -13173,6 +13425,53 @@
"node": ">=8"
}
},
"node_modules/shelljs": {
"version": "0.8.5",
"resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
"integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
"dev": true,
"dependencies": {
"glob": "^7.0.0",
"interpret": "^1.0.0",
"rechoir": "^0.6.2"
},
"bin": {
"shjs": "bin/shjs"
},
"engines": {
"node": ">=4"
}
},
"node_modules/shelljs.exec": {
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/shelljs.exec/-/shelljs.exec-1.1.8.tgz",
"integrity": "sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==",
"dev": true,
"engines": {
"node": ">= 4.0.0"
}
},
"node_modules/shelljs/node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"deprecated": "Glob versions prior to v9 are no longer supported",
"dev": true,
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/shimmer": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz",
@ -15906,6 +16205,44 @@
"engines": {
"node": ">= 14"
}
},
"node_modules/zip-stream/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/zip-stream/node_modules/readable-stream": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
"dependencies": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
}
}
}

View file

@ -67,10 +67,13 @@
"ioredis": "^5.3.2",
"joi": "^17.10.0",
"js-yaml": "^4.1.0",
"kysely": "^0.27.3",
"kysely-postgres-js": "^2.0.0",
"lodash": "^4.17.21",
"luxon": "^3.4.2",
"nest-commander": "^3.11.1",
"nestjs-cls": "^4.3.0",
"nestjs-kysely": "^1.0.0",
"nestjs-otel": "^6.0.0",
"nodemailer": "^6.9.13",
"openid-client": "^5.4.3",
@ -124,6 +127,7 @@
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-unicorn": "^56.0.1",
"globals": "^15.9.0",
"kysely-codegen": "^0.16.3",
"mock-fs": "^5.2.0",
"pngjs": "^7.0.0",
"prettier": "^3.0.2",

View file

@ -4,6 +4,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE, ModuleRef } from '@ne
import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ClsModule } from 'nestjs-cls';
import { KyselyModule } from 'nestjs-kysely';
import { OpenTelemetryModule } from 'nestjs-otel';
import { commands } from 'src/commands';
import { IWorker } from 'src/constants';
@ -48,7 +49,7 @@ const imports = [
inject: [ModuleRef],
useFactory: (moduleRef: ModuleRef) => {
return {
...database.config,
...database.config.typeorm,
poolErrorHandler: (error) => {
moduleRef.get(DatabaseService, { strict: false }).handleConnectionError(error);
},
@ -56,6 +57,7 @@ const imports = [
},
}),
TypeOrmModule.forFeature(entities),
KyselyModule.forRoot(database.config.kysely),
];
class BaseModule implements OnModuleInit, OnModuleDestroy {

View file

@ -8,4 +8,4 @@ const { database } = new ConfigRepository().getEnv();
*
* this export is ONLY to be used for TypeORM commands in package.json#scripts
*/
export const dataSource = new DataSource({ ...database.config, host: 'localhost' });
export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' });

View file

@ -4,6 +4,7 @@ import { Reflector } from '@nestjs/core';
import { SchedulerRegistry } from '@nestjs/schedule';
import { Test } from '@nestjs/testing';
import { TypeOrmModule } from '@nestjs/typeorm';
import { KyselyModule } from 'nestjs-kysely';
import { OpenTelemetryModule } from 'nestjs-otel';
import { mkdir, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
@ -73,13 +74,23 @@ class SqlGenerator {
await rm(this.options.targetDir, { force: true, recursive: true });
await mkdir(this.options.targetDir);
process.env.DB_HOSTNAME = 'localhost';
const { database, otel } = new ConfigRepository().getEnv();
const moduleFixture = await Test.createTestingModule({
imports: [
KyselyModule.forRoot({
...database.config.kysely,
log: (event) => {
if (event.level === 'query') {
this.sqlLogger.logQuery(event.query.sql);
} else if (event.level === 'error') {
this.sqlLogger.logQueryError(event.error as Error, event.query.sql);
}
},
}),
TypeOrmModule.forRoot({
...database.config,
host: 'localhost',
...database.config.typeorm,
entities,
logging: ['query'],
logger: this.sqlLogger,

View file

@ -7,6 +7,10 @@ export const POSTGRES_VERSION_RANGE = '>=14.0.0';
export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';
export const VECTOR_VERSION_RANGE = '>=0.5 <1';
export const ASSET_FILE_CONFLICT_KEYS = ['assetId', 'type'] as const;
export const EXIF_CONFLICT_KEYS = ['assetId'] as const;
export const JOB_STATUS_CONFLICT_KEYS = ['assetId'] as const;
export const NEXT_RELEASE = 'NEXT_RELEASE';
export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle';
export const DEPRECATED_IN_PREFIX = 'This property was deprecated in ';

439
server/src/db.d.ts vendored Normal file
View file

@ -0,0 +1,439 @@
/**
* This file was generated by kysely-codegen.
* Please do not edit it manually.
*/
import type { ColumnType } from 'kysely';
export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[] ? U[] : ArrayTypeImpl<T>;
export type ArrayTypeImpl<T> = T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S[], I[], U[]> : T[];
export type AssetsStatusEnum = 'active' | 'deleted' | 'trashed';
export type Generated<T> =
T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S, I | undefined, U> : ColumnType<T, T | undefined, T>;
export type Int8 = ColumnType<string, bigint | number | string, bigint | number | string>;
export type Json = JsonValue;
export type JsonArray = JsonValue[];
export type JsonObject = {
[x: string]: JsonValue | undefined;
};
export type JsonPrimitive = boolean | number | string | null;
export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
export type Sourcetype = 'exif' | 'machine-learning';
export type Timestamp = ColumnType<Date, Date | string, Date | string>;
export interface Activity {
albumId: string;
assetId: string | null;
comment: string | null;
createdAt: Generated<Timestamp>;
id: Generated<string>;
isLiked: Generated<boolean>;
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface Albums {
albumName: Generated<string>;
/**
* Asset ID to be used as thumbnail
*/
albumThumbnailAssetId: string | null;
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
description: Generated<string>;
id: Generated<string>;
isActivityEnabled: Generated<boolean>;
order: Generated<string>;
ownerId: string;
updatedAt: Generated<Timestamp>;
}
export interface AlbumsAssetsAssets {
albumsId: string;
assetsId: string;
}
export interface AlbumsSharedUsersUsers {
albumsId: string;
role: Generated<string>;
usersId: string;
}
export interface ApiKeys {
createdAt: Generated<Timestamp>;
id: Generated<string>;
key: string;
name: string;
permissions: string[];
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface AssetFaces {
assetId: string;
boundingBoxX1: Generated<number>;
boundingBoxX2: Generated<number>;
boundingBoxY1: Generated<number>;
boundingBoxY2: Generated<number>;
id: Generated<string>;
imageHeight: Generated<number>;
imageWidth: Generated<number>;
personId: string | null;
sourceType: Generated<Sourcetype>;
}
export interface AssetFiles {
assetId: string;
createdAt: Generated<Timestamp>;
id: Generated<string>;
path: string;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface AssetJobStatus {
assetId: string;
duplicatesDetectedAt: Timestamp | null;
facesRecognizedAt: Timestamp | null;
metadataExtractedAt: Timestamp | null;
previewAt: Timestamp | null;
thumbnailAt: Timestamp | null;
}
export interface Assets {
checksum: Buffer;
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
deviceAssetId: string;
deviceId: string;
duplicateId: string | null;
duration: string | null;
encodedVideoPath: Generated<string | null>;
fileCreatedAt: Timestamp;
fileModifiedAt: Timestamp;
id: Generated<string>;
isArchived: Generated<boolean>;
isExternal: Generated<boolean>;
isFavorite: Generated<boolean>;
isOffline: Generated<boolean>;
isVisible: Generated<boolean>;
libraryId: string | null;
livePhotoVideoId: string | null;
localDateTime: Timestamp;
originalFileName: string;
originalPath: string;
ownerId: string;
sidecarPath: string | null;
stackId: string | null;
status: Generated<AssetsStatusEnum>;
thumbhash: Buffer | null;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface AssetStack {
id: Generated<string>;
ownerId: string;
primaryAssetId: string;
}
export interface Audit {
action: string;
createdAt: Generated<Timestamp>;
entityId: string;
entityType: string;
id: Generated<number>;
ownerId: string;
}
export interface Exif {
assetId: string;
autoStackId: string | null;
bitsPerSample: number | null;
city: string | null;
colorspace: string | null;
country: string | null;
dateTimeOriginal: Timestamp | null;
description: Generated<string>;
exifImageHeight: number | null;
exifImageWidth: number | null;
exposureTime: string | null;
fileSizeInByte: Int8 | null;
fNumber: number | null;
focalLength: number | null;
fps: number | null;
iso: number | null;
latitude: number | null;
lensModel: string | null;
livePhotoCID: string | null;
longitude: number | null;
make: string | null;
model: string | null;
modifyDate: Timestamp | null;
orientation: string | null;
profileDescription: string | null;
projectionType: string | null;
rating: number | null;
state: string | null;
timeZone: string | null;
}
export interface FaceSearch {
embedding: string;
faceId: string;
}
export interface GeodataPlaces {
admin1Code: string | null;
admin1Name: string | null;
admin2Code: string | null;
admin2Name: string | null;
alternateNames: string | null;
countryCode: string;
earthCoord: Generated<string | null>;
id: number;
latitude: number;
longitude: number;
modificationDate: Timestamp;
name: string;
}
export interface Libraries {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
exclusionPatterns: string[];
id: Generated<string>;
importPaths: string[];
name: string;
ownerId: string;
refreshedAt: Timestamp | null;
updatedAt: Generated<Timestamp>;
}
export interface Memories {
createdAt: Generated<Timestamp>;
data: Json;
deletedAt: Timestamp | null;
id: Generated<string>;
isSaved: Generated<boolean>;
memoryAt: Timestamp;
ownerId: string;
seenAt: Timestamp | null;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface MemoriesAssetsAssets {
assetsId: string;
memoriesId: string;
}
export interface Migrations {
id: Generated<number>;
name: string;
timestamp: Int8;
}
export interface MoveHistory {
entityId: string;
id: Generated<string>;
newPath: string;
oldPath: string;
pathType: string;
}
export interface NaturalearthCountries {
admin: string;
admin_a3: string;
coordinates: string;
id: Generated<number>;
type: string;
}
export interface Partners {
createdAt: Generated<Timestamp>;
inTimeline: Generated<boolean>;
sharedById: string;
sharedWithId: string;
updatedAt: Generated<Timestamp>;
}
export interface Person {
birthDate: Timestamp | null;
createdAt: Generated<Timestamp>;
faceAssetId: string | null;
id: Generated<string>;
isHidden: Generated<boolean>;
name: Generated<string>;
ownerId: string;
thumbnailPath: Generated<string>;
updatedAt: Generated<Timestamp>;
}
export interface Sessions {
createdAt: Generated<Timestamp>;
deviceOS: Generated<string>;
deviceType: Generated<string>;
id: Generated<string>;
token: string;
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface SharedLinkAsset {
assetsId: string;
sharedLinksId: string;
}
export interface SharedLinks {
albumId: string | null;
allowDownload: Generated<boolean>;
allowUpload: Generated<boolean>;
createdAt: Generated<Timestamp>;
description: string | null;
expiresAt: Timestamp | null;
id: Generated<string>;
key: Buffer;
password: string | null;
showExif: Generated<boolean>;
type: string;
userId: string;
}
export interface SmartInfo {
assetId: string;
objects: string[] | null;
smartInfoTextSearchableColumn: Generated<string>;
tags: string[] | null;
}
export interface SmartSearch {
assetId: string;
embedding: string;
}
export interface SocketIoAttachments {
created_at: Generated<Timestamp | null>;
id: Generated<Int8>;
payload: Buffer | null;
}
export interface SystemConfig {
key: string;
value: string | null;
}
export interface SystemMetadata {
key: string;
value: Json;
}
export interface TagAsset {
assetsId: string;
tagsId: string;
}
export interface Tags {
color: string | null;
createdAt: Generated<Timestamp>;
id: Generated<string>;
parentId: string | null;
updatedAt: Generated<Timestamp>;
userId: string;
value: string;
}
export interface TagsClosure {
id_ancestor: string;
id_descendant: string;
}
export interface UserMetadata {
key: string;
userId: string;
value: Json;
}
export interface Users {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
email: string;
id: Generated<string>;
isAdmin: Generated<boolean>;
name: Generated<string>;
oauthId: Generated<string>;
password: Generated<string>;
profileChangedAt: Generated<Timestamp>;
profileImagePath: Generated<string>;
quotaSizeInBytes: Int8 | null;
quotaUsageInBytes: Generated<Int8>;
shouldChangePassword: Generated<boolean>;
status: Generated<string>;
storageLabel: string | null;
updatedAt: Generated<Timestamp>;
}
export interface VectorsPgVectorIndexStat {
idx_growing: ArrayType<Int8> | null;
idx_indexing: boolean | null;
idx_options: string | null;
idx_sealed: ArrayType<Int8> | null;
idx_size: Int8 | null;
idx_status: string | null;
idx_tuples: Int8 | null;
idx_write: Int8 | null;
indexname: string | null;
indexrelid: number | null;
tablename: string | null;
tablerelid: number | null;
}
export interface DB {
activity: Activity;
albums: Albums;
albums_assets_assets: AlbumsAssetsAssets;
albums_shared_users_users: AlbumsSharedUsersUsers;
api_keys: ApiKeys;
asset_faces: AssetFaces;
asset_files: AssetFiles;
asset_job_status: AssetJobStatus;
asset_stack: AssetStack;
assets: Assets;
audit: Audit;
exif: Exif;
face_search: FaceSearch;
geodata_places: GeodataPlaces;
libraries: Libraries;
memories: Memories;
memories_assets_assets: MemoriesAssetsAssets;
migrations: Migrations;
move_history: MoveHistory;
naturalearth_countries: NaturalearthCountries;
partners: Partners;
person: Person;
sessions: Sessions;
shared_link__asset: SharedLinkAsset;
shared_links: SharedLinks;
smart_info: SmartInfo;
smart_search: SmartSearch;
socket_io_attachments: SocketIoAttachments;
system_config: SystemConfig;
system_metadata: SystemMetadata;
tag_asset: TagAsset;
tags: Tags;
tags_closure: TagsClosure;
user_metadata: UserMetadata;
users: Users;
'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat;
}

View file

@ -97,10 +97,19 @@ const mapStack = (entity: AssetEntity) => {
return {
id: entity.stack.id,
primaryAssetId: entity.stack.primaryAssetId,
assetCount: entity.stack.assetCount ?? entity.stack.assets.length,
assetCount: entity.stack.assetCount ?? entity.stack.assets.length + 1,
};
};
// if an asset is jsonified in the DB before being returned, its buffer fields will be hex-encoded strings
const hexOrBufferToBase64 = (encoded: string | Buffer) => {
if (typeof encoded === 'string') {
return Buffer.from(encoded.slice(2), 'hex').toString('base64');
}
return encoded.toString('base64');
};
export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto {
const { stripMetadata = false, withStack = false } = options;
@ -129,7 +138,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
originalPath: entity.originalPath,
originalFileName: entity.originalFileName,
originalMimeType: mimeTypes.lookup(entity.originalFileName),
thumbhash: entity.thumbhash?.toString('base64') ?? null,
thumbhash: entity.thumbhash ? hexOrBufferToBase64(entity.thumbhash) : null,
fileCreatedAt: entity.fileCreatedAt,
fileModifiedAt: entity.fileModifiedAt,
localDateTime: entity.localDateTime,
@ -143,7 +152,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
tags: entity.tags?.map((tag) => mapTag(tag)),
people: peopleWithFaces(entity.faces),
unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)),
checksum: entity.checksum.toString('base64'),
checksum: hexOrBufferToBase64(entity.checksum),
stack: withStack ? mapStack(entity) : undefined,
isOffline: entity.isOffline,
hasMetadata: true,

View file

@ -1,5 +1,4 @@
import { IsNotEmpty } from 'class-validator';
import { groupBy, sortBy } from 'lodash';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { ValidateUUID } from 'src/validation';
@ -13,16 +12,3 @@ export class ResolveDuplicatesDto {
@ValidateUUID({ each: true })
assetIds!: string[];
}
export function mapDuplicateResponse(assets: AssetResponseDto[]): DuplicateResponseDto[] {
const result = [];
const grouped = groupBy(assets, (a) => a.duplicateId);
for (const [duplicateId, unsortedAssets] of Object.entries(grouped)) {
const assets = sortBy(unsortedAssets, (asset) => asset.localDateTime);
result.push({ duplicateId, assets });
}
return result;
}

View file

@ -162,7 +162,7 @@ export class MetadataSearchDto extends RandomSearchDto {
@IsEnum(AssetOrder)
@Optional()
@ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder })
@ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder, default: AssetOrder.DESC })
order?: AssetOrder;
@IsInt()

View file

@ -1,3 +1,6 @@
import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, Selectable, SelectQueryBuilder, sql } from 'kysely';
import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { Assets, DB } from 'src/db';
import { AlbumEntity } from 'src/entities/album.entity';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetFileEntity } from 'src/entities/asset-files.entity';
@ -9,7 +12,10 @@ import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { StackEntity } from 'src/entities/stack.entity';
import { TagEntity } from 'src/entities/tag.entity';
import { UserEntity } from 'src/entities/user.entity';
import { AssetStatus, AssetType } from 'src/enum';
import { AssetFileType, AssetStatus, AssetType } from 'src/enum';
import { TimeBucketSize } from 'src/interfaces/asset.interface';
import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface';
import { anyUuid, asUuid } from 'src/utils/database';
import {
Column,
CreateDateColumn,
@ -38,8 +44,8 @@ export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum';
unique: true,
where: '"libraryId" IS NOT NULL',
})
@Index('IDX_day_of_month', { synchronize: false })
@Index('IDX_month', { synchronize: false })
@Index('idx_local_date_time', { synchronize: false })
@Index('idx_local_date_time_month', { synchronize: false })
@Index('IDX_originalPath_libraryId', ['originalPath', 'libraryId'])
@Index('IDX_asset_id_stackId', ['id', 'stackId'])
@Index('idx_originalFileName_trigram', { synchronize: false })
@ -173,3 +179,257 @@ export class AssetEntity {
@Column({ type: 'uuid', nullable: true })
duplicateId!: string | null;
}
export function withExif<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {
return qb
.leftJoin('exif', 'assets.id', 'exif.assetId')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'));
}
export function withExifInner<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {
return qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'));
}
export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {
return qb
.leftJoin('smart_search', 'assets.id', 'smart_search.assetId')
.select(sql<number[]>`smart_search.embedding`.as('embedding'));
}
export function withFaces(eb: ExpressionBuilder<DB, 'assets'>) {
return jsonArrayFrom(eb.selectFrom('asset_faces').selectAll().whereRef('asset_faces.assetId', '=', 'assets.id')).as(
'faces',
);
}
export function withFiles(eb: ExpressionBuilder<DB, 'assets'>, type?: AssetFileType) {
return jsonArrayFrom(
eb
.selectFrom('asset_files')
.selectAll()
.whereRef('asset_files.assetId', '=', 'assets.id')
.$if(!!type, (qb) => qb.where('type', '=', type!)),
).as('files');
}
export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'assets'>) {
return eb
.selectFrom('asset_faces')
.leftJoin('person', 'person.id', 'asset_faces.personId')
.whereRef('asset_faces.assetId', '=', 'assets.id')
.select((eb) =>
eb
.fn('jsonb_agg', [
eb
.case()
.when('person.id', 'is not', null)
.then(
eb.fn('jsonb_insert', [
eb.fn('to_jsonb', [eb.table('asset_faces')]),
sql`'{person}'::text[]`,
eb.fn('to_jsonb', [eb.table('person')]),
]),
)
.else(eb.fn('to_jsonb', [eb.table('asset_faces')]))
.end(),
])
.as('faces'),
)
.as('faces');
}
/** Adds a `has_people` CTE that can be inner joined on to filter out assets */
export function hasPeopleCte(db: Kysely<DB>, personIds: string[]) {
return db.with('has_people', (qb) =>
qb
.selectFrom('asset_faces')
.select('assetId')
.where('personId', '=', anyUuid(personIds!))
.groupBy('assetId')
.having((eb) => eb.fn.count('personId'), '>=', personIds.length),
);
}
export function hasPeople(db: Kysely<DB>, personIds?: string[]) {
return personIds && personIds.length > 0
? hasPeopleCte(db, personIds).selectFrom('assets').innerJoin('has_people', 'has_people.assetId', 'assets.id')
: db.selectFrom('assets');
}
export function withOwner(eb: ExpressionBuilder<DB, 'assets'>) {
return jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner');
}
export function withLibrary(eb: ExpressionBuilder<DB, 'assets'>) {
return jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as(
'library',
);
}
export function withStackedAssets<O>(qb: SelectQueryBuilder<DB, 'assets' | 'asset_stack', O>) {
return qb
.innerJoinLateral(
(eb: ExpressionBuilder<DB, 'assets' | 'asset_stack'>) =>
eb
.selectFrom('assets as stacked')
.select((eb) => eb.fn<Selectable<Assets>[]>('array_agg', [eb.table('stacked')]).as('assets'))
.whereRef('asset_stack.id', '=', 'stacked.stackId')
.whereRef('asset_stack.primaryAssetId', '!=', 'stacked.id')
.as('s'),
(join) =>
join.on((eb) =>
eb.or([eb('asset_stack.primaryAssetId', '=', eb.ref('assets.id')), eb('assets.stackId', 'is', null)]),
),
)
.select('s.assets');
}
export function withStack<O>(
qb: SelectQueryBuilder<DB, 'assets', O>,
{ assets, count }: { assets: boolean; count: boolean },
) {
return qb
.leftJoinLateral(
(eb) =>
eb
.selectFrom('asset_stack')
.selectAll('asset_stack')
.whereRef('assets.stackId', '=', 'asset_stack.id')
.$if(assets, withStackedAssets)
.$if(count, (qb) =>
// There is no `selectNoFrom` method for expression builders
qb.select(
sql`(select count(*) as "assetCount" where "asset_stack"."id" = "assets"."stackId")`.as('assetCount'),
),
)
.as('stacked_assets'),
(join) => join.onTrue(),
)
.select((eb) => eb.fn('to_jsonb', [eb.table('stacked_assets')]).as('stack'));
}
export function withAlbums<O>(qb: SelectQueryBuilder<DB, 'assets', O>, { albumId }: { albumId?: string }) {
return qb
.select((eb) =>
jsonArrayFrom(
eb
.selectFrom('albums')
.selectAll()
.innerJoin('albums_assets_assets', (join) =>
join
.onRef('albums.id', '=', 'albums_assets_assets.albumsId')
.onRef('assets.id', '=', 'albums_assets_assets.assetsId'),
)
.whereRef('albums.id', '=', 'albums_assets_assets.albumsId')
.$if(!!albumId, (qb) => qb.where('albums.id', '=', asUuid(albumId!))),
).as('albums'),
)
.$if(!!albumId, (qb) =>
qb.where((eb) =>
eb.exists((eb) =>
eb
.selectFrom('albums_assets_assets')
.whereRef('albums_assets_assets.assetsId', '=', 'assets.id')
.where('albums_assets_assets.albumsId', '=', asUuid(albumId!)),
),
),
);
}
export function withTags(eb: ExpressionBuilder<DB, 'assets'>) {
return jsonArrayFrom(
eb
.selectFrom('tags')
.selectAll('tags')
.innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
.whereRef('assets.id', '=', 'tag_asset.assetsId'),
).as('tags');
}
export function truncatedDate<O>(size: TimeBucketSize) {
return sql<O>`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`;
}
const joinDeduplicationPlugin = new DeduplicateJoinsPlugin();
/** TODO: This should only be used for search-related queries, not as a general purpose query builder */
export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuilderOptions) {
options.isArchived ??= options.withArchived ? undefined : false;
options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore);
return hasPeople(kysely.withPlugin(joinDeduplicationPlugin), options.personIds)
.selectAll('assets')
.$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!))
.$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!))
.$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!))
.$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!))
.$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!))
.$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!))
.$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!))
.$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!))
.$if(options.city !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.city', options.city === null ? 'is' : '=', options.city!),
)
.$if(options.state !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.state', options.state === null ? 'is' : '=', options.state!),
)
.$if(options.country !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.country', options.country === null ? 'is' : '=', options.country!),
)
.$if(options.make !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.make', options.make === null ? 'is' : '=', options.make!),
)
.$if(options.model !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.model', options.model === null ? 'is' : '=', options.model!),
)
.$if(options.lensModel !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!),
)
.$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!))
.$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!))
.$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!))
.$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!)))
.$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!)))
.$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!)))
.$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!))
.$if(!!options.originalPath, (qb) => qb.where('assets.originalPath', '=', options.originalPath!))
.$if(!!options.originalFileName, (qb) =>
qb.where(
sql`f_unaccent(assets."originalFileName")`,
'ilike',
sql`'%' || f_unaccent(${options.originalFileName}) || '%'`,
),
)
.$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!))
.$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!))
.$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!))
.$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!))
.$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!))
.$if(options.isEncoded !== undefined, (qb) =>
qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null),
)
.$if(options.isMotion !== undefined, (qb) =>
qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null),
)
.$if(!!options.isNotInAlbum, (qb) =>
qb.where((eb) =>
eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))),
),
)
.$if(!!options.withExif, withExifInner)
.$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))
.$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
}

View file

@ -1,5 +1,4 @@
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { asVector } from 'src/utils/database';
import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm';
@Entity('face_search', { synchronize: false })
@ -15,7 +14,7 @@ export class FaceSearchEntity {
@Column({
type: 'float4',
array: true,
transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v) },
transformer: { from: JSON.parse, to: (v) => `[${v}]` },
})
embedding!: number[];
}

View file

@ -11,6 +11,6 @@ export class SmartSearchEntity {
assetId!: string;
@Index('clip_index', { synchronize: false })
@Column({ type: 'float4', array: true, transformer: { from: (v) => JSON.parse(v), to: (v) => v } })
@Column({ type: 'float4', array: true, transformer: { from: JSON.parse, to: (v) => v } })
embedding!: number[];
}

View file

@ -1,10 +1,9 @@
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { Insertable, Updateable } from 'kysely';
import { AssetFiles, AssetJobStatus, Assets, Exif } from 'src/db';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
import { Paginated, PaginationOptions } from 'src/utils/pagination';
import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
export type AssetStats = Record<AssetType, number>;
@ -66,43 +65,6 @@ export interface TimeBucketItem {
count: number;
}
export type AssetCreate = Pick<
AssetEntity,
| 'deviceAssetId'
| 'ownerId'
| 'libraryId'
| 'deviceId'
| 'type'
| 'originalPath'
| 'fileCreatedAt'
| 'localDateTime'
| 'fileModifiedAt'
| 'checksum'
| 'originalFileName'
> &
Partial<AssetEntity>;
export type AssetWithoutRelations = Omit<
AssetEntity,
| 'livePhotoVideo'
| 'stack'
| 'albums'
| 'faces'
| 'owner'
| 'library'
| 'exifInfo'
| 'sharedLinks'
| 'smartSearch'
| 'tags'
>;
type AssetUpdateWithoutRelations = Pick<AssetWithoutRelations, 'id'> & Partial<AssetWithoutRelations>;
type AssetUpdateWithLivePhotoRelation = Pick<AssetWithoutRelations, 'id'> & Pick<AssetEntity, 'livePhotoVideo'>;
export type AssetUpdateOptions = AssetUpdateWithoutRelations | AssetUpdateWithLivePhotoRelation;
export type AssetUpdateAllOptions = Omit<Partial<AssetWithoutRelations>, 'id'>;
export interface MonthDay {
day: number;
month: number;
@ -113,12 +75,6 @@ export interface AssetExploreFieldOptions {
minAssetsPerField: number;
}
export interface AssetExploreOptions extends AssetExploreFieldOptions {
relation: keyof AssetEntity;
relatedField: string;
unnest?: boolean;
}
export interface AssetFullSyncOptions {
ownerId: string;
lastId?: string;
@ -144,8 +100,30 @@ export interface UpsertFileOptions {
path: string;
}
export interface AssetGetByChecksumOptions {
ownerId: string;
checksum: Buffer;
libraryId?: string;
}
export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
export interface GetByIdsRelations {
exifInfo?: boolean;
faces?: { person?: boolean };
files?: boolean;
library?: boolean;
owner?: boolean;
smartSearch?: boolean;
stack?: { assets?: boolean };
tags?: boolean;
}
export interface DuplicateGroup {
duplicateId: string;
assets: AssetEntity[];
}
export interface DayOfYearAssets {
yearsAgo: number;
assets: AssetEntity[];
@ -154,47 +132,39 @@ export interface DayOfYearAssets {
export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
create(asset: AssetCreate): Promise<AssetEntity>;
getByIds(
ids: string[],
relations?: FindOptionsRelations<AssetEntity>,
select?: FindOptionsSelect<AssetEntity>,
): Promise<AssetEntity[]>;
create(asset: Insertable<Assets>): Promise<AssetEntity>;
getByIds(ids: string[], relations?: GetByIdsRelations): Promise<AssetEntity[]>;
getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<DayOfYearAssets[]>;
getByChecksum(options: { ownerId: string; checksum: Buffer; libraryId?: string }): Promise<AssetEntity | null>;
getByChecksum(options: AssetGetByChecksumOptions): Promise<AssetEntity | undefined>;
getByChecksums(userId: string, checksums: Buffer[]): Promise<AssetEntity[]>;
getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByDeviceIds(ownerId: string, deviceId: string, deviceAssetIds: string[]): Promise<string[]>;
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
getById(
id: string,
relations?: FindOptionsRelations<AssetEntity>,
order?: FindOptionsOrder<AssetEntity>,
): Promise<AssetEntity | null>;
getById(id: string, relations?: GetByIdsRelations): Promise<AssetEntity | undefined>;
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getRandom(userIds: string[], count: number): Promise<AssetEntity[]>;
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | undefined>;
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | undefined>;
deleteAll(ownerId: string): Promise<void>;
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getLivePhotoCount(motionId: string): Promise<number>;
updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>;
updateAll(ids: string[], options: Updateable<Assets>): Promise<void>;
updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>;
update(asset: AssetUpdateOptions): Promise<void>;
update(asset: Updateable<Assets> & { id: string }): Promise<AssetEntity>;
remove(asset: AssetEntity): Promise<void>;
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null>;
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | undefined>;
getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
upsertJobStatus(...jobStatus: Partial<AssetJobStatusEntity>[]): Promise<void>;
upsertExif(exif: Insertable<Exif>): Promise<void>;
upsertJobStatus(...jobStatus: Insertable<AssetJobStatus>[]): Promise<void>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
getDuplicates(userId: string): Promise<DuplicateGroup[]>;
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
upsertFile(file: UpsertFileOptions): Promise<void>;
upsertFiles(files: UpsertFileOptions[]): Promise<void>;
upsertFile(options: Insertable<AssetFiles>): Promise<void>;
upsertFiles(options: Insertable<AssetFiles>[]): Promise<void>;
}

View file

@ -1,6 +1,7 @@
import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis';
import { KyselyConfig } from 'kysely';
import { ClsModuleOptions } from 'nestjs-cls';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum';
@ -42,7 +43,7 @@ export interface EnvData {
};
database: {
config: PostgresConnectionOptions & DatabaseConnectionParams;
config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: KyselyConfig };
skipMigrations: boolean;
vectorExtension: VectorExtension;
};

View file

@ -59,6 +59,7 @@ export interface VectorUpdateResult {
export const IDatabaseRepository = 'IDatabaseRepository';
export interface IDatabaseRepository {
init(): void;
reconnect(): Promise<boolean>;
getExtensionVersion(extension: DatabaseExtension): Promise<ExtensionVersion>;
getExtensionVersionRange(extension: VectorExtension): string;

View file

@ -1,4 +1,3 @@
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { AssetStatus, AssetType } from 'src/enum';
@ -114,7 +113,7 @@ export interface SearchPeopleOptions {
}
export interface SearchOrderOptions {
orderDirection?: 'ASC' | 'DESC';
orderDirection?: 'asc' | 'desc';
}
export interface SearchPaginationOptions {
@ -148,20 +147,21 @@ export type SmartSearchOptions = SearchDateOptions &
export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
hasPerson?: boolean;
numResults: number;
maxDistance?: number;
maxDistance: number;
}
export interface AssetDuplicateSearch {
assetId: string;
embedding: number[];
maxDistance?: number;
maxDistance: number;
type: AssetType;
userIds: string[];
}
export interface FaceSearchResult {
distance: number;
face: AssetFaceEntity;
id: string;
personId: string | null;
}
export interface AssetDuplicateResult {

View file

@ -0,0 +1,25 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddTimeBucketIndices1734574016301 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE INDEX idx_local_date_time_month ON public.assets ((date_trunc('MONTH', "localDateTime" at time zone 'UTC') at time zone 'UTC'))`,
);
await queryRunner.query(
`CREATE INDEX idx_local_date_time ON public.assets ((("localDateTime" at time zone 'UTC')::date))`,
);
await queryRunner.query(`DROP INDEX "IDX_day_of_month"`);
await queryRunner.query(`DROP INDEX "IDX_month"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX idx_local_date_time_month`);
await queryRunner.query(`DROP INDEX idx_local_date_time`);
await queryRunner.query(
`CREATE INDEX "IDX_day_of_month" ON assets (EXTRACT(DAY FROM "localDateTime" AT TIME ZONE 'UTC'))`,
);
await queryRunner.query(
`CREATE INDEX "IDX_month" ON assets (EXTRACT(MONTH FROM "localDateTime" AT TIME ZONE 'UTC'))`,
);
}
}

View file

@ -450,19 +450,6 @@ WHERE
ORDER BY
"AlbumEntity"."createdAt" DESC
-- AlbumRepository.removeAsset
DELETE FROM "albums_assets_assets"
WHERE
"albums_assets_assets"."assetsId" = $1
-- AlbumRepository.removeAssetIds
DELETE FROM "albums_assets_assets"
WHERE
(
"albumsId" = $1
AND "assetsId" IN ($2)
)
-- AlbumRepository.getAssetIds
SELECT
"albums_assets"."assetsId" AS "assetId"
@ -471,52 +458,3 @@ FROM
WHERE
"albums_assets"."albumsId" = $1
AND "albums_assets"."assetsId" IN ($2)
-- AlbumRepository.addAssetIds
INSERT INTO
"albums_assets_assets" ("albumsId", "assetsId")
VALUES
($1, $2)
-- AlbumRepository.updateThumbnails
UPDATE "albums"
SET
"albumThumbnailAssetId" = (
SELECT
"album_assets"."assetsId"
FROM
"albums_assets_assets" "album_assets"
INNER JOIN "assets" "assets" ON "album_assets"."assetsId" = "assets"."id"
AND "assets"."deletedAt" IS NULL
WHERE
"album_assets"."albumsId" = "albums"."id"
ORDER BY
"assets"."fileCreatedAt" DESC
LIMIT
1
),
"updatedAt" = CURRENT_TIMESTAMP
WHERE
"albums"."albumThumbnailAssetId" IS NULL
AND EXISTS (
SELECT
1
FROM
"albums_assets_assets" "album_assets"
INNER JOIN "assets" "assets" ON "album_assets"."assetsId" = "assets"."id"
AND "assets"."deletedAt" IS NULL
WHERE
"album_assets"."albumsId" = "albums"."id"
)
OR "albums"."albumThumbnailAssetId" IS NOT NULL
AND NOT EXISTS (
SELECT
1
FROM
"albums_assets_assets" "album_assets"
INNER JOIN "assets" "assets" ON "album_assets"."assetsId" = "assets"."id"
AND "assets"."deletedAt" IS NULL
WHERE
"album_assets"."albumsId" = "albums"."id"
AND "albums"."albumThumbnailAssetId" = "album_assets"."assetsId"
)

File diff suppressed because it is too large Load diff

View file

@ -8,17 +8,3 @@ FROM
WHERE
"memories_assets"."memoriesId" = $1
AND "memories_assets"."assetsId" IN ($2)
-- MemoryRepository.addAssetIds
INSERT INTO
"memories_assets_assets" ("memoriesId", "assetsId")
VALUES
($1, $2)
-- MemoryRepository.removeAssetIds
DELETE FROM "memories_assets_assets"
WHERE
(
"memoriesId" = $1
AND "assetsId" IN ($2)
)

View file

@ -1,641 +1,268 @@
-- NOTE: This file is auto generated by ./sql-generator
-- SearchRepository.searchMetadata
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_fileCreatedAt"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_fileCreatedAt" DESC,
"asset_id" ASC
LIMIT
101
select
"assets".*
from
"assets"
inner join "exif" on "assets"."id" = "exif"."assetId"
where
"assets"."fileCreatedAt" >= $1
and "exif"."lensModel" = $2
and "assets"."ownerId" = any ($3::uuid [])
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
order by
"assets"."fileCreatedAt" desc
limit
$6
offset
$7
-- SearchRepository.searchRandom
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" > $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" < $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
(
select
"assets".*
from
"assets"
inner join "exif" on "assets"."id" = "exif"."assetId"
where
"assets"."fileCreatedAt" >= $1
and "exif"."lensModel" = $2
and "assets"."ownerId" = any ($3::uuid [])
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
and "assets"."id" < $6
order by
"assets"."id"
limit
$7
)
union all
(
select
"assets".*
from
"assets"
inner join "exif" on "assets"."id" = "exif"."assetId"
where
"assets"."fileCreatedAt" >= $8
and "exif"."lensModel" = $9
and "assets"."ownerId" = any ($10::uuid [])
and "assets"."isFavorite" = $11
and "assets"."isArchived" = $12
and "assets"."deletedAt" is null
and "assets"."id" > $13
order by
"assets"."id"
limit
$14
)
-- SearchRepository.searchSmart
START TRANSACTION
SET
LOCAL vectors.hnsw_ef_search = 200;
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND 1 = 1
AND (
"asset"."isFavorite" = $3
AND "asset"."isArchived" = $4
)
AND "asset"."ownerId" IN ($5)
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
"search"."embedding" <= > $6 ASC
LIMIT
201
COMMIT
select
"assets".*
from
"assets"
inner join "exif" on "assets"."id" = "exif"."assetId"
inner join "smart_search" on "assets"."id" = "smart_search"."assetId"
where
"assets"."fileCreatedAt" >= $1
and "exif"."lensModel" = $2
and "assets"."ownerId" = any ($3::uuid [])
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
order by
smart_search.embedding <= > $6::vector
limit
$7
offset
$8
-- SearchRepository.searchDuplicates
WITH
"cte" AS (
SELECT
"asset"."duplicateId" AS "duplicateId",
"search"."assetId" AS "assetId",
"search"."embedding" <= > $1 AS "distance"
FROM
"assets" "asset"
INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
WHERE
(
"asset"."ownerId" IN ($2)
AND "asset"."id" != $3
AND "asset"."isVisible" = $4
AND "asset"."type" = $5
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
"search"."embedding" <= > $1 ASC
LIMIT
64
with
"cte" as (
select
"assets"."id" as "assetId",
"assets"."duplicateId",
smart_search.embedding <= > $1::vector as "distance"
from
"assets"
inner join "smart_search" on "assets"."id" = "smart_search"."assetId"
where
"assets"."ownerId" = any ($2::uuid [])
and "assets"."deletedAt" is null
and "assets"."isVisible" = $3
and "assets"."type" = $4
and "assets"."id" != $5::uuid
order by
smart_search.embedding <= > $6::vector
limit
$7
)
SELECT
res.*
FROM
"cte" "res"
WHERE
res.distance <= $6
select
*
from
"cte"
where
"cte"."distance" <= $8
-- SearchRepository.searchFaces
START TRANSACTION
SET
LOCAL vectors.hnsw_ef_search = 100;
WITH
"cte" AS (
SELECT
"faces"."id" AS "id",
"faces"."assetId" AS "assetId",
"faces"."personId" AS "personId",
"faces"."imageWidth" AS "imageWidth",
"faces"."imageHeight" AS "imageHeight",
"faces"."boundingBoxX1" AS "boundingBoxX1",
"faces"."boundingBoxY1" AS "boundingBoxY1",
"faces"."boundingBoxX2" AS "boundingBoxX2",
"faces"."boundingBoxY2" AS "boundingBoxY2",
"faces"."sourceType" AS "sourceType",
"search"."embedding" <= > $1 AS "distance"
FROM
"asset_faces" "faces"
INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
AND ("asset"."deletedAt" IS NULL)
INNER JOIN "face_search" "search" ON "search"."faceId" = "faces"."id"
WHERE
"asset"."ownerId" IN ($2)
ORDER BY
"search"."embedding" <= > $1 ASC
LIMIT
64
with
"cte" as (
select
"asset_faces"."id",
"asset_faces"."personId",
face_search.embedding <= > $1::vector as "distance"
from
"asset_faces"
inner join "assets" on "assets"."id" = "asset_faces"."assetId"
inner join "face_search" on "face_search"."faceId" = "asset_faces"."id"
where
"assets"."ownerId" = any ($2::uuid [])
and "assets"."deletedAt" is null
order by
face_search.embedding <= > $3::vector
limit
$4
)
SELECT
res.*
FROM
"cte" "res"
WHERE
res.distance <= $3
ORDER BY
res.distance ASC
COMMIT
select
*
from
"cte"
where
"cte"."distance" <= $5
-- SearchRepository.searchPlaces
SELECT
"geoplaces"."id" AS "geoplaces_id",
"geoplaces"."name" AS "geoplaces_name",
"geoplaces"."longitude" AS "geoplaces_longitude",
"geoplaces"."latitude" AS "geoplaces_latitude",
"geoplaces"."countryCode" AS "geoplaces_countryCode",
"geoplaces"."admin1Code" AS "geoplaces_admin1Code",
"geoplaces"."admin2Code" AS "geoplaces_admin2Code",
"geoplaces"."admin1Name" AS "geoplaces_admin1Name",
"geoplaces"."admin2Name" AS "geoplaces_admin2Name",
"geoplaces"."alternateNames" AS "geoplaces_alternateNames",
"geoplaces"."modificationDate" AS "geoplaces_modificationDate"
FROM
"geodata_places" "geoplaces"
WHERE
select
*
from
"geodata_places"
where
f_unaccent (name) %>> f_unaccent ($1)
OR f_unaccent ("admin2Name") %>> f_unaccent ($1)
OR f_unaccent ("admin1Name") %>> f_unaccent ($1)
OR f_unaccent ("alternateNames") %>> f_unaccent ($1)
ORDER BY
COALESCE(f_unaccent (name) <->>> f_unaccent ($1), 0.1) + COALESCE(
f_unaccent ("admin2Name") <->>> f_unaccent ($1),
or f_unaccent ("admin2Name") %>> f_unaccent ($2)
or f_unaccent ("admin1Name") %>> f_unaccent ($3)
or f_unaccent ("alternateNames") %>> f_unaccent ($4)
order by
coalesce(f_unaccent (name) <->>> f_unaccent ($5), 0.1) + coalesce(
f_unaccent ("admin2Name") <->>> f_unaccent ($6),
0.1
) + COALESCE(
f_unaccent ("admin1Name") <->>> f_unaccent ($1),
) + coalesce(
f_unaccent ("admin1Name") <->>> f_unaccent ($7),
0.1
) + COALESCE(
f_unaccent ("alternateNames") <->>> f_unaccent ($1),
) + coalesce(
f_unaccent ("alternateNames") <->>> f_unaccent ($8),
0.1
) ASC
LIMIT
20
)
limit
$9
-- SearchRepository.getAssetsByCity
WITH RECURSIVE
cte AS (
with recursive
"cte" as (
(
SELECT
city,
select
"city",
"assetId"
FROM
exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE
"ownerId" = ANY ($1::uuid [])
AND "isVisible" = $2
AND "isArchived" = $3
AND type = $4
ORDER BY
city
LIMIT
1
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"assets"."ownerId" = any ($1::uuid [])
and "assets"."isVisible" = $2
and "assets"."isArchived" = $3
and "assets"."type" = $4
and "assets"."deletedAt" is null
order by
"city"
limit
$5
)
union all
(
select
"l"."city",
"l"."assetId"
from
"cte"
inner join lateral (
select
"city",
"assetId"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"assets"."ownerId" = any ($6::uuid [])
and "assets"."isVisible" = $7
and "assets"."isArchived" = $8
and "assets"."type" = $9
and "assets"."deletedAt" is null
and "exif"."city" > "cte"."city"
order by
"city"
limit
$10
) as "l" on true
)
UNION ALL
SELECT
l.city,
l."assetId"
FROM
cte c,
LATERAL (
SELECT
city,
"assetId"
FROM
exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE
city > c.city
AND "ownerId" = ANY ($1::uuid [])
AND "isVisible" = $2
AND "isArchived" = $3
AND type = $4
ORDER BY
city
LIMIT
1
) l
)
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"exif"."assetId" AS "exif_assetId",
"exif"."description" AS "exif_description",
"exif"."exifImageWidth" AS "exif_exifImageWidth",
"exif"."exifImageHeight" AS "exif_exifImageHeight",
"exif"."fileSizeInByte" AS "exif_fileSizeInByte",
"exif"."orientation" AS "exif_orientation",
"exif"."dateTimeOriginal" AS "exif_dateTimeOriginal",
"exif"."modifyDate" AS "exif_modifyDate",
"exif"."timeZone" AS "exif_timeZone",
"exif"."latitude" AS "exif_latitude",
"exif"."longitude" AS "exif_longitude",
"exif"."projectionType" AS "exif_projectionType",
"exif"."city" AS "exif_city",
"exif"."livePhotoCID" AS "exif_livePhotoCID",
"exif"."autoStackId" AS "exif_autoStackId",
"exif"."state" AS "exif_state",
"exif"."country" AS "exif_country",
"exif"."make" AS "exif_make",
"exif"."model" AS "exif_model",
"exif"."lensModel" AS "exif_lensModel",
"exif"."fNumber" AS "exif_fNumber",
"exif"."focalLength" AS "exif_focalLength",
"exif"."iso" AS "exif_iso",
"exif"."exposureTime" AS "exif_exposureTime",
"exif"."profileDescription" AS "exif_profileDescription",
"exif"."colorspace" AS "exif_colorspace",
"exif"."bitsPerSample" AS "exif_bitsPerSample",
"exif"."rating" AS "exif_rating",
"exif"."fps" AS "exif_fps"
FROM
"assets" "asset"
INNER JOIN "exif" "exif" ON "exif"."assetId" = "asset"."id"
INNER JOIN cte ON asset.id = cte."assetId"
ORDER BY
exif.city
-- SearchRepository.getCountries
SELECT DISTINCT
ON ("exif"."country") "exif"."country" AS "country"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."country" != ''
AND "exif"."country" IS NOT NULL
select
"assets".*,
to_jsonb("exif") as "exifInfo"
from
"assets"
inner join "exif" on "assets"."id" = "exif"."assetId"
inner join "cte" on "assets"."id" = "cte"."assetId"
order by
"exif"."city"
-- SearchRepository.getStates
SELECT DISTINCT
ON ("exif"."state") "exif"."state" AS "state"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."state" != ''
AND "exif"."state" IS NOT NULL
select distinct
on ("state") "state"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"ownerId" = any ($1::uuid [])
and "isVisible" = $2
and "deletedAt" is null
and "state" is not null
-- SearchRepository.getCities
SELECT DISTINCT
ON ("exif"."city") "exif"."city" AS "city"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."city" != ''
AND "exif"."city" IS NOT NULL
select distinct
on ("city") "city"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"ownerId" = any ($1::uuid [])
and "isVisible" = $2
and "deletedAt" is null
and "city" is not null
-- SearchRepository.getCameraMakes
SELECT DISTINCT
ON ("exif"."make") "exif"."make" AS "make"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."make" != ''
AND "exif"."make" IS NOT NULL
select distinct
on ("make") "make"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"ownerId" = any ($1::uuid [])
and "isVisible" = $2
and "deletedAt" is null
and "make" is not null
-- SearchRepository.getCameraModels
SELECT DISTINCT
ON ("exif"."model") "exif"."model" AS "model"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."model" != ''
AND "exif"."model" IS NOT NULL
select distinct
on ("model") "model"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"ownerId" = any ($1::uuid [])
and "isVisible" = $2
and "deletedAt" is null
and "model" is not null

View file

@ -8,23 +8,3 @@ FROM
WHERE
"tag_asset"."tagsId" = $1
AND "tag_asset"."assetsId" IN ($2)
-- TagRepository.addAssetIds
INSERT INTO
"tag_asset" ("assetsId", "tagsId")
VALUES
($1, $2)
-- TagRepository.removeAssetIds
DELETE FROM "tag_asset"
WHERE
(
"tagsId" = $1
AND "assetsId" IN ($2)
)
-- TagRepository.upsertAssetIds
INSERT INTO
"tag_asset" ("assetsId", "tagsId")
VALUES
($1, $2)

View file

@ -1,79 +1,29 @@
-- NOTE: This file is auto generated by ./sql-generator
-- ViewRepository.getUniqueOriginalPaths
select distinct
substring("assets"."originalPath", $1) as "directoryPath"
from
"assets"
where
"ownerId" = $2::uuid
and "isVisible" = $3
and "isArchived" = $4
and "deletedAt" is null
-- ViewRepository.getAssetsByOriginalPath
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"exifInfo"."assetId" AS "exifInfo_assetId",
"exifInfo"."description" AS "exifInfo_description",
"exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
"exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
"exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
"exifInfo"."orientation" AS "exifInfo_orientation",
"exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
"exifInfo"."modifyDate" AS "exifInfo_modifyDate",
"exifInfo"."timeZone" AS "exifInfo_timeZone",
"exifInfo"."latitude" AS "exifInfo_latitude",
"exifInfo"."longitude" AS "exifInfo_longitude",
"exifInfo"."projectionType" AS "exifInfo_projectionType",
"exifInfo"."city" AS "exifInfo_city",
"exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
"exifInfo"."autoStackId" AS "exifInfo_autoStackId",
"exifInfo"."state" AS "exifInfo_state",
"exifInfo"."country" AS "exifInfo_country",
"exifInfo"."make" AS "exifInfo_make",
"exifInfo"."model" AS "exifInfo_model",
"exifInfo"."lensModel" AS "exifInfo_lensModel",
"exifInfo"."fNumber" AS "exifInfo_fNumber",
"exifInfo"."focalLength" AS "exifInfo_focalLength",
"exifInfo"."iso" AS "exifInfo_iso",
"exifInfo"."exposureTime" AS "exifInfo_exposureTime",
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
"exifInfo"."colorspace" AS "exifInfo_colorspace",
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."rating" AS "exifInfo_rating",
"exifInfo"."fps" AS "exifInfo_fps"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
WHERE
(
(
"asset"."isVisible" = $1
AND "asset"."isArchived" = $2
AND "asset"."ownerId" = $3
)
AND (
"asset"."originalPath" LIKE $4
AND "asset"."originalPath" NOT LIKE $5
)
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
regexp_replace("asset"."originalPath", '.*/(.+)', '\1') ASC
select
"assets".*,
to_jsonb("exif") as "exifInfo"
from
"assets"
left join "exif" on "assets"."id" = "exif"."assetId"
where
"ownerId" = $1::uuid
and "isVisible" = $2
and "isArchived" = $3
and "deletedAt" is null
and "originalPath" like $4
and "originalPath" not like $5
order by
regexp_replace("assets"."originalPath", $6, $7) asc

View file

@ -153,7 +153,6 @@ export class AlbumRepository implements IAlbumRepository {
await this.repository.delete({ ownerId: userId });
}
@GenerateSql({ params: [DummyValue.UUID] })
async removeAsset(assetId: string): Promise<void> {
// Using dataSource, because there is no direct access to albums_assets_assets.
await this.dataSource
@ -164,7 +163,6 @@ export class AlbumRepository implements IAlbumRepository {
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
@Chunked({ paramIndex: 1 })
async removeAssetIds(albumId: string, assetIds: string[]): Promise<void> {
if (assetIds.length === 0) {
@ -207,7 +205,6 @@ export class AlbumRepository implements IAlbumRepository {
return new Set(results.map(({ assetId }) => assetId));
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
async addAssetIds(albumId: string, assetIds: string[]): Promise<void> {
await this.addAssets(this.dataSource.manager, albumId, assetIds);
}
@ -272,7 +269,6 @@ export class AlbumRepository implements IAlbumRepository {
*
* @returns Amount of updated album thumbnails or undefined when unknown
*/
@GenerateSql()
async updateThumbnails(): Promise<number | undefined> {
// Subquery for getting a new thumbnail.

File diff suppressed because it is too large Load diff

View file

@ -1,3 +1,4 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import { ImmichTelemetry } from 'src/enum';
import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository';
@ -79,14 +80,20 @@ describe('getEnv', () => {
it('should use defaults', () => {
const { database } = getEnv();
expect(database).toEqual({
config: expect.objectContaining({
type: 'postgres',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
config: {
kysely: {
dialect: expect.any(PostgresJSDialect),
log: ['error'],
},
typeorm: expect.objectContaining({
type: 'postgres',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
},
skipMigrations: false,
vectorExtension: 'vectors',
});

View file

@ -2,8 +2,10 @@ import { Inject, Injectable, Optional } from '@nestjs/common';
import { plainToInstance } from 'class-transformer';
import { validateSync } from 'class-validator';
import { Request, Response } from 'express';
import { PostgresJSDialect } from 'kysely-postgres-js';
import { CLS_ID } from 'nestjs-cls';
import { join, resolve } from 'node:path';
import postgres from 'postgres';
import { citiesFile, excludePaths, IWorker } from 'src/constants';
import { Telemetry } from 'src/decorators';
import { EnvDto } from 'src/dtos/env.dto';
@ -96,6 +98,33 @@ const getEnv = (): EnvData => {
}
}
const driverOptions = {
max: 10,
types: {
date: {
to: 1184,
from: [1082, 1114, 1184],
serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x),
parse: (x: string) => new Date(x),
},
bigint: {
to: 20,
from: [20],
parse: (value: string) => Number.parseInt(value),
serialize: (value: number) => value.toString(),
},
},
};
const parts = {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
} as const;
return {
host: dto.IMMICH_HOST,
port: dto.IMMICH_PORT || 2283,
@ -150,24 +179,23 @@ const getEnv = (): EnvData => {
database: {
config: {
type: 'postgres',
entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'],
migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(databaseUrl
? { connectionType: 'url', url: databaseUrl }
: {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
}),
typeorm: {
type: 'postgres',
entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'],
migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts),
},
kysely: {
dialect: new PostgresJSDialect({
postgres: databaseUrl ? postgres(databaseUrl, driverOptions) : postgres({ ...parts, ...driverOptions }),
}),
log: ['error'] as const,
},
},
skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,

View file

@ -1,8 +1,10 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import AsyncLock from 'async-lock';
import { sql } from 'kysely';
import semver from 'semver';
import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
import { DB } from 'src/db';
import { IConfigRepository } from 'src/interfaces/config.interface';
import {
DatabaseExtension,
@ -15,13 +17,14 @@ import {
VectorUpdateResult,
} from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { UPSERT_COLUMNS } from 'src/utils/database';
import { isValidInteger } from 'src/validation';
import { DataSource, EntityManager, QueryRunner } from 'typeorm';
import { DataSource, EntityManager, EntityMetadata, QueryRunner } from 'typeorm';
@Injectable()
export class DatabaseRepository implements IDatabaseRepository {
private vectorExtension: VectorExtension;
readonly asyncLock = new AsyncLock();
private readonly asyncLock = new AsyncLock();
constructor(
@InjectDataSource() private dataSource: DataSource,
@ -32,6 +35,13 @@ export class DatabaseRepository implements IDatabaseRepository {
this.logger.setContext(DatabaseRepository.name);
}
init() {
for (const metadata of this.dataSource.entityMetadatas) {
const table = metadata.tableName as keyof DB;
UPSERT_COLUMNS[table] = this.getUpsertColumns(metadata);
}
}
async reconnect() {
try {
if (this.dataSource.isInitialized) {
@ -249,4 +259,10 @@ export class DatabaseRepository implements IDatabaseRepository {
private async releaseLock(lock: DatabaseLock, queryRunner: QueryRunner): Promise<void> {
return queryRunner.query('SELECT pg_advisory_unlock($1)', [lock]);
}
private getUpsertColumns(metadata: EntityMetadata) {
return Object.fromEntries(
metadata.ownColumns.map((column) => [column.propertyName, sql<string>`excluded.${sql.ref(column.propertyName)}`]),
) as any;
}
}

View file

@ -64,7 +64,6 @@ export class MemoryRepository implements IMemoryRepository {
return new Set(results.map(({ assetId }) => assetId));
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
async addAssetIds(id: string, assetIds: string[]): Promise<void> {
await this.dataSource
.createQueryBuilder()
@ -74,7 +73,6 @@ export class MemoryRepository implements IMemoryRepository {
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
@Chunked({ paramIndex: 1 })
async removeAssetIds(id: string, assetIds: string[]): Promise<void> {
await this.dataSource

View file

@ -1,22 +1,17 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Kysely, OrderByDirectionExpression, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { randomUUID } from 'node:crypto';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { AssetType, PaginationMode } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';
import { AssetType } from 'src/enum';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
AssetDuplicateResult,
AssetDuplicateSearch,
AssetSearchOptions,
FaceEmbeddingSearch,
FaceSearchResult,
GetCameraMakesOptions,
GetCameraModelsOptions,
GetCitiesOptions,
@ -25,40 +20,17 @@ import {
SearchPaginationOptions,
SmartSearchOptions,
} from 'src/interfaces/search.interface';
import { asVector, searchAssetBuilder } from 'src/utils/database';
import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination';
import { anyUuid, asUuid, asVector } from 'src/utils/database';
import { Paginated } from 'src/utils/pagination';
import { isValidInteger } from 'src/validation';
import { Repository } from 'typeorm';
@Injectable()
export class SearchRepository implements ISearchRepository {
private vectorExtension: VectorExtension;
private faceColumns: string[];
private assetsByCityQuery: string;
constructor(
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(IConfigRepository) configRepository: IConfigRepository,
@InjectKysely() private db: Kysely<DB>,
) {
this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(SearchRepository.name);
this.faceColumns = this.assetFaceRepository.manager.connection
.getMetadata(AssetFaceEntity)
.ownColumns.map((column) => column.propertyName)
.filter((propertyName) => propertyName !== 'embedding');
this.assetsByCityQuery =
assetsByCityCte +
this.assetRepository
.createQueryBuilder('asset')
.innerJoinAndSelect('asset.exifInfo', 'exif')
.withDeleted()
.getQuery() +
' INNER JOIN cte ON asset.id = cte."assetId" ORDER BY exif.city';
}
@GenerateSql({
@ -74,14 +46,15 @@ export class SearchRepository implements ISearchRepository {
],
})
async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> {
let builder = this.assetRepository.createQueryBuilder('asset');
builder = searchAssetBuilder(builder, options).orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE,
skip: (pagination.page - 1) * pagination.size,
take: pagination.size,
});
const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirectionExpression;
const items = await searchAssetBuilder(this.db, options)
.orderBy('assets.fileCreatedAt', orderDirection)
.limit(pagination.size + 1)
.offset((pagination.page - 1) * pagination.size)
.execute();
const hasNextPage = items.length > pagination.size;
items.splice(pagination.size);
return { items: items as any as AssetEntity[], hasNextPage };
}
@GenerateSql({
@ -96,21 +69,12 @@ export class SearchRepository implements ISearchRepository {
},
],
})
async searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]> {
const builder1 = searchAssetBuilder(this.assetRepository.createQueryBuilder('asset'), options);
const builder2 = builder1.clone();
searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]> {
const uuid = randomUUID();
builder1.andWhere('asset.id > :uuid', { uuid }).orderBy('asset.id').take(size);
builder2.andWhere('asset.id < :uuid', { uuid }).orderBy('asset.id').take(size);
const [assets1, assets2] = await Promise.all([builder1.getMany(), builder2.getMany()]);
const missingCount = size - assets1.length;
for (let i = 0; i < missingCount && i < assets2.length; i++) {
assets1.push(assets2[i]);
}
return assets1;
const builder = searchAssetBuilder(this.db, options);
const lessThan = builder.where('assets.id', '<', uuid).orderBy('assets.id').limit(size);
const greaterThan = builder.where('assets.id', '>', uuid).orderBy('assets.id').limit(size);
return sql`${lessThan} union all ${greaterThan}`.execute(this.db) as any as Promise<AssetEntity[]>;
}
@GenerateSql({
@ -126,76 +90,58 @@ export class SearchRepository implements ISearchRepository {
},
],
})
async searchSmart(
pagination: SearchPaginationOptions,
{ embedding, userIds, ...options }: SmartSearchOptions,
): Paginated<AssetEntity> {
let results: PaginationResult<AssetEntity> = { items: [], hasNextPage: false };
async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity> {
if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) {
throw new Error(`Invalid value for 'size': ${pagination.size}`);
}
await this.assetRepository.manager.transaction(async (manager) => {
let builder = manager.createQueryBuilder(AssetEntity, 'asset');
builder = searchAssetBuilder(builder, options);
builder
.innerJoin('asset.smartSearch', 'search')
.andWhere('asset.ownerId IN (:...userIds )')
.orderBy('search.embedding <=> :embedding')
.setParameters({ userIds, embedding: asVector(embedding) });
const items = (await searchAssetBuilder(this.db, options)
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
.orderBy(sql`smart_search.embedding <=> ${asVector(options.embedding)}`)
.limit(pagination.size + 1)
.offset((pagination.page - 1) * pagination.size)
.execute()) as any as AssetEntity[];
const runtimeConfig = this.getRuntimeConfig(pagination.size);
if (runtimeConfig) {
await manager.query(runtimeConfig);
}
results = await paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.LIMIT_OFFSET,
skip: (pagination.page - 1) * pagination.size,
take: pagination.size,
});
});
return results;
const hasNextPage = items.length > pagination.size;
items.splice(pagination.size);
return { items, hasNextPage };
}
@GenerateSql({
params: [
{
assetId: DummyValue.UUID,
embedding: Array.from({ length: 512 }, Math.random),
maxDistance: 0.6,
type: AssetType.IMAGE,
userIds: [DummyValue.UUID],
},
],
})
searchDuplicates({
assetId,
embedding,
maxDistance,
type,
userIds,
}: AssetDuplicateSearch): Promise<AssetDuplicateResult[]> {
const cte = this.assetRepository.createQueryBuilder('asset');
cte
.select('search.assetId', 'assetId')
.addSelect('asset.duplicateId', 'duplicateId')
.addSelect(`search.embedding <=> :embedding`, 'distance')
.innerJoin('asset.smartSearch', 'search')
.where('asset.ownerId IN (:...userIds )')
.andWhere('asset.id != :assetId')
.andWhere('asset.isVisible = :isVisible')
.andWhere('asset.type = :type')
.orderBy('search.embedding <=> :embedding')
.limit(64)
.setParameters({ assetId, embedding: asVector(embedding), isVisible: true, type, userIds });
const builder = this.assetRepository.manager
.createQueryBuilder()
.addCommonTableExpression(cte, 'cte')
.from('cte', 'res')
.select('res.*');
if (maxDistance) {
builder.where('res.distance <= :maxDistance', { maxDistance });
}
return builder.getRawMany() as Promise<AssetDuplicateResult[]>;
searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) {
const vector = asVector(embedding);
return this.db
.with('cte', (qb) =>
qb
.selectFrom('assets')
.select([
'assets.id as assetId',
'assets.duplicateId',
sql<number>`smart_search.embedding <=> ${vector}`.as('distance'),
])
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
.where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.deletedAt', 'is', null)
.where('assets.isVisible', '=', true)
.where('assets.type', '=', type)
.where('assets.id', '!=', asUuid(assetId))
.orderBy(sql`smart_search.embedding <=> ${vector}`)
.limit(64),
)
.selectFrom('cte')
.selectAll()
.where('cte.distance', '<=', maxDistance as number)
.execute();
}
@GenerateSql({
@ -208,120 +154,131 @@ export class SearchRepository implements ISearchRepository {
},
],
})
async searchFaces({
userIds,
embedding,
numResults,
maxDistance,
hasPerson,
}: FaceEmbeddingSearch): Promise<FaceSearchResult[]> {
if (!isValidInteger(numResults, { min: 1 })) {
searchFaces({ userIds, embedding, numResults, maxDistance, hasPerson }: FaceEmbeddingSearch) {
if (!isValidInteger(numResults, { min: 1, max: 1000 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`);
}
// setting this too low messes with prefilter recall
numResults = Math.max(numResults, 64);
let results: Array<AssetFaceEntity & { distance: number }> = [];
await this.assetRepository.manager.transaction(async (manager) => {
const cte = manager
.createQueryBuilder(AssetFaceEntity, 'faces')
.select('search.embedding <=> :embedding', 'distance')
.innerJoin('faces.asset', 'asset')
.innerJoin('faces.faceSearch', 'search')
.where('asset.ownerId IN (:...userIds )')
.orderBy('search.embedding <=> :embedding')
.setParameters({ userIds, embedding: asVector(embedding) });
cte.limit(numResults);
if (hasPerson) {
cte.andWhere('faces."personId" IS NOT NULL');
}
for (const col of this.faceColumns) {
cte.addSelect(`faces.${col}`, col);
}
const runtimeConfig = this.getRuntimeConfig(numResults);
if (runtimeConfig) {
await manager.query(runtimeConfig);
}
results = await manager
.createQueryBuilder()
.select('res.*')
.addCommonTableExpression(cte, 'cte')
.from('cte', 'res')
.where('res.distance <= :maxDistance', { maxDistance })
.orderBy('res.distance')
.getRawMany();
});
return results.map((row) => ({
face: this.assetFaceRepository.create(row),
distance: row.distance,
}));
const vector = asVector(embedding);
return this.db
.with('cte', (qb) =>
qb
.selectFrom('asset_faces')
.select([
'asset_faces.id',
'asset_faces.personId',
sql<number>`face_search.embedding <=> ${vector}`.as('distance'),
])
.innerJoin('assets', 'assets.id', 'asset_faces.assetId')
.innerJoin('face_search', 'face_search.faceId', 'asset_faces.id')
.where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.deletedAt', 'is', null)
.$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null))
.orderBy(sql`face_search.embedding <=> ${vector}`)
.limit(numResults),
)
.selectFrom('cte')
.selectAll()
.where('cte.distance', '<=', maxDistance)
.execute();
}
@GenerateSql({ params: [DummyValue.STRING] })
async searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]> {
return await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.where(`f_unaccent(name) %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("admin2Name") %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("admin1Name") %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("alternateNames") %>> f_unaccent(:placeName)`)
searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]> {
return this.db
.selectFrom('geodata_places')
.selectAll()
.where(
() =>
// kysely doesn't support trigram %>> or <->>> operators
sql`
f_unaccent(name) %>> f_unaccent(${placeName}) or
f_unaccent("admin2Name") %>> f_unaccent(${placeName}) or
f_unaccent("admin1Name") %>> f_unaccent(${placeName}) or
f_unaccent("alternateNames") %>> f_unaccent(${placeName})
`,
)
.orderBy(
`
COALESCE(f_unaccent(name) <->>> f_unaccent(:placeName), 0.1) +
COALESCE(f_unaccent("admin2Name") <->>> f_unaccent(:placeName), 0.1) +
COALESCE(f_unaccent("admin1Name") <->>> f_unaccent(:placeName), 0.1) +
COALESCE(f_unaccent("alternateNames") <->>> f_unaccent(:placeName), 0.1)
sql`
coalesce(f_unaccent(name) <->>> f_unaccent(${placeName}), 0.1) +
coalesce(f_unaccent("admin2Name") <->>> f_unaccent(${placeName}), 0.1) +
coalesce(f_unaccent("admin1Name") <->>> f_unaccent(${placeName}), 0.1) +
coalesce(f_unaccent("alternateNames") <->>> f_unaccent(${placeName}), 0.1)
`,
)
.setParameters({ placeName })
.limit(20)
.getMany();
.execute() as Promise<GeodataPlacesEntity[]>;
}
@GenerateSql({ params: [[DummyValue.UUID]] })
async getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> {
const parameters = [userIds, true, false, AssetType.IMAGE];
const rawRes = await this.assetRepository.query(this.assetsByCityQuery, parameters);
getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> {
return this.db
.withRecursive('cte', (qb) => {
const base = qb
.selectFrom('exif')
.select(['city', 'assetId'])
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.isVisible', '=', true)
.where('assets.isArchived', '=', false)
.where('assets.type', '=', 'IMAGE')
.where('assets.deletedAt', 'is', null)
.orderBy('city')
.limit(1);
const items: AssetEntity[] = [];
for (const res of rawRes) {
const item = { exifInfo: {} as Record<string, any> } as Record<string, any>;
for (const [key, value] of Object.entries(res)) {
if (key.startsWith('exif_')) {
item.exifInfo[key.replace('exif_', '')] = value;
} else {
item[key.replace('asset_', '')] = value;
}
}
items.push(item as AssetEntity);
}
const recursive = qb
.selectFrom('cte')
.select(['l.city', 'l.assetId'])
.innerJoinLateral(
(qb) =>
qb
.selectFrom('exif')
.select(['city', 'assetId'])
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.isVisible', '=', true)
.where('assets.isArchived', '=', false)
.where('assets.type', '=', 'IMAGE')
.where('assets.deletedAt', 'is', null)
.whereRef('exif.city', '>', 'cte.city')
.orderBy('city')
.limit(1)
.as('l'),
(join) => join.onTrue(),
);
return items;
return sql<{ city: string; assetId: string }>`(${base} union all ${recursive})`;
})
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.innerJoin('cte', 'assets.id', 'cte.assetId')
.selectAll('assets')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'))
.orderBy('exif.city')
.execute() as any as Promise<AssetEntity[]>;
}
async upsert(assetId: string, embedding: number[]): Promise<void> {
await this.smartSearchRepository.upsert(
{ assetId, embedding: () => asVector(embedding, true) },
{ conflictPaths: ['assetId'] },
);
const vector = asVector(embedding);
await this.db
.insertInto('smart_search')
.values({ assetId: asUuid(assetId), embedding: vector } as any)
.onConflict((oc) => oc.column('assetId').doUpdateSet({ embedding: vector } as any))
.execute();
}
async getDimensionSize(): Promise<number> {
const res = await this.smartSearchRepository.manager.query(`
SELECT atttypmod as dimsize
FROM pg_attribute f
JOIN pg_class c ON c.oid = f.attrelid
WHERE c.relkind = 'r'::char
AND f.attnum > 0
AND c.relname = 'smart_search'
AND f.attname = 'embedding'`);
const { rows } = await sql<{ dimsize: number }>`
select atttypmod as dimsize
from pg_attribute f
join pg_class c ON c.oid = f.attrelid
where c.relkind = 'r'::char
and f.attnum > 0
and c.relname = 'smart_search'
and f.attname = 'embedding'
`.execute(this.db);
const dimSize = res[0]['dimsize'];
const dimSize = rows[0]['dimsize'];
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Could not retrieve CLIP dimension size`);
}
@ -333,146 +290,71 @@ export class SearchRepository implements ISearchRepository {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
return this.smartSearchRepository.manager.transaction(async (manager) => {
await manager.clear(SmartSearchEntity);
await manager.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE vector(${dimSize})`);
await manager.query(`REINDEX INDEX clip_index`);
return this.db.transaction().execute(async (trx) => {
await sql`truncate ${sql.table('smart_search')}`.execute(trx);
await trx.schema
.alterTable('smart_search')
.alterColumn('embedding', (col) => col.setDataType(sql.lit(`vector(${dimSize})`)))
.execute();
await sql`reindex index clip_index`.execute(trx);
});
}
async deleteAllSearchEmbeddings(): Promise<void> {
return this.smartSearchRepository.clear();
await sql`truncate ${sql.table('smart_search')}`.execute(this.db);
}
@GenerateSql({ params: [[DummyValue.UUID]] })
async getCountries(userIds: string[]): Promise<string[]> {
const query = this.exifRepository
.createQueryBuilder('exif')
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.country != ''`)
.andWhere('exif.country IS NOT NULL')
.select('exif.country', 'country')
.distinctOn(['exif.country']);
const results = await query.getRawMany<{ country: string }>();
return results.map(({ country }) => country);
const res = await this.getExifField('country', userIds).execute();
return res.map((row) => row.country!);
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getStates(userIds: string[], { country }: GetStatesOptions): Promise<string[]> {
const query = this.exifRepository
.createQueryBuilder('exif')
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.state != ''`)
.andWhere('exif.state IS NOT NULL')
.select('exif.state', 'state')
.distinctOn(['exif.state']);
const res = await this.getExifField('state', userIds)
.$if(!!country, (qb) => qb.where('country', '=', country!))
.execute();
if (country) {
query.andWhere('exif.country = :country', { country });
}
const result = await query.getRawMany<{ state: string }>();
return result.map(({ state }) => state);
return res.map((row) => row.state!);
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING, DummyValue.STRING] })
async getCities(userIds: string[], { country, state }: GetCitiesOptions): Promise<string[]> {
const query = this.exifRepository
.createQueryBuilder('exif')
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.city != ''`)
.andWhere('exif.city IS NOT NULL')
.select('exif.city', 'city')
.distinctOn(['exif.city']);
const res = await this.getExifField('city', userIds)
.$if(!!country, (qb) => qb.where('country', '=', country!))
.$if(!!state, (qb) => qb.where('state', '=', state!))
.execute();
if (country) {
query.andWhere('exif.country = :country', { country });
}
if (state) {
query.andWhere('exif.state = :state', { state });
}
const results = await query.getRawMany<{ city: string }>();
return results.map(({ city }) => city);
return res.map((row) => row.city!);
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getCameraMakes(userIds: string[], { model }: GetCameraMakesOptions): Promise<string[]> {
const query = this.exifRepository
.createQueryBuilder('exif')
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.make != ''`)
.andWhere('exif.make IS NOT NULL')
.select('exif.make', 'make')
.distinctOn(['exif.make']);
const res = await this.getExifField('make', userIds)
.$if(!!model, (qb) => qb.where('model', '=', model!))
.execute();
if (model) {
query.andWhere('exif.model = :model', { model });
}
const results = await query.getRawMany<{ make: string }>();
return results.map(({ make }) => make);
return res.map((row) => row.make!);
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getCameraModels(userIds: string[], { make }: GetCameraModelsOptions): Promise<string[]> {
const query = this.exifRepository
.createQueryBuilder('exif')
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.model != ''`)
.andWhere('exif.model IS NOT NULL')
.select('exif.model', 'model')
.distinctOn(['exif.model']);
const res = await this.getExifField('model', userIds)
.$if(!!make, (qb) => qb.where('make', '=', make!))
.execute();
if (make) {
query.andWhere('exif.make = :make', { make });
}
const results = await query.getRawMany<{ model: string }>();
return results.map(({ model }) => model);
return res.map((row) => row.model!);
}
private getRuntimeConfig(numResults?: number): string | undefined {
if (this.vectorExtension === DatabaseExtension.VECTOR) {
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
}
if (numResults && numResults !== 100) {
return `SET LOCAL vectors.hnsw_ef_search = ${Math.max(numResults, 100)};`;
}
private getExifField<K extends 'city' | 'state' | 'country' | 'make' | 'model'>(field: K, userIds: string[]) {
return this.db
.selectFrom('exif')
.select(field)
.distinctOn(field)
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('ownerId', '=', anyUuid(userIds))
.where('isVisible', '=', true)
.where('deletedAt', 'is', null)
.where(field, 'is not', null);
}
}
// the performance difference between this and the normal way is too huge to ignore, e.g. 3s vs 4ms
const assetsByCityCte = `
WITH RECURSIVE cte AS (
(
SELECT city, "assetId"
FROM exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
ORDER BY city
LIMIT 1
)
UNION ALL
SELECT l.city, l."assetId"
FROM cte c
, LATERAL (
SELECT city, "assetId"
FROM exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE city > c.city AND "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
ORDER BY city
LIMIT 1
) l
)
`;

View file

@ -108,7 +108,6 @@ export class TagRepository implements ITagRepository {
return new Set(results.map(({ assetId }) => assetId));
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
async addAssetIds(tagId: string, assetIds: string[]): Promise<void> {
if (assetIds.length === 0) {
return;
@ -122,7 +121,6 @@ export class TagRepository implements ITagRepository {
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
@Chunked({ paramIndex: 1 })
async removeAssetIds(tagId: string, assetIds: string[]): Promise<void> {
if (assetIds.length === 0) {
@ -140,7 +138,6 @@ export class TagRepository implements ITagRepository {
.execute();
}
@GenerateSql({ params: [[{ assetId: DummyValue.UUID, tagId: DummyValue.UUID }]] })
@Chunked()
async upsertAssetIds(items: AssetTagItem[]): Promise<AssetTagItem[]> {
if (items.length === 0) {

View file

@ -1,48 +1,47 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetEntity } from 'src/entities/asset.entity';
import { AssetEntity, withExif } from 'src/entities/asset.entity';
import { IViewRepository } from 'src/interfaces/view.interface';
import { Brackets, Repository } from 'typeorm';
import { asUuid } from 'src/utils/database';
export class ViewRepository implements IViewRepository {
constructor(@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>) {}
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
async getUniqueOriginalPaths(userId: string): Promise<string[]> {
const results = await this.assetRepository
.createQueryBuilder('asset')
.where({
isVisible: true,
isArchived: false,
ownerId: userId,
})
.select("DISTINCT substring(asset.originalPath FROM '^(.*/)[^/]*$')", 'directoryPath')
.getRawMany();
const results = await this.db
.selectFrom('assets')
.select((eb) => eb.fn<string>('substring', ['assets.originalPath', eb.val('^(.*/)[^/]*$')]).as('directoryPath'))
.distinct()
.where('ownerId', '=', asUuid(userId))
.where('isVisible', '=', true)
.where('isArchived', '=', false)
.where('deletedAt', 'is', null)
.execute();
return results.map((row: { directoryPath: string }) => row.directoryPath.replaceAll(/^\/|\/$/g, ''));
return results.map((row) => row.directoryPath.replaceAll(/^\/|\/$/g, ''));
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
async getAssetsByOriginalPath(userId: string, partialPath: string): Promise<AssetEntity[]> {
const normalizedPath = partialPath.replaceAll(/^\/|\/$/g, '');
const assets = await this.assetRepository
.createQueryBuilder('asset')
.where({
isVisible: true,
isArchived: false,
ownerId: userId,
})
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.andWhere(
new Brackets((qb) => {
qb.where('asset.originalPath LIKE :likePath', { likePath: `%${normalizedPath}/%` }).andWhere(
'asset.originalPath NOT LIKE :notLikePath',
{ notLikePath: `%${normalizedPath}/%/%` },
);
}),
)
.orderBy(String.raw`regexp_replace(asset.originalPath, '.*/(.+)', '\1')`, 'ASC')
.getMany();
return assets;
return this.db
.selectFrom('assets')
.selectAll('assets')
.$call(withExif)
.where('ownerId', '=', asUuid(userId))
.where('isVisible', '=', true)
.where('isArchived', '=', false)
.where('deletedAt', 'is', null)
.where('originalPath', 'like', `%${normalizedPath}/%`)
.where('originalPath', 'not like', `%${normalizedPath}/%/%`)
.orderBy(
(eb) => eb.fn('regexp_replace', ['assets.originalPath', eb.val('.*/(.+)'), eb.val(String.raw`\1`)]),
'asc',
)
.execute() as any as Promise<AssetEntity[]>;
}
}

View file

@ -23,7 +23,6 @@ import { fileStub } from 'test/fixtures/file.stub';
import { userStub } from 'test/fixtures/user.stub';
import { IAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newTestService } from 'test/utils';
import { QueryFailedError } from 'typeorm';
import { Mocked } from 'vitest';
const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
@ -384,8 +383,8 @@ describe(AssetMediaService.name, () => {
originalName: 'asset_1.jpeg',
size: 0,
};
const error = new QueryFailedError('', [], new Error('unique key violation'));
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT;
const error = new Error('unique key violation');
(error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.create.mockRejectedValue(error);
assetMock.getUploadAssetIdByChecksum.mockResolvedValue(assetEntity.id);
@ -411,8 +410,8 @@ describe(AssetMediaService.name, () => {
originalName: 'asset_1.jpeg',
size: 0,
};
const error = new QueryFailedError('', [], new Error('unique key violation'));
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT;
const error = new Error('unique key violation');
(error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.create.mockRejectedValue(error);
@ -494,7 +493,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset is not found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(null);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(NotFoundException);
@ -526,7 +524,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset does not exist', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id]));
assetMock.getById.mockResolvedValue(null);
await expect(
sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.PREVIEW }),
@ -632,7 +629,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset does not exist', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id]));
assetMock.getById.mockResolvedValue(null);
await expect(sut.playbackVideo(authStub.admin, assetStub.image.id)).rejects.toBeInstanceOf(NotFoundException);
});
@ -684,8 +680,6 @@ describe(AssetMediaService.name, () => {
describe('replaceAsset', () => {
it('should error when update photo does not exist', async () => {
assetMock.getById.mockResolvedValueOnce(null);
await expect(sut.replaceAsset(authStub.user1, 'id', replaceDto, fileStub.photo)).rejects.toThrow(
'Not found or no asset.update access',
);
@ -799,8 +793,8 @@ describe(AssetMediaService.name, () => {
it('should handle a photo with sidecar to duplicate photo ', async () => {
const updatedFile = fileStub.photo;
const error = new QueryFailedError('', [], new Error('unique key violation'));
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT;
const error = new Error('unique key violation');
(error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.update.mockRejectedValue(error);
assetMock.getById.mockResolvedValueOnce(sidecarAsset);

View file

@ -30,7 +30,6 @@ import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util';
import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types';
import { fromChecksum } from 'src/utils/request';
import { QueryFailedError } from 'typeorm';
export interface UploadRequest {
auth: AuthDto | null;
fieldName: UploadFieldName;
@ -302,7 +301,7 @@ export class AssetMediaService extends BaseService {
});
// handle duplicates with a success response
if (error instanceof QueryFailedError && (error as any).constraint === ASSET_CHECKSUM_CONSTRAINT) {
if (error.constraint_name === ASSET_CHECKSUM_CONSTRAINT) {
const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, file.checksum);
if (!duplicateId) {
this.logger.error(`Error locating duplicate for checksum constraint`);
@ -343,7 +342,7 @@ export class AssetMediaService extends BaseService {
localDateTime: dto.fileCreatedAt,
duration: dto.duration || null,
livePhotoVideo: null,
livePhotoVideoId: null,
sidecarPath: sidecarPath || null,
});

View file

@ -51,9 +51,7 @@ describe(AssetService.name, () => {
});
const mockGetById = (assets: AssetEntity[]) => {
assetMock.getById.mockImplementation((assetId) =>
Promise.resolve(assets.find((asset) => asset.id === assetId) ?? null),
);
assetMock.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId)));
};
beforeEach(() => {
@ -250,27 +248,34 @@ describe(AssetService.name, () => {
it('should update the asset', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { isFavorite: true });
expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-1', isFavorite: true });
});
it('should update the exif description', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { description: 'Test description' });
expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' });
});
it('should update the exif rating', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image);
assetMock.getById.mockResolvedValueOnce(assetStub.image);
assetMock.update.mockResolvedValueOnce(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { rating: 3 });
expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 });
});
it('should fail linking a live video if the motion part could not be found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValue(null);
await expect(
sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, {
@ -339,6 +344,7 @@ describe(AssetService.name, () => {
isVisible: true,
});
assetMock.getById.mockResolvedValueOnce(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, {
livePhotoVideoId: assetStub.livePhotoMotionAsset.id,
@ -366,7 +372,7 @@ describe(AssetService.name, () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoStillAsset);
assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoMotionAsset);
assetMock.getById.mockResolvedValueOnce(assetStub.image);
assetMock.update.mockResolvedValueOnce(assetStub.image);
await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null });
@ -383,15 +389,15 @@ describe(AssetService.name, () => {
it('should fail unlinking a live video if the asset could not be found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValue(null);
// eslint-disable-next-line unicorn/no-useless-undefined
assetMock.getById.mockResolvedValueOnce(undefined);
await expect(
sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }),
).rejects.toBeInstanceOf(BadRequestException);
expect(assetMock.update).not.toHaveBeenCalled();
expect(assetMock.update).not.toHaveBeenCalledWith();
expect(eventMock.emit).not.toHaveBeenCalledWith();
expect(eventMock.emit).not.toHaveBeenCalled();
});
});

View file

@ -74,29 +74,13 @@ export class AssetService extends BaseService {
async get(auth: AuthDto, id: string): Promise<AssetResponseDto | SanitizedAssetResponseDto> {
await this.requireAccess({ auth, permission: Permission.ASSET_READ, ids: [id] });
const asset = await this.assetRepository.getById(
id,
{
exifInfo: true,
sharedLinks: true,
tags: true,
owner: true,
faces: {
person: true,
},
stack: {
assets: {
exifInfo: true,
},
},
files: true,
},
{
faces: {
boundingBoxX1: 'ASC',
},
},
);
const asset = await this.assetRepository.getById(id, {
exifInfo: true,
owner: true,
faces: { person: true },
stack: { assets: true },
tags: true,
});
if (!asset) {
throw new BadRequestException('Asset not found');
@ -137,22 +121,12 @@ export class AssetService extends BaseService {
await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude, rating });
await this.assetRepository.update({ id, ...rest });
const asset = await this.assetRepository.update({ id, ...rest });
if (previousMotion) {
await onAfterUnlink(repos, { userId: auth.user.id, livePhotoVideoId: previousMotion.id });
}
const asset = await this.assetRepository.getById(id, {
exifInfo: true,
owner: true,
tags: true,
faces: {
person: true,
},
files: true,
});
if (!asset) {
throw new BadRequestException('Asset not found');
}
@ -202,9 +176,7 @@ export class AssetService extends BaseService {
const { id, deleteOnDisk } = job;
const asset = await this.assetRepository.getById(id, {
faces: {
person: true,
},
faces: { person: true },
library: true,
stack: { assets: true },
exifInfo: true,

View file

@ -71,10 +71,8 @@ export class BackupService extends BaseService {
@OnJob({ name: JobName.BACKUP_DATABASE, queue: QueueName.BACKUP_DATABASE })
async handleBackupDatabase(): Promise<JobStatus> {
this.logger.debug(`Database Backup Started`);
const {
database: { config },
} = this.configRepository.getEnv();
const { database } = this.configRepository.getEnv();
const config = database.config.typeorm;
const isUrlConnection = config.connectionType === 'url';

View file

@ -1,3 +1,4 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import { IConfigRepository } from 'src/interfaces/config.interface';
import {
DatabaseExtension,
@ -61,13 +62,19 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
kysely: {
dialect: expect.any(PostgresJSDialect),
log: ['error'],
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
},
skipMigrations: false,
vectorExtension: extension,
@ -291,13 +298,19 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
kysely: {
dialect: expect.any(PostgresJSDialect),
log: ['error'],
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
},
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTORS,
@ -315,13 +328,19 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
kysely: {
dialect: expect.any(PostgresJSDialect),
log: ['error'],
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
},
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR,

View file

@ -113,6 +113,7 @@ export class DatabaseService extends BaseService {
if (!database.skipMigrations) {
await this.databaseRepository.runMigrations();
}
this.databaseRepository.init();
});
}

View file

@ -31,7 +31,12 @@ describe(SearchService.name, () => {
describe('getDuplicates', () => {
it('should get duplicates', async () => {
assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe, assetStub.hasDupe]);
assetMock.getDuplicates.mockResolvedValue([
{
duplicateId: assetStub.hasDupe.duplicateId!,
assets: [assetStub.hasDupe, assetStub.hasDupe],
},
]);
await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([
{
duplicateId: assetStub.hasDupe.duplicateId,
@ -42,12 +47,6 @@ describe(SearchService.name, () => {
},
]);
});
it('should update assets with duplicateId', async () => {
assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe]);
await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([]);
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.hasDupe.id], { duplicateId: null });
});
});
describe('handleQueueSearchDuplicates', () => {

View file

@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
import { OnJob } from 'src/decorators';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { DuplicateResponseDto, mapDuplicateResponse } from 'src/dtos/duplicate.dto';
import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { WithoutProperty } from 'src/interfaces/asset.interface';
import { JOBS_ASSET_PAGINATION_SIZE, JobName, JobOf, JobStatus, QueueName } from 'src/interfaces/job.interface';
@ -15,25 +15,11 @@ import { usePagination } from 'src/utils/pagination';
@Injectable()
export class DuplicateService extends BaseService {
async getDuplicates(auth: AuthDto): Promise<DuplicateResponseDto[]> {
const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] });
const uniqueAssetIds: string[] = [];
const duplicates = mapDuplicateResponse(res.map((a) => mapAsset(a, { auth, withStack: true }))).filter(
(duplicate) => {
if (duplicate.assets.length === 1) {
uniqueAssetIds.push(duplicate.assets[0].id);
return false;
}
return true;
},
);
if (uniqueAssetIds.length > 0) {
try {
await this.assetRepository.updateAll(uniqueAssetIds, { duplicateId: null });
} catch (error: any) {
this.logger.error(`Failed to remove duplicateId from assets: ${error.message}`);
}
}
return duplicates;
const duplicates = await this.assetRepository.getDuplicates(auth.user.id);
return duplicates.map(({ duplicateId, assets }) => ({
duplicateId,
assets: assets.map((asset) => mapAsset(asset, { auth })),
}));
}
@OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION })

View file

@ -256,8 +256,6 @@ describe(LibraryService.name, () => {
exclusionPatterns: [],
};
assetMock.getById.mockResolvedValue(null);
await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED);
expect(assetMock.remove).not.toHaveBeenCalled();
@ -394,7 +392,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg',
};
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image);
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
@ -439,7 +436,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/video.mp4',
};
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.video);
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
@ -484,7 +480,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg',
};
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image);
libraryMock.get.mockResolvedValue({ ...libraryStub.externalLibrary1, deletedAt: new Date() });
@ -550,7 +545,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg',
};
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image);
await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED);
@ -569,7 +563,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg',
};
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image);
await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED);

View file

@ -2,6 +2,7 @@ import { BinaryField, ExifDateTime } from 'exiftool-vendored';
import { randomBytes } from 'node:crypto';
import { Stats } from 'node:fs';
import { constants } from 'node:fs/promises';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum';
import { IAlbumRepository } from 'src/interfaces/album.interface';
@ -200,7 +201,6 @@ describe(MetadataService.name, () => {
exifInfo: { livePhotoCID: 'CID' } as ExifEntity,
},
]);
assetMock.findLivePhotoMatch.mockResolvedValue(null);
await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(
JobStatus.SKIPPED,
@ -579,7 +579,6 @@ describe(MetadataService.name, () => {
EmbeddedVideoType: 'MotionPhoto_Data',
});
cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512);
@ -624,7 +623,6 @@ describe(MetadataService.name, () => {
EmbeddedVideoType: 'MotionPhoto_Data',
});
cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512);
@ -670,7 +668,6 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1,
});
cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512);
@ -716,8 +713,9 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1,
});
cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockImplementation((asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }));
assetMock.create.mockImplementation(
(asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise<AssetEntity>,
);
const video = randomBytes(512);
storageMock.readFile.mockResolvedValue(video);
@ -789,7 +787,6 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1,
});
cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
const video = randomBytes(512);
storageMock.readFile.mockResolvedValue(video);

View file

@ -1,16 +1,17 @@
import { Injectable } from '@nestjs/common';
import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored';
import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
import { Insertable } from 'kysely';
import _ from 'lodash';
import { Duration } from 'luxon';
import { constants } from 'node:fs/promises';
import path from 'node:path';
import { SystemConfig } from 'src/config';
import { StorageCore } from 'src/cores/storage.core';
import { Exif } from 'src/db';
import { OnEvent, OnJob } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { PersonEntity } from 'src/entities/person.entity';
import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum';
import { WithoutProperty } from 'src/interfaces/asset.interface';
@ -166,7 +167,7 @@ export class MetadataService extends BaseService {
const { width, height } = this.getImageDimensions(exifTags);
const exifData: Partial<ExifEntity> = {
const exifData: Insertable<Exif> = {
assetId: asset.id,
// dates

View file

@ -728,11 +728,13 @@ describe(PersonService.name, () => {
assetId: assetStub.image.id,
facesRecognizedAt: expect.any(Date),
});
expect(assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt?.getTime()).toBeGreaterThan(start);
const facesRecognizedAt = assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt as Date;
expect(facesRecognizedAt.getTime()).toBeGreaterThan(start);
});
it('should create a face with no person and queue recognition job', async () => {
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
searchMock.searchFaces.mockResolvedValue([{ ...faceStub.face1, distance: 0.7 }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleDetectFaces({ id: assetStub.image.id });
@ -840,10 +842,10 @@ describe(PersonService.name, () => {
}
const faces = [
{ face: faceStub.noPerson1, distance: 0 },
{ face: faceStub.primaryFace1, distance: 0.2 },
{ face: faceStub.noPerson2, distance: 0.3 },
{ face: faceStub.face1, distance: 0.4 },
{ ...faceStub.noPerson1, distance: 0 },
{ ...faceStub.primaryFace1, distance: 0.2 },
{ ...faceStub.noPerson2, distance: 0.3 },
{ ...faceStub.face1, distance: 0.4 },
] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
@ -867,8 +869,8 @@ describe(PersonService.name, () => {
it('should create a new person if the face is a core point with no person', async () => {
const faces = [
{ face: faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.3 },
{ ...faceStub.noPerson1, distance: 0 },
{ ...faceStub.noPerson2, distance: 0.3 },
] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
@ -889,7 +891,7 @@ describe(PersonService.name, () => {
});
it('should not queue face with no matches', async () => {
const faces = [{ face: faceStub.noPerson1, distance: 0 }] as FaceSearchResult[];
const faces = [{ ...faceStub.noPerson1, distance: 0 }] as FaceSearchResult[];
searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
@ -905,8 +907,8 @@ describe(PersonService.name, () => {
it('should defer non-core faces to end of queue', async () => {
const faces = [
{ face: faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.4 },
{ ...faceStub.noPerson1, distance: 0 },
{ ...faceStub.noPerson2, distance: 0.4 },
] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });
@ -927,8 +929,8 @@ describe(PersonService.name, () => {
it('should not assign person to deferred non-core face with no matching person', async () => {
const faces = [
{ face: faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.4 },
{ ...faceStub.noPerson1, distance: 0 },
{ ...faceStub.noPerson2, distance: 0.4 },
] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });

View file

@ -261,7 +261,7 @@ export class PersonService extends BaseService {
return force === false
? this.assetRepository.getWithout(pagination, WithoutProperty.FACES)
: this.assetRepository.getAll(pagination, {
orderDirection: 'DESC',
orderDirection: 'desc',
withFaces: true,
withArchived: true,
isVisible: true,
@ -288,13 +288,7 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED;
}
const relations = {
exifInfo: true,
faces: {
person: false,
},
files: true,
};
const relations = { exifInfo: true, faces: { person: false }, files: true };
const [asset] = await this.assetRepository.getByIds([id], relations);
const { previewFile } = getAssetFiles(asset.files);
if (!asset || !previewFile) {
@ -491,7 +485,7 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED;
}
let personId = matches.find((match) => match.face.personId)?.face.personId;
let personId = matches.find((match) => match.personId)?.personId;
if (!personId) {
const matchWithPerson = await this.searchRepository.searchFaces({
userIds: [face.asset.ownerId],
@ -502,7 +496,7 @@ export class PersonService extends BaseService {
});
if (matchWithPerson.length > 0) {
personId = matchWithPerson[0].face.personId;
personId = matchWithPerson[0].personId;
}
}

View file

@ -45,11 +45,11 @@ describe(SearchService.name, () => {
it('should get assets by city and tag', async () => {
assetMock.getAssetIdByCity.mockResolvedValue({
fieldName: 'exifInfo.city',
items: [{ value: 'Paris', data: assetStub.image.id }],
items: [{ value: 'test-city', data: assetStub.withLocation.id }],
});
assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.image, assetStub.imageFrom2015]);
assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]);
const expectedResponse = [
{ fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
{ fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] },
];
const result = await sut.getExploreData(authStub.user1);

View file

@ -34,16 +34,10 @@ export class SearchService extends BaseService {
async getExploreData(auth: AuthDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
const options = { maxFields: 12, minAssetsPerField: 5 };
const result = await this.assetRepository.getAssetIdByCity(auth.user.id, options);
const results = [result];
const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data)));
const assets = await this.assetRepository.getByIdsWithAllRelations([...assetIds]);
const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
return results.map(({ fieldName, items }) => ({
fieldName,
items: items.map(({ value, data }) => ({ value, data: assetMap.get(data) as AssetResponseDto })),
}));
const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options);
const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data));
const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) }));
return [{ fieldName: cities.fieldName, items }];
}
async searchMetadata(auth: AuthDto, dto: MetadataSearchDto): Promise<SearchResponseDto> {
@ -57,14 +51,13 @@ export class SearchService extends BaseService {
const page = dto.page ?? 1;
const size = dto.size || 250;
const enumToOrder = { [AssetOrder.ASC]: 'ASC', [AssetOrder.DESC]: 'DESC' } as const;
const { hasNextPage, items } = await this.searchRepository.searchMetadata(
{ page, size },
{
...dto,
checksum,
userIds,
orderDirection: dto.order ? enumToOrder[dto.order] : 'DESC',
orderDirection: dto.order ?? AssetOrder.DESC,
},
);

View file

@ -61,12 +61,15 @@ describe(TimelineService.name, () => {
userId: authStub.admin.user.id,
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
userIds: [authStub.admin.user.id],
});
expect(assetMock.getTimeBucket).toHaveBeenCalledWith(
'bucket',
expect.objectContaining({
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
userIds: [authStub.admin.user.id],
}),
);
});
it('should include partner shared assets', async () => {
@ -143,11 +146,14 @@ describe(TimelineService.name, () => {
userId: authStub.admin.user.id,
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
});
expect(assetMock.getTimeBucket).toHaveBeenCalledWith(
'bucket',
expect.objectContaining({
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
}),
);
});
it('should throw an error if withParners is true and isArchived true or undefined', async () => {

View file

@ -1,8 +1,7 @@
import _ from 'lodash';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface';
import { Between, IsNull, LessThanOrEqual, MoreThanOrEqual, Not, SelectQueryBuilder } from 'typeorm';
import { Expression, RawBuilder, sql, ValueExpression } from 'kysely';
import { InsertObject } from 'node_modules/kysely/dist/cjs';
import { DB } from 'src/db';
import { Between, LessThanOrEqual, MoreThanOrEqual } from 'typeorm';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
@ -18,131 +17,42 @@ export function OptionalBetween<T>(from?: T, to?: T) {
}
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
// populated by the database repository at bootstrap
export const UPSERT_COLUMNS = {} as { [T in keyof DB]: { [K in keyof DB[T]]: RawBuilder<unknown> } };
export function searchAssetBuilder(
builder: SelectQueryBuilder<AssetEntity>,
options: AssetSearchBuilderOptions,
): SelectQueryBuilder<AssetEntity> {
builder.andWhere(
_.omitBy(
{
createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
},
_.isUndefined,
),
);
const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
const hasExifQuery = Object.keys(exifInfo).length > 0;
if (options.withExif && !hasExifQuery) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
}
if (hasExifQuery) {
if (options.withExif) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
} else {
builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
}
for (const [key, value] of Object.entries(exifInfo)) {
if (value === null) {
builder.andWhere(`exifInfo.${key} IS NULL`);
} else {
builder.andWhere(`exifInfo.${key} = :${key}`, { [key]: value });
}
/** Generates the columns for an upsert statement, excluding the conflict keys.
* Assumes that all entries have the same keys. */
export function mapUpsertColumns<T extends keyof DB>(
table: T,
entry: InsertObject<DB, T>,
conflictKeys: readonly (keyof DB[T])[],
) {
const columns = UPSERT_COLUMNS[table] as { [K in keyof DB[T]]: RawBuilder<unknown> };
const upsertColumns: Partial<Record<keyof typeof entry, RawBuilder<unknown>>> = {};
for (const entryColumn in entry) {
if (!conflictKeys.includes(entryColumn as keyof DB[T])) {
upsertColumns[entryColumn as keyof typeof entry] = columns[entryColumn as keyof DB[T]];
}
}
const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId']);
if (id.libraryId === null) {
id.libraryId = IsNull() as unknown as string;
}
builder.andWhere(_.omitBy(id, _.isUndefined));
if (options.userIds) {
builder.andWhere(`${builder.alias}.ownerId IN (:...userIds)`, { userIds: options.userIds });
}
const path = _.pick(options, ['encodedVideoPath', 'originalPath']);
builder.andWhere(_.omitBy(path, _.isUndefined));
if (options.originalFileName) {
builder.andWhere(`f_unaccent(${builder.alias}.originalFileName) ILIKE f_unaccent(:originalFileName)`, {
originalFileName: `%${options.originalFileName}%`,
});
}
const status = _.pick(options, ['isFavorite', 'isVisible', 'type']);
const {
isArchived,
isEncoded,
isMotion,
withArchived,
isNotInAlbum,
withFaces,
withPeople,
personIds,
withStacked,
trashedAfter,
trashedBefore,
} = options;
builder.andWhere(
_.omitBy(
{
...status,
isArchived: isArchived ?? (withArchived ? undefined : false),
encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
},
_.isUndefined,
),
);
if (isNotInAlbum) {
builder
.leftJoin(`${builder.alias}.albums`, 'albums')
.andWhere('albums.id IS NULL')
.andWhere(`${builder.alias}.isVisible = true`);
}
if (withFaces || withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
}
if (withPeople) {
builder.leftJoinAndSelect('faces.person', 'person');
}
if (personIds && personIds.length > 0) {
const cte = builder
.createQueryBuilder()
.select('faces."assetId"')
.from(AssetFaceEntity, 'faces')
.where('faces."personId" IN (:...personIds)', { personIds })
.groupBy(`faces."assetId"`)
.having(`COUNT(DISTINCT faces."personId") = :personCount`, { personCount: personIds.length });
builder.addCommonTableExpression(cte, 'face_ids').innerJoin('face_ids', 'a', 'a."assetId" = asset.id');
builder.getQuery(); // typeorm mixes up parameters without this (੭ °ཀ°)੭
}
if (withStacked) {
builder.leftJoinAndSelect(`${builder.alias}.stack`, 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
}
const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
if (withDeleted) {
builder.withDeleted();
}
return builder;
return upsertColumns as Expand<Record<keyof typeof entry, ValueExpression<DB, T, any>>>;
}
export const asUuid = (id: string | Expression<string>) => sql<string>`${id}::uuid`;
export const anyUuid = (ids: string[]) => sql<string>`any(${`{${ids}}`}::uuid[])`;
export const asVector = (embedding: number[]) => sql<number[]>`${`[${embedding}]`}::vector`;
/**
* Mainly for type debugging to make VS Code display a more useful tooltip.
* Source: https://stackoverflow.com/a/69288824
*/
export type Expand<T> = T extends infer O ? { [K in keyof O]: O[K] } : never;
/** Recursive version of {@link Expand} from the same source. */
export type ExpandRecursively<T> = T extends object
? T extends infer O
? { [K in keyof O]: ExpandRecursively<O[K]> }
: never
: T;

View file

@ -33,7 +33,10 @@ export async function* usePagination<T>(
}
}
function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
export function paginationHelper<Entity extends ObjectLiteral>(
items: Entity[],
take: number,
): PaginationResult<Entity> {
const hasNextPage = items.length > take;
items.splice(take);

View file

@ -1,3 +1,5 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import postgres from 'postgres';
import { ImmichEnvironment, ImmichWorker } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface';
@ -21,16 +23,24 @@ const envData: EnvData = {
database: {
config: {
connectionType: 'parts',
database: 'immich',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
synchronize: false,
migrationsRun: true,
kysely: {
dialect: new PostgresJSDialect({
postgres: postgres({ database: 'immich', host: 'database', port: 5432 }),
}),
log: ['error'],
},
typeorm: {
connectionType: 'parts',
database: 'immich',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
synchronize: false,
migrationsRun: true,
},
},
skipMigrations: false,

View file

@ -3,6 +3,7 @@ import { Mocked, vitest } from 'vitest';
export const newDatabaseRepositoryMock = (): Mocked<IDatabaseRepository> => {
return {
init: vitest.fn(),
reconnect: vitest.fn(),
getExtensionVersion: vitest.fn(),
getExtensionVersionRange: vitest.fn(),

View file

@ -19,7 +19,8 @@
"preserveWatchOutput": true,
"baseUrl": "./",
"jsx": "react",
"types": ["vitest/globals"]
"types": ["vitest/globals"],
"noErrorTruncation": true
},
"exclude": ["dist", "node_modules", "upload"]
}