1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-09 13:26:47 +01:00
This commit is contained in:
mertalev 2024-12-18 15:27:31 -05:00
parent 007caa26bd
commit 38a82d39d3
No known key found for this signature in database
GPG key ID: 3A2B5BFC678DBC80
48 changed files with 2702 additions and 3164 deletions

View file

@ -538,7 +538,7 @@ describe('/asset', () => {
expect(body).toMatchObject({ expect(body).toMatchObject({
id: user1Assets[0].id, id: user1Assets[0].id,
exifInfo: expect.objectContaining({ exifInfo: expect.objectContaining({
dateTimeOriginal: '2023-11-20T01:11:00.000Z', dateTimeOriginal: '2023-11-20T01:11:00+00:00',
}), }),
}); });
expect(status).toEqual(200); expect(status).toEqual(200);
@ -608,7 +608,7 @@ describe('/asset', () => {
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const assetInfo = await utils.getAssetInfo(user1.accessToken, id); const assetInfo = await utils.getAssetInfo(user1.accessToken, id);
expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52.000Z'); expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52+00:00');
const { status, body } = await request(app) const { status, body } = await request(app)
.put(`/assets/${id}`) .put(`/assets/${id}`)
@ -618,7 +618,7 @@ describe('/asset', () => {
expect(body).toMatchObject({ expect(body).toMatchObject({
id, id,
exifInfo: expect.objectContaining({ exifInfo: expect.objectContaining({
dateTimeOriginal: '2023-11-20T01:11:00.000Z', dateTimeOriginal: '2023-11-20T01:11:00+00:00',
}), }),
}); });
expect(status).toEqual(200); expect(status).toEqual(200);
@ -953,8 +953,6 @@ describe('/asset', () => {
exifImageHeight: 1080, exifImageHeight: 1080,
exifImageWidth: 1617, exifImageWidth: 1617,
fileSizeInByte: 862_424, fileSizeInByte: 862_424,
latitude: null,
longitude: null,
}, },
}, },
}, },
@ -964,11 +962,9 @@ describe('/asset', () => {
type: AssetTypeEnum.Image, type: AssetTypeEnum.Image,
originalFileName: 'el_torcal_rocks.jpg', originalFileName: 'el_torcal_rocks.jpg',
exifInfo: { exifInfo: {
dateTimeOriginal: '2012-08-05T11:39:59.000Z', dateTimeOriginal: '2012-08-05T11:39:59+00:00',
exifImageWidth: 512, exifImageWidth: 512,
exifImageHeight: 341, exifImageHeight: 341,
latitude: null,
longitude: null,
focalLength: 75, focalLength: 75,
iso: 200, iso: 200,
fNumber: 11, fNumber: 11,
@ -976,7 +972,6 @@ describe('/asset', () => {
fileSizeInByte: 53_493, fileSizeInByte: 53_493,
make: 'SONY', make: 'SONY',
model: 'DSLR-A550', model: 'DSLR-A550',
orientation: null,
description: 'SONY DSC', description: 'SONY DSC',
}, },
}, },
@ -991,8 +986,6 @@ describe('/asset', () => {
exifImageHeight: 1080, exifImageHeight: 1080,
exifImageWidth: 1440, exifImageWidth: 1440,
fileSizeInByte: 1_780_777, fileSizeInByte: 1_780_777,
latitude: null,
longitude: null,
}, },
}, },
}, },
@ -1003,7 +996,7 @@ describe('/asset', () => {
originalFileName: 'IMG_2682.heic', originalFileName: 'IMG_2682.heic',
fileCreatedAt: '2019-03-21T16:04:22.348Z', fileCreatedAt: '2019-03-21T16:04:22.348Z',
exifInfo: { exifInfo: {
dateTimeOriginal: '2019-03-21T16:04:22.348Z', dateTimeOriginal: '2019-03-21T16:04:22.348+00:00',
exifImageWidth: 4032, exifImageWidth: 4032,
exifImageHeight: 3024, exifImageHeight: 3024,
latitude: 41.2203, latitude: 41.2203,
@ -1028,8 +1021,6 @@ describe('/asset', () => {
exifInfo: { exifInfo: {
exifImageWidth: 800, exifImageWidth: 800,
exifImageHeight: 800, exifImageHeight: 800,
latitude: null,
longitude: null,
fileSizeInByte: 25_408, fileSizeInByte: 25_408,
}, },
}, },
@ -1048,9 +1039,7 @@ describe('/asset', () => {
focalLength: 18, focalLength: 18,
iso: 100, iso: 100,
fileSizeInByte: 9_057_784, fileSizeInByte: 9_057_784,
dateTimeOriginal: '2010-07-20T17:27:12.000Z', dateTimeOriginal: '2010-07-20T17:27:12+00:00',
latitude: null,
longitude: null,
orientation: '1', orientation: '1',
}, },
}, },
@ -1069,9 +1058,7 @@ describe('/asset', () => {
focalLength: 85, focalLength: 85,
iso: 200, iso: 200,
fileSizeInByte: 15_856_335, fileSizeInByte: 15_856_335,
dateTimeOriginal: '2016-09-22T21:10:29.060Z', dateTimeOriginal: '2016-09-22T21:10:29.06+00:00',
latitude: null,
longitude: null,
orientation: '1', orientation: '1',
timeZone: 'UTC-4', timeZone: 'UTC-4',
}, },
@ -1093,9 +1080,7 @@ describe('/asset', () => {
focalLength: 35, focalLength: 35,
iso: 400, iso: 400,
fileSizeInByte: 19_587_072, fileSizeInByte: 19_587_072,
dateTimeOriginal: '2018-05-10T08:42:37.842Z', dateTimeOriginal: '2018-05-10T08:42:37.842+00:00',
latitude: null,
longitude: null,
orientation: '1', orientation: '1',
}, },
}, },
@ -1117,9 +1102,7 @@ describe('/asset', () => {
iso: 100, iso: 100,
lensModel: 'E PZ 18-105mm F4 G OSS', lensModel: 'E PZ 18-105mm F4 G OSS',
fileSizeInByte: 25_001_984, fileSizeInByte: 25_001_984,
dateTimeOriginal: '2016-09-27T10:51:44.000Z', dateTimeOriginal: '2016-09-27T10:51:44+00:00',
latitude: null,
longitude: null,
orientation: '1', orientation: '1',
}, },
}, },
@ -1141,9 +1124,7 @@ describe('/asset', () => {
iso: 100, iso: 100,
lensModel: 'E 25mm F2', lensModel: 'E 25mm F2',
fileSizeInByte: 49_512_448, fileSizeInByte: 49_512_448,
dateTimeOriginal: '2016-01-08T14:08:01.000Z', dateTimeOriginal: '2016-01-08T14:08:01+00:00',
latitude: null,
longitude: null,
orientation: '1', orientation: '1',
}, },
}, },
@ -1165,7 +1146,7 @@ describe('/asset', () => {
iso: 80, iso: 80,
lensModel: null, lensModel: null,
fileSizeInByte: 11_113_617, fileSizeInByte: 11_113_617,
dateTimeOriginal: '2015-12-27T09:55:40.000Z', dateTimeOriginal: '2015-12-27T09:55:40+00:00',
latitude: null, latitude: null,
longitude: null, longitude: null,
orientation: '1', orientation: '1',
@ -1189,7 +1170,7 @@ describe('/asset', () => {
iso: 160, iso: 160,
lensModel: null, lensModel: null,
fileSizeInByte: 13_551_312, fileSizeInByte: 13_551_312,
dateTimeOriginal: '2024-10-12T21:01:01.000Z', dateTimeOriginal: '2024-10-12T21:01:01+00:00',
latitude: null, latitude: null,
longitude: null, longitude: null,
orientation: '6', orientation: '6',
@ -1203,7 +1184,7 @@ describe('/asset', () => {
originalFileName: 'Ricoh_GR3-450.DNG', originalFileName: 'Ricoh_GR3-450.DNG',
fileCreatedAt: '2024-06-08T13:48:39.000Z', fileCreatedAt: '2024-06-08T13:48:39.000Z',
exifInfo: { exifInfo: {
dateTimeOriginal: '2024-06-08T13:48:39.000Z', dateTimeOriginal: '2024-06-08T13:48:39+00:00',
exifImageHeight: 4064, exifImageHeight: 4064,
exifImageWidth: 6112, exifImageWidth: 6112,
exposureTime: '1/400', exposureTime: '1/400',

View file

@ -151,7 +151,7 @@ describe('/timeline', () => {
it('should require authentication', async () => { it('should require authentication', async () => {
const { status, body } = await request(app).get('/timeline/bucket').query({ const { status, body } = await request(app).get('/timeline/bucket').query({
size: TimeBucketSize.Month, size: TimeBucketSize.Month,
timeBucket: '1900-01-01T00:00:00.000Z', timeBucket: '1900-01-01',
}); });
expect(status).toBe(401); expect(status).toBe(401);
@ -161,7 +161,7 @@ describe('/timeline', () => {
it('should handle 5 digit years', async () => { it('should handle 5 digit years', async () => {
const { status, body } = await request(app) const { status, body } = await request(app)
.get('/timeline/bucket') .get('/timeline/bucket')
.query({ size: TimeBucketSize.Month, timeBucket: '+012345-01-01T00:00:00.000Z' }) .query({ size: TimeBucketSize.Month, timeBucket: '012345-01-01' })
.set('Authorization', `Bearer ${timeBucketUser.accessToken}`); .set('Authorization', `Bearer ${timeBucketUser.accessToken}`);
expect(status).toBe(200); expect(status).toBe(200);
@ -183,7 +183,7 @@ describe('/timeline', () => {
const { status, body } = await request(app) const { status, body } = await request(app)
.get('/timeline/bucket') .get('/timeline/bucket')
.set('Authorization', `Bearer ${timeBucketUser.accessToken}`) .set('Authorization', `Bearer ${timeBucketUser.accessToken}`)
.query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10T00:00:00.000Z' }); .query({ size: TimeBucketSize.Month, timeBucket: '1970-02-10' });
expect(status).toBe(200); expect(status).toBe(200);
expect(body).toEqual([]); expect(body).toEqual([]);

View file

@ -8420,6 +8420,7 @@
"type": "string" "type": "string"
}, },
"AssetOrder": { "AssetOrder": {
"default": "desc",
"enum": [ "enum": [
"asc", "asc",
"desc" "desc"

337
server/package-lock.json generated
View file

@ -42,10 +42,13 @@
"ioredis": "^5.3.2", "ioredis": "^5.3.2",
"joi": "^17.10.0", "joi": "^17.10.0",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"kysely": "^0.27.3",
"kysely-postgres-js": "^2.0.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"luxon": "^3.4.2", "luxon": "^3.4.2",
"nest-commander": "^3.11.1", "nest-commander": "^3.11.1",
"nestjs-cls": "^4.3.0", "nestjs-cls": "^4.3.0",
"nestjs-kysely": "^1.0.0",
"nestjs-otel": "^6.0.0", "nestjs-otel": "^6.0.0",
"nodemailer": "^6.9.13", "nodemailer": "^6.9.13",
"openid-client": "^5.4.3", "openid-client": "^5.4.3",
@ -99,6 +102,7 @@
"eslint-plugin-prettier": "^5.1.3", "eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-unicorn": "^56.0.1", "eslint-plugin-unicorn": "^56.0.1",
"globals": "^15.9.0", "globals": "^15.9.0",
"kysely-codegen": "^0.16.3",
"mock-fs": "^5.2.0", "mock-fs": "^5.2.0",
"pngjs": "^7.0.0", "pngjs": "^7.0.0",
"prettier": "^3.0.2", "prettier": "^3.0.2",
@ -9176,6 +9180,102 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/git-diff": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/git-diff/-/git-diff-2.0.6.tgz",
"integrity": "sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==",
"dev": true,
"dependencies": {
"chalk": "^2.3.2",
"diff": "^3.5.0",
"loglevel": "^1.6.1",
"shelljs": "^0.8.1",
"shelljs.exec": "^1.1.7"
},
"engines": {
"node": ">= 4.8.0"
}
},
"node_modules/git-diff/node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"dependencies": {
"color-convert": "^1.9.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dev": true,
"dependencies": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"dependencies": {
"color-name": "1.1.3"
}
},
"node_modules/git-diff/node_modules/color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
"dev": true
},
"node_modules/git-diff/node_modules/diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true,
"engines": {
"node": ">=0.3.1"
}
},
"node_modules/git-diff/node_modules/escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
"dev": true,
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/git-diff/node_modules/has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
"dev": true,
"engines": {
"node": ">=4"
}
},
"node_modules/git-diff/node_modules/supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"dev": true,
"dependencies": {
"has-flag": "^3.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/glob": { "node_modules/glob": {
"version": "10.4.5", "version": "10.4.5",
"resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@ -9617,6 +9717,15 @@
"node": ">=12.0.0" "node": ">=12.0.0"
} }
}, },
"node_modules/interpret": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
"integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
"dev": true,
"engines": {
"node": ">= 0.10"
}
},
"node_modules/ioredis": { "node_modules/ioredis": {
"version": "5.4.1", "version": "5.4.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz",
@ -10048,6 +10157,100 @@
"json-buffer": "3.0.1" "json-buffer": "3.0.1"
} }
}, },
"node_modules/kysely": {
"version": "0.27.4",
"resolved": "https://registry.npmjs.org/kysely/-/kysely-0.27.4.tgz",
"integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/kysely-codegen": {
"version": "0.16.3",
"resolved": "https://registry.npmjs.org/kysely-codegen/-/kysely-codegen-0.16.3.tgz",
"integrity": "sha512-SOOF3AhrsjREJuRewXmKl0nb6CkEzpP7VavHXzWdfIdIdfoJnlWlozuZhgMsYoIFmzL8aG4skvKGXF/dF3mbwg==",
"dev": true,
"dependencies": {
"chalk": "4.1.2",
"dotenv": "^16.4.5",
"dotenv-expand": "^11.0.6",
"git-diff": "^2.0.6",
"micromatch": "^4.0.8",
"minimist": "^1.2.8",
"pluralize": "^8.0.0"
},
"bin": {
"kysely-codegen": "dist/cli/bin.js"
},
"peerDependencies": {
"@libsql/kysely-libsql": "^0.3.0",
"@tediousjs/connection-string": "^0.5.0",
"better-sqlite3": ">=7.6.2",
"kysely": "^0.27.0",
"kysely-bun-sqlite": "^0.3.2",
"kysely-bun-worker": "^0.5.3",
"mysql2": "^2.3.3 || ^3.0.0",
"pg": "^8.8.0",
"tarn": "^3.0.0",
"tedious": "^18.0.0"
},
"peerDependenciesMeta": {
"@libsql/kysely-libsql": {
"optional": true
},
"@tediousjs/connection-string": {
"optional": true
},
"better-sqlite3": {
"optional": true
},
"kysely": {
"optional": false
},
"kysely-bun-sqlite": {
"optional": true
},
"kysely-bun-worker": {
"optional": true
},
"mysql2": {
"optional": true
},
"pg": {
"optional": true
},
"tarn": {
"optional": true
},
"tedious": {
"optional": true
}
}
},
"node_modules/kysely-codegen/node_modules/dotenv-expand": {
"version": "11.0.6",
"resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.6.tgz",
"integrity": "sha512-8NHi73otpWsZGBSZwwknTXS5pqMOrk9+Ssrna8xCaxkzEpU9OTf9R5ArQGVw03//Zmk9MOwLPng9WwndvpAJ5g==",
"dev": true,
"dependencies": {
"dotenv": "^16.4.4"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/kysely-postgres-js": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/kysely-postgres-js/-/kysely-postgres-js-2.0.0.tgz",
"integrity": "sha512-R1tWx6/x3tSatWvsmbHJxpBZYhNNxcnMw52QzZaHKg7ZOWtHib4iZyEaw4gb2hNKVctWQ3jfMxZT/ZaEMK6kBQ==",
"peerDependencies": {
"kysely": ">= 0.24.0 < 1",
"postgres": ">= 3.4.0 < 4"
}
},
"node_modules/lazystream": { "node_modules/lazystream": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz",
@ -10221,6 +10424,19 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/loglevel": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz",
"integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==",
"dev": true,
"engines": {
"node": ">= 0.6.0"
},
"funding": {
"type": "tidelift",
"url": "https://tidelift.com/funding/github/npm/loglevel"
}
},
"node_modules/long": { "node_modules/long": {
"version": "5.2.3", "version": "5.2.3",
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
@ -10825,6 +11041,17 @@
"rxjs": ">= 7" "rxjs": ">= 7"
} }
}, },
"node_modules/nestjs-kysely": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/nestjs-kysely/-/nestjs-kysely-1.0.0.tgz",
"integrity": "sha512-0XwXm3H/sGQ8DhOY1UOMuosdgii+qePJpbdlWzRPP7ueP/Z/JGnxnu2jUHurFGS+bMGPipGnk9ynIkJuM05HUw==",
"peerDependencies": {
"@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0",
"@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0",
"kysely": "0.x",
"reflect-metadata": "^0.1.13 || ^0.2.2"
}
},
"node_modules/nestjs-otel": { "node_modules/nestjs-otel": {
"version": "6.1.1", "version": "6.1.1",
"resolved": "https://registry.npmjs.org/nestjs-otel/-/nestjs-otel-6.1.1.tgz", "resolved": "https://registry.npmjs.org/nestjs-otel/-/nestjs-otel-6.1.1.tgz",
@ -11736,6 +11963,19 @@
"license": "MIT", "license": "MIT",
"peer": true "peer": true
}, },
"node_modules/postgres": {
"version": "3.4.4",
"resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz",
"integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==",
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/porsager"
}
},
"node_modules/postgres-array": { "node_modules/postgres-array": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
@ -12490,6 +12730,18 @@
"url": "https://github.com/sponsors/jonschlinkert" "url": "https://github.com/sponsors/jonschlinkert"
} }
}, },
"node_modules/rechoir": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
"integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
"dev": true,
"dependencies": {
"resolve": "^1.1.6"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/redis-errors": { "node_modules/redis-errors": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
@ -13173,6 +13425,53 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/shelljs": {
"version": "0.8.5",
"resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
"integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
"dev": true,
"dependencies": {
"glob": "^7.0.0",
"interpret": "^1.0.0",
"rechoir": "^0.6.2"
},
"bin": {
"shjs": "bin/shjs"
},
"engines": {
"node": ">=4"
}
},
"node_modules/shelljs.exec": {
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/shelljs.exec/-/shelljs.exec-1.1.8.tgz",
"integrity": "sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==",
"dev": true,
"engines": {
"node": ">= 4.0.0"
}
},
"node_modules/shelljs/node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"deprecated": "Glob versions prior to v9 are no longer supported",
"dev": true,
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/shimmer": { "node_modules/shimmer": {
"version": "1.2.1", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz",
@ -15906,6 +16205,44 @@
"engines": { "engines": {
"node": ">= 14" "node": ">= 14"
} }
},
"node_modules/zip-stream/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/zip-stream/node_modules/readable-stream": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
"dependencies": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
} }
} }
} }

View file

@ -67,10 +67,13 @@
"ioredis": "^5.3.2", "ioredis": "^5.3.2",
"joi": "^17.10.0", "joi": "^17.10.0",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"kysely": "^0.27.3",
"kysely-postgres-js": "^2.0.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"luxon": "^3.4.2", "luxon": "^3.4.2",
"nest-commander": "^3.11.1", "nest-commander": "^3.11.1",
"nestjs-cls": "^4.3.0", "nestjs-cls": "^4.3.0",
"nestjs-kysely": "^1.0.0",
"nestjs-otel": "^6.0.0", "nestjs-otel": "^6.0.0",
"nodemailer": "^6.9.13", "nodemailer": "^6.9.13",
"openid-client": "^5.4.3", "openid-client": "^5.4.3",
@ -124,6 +127,7 @@
"eslint-plugin-prettier": "^5.1.3", "eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-unicorn": "^56.0.1", "eslint-plugin-unicorn": "^56.0.1",
"globals": "^15.9.0", "globals": "^15.9.0",
"kysely-codegen": "^0.16.3",
"mock-fs": "^5.2.0", "mock-fs": "^5.2.0",
"pngjs": "^7.0.0", "pngjs": "^7.0.0",
"prettier": "^3.0.2", "prettier": "^3.0.2",

View file

@ -4,6 +4,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE, ModuleRef } from '@ne
import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule'; import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
import { ClsModule } from 'nestjs-cls'; import { ClsModule } from 'nestjs-cls';
import { KyselyModule } from 'nestjs-kysely';
import { OpenTelemetryModule } from 'nestjs-otel'; import { OpenTelemetryModule } from 'nestjs-otel';
import { commands } from 'src/commands'; import { commands } from 'src/commands';
import { IWorker } from 'src/constants'; import { IWorker } from 'src/constants';
@ -48,7 +49,7 @@ const imports = [
inject: [ModuleRef], inject: [ModuleRef],
useFactory: (moduleRef: ModuleRef) => { useFactory: (moduleRef: ModuleRef) => {
return { return {
...database.config, ...database.config.typeorm,
poolErrorHandler: (error) => { poolErrorHandler: (error) => {
moduleRef.get(DatabaseService, { strict: false }).handleConnectionError(error); moduleRef.get(DatabaseService, { strict: false }).handleConnectionError(error);
}, },
@ -56,6 +57,7 @@ const imports = [
}, },
}), }),
TypeOrmModule.forFeature(entities), TypeOrmModule.forFeature(entities),
KyselyModule.forRoot(database.config.kysely),
]; ];
class BaseModule implements OnModuleInit, OnModuleDestroy { class BaseModule implements OnModuleInit, OnModuleDestroy {

View file

@ -8,4 +8,4 @@ const { database } = new ConfigRepository().getEnv();
* *
* this export is ONLY to be used for TypeORM commands in package.json#scripts * this export is ONLY to be used for TypeORM commands in package.json#scripts
*/ */
export const dataSource = new DataSource({ ...database.config, host: 'localhost' }); export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' });

View file

@ -4,6 +4,7 @@ import { Reflector } from '@nestjs/core';
import { SchedulerRegistry } from '@nestjs/schedule'; import { SchedulerRegistry } from '@nestjs/schedule';
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
import { KyselyModule } from 'nestjs-kysely';
import { OpenTelemetryModule } from 'nestjs-otel'; import { OpenTelemetryModule } from 'nestjs-otel';
import { mkdir, rm, writeFile } from 'node:fs/promises'; import { mkdir, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path'; import { join } from 'node:path';
@ -73,13 +74,23 @@ class SqlGenerator {
await rm(this.options.targetDir, { force: true, recursive: true }); await rm(this.options.targetDir, { force: true, recursive: true });
await mkdir(this.options.targetDir); await mkdir(this.options.targetDir);
process.env.DB_HOSTNAME = 'localhost';
const { database, otel } = new ConfigRepository().getEnv(); const { database, otel } = new ConfigRepository().getEnv();
const moduleFixture = await Test.createTestingModule({ const moduleFixture = await Test.createTestingModule({
imports: [ imports: [
KyselyModule.forRoot({
...database.config.kysely,
log: (event) => {
if (event.level === 'query') {
this.sqlLogger.logQuery(event.query.sql);
} else if (event.level === 'error') {
this.sqlLogger.logQueryError(event.error as Error, event.query.sql);
}
},
}),
TypeOrmModule.forRoot({ TypeOrmModule.forRoot({
...database.config, ...database.config.typeorm,
host: 'localhost',
entities, entities,
logging: ['query'], logging: ['query'],
logger: this.sqlLogger, logger: this.sqlLogger,

View file

@ -7,6 +7,10 @@ export const POSTGRES_VERSION_RANGE = '>=14.0.0';
export const VECTORS_VERSION_RANGE = '>=0.2 <0.4'; export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';
export const VECTOR_VERSION_RANGE = '>=0.5 <1'; export const VECTOR_VERSION_RANGE = '>=0.5 <1';
export const ASSET_FILE_CONFLICT_KEYS = ['assetId', 'type'] as const;
export const EXIF_CONFLICT_KEYS = ['assetId'] as const;
export const JOB_STATUS_CONFLICT_KEYS = ['assetId'] as const;
export const NEXT_RELEASE = 'NEXT_RELEASE'; export const NEXT_RELEASE = 'NEXT_RELEASE';
export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle'; export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle';
export const DEPRECATED_IN_PREFIX = 'This property was deprecated in '; export const DEPRECATED_IN_PREFIX = 'This property was deprecated in ';

439
server/src/db.d.ts vendored Normal file
View file

@ -0,0 +1,439 @@
/**
* This file was generated by kysely-codegen.
* Please do not edit it manually.
*/
import type { ColumnType } from 'kysely';
export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[] ? U[] : ArrayTypeImpl<T>;
export type ArrayTypeImpl<T> = T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S[], I[], U[]> : T[];
export type AssetsStatusEnum = 'active' | 'deleted' | 'trashed';
export type Generated<T> =
T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S, I | undefined, U> : ColumnType<T, T | undefined, T>;
export type Int8 = ColumnType<string, bigint | number | string, bigint | number | string>;
export type Json = JsonValue;
export type JsonArray = JsonValue[];
export type JsonObject = {
[x: string]: JsonValue | undefined;
};
export type JsonPrimitive = boolean | number | string | null;
export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
export type Sourcetype = 'exif' | 'machine-learning';
export type Timestamp = ColumnType<Date, Date | string, Date | string>;
export interface Activity {
albumId: string;
assetId: string | null;
comment: string | null;
createdAt: Generated<Timestamp>;
id: Generated<string>;
isLiked: Generated<boolean>;
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface Albums {
albumName: Generated<string>;
/**
* Asset ID to be used as thumbnail
*/
albumThumbnailAssetId: string | null;
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
description: Generated<string>;
id: Generated<string>;
isActivityEnabled: Generated<boolean>;
order: Generated<string>;
ownerId: string;
updatedAt: Generated<Timestamp>;
}
export interface AlbumsAssetsAssets {
albumsId: string;
assetsId: string;
}
export interface AlbumsSharedUsersUsers {
albumsId: string;
role: Generated<string>;
usersId: string;
}
export interface ApiKeys {
createdAt: Generated<Timestamp>;
id: Generated<string>;
key: string;
name: string;
permissions: string[];
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface AssetFaces {
assetId: string;
boundingBoxX1: Generated<number>;
boundingBoxX2: Generated<number>;
boundingBoxY1: Generated<number>;
boundingBoxY2: Generated<number>;
id: Generated<string>;
imageHeight: Generated<number>;
imageWidth: Generated<number>;
personId: string | null;
sourceType: Generated<Sourcetype>;
}
export interface AssetFiles {
assetId: string;
createdAt: Generated<Timestamp>;
id: Generated<string>;
path: string;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface AssetJobStatus {
assetId: string;
duplicatesDetectedAt: Timestamp | null;
facesRecognizedAt: Timestamp | null;
metadataExtractedAt: Timestamp | null;
previewAt: Timestamp | null;
thumbnailAt: Timestamp | null;
}
export interface Assets {
checksum: Buffer;
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
deviceAssetId: string;
deviceId: string;
duplicateId: string | null;
duration: string | null;
encodedVideoPath: Generated<string | null>;
fileCreatedAt: Timestamp;
fileModifiedAt: Timestamp;
id: Generated<string>;
isArchived: Generated<boolean>;
isExternal: Generated<boolean>;
isFavorite: Generated<boolean>;
isOffline: Generated<boolean>;
isVisible: Generated<boolean>;
libraryId: string | null;
livePhotoVideoId: string | null;
localDateTime: Timestamp;
originalFileName: string;
originalPath: string;
ownerId: string;
sidecarPath: string | null;
stackId: string | null;
status: Generated<AssetsStatusEnum>;
thumbhash: Buffer | null;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface AssetStack {
id: Generated<string>;
ownerId: string;
primaryAssetId: string;
}
export interface Audit {
action: string;
createdAt: Generated<Timestamp>;
entityId: string;
entityType: string;
id: Generated<number>;
ownerId: string;
}
export interface Exif {
assetId: string;
autoStackId: string | null;
bitsPerSample: number | null;
city: string | null;
colorspace: string | null;
country: string | null;
dateTimeOriginal: Timestamp | null;
description: Generated<string>;
exifImageHeight: number | null;
exifImageWidth: number | null;
exposureTime: string | null;
fileSizeInByte: Int8 | null;
fNumber: number | null;
focalLength: number | null;
fps: number | null;
iso: number | null;
latitude: number | null;
lensModel: string | null;
livePhotoCID: string | null;
longitude: number | null;
make: string | null;
model: string | null;
modifyDate: Timestamp | null;
orientation: string | null;
profileDescription: string | null;
projectionType: string | null;
rating: number | null;
state: string | null;
timeZone: string | null;
}
export interface FaceSearch {
embedding: string;
faceId: string;
}
export interface GeodataPlaces {
admin1Code: string | null;
admin1Name: string | null;
admin2Code: string | null;
admin2Name: string | null;
alternateNames: string | null;
countryCode: string;
earthCoord: Generated<string | null>;
id: number;
latitude: number;
longitude: number;
modificationDate: Timestamp;
name: string;
}
export interface Libraries {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
exclusionPatterns: string[];
id: Generated<string>;
importPaths: string[];
name: string;
ownerId: string;
refreshedAt: Timestamp | null;
updatedAt: Generated<Timestamp>;
}
export interface Memories {
createdAt: Generated<Timestamp>;
data: Json;
deletedAt: Timestamp | null;
id: Generated<string>;
isSaved: Generated<boolean>;
memoryAt: Timestamp;
ownerId: string;
seenAt: Timestamp | null;
type: string;
updatedAt: Generated<Timestamp>;
}
export interface MemoriesAssetsAssets {
assetsId: string;
memoriesId: string;
}
export interface Migrations {
id: Generated<number>;
name: string;
timestamp: Int8;
}
export interface MoveHistory {
entityId: string;
id: Generated<string>;
newPath: string;
oldPath: string;
pathType: string;
}
export interface NaturalearthCountries {
admin: string;
admin_a3: string;
coordinates: string;
id: Generated<number>;
type: string;
}
export interface Partners {
createdAt: Generated<Timestamp>;
inTimeline: Generated<boolean>;
sharedById: string;
sharedWithId: string;
updatedAt: Generated<Timestamp>;
}
export interface Person {
birthDate: Timestamp | null;
createdAt: Generated<Timestamp>;
faceAssetId: string | null;
id: Generated<string>;
isHidden: Generated<boolean>;
name: Generated<string>;
ownerId: string;
thumbnailPath: Generated<string>;
updatedAt: Generated<Timestamp>;
}
export interface Sessions {
createdAt: Generated<Timestamp>;
deviceOS: Generated<string>;
deviceType: Generated<string>;
id: Generated<string>;
token: string;
updatedAt: Generated<Timestamp>;
userId: string;
}
export interface SharedLinkAsset {
assetsId: string;
sharedLinksId: string;
}
export interface SharedLinks {
albumId: string | null;
allowDownload: Generated<boolean>;
allowUpload: Generated<boolean>;
createdAt: Generated<Timestamp>;
description: string | null;
expiresAt: Timestamp | null;
id: Generated<string>;
key: Buffer;
password: string | null;
showExif: Generated<boolean>;
type: string;
userId: string;
}
export interface SmartInfo {
assetId: string;
objects: string[] | null;
smartInfoTextSearchableColumn: Generated<string>;
tags: string[] | null;
}
export interface SmartSearch {
assetId: string;
embedding: string;
}
export interface SocketIoAttachments {
created_at: Generated<Timestamp | null>;
id: Generated<Int8>;
payload: Buffer | null;
}
export interface SystemConfig {
key: string;
value: string | null;
}
export interface SystemMetadata {
key: string;
value: Json;
}
export interface TagAsset {
assetsId: string;
tagsId: string;
}
export interface Tags {
color: string | null;
createdAt: Generated<Timestamp>;
id: Generated<string>;
parentId: string | null;
updatedAt: Generated<Timestamp>;
userId: string;
value: string;
}
export interface TagsClosure {
id_ancestor: string;
id_descendant: string;
}
export interface UserMetadata {
key: string;
userId: string;
value: Json;
}
export interface Users {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
email: string;
id: Generated<string>;
isAdmin: Generated<boolean>;
name: Generated<string>;
oauthId: Generated<string>;
password: Generated<string>;
profileChangedAt: Generated<Timestamp>;
profileImagePath: Generated<string>;
quotaSizeInBytes: Int8 | null;
quotaUsageInBytes: Generated<Int8>;
shouldChangePassword: Generated<boolean>;
status: Generated<string>;
storageLabel: string | null;
updatedAt: Generated<Timestamp>;
}
export interface VectorsPgVectorIndexStat {
idx_growing: ArrayType<Int8> | null;
idx_indexing: boolean | null;
idx_options: string | null;
idx_sealed: ArrayType<Int8> | null;
idx_size: Int8 | null;
idx_status: string | null;
idx_tuples: Int8 | null;
idx_write: Int8 | null;
indexname: string | null;
indexrelid: number | null;
tablename: string | null;
tablerelid: number | null;
}
export interface DB {
activity: Activity;
albums: Albums;
albums_assets_assets: AlbumsAssetsAssets;
albums_shared_users_users: AlbumsSharedUsersUsers;
api_keys: ApiKeys;
asset_faces: AssetFaces;
asset_files: AssetFiles;
asset_job_status: AssetJobStatus;
asset_stack: AssetStack;
assets: Assets;
audit: Audit;
exif: Exif;
face_search: FaceSearch;
geodata_places: GeodataPlaces;
libraries: Libraries;
memories: Memories;
memories_assets_assets: MemoriesAssetsAssets;
migrations: Migrations;
move_history: MoveHistory;
naturalearth_countries: NaturalearthCountries;
partners: Partners;
person: Person;
sessions: Sessions;
shared_link__asset: SharedLinkAsset;
shared_links: SharedLinks;
smart_info: SmartInfo;
smart_search: SmartSearch;
socket_io_attachments: SocketIoAttachments;
system_config: SystemConfig;
system_metadata: SystemMetadata;
tag_asset: TagAsset;
tags: Tags;
tags_closure: TagsClosure;
user_metadata: UserMetadata;
users: Users;
'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat;
}

View file

@ -97,10 +97,19 @@ const mapStack = (entity: AssetEntity) => {
return { return {
id: entity.stack.id, id: entity.stack.id,
primaryAssetId: entity.stack.primaryAssetId, primaryAssetId: entity.stack.primaryAssetId,
assetCount: entity.stack.assetCount ?? entity.stack.assets.length, assetCount: entity.stack.assetCount ?? entity.stack.assets.length + 1,
}; };
}; };
// if an asset is jsonified in the DB before being returned, its buffer fields will be hex-encoded strings
const hexOrBufferToBase64 = (encoded: string | Buffer) => {
if (typeof encoded === 'string') {
return Buffer.from(encoded.slice(2), 'hex').toString('base64');
}
return encoded.toString('base64');
};
export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto { export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto {
const { stripMetadata = false, withStack = false } = options; const { stripMetadata = false, withStack = false } = options;
@ -129,7 +138,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
originalPath: entity.originalPath, originalPath: entity.originalPath,
originalFileName: entity.originalFileName, originalFileName: entity.originalFileName,
originalMimeType: mimeTypes.lookup(entity.originalFileName), originalMimeType: mimeTypes.lookup(entity.originalFileName),
thumbhash: entity.thumbhash?.toString('base64') ?? null, thumbhash: entity.thumbhash ? hexOrBufferToBase64(entity.thumbhash) : null,
fileCreatedAt: entity.fileCreatedAt, fileCreatedAt: entity.fileCreatedAt,
fileModifiedAt: entity.fileModifiedAt, fileModifiedAt: entity.fileModifiedAt,
localDateTime: entity.localDateTime, localDateTime: entity.localDateTime,
@ -143,7 +152,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
tags: entity.tags?.map((tag) => mapTag(tag)), tags: entity.tags?.map((tag) => mapTag(tag)),
people: peopleWithFaces(entity.faces), people: peopleWithFaces(entity.faces),
unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)), unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)),
checksum: entity.checksum.toString('base64'), checksum: hexOrBufferToBase64(entity.checksum),
stack: withStack ? mapStack(entity) : undefined, stack: withStack ? mapStack(entity) : undefined,
isOffline: entity.isOffline, isOffline: entity.isOffline,
hasMetadata: true, hasMetadata: true,

View file

@ -1,5 +1,4 @@
import { IsNotEmpty } from 'class-validator'; import { IsNotEmpty } from 'class-validator';
import { groupBy, sortBy } from 'lodash';
import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { ValidateUUID } from 'src/validation'; import { ValidateUUID } from 'src/validation';
@ -13,16 +12,3 @@ export class ResolveDuplicatesDto {
@ValidateUUID({ each: true }) @ValidateUUID({ each: true })
assetIds!: string[]; assetIds!: string[];
} }
export function mapDuplicateResponse(assets: AssetResponseDto[]): DuplicateResponseDto[] {
const result = [];
const grouped = groupBy(assets, (a) => a.duplicateId);
for (const [duplicateId, unsortedAssets] of Object.entries(grouped)) {
const assets = sortBy(unsortedAssets, (asset) => asset.localDateTime);
result.push({ duplicateId, assets });
}
return result;
}

View file

@ -162,7 +162,7 @@ export class MetadataSearchDto extends RandomSearchDto {
@IsEnum(AssetOrder) @IsEnum(AssetOrder)
@Optional() @Optional()
@ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder }) @ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder, default: AssetOrder.DESC })
order?: AssetOrder; order?: AssetOrder;
@IsInt() @IsInt()

View file

@ -1,3 +1,6 @@
import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, Selectable, SelectQueryBuilder, sql } from 'kysely';
import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { Assets, DB } from 'src/db';
import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumEntity } from 'src/entities/album.entity';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetFileEntity } from 'src/entities/asset-files.entity'; import { AssetFileEntity } from 'src/entities/asset-files.entity';
@ -9,7 +12,9 @@ import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { StackEntity } from 'src/entities/stack.entity'; import { StackEntity } from 'src/entities/stack.entity';
import { TagEntity } from 'src/entities/tag.entity'; import { TagEntity } from 'src/entities/tag.entity';
import { UserEntity } from 'src/entities/user.entity'; import { UserEntity } from 'src/entities/user.entity';
import { AssetStatus, AssetType } from 'src/enum'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum';
import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface';
import { anyUuid, asUuid } from 'src/utils/database';
import { import {
Column, Column,
CreateDateColumn, CreateDateColumn,
@ -38,8 +43,8 @@ export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum';
unique: true, unique: true,
where: '"libraryId" IS NOT NULL', where: '"libraryId" IS NOT NULL',
}) })
@Index('IDX_day_of_month', { synchronize: false }) @Index('idx_local_date_time', { synchronize: false })
@Index('IDX_month', { synchronize: false }) @Index('idx_local_date_time_month', { synchronize: false })
@Index('IDX_originalPath_libraryId', ['originalPath', 'libraryId']) @Index('IDX_originalPath_libraryId', ['originalPath', 'libraryId'])
@Index('IDX_asset_id_stackId', ['id', 'stackId']) @Index('IDX_asset_id_stackId', ['id', 'stackId'])
@Index('idx_originalFileName_trigram', { synchronize: false }) @Index('idx_originalFileName_trigram', { synchronize: false })
@ -173,3 +178,247 @@ export class AssetEntity {
@Column({ type: 'uuid', nullable: true }) @Column({ type: 'uuid', nullable: true })
duplicateId!: string | null; duplicateId!: string | null;
} }
export const withExif = <O>(qb: SelectQueryBuilder<DB, 'assets', O>) => {
return qb
.leftJoin('exif', 'assets.id', 'exif.assetId')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'));
};
export const withExifInner = <O>(qb: SelectQueryBuilder<DB, 'assets', O>) => {
return qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'));
};
export const withSmartSearch = <O>(qb: SelectQueryBuilder<DB, 'assets', O>, options?: { inner: boolean }) => {
const join = options?.inner
? qb.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
: qb.leftJoin('smart_search', 'assets.id', 'smart_search.assetId');
return join.select(sql<number[]>`smart_search.embedding`.as('embedding'));
};
export const withFaces = (eb: ExpressionBuilder<DB, 'assets'>) =>
jsonArrayFrom(eb.selectFrom('asset_faces').selectAll().whereRef('asset_faces.assetId', '=', 'assets.id')).as('faces');
export const withFiles = (eb: ExpressionBuilder<DB, 'assets'>, type?: AssetFileType) =>
jsonArrayFrom(
eb
.selectFrom('asset_files')
.selectAll()
.whereRef('asset_files.assetId', '=', 'assets.id')
.$if(!!type, (qb) => qb.where('type', '=', type!)),
).as('files');
export const withFacesAndPeople = (eb: ExpressionBuilder<DB, 'assets'>) =>
eb
.selectFrom('asset_faces')
.leftJoin('person', 'person.id', 'asset_faces.personId')
.whereRef('asset_faces.assetId', '=', 'assets.id')
.select((eb) =>
eb
.fn('jsonb_agg', [
eb
.case()
.when('person.id', 'is not', null)
.then(
eb.fn('jsonb_insert', [
eb.fn('to_jsonb', [eb.table('asset_faces')]),
sql`'{person}'::text[]`,
eb.fn('to_jsonb', [eb.table('person')]),
]),
)
.else(eb.fn('to_jsonb', [eb.table('asset_faces')]))
.end(),
])
.as('faces'),
)
.as('faces');
/** Adds a `has_people` CTE that can be inner joined on to filter out assets */
export const hasPeopleCte = (db: Kysely<DB>, personIds: string[]) =>
db.with('has_people', (qb) =>
qb
.selectFrom('asset_faces')
.select('assetId')
.where('personId', '=', anyUuid(personIds!))
.groupBy('assetId')
.having((eb) => eb.fn.count('personId'), '>=', personIds.length),
);
export const hasPeople = (db: Kysely<DB>, personIds?: string[]) =>
personIds && personIds.length > 0
? hasPeopleCte(db, personIds).selectFrom('assets').innerJoin('has_people', 'has_people.assetId', 'assets.id')
: db.selectFrom('assets');
export const withOwner = (eb: ExpressionBuilder<DB, 'assets'>) =>
jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner');
export const withLibrary = (eb: ExpressionBuilder<DB, 'assets'>) =>
jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as(
'library',
);
type Stacked = SelectQueryBuilder<
DB & { stacked: Selectable<Assets> },
'assets' | 'asset_stack' | 'stacked',
{ assets: Selectable<Assets>[] }
>;
type StackExpression = (eb: Stacked) => Stacked;
export const withStack = <O>(
qb: SelectQueryBuilder<DB, 'assets', O>,
{ assets }: { assets?: boolean | StackExpression },
) =>
qb
.leftJoinLateral(
(eb) =>
eb
.selectFrom('asset_stack')
.selectAll('asset_stack')
.whereRef('assets.stackId', '=', 'asset_stack.id')
.$if(!!assets, (qb) =>
qb
.innerJoinLateral(
(eb: ExpressionBuilder<DB, 'assets' | 'asset_stack'>) =>
eb
.selectFrom('assets as stacked')
.select((eb) => eb.fn<Selectable<Assets>[]>('array_agg', [eb.table('stacked')]).as('assets'))
.whereRef('asset_stack.id', '=', 'stacked.stackId')
.whereRef('asset_stack.primaryAssetId', '!=', 'stacked.id')
.$if(typeof assets === 'function', assets as StackExpression)
.as('s'),
(join) =>
join.on((eb) =>
eb.or([
eb('asset_stack.primaryAssetId', '=', eb.ref('assets.id')),
eb('assets.stackId', 'is', null),
]),
),
)
.select('s.assets'),
)
.as('stacked_assets'),
(join) => join.onTrue(),
)
.select((eb) => eb.fn('to_jsonb', [eb.table('stacked_assets')]).as('stack'));
export const withAlbums = <O>(qb: SelectQueryBuilder<DB, 'assets', O>, { albumId }: { albumId?: string }) => {
return qb
.select((eb) =>
jsonArrayFrom(
eb
.selectFrom('albums')
.selectAll()
.innerJoin('albums_assets_assets', (join) =>
join
.onRef('albums.id', '=', 'albums_assets_assets.albumsId')
.onRef('assets.id', '=', 'albums_assets_assets.assetsId'),
)
.whereRef('albums.id', '=', 'albums_assets_assets.albumsId')
.$if(!!albumId, (qb) => qb.where('albums.id', '=', asUuid(albumId!))),
).as('albums'),
)
.$if(!!albumId, (qb) =>
qb.where((eb) =>
eb.exists((eb) =>
eb
.selectFrom('albums_assets_assets')
.whereRef('albums_assets_assets.assetsId', '=', 'assets.id')
.where('albums_assets_assets.albumsId', '=', asUuid(albumId!)),
),
),
);
};
export const withTags = (eb: ExpressionBuilder<DB, 'assets'>) =>
jsonArrayFrom(
eb
.selectFrom('tags')
.selectAll('tags')
.innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
.whereRef('assets.id', '=', 'tag_asset.assetsId'),
).as('tags');
const joinDeduplicationPlugin = new DeduplicateJoinsPlugin();
/** TODO: This should only be used for search-related queries, not as a general purpose query builder */
export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuilderOptions) {
options.isArchived ??= options.withArchived ? undefined : false;
options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore);
return hasPeople(kysely.withPlugin(joinDeduplicationPlugin), options.personIds)
.selectAll('assets')
.$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!))
.$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!))
.$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!))
.$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!))
.$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!))
.$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!))
.$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!))
.$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!))
.$if(options.city !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.city', options.city === null ? 'is' : '=', options.city!),
)
.$if(options.state !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.state', options.state === null ? 'is' : '=', options.state!),
)
.$if(options.country !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.country', options.country === null ? 'is' : '=', options.country!),
)
.$if(options.make !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.make', options.make === null ? 'is' : '=', options.make!),
)
.$if(options.model !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.model', options.model === null ? 'is' : '=', options.model!),
)
.$if(options.lensModel !== undefined, (qb) =>
qb
.innerJoin('exif', 'assets.id', 'exif.assetId')
.where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!),
)
.$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!))
.$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!))
.$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!))
.$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!)))
.$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!)))
.$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!)))
.$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!))
.$if(!!options.originalPath, (qb) => qb.where('assets.originalPath', '=', options.originalPath!))
.$if(!!options.originalFileName, (qb) =>
qb.where(
sql`f_unaccent(assets."originalFileName")`,
'ilike',
sql`'%' || f_unaccent(${options.originalFileName}) || '%'`,
),
)
.$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!))
.$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!))
.$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!))
.$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!))
.$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!))
.$if(options.isEncoded !== undefined, (qb) =>
qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null),
)
.$if(options.isMotion !== undefined, (qb) =>
qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null),
)
.$if(!!options.isNotInAlbum, (qb) =>
qb.where((eb) =>
eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))),
),
)
.$if(!!options.withExif, withExifInner)
.$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))
.$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
}

View file

@ -1,5 +1,4 @@
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { asVector } from 'src/utils/database';
import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm';
@Entity('face_search', { synchronize: false }) @Entity('face_search', { synchronize: false })
@ -15,7 +14,7 @@ export class FaceSearchEntity {
@Column({ @Column({
type: 'float4', type: 'float4',
array: true, array: true,
transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v) }, transformer: { from: (v) => JSON.parse(v), to: (v) => `[${v}]` },
}) })
embedding!: number[]; embedding!: number[];
} }

View file

@ -1,10 +1,9 @@
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { Insertable, Updateable } from 'kysely';
import { AssetFiles, AssetJobStatus, Assets, Exif } from 'src/db';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
import { Paginated, PaginationOptions } from 'src/utils/pagination'; import { Paginated, PaginationOptions } from 'src/utils/pagination';
import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
export type AssetStats = Record<AssetType, number>; export type AssetStats = Record<AssetType, number>;
@ -66,43 +65,6 @@ export interface TimeBucketItem {
count: number; count: number;
} }
export type AssetCreate = Pick<
AssetEntity,
| 'deviceAssetId'
| 'ownerId'
| 'libraryId'
| 'deviceId'
| 'type'
| 'originalPath'
| 'fileCreatedAt'
| 'localDateTime'
| 'fileModifiedAt'
| 'checksum'
| 'originalFileName'
> &
Partial<AssetEntity>;
export type AssetWithoutRelations = Omit<
AssetEntity,
| 'livePhotoVideo'
| 'stack'
| 'albums'
| 'faces'
| 'owner'
| 'library'
| 'exifInfo'
| 'sharedLinks'
| 'smartSearch'
| 'tags'
>;
type AssetUpdateWithoutRelations = Pick<AssetWithoutRelations, 'id'> & Partial<AssetWithoutRelations>;
type AssetUpdateWithLivePhotoRelation = Pick<AssetWithoutRelations, 'id'> & Pick<AssetEntity, 'livePhotoVideo'>;
export type AssetUpdateOptions = AssetUpdateWithoutRelations | AssetUpdateWithLivePhotoRelation;
export type AssetUpdateAllOptions = Omit<Partial<AssetWithoutRelations>, 'id'>;
export interface MonthDay { export interface MonthDay {
day: number; day: number;
month: number; month: number;
@ -113,12 +75,6 @@ export interface AssetExploreFieldOptions {
minAssetsPerField: number; minAssetsPerField: number;
} }
export interface AssetExploreOptions extends AssetExploreFieldOptions {
relation: keyof AssetEntity;
relatedField: string;
unnest?: boolean;
}
export interface AssetFullSyncOptions { export interface AssetFullSyncOptions {
ownerId: string; ownerId: string;
lastId?: string; lastId?: string;
@ -144,8 +100,30 @@ export interface UpsertFileOptions {
path: string; path: string;
} }
export interface AssetGetByChecksumOptions {
ownerId: string;
checksum: Buffer;
libraryId?: string;
}
export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>; export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
export interface GetByIdsRelations {
exifInfo?: boolean;
faces?: { person?: boolean };
files?: boolean;
library?: boolean;
owner?: boolean;
smartSearch?: boolean;
stack?: { assets?: boolean };
tags?: boolean;
}
export interface DuplicateGroup {
duplicateId: string;
assets: AssetEntity[];
}
export interface DayOfYearAssets { export interface DayOfYearAssets {
yearsAgo: number; yearsAgo: number;
assets: AssetEntity[]; assets: AssetEntity[];
@ -154,47 +132,39 @@ export interface DayOfYearAssets {
export const IAssetRepository = 'IAssetRepository'; export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository { export interface IAssetRepository {
create(asset: AssetCreate): Promise<AssetEntity>; create(asset: Insertable<Assets>): Promise<AssetEntity>;
getByIds( getByIds(ids: string[], relations?: GetByIdsRelations): Promise<AssetEntity[]>;
ids: string[],
relations?: FindOptionsRelations<AssetEntity>,
select?: FindOptionsSelect<AssetEntity>,
): Promise<AssetEntity[]>;
getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>; getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<DayOfYearAssets[]>; getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<DayOfYearAssets[]>;
getByChecksum(options: { ownerId: string; checksum: Buffer; libraryId?: string }): Promise<AssetEntity | null>; getByChecksum(options: AssetGetByChecksumOptions): Promise<AssetEntity | undefined>;
getByChecksums(userId: string, checksums: Buffer[]): Promise<AssetEntity[]>; getByChecksums(userId: string, checksums: Buffer[]): Promise<AssetEntity[]>;
getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>; getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>; getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByDeviceIds(ownerId: string, deviceId: string, deviceAssetIds: string[]): Promise<string[]>; getByDeviceIds(ownerId: string, deviceId: string, deviceAssetIds: string[]): Promise<string[]>;
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>; getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
getById( getById(id: string, relations?: GetByIdsRelations): Promise<AssetEntity | undefined>;
id: string,
relations?: FindOptionsRelations<AssetEntity>,
order?: FindOptionsOrder<AssetEntity>,
): Promise<AssetEntity | null>;
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>; getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getRandom(userIds: string[], count: number): Promise<AssetEntity[]>; getRandom(userIds: string[], count: number): Promise<AssetEntity[]>;
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>; getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | undefined>;
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>; getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | undefined>;
deleteAll(ownerId: string): Promise<void>; deleteAll(ownerId: string): Promise<void>;
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>; getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>; getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getLivePhotoCount(motionId: string): Promise<number>; getLivePhotoCount(motionId: string): Promise<number>;
updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>; updateAll(ids: string[], options: Updateable<Assets>): Promise<void>;
updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>; updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>;
update(asset: AssetUpdateOptions): Promise<void>; update(asset: Updateable<Assets> & { id: string }): Promise<AssetEntity>;
remove(asset: AssetEntity): Promise<void>; remove(asset: AssetEntity): Promise<void>;
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null>; findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | undefined>;
getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>; getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>; getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>; getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
upsertExif(exif: Partial<ExifEntity>): Promise<void>; upsertExif(exif: Insertable<Exif>): Promise<void>;
upsertJobStatus(...jobStatus: Partial<AssetJobStatusEntity>[]): Promise<void>; upsertJobStatus(...jobStatus: Insertable<AssetJobStatus>[]): Promise<void>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>; getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>; getDuplicates(userId: string): Promise<DuplicateGroup[]>;
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>; getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>; getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
upsertFile(file: UpsertFileOptions): Promise<void>; upsertFile(options: Insertable<AssetFiles>): Promise<void>;
upsertFiles(files: UpsertFileOptions[]): Promise<void>; upsertFiles(options: Insertable<AssetFiles>[]): Promise<void>;
} }

View file

@ -1,6 +1,7 @@
import { RegisterQueueOptions } from '@nestjs/bullmq'; import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq'; import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis'; import { RedisOptions } from 'ioredis';
import { KyselyConfig } from 'kysely';
import { ClsModuleOptions } from 'nestjs-cls'; import { ClsModuleOptions } from 'nestjs-cls';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum'; import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum';
@ -42,7 +43,7 @@ export interface EnvData {
}; };
database: { database: {
config: PostgresConnectionOptions & DatabaseConnectionParams; config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: KyselyConfig };
skipMigrations: boolean; skipMigrations: boolean;
vectorExtension: VectorExtension; vectorExtension: VectorExtension;
}; };

View file

@ -1,4 +1,3 @@
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { AssetStatus, AssetType } from 'src/enum'; import { AssetStatus, AssetType } from 'src/enum';
@ -114,7 +113,7 @@ export interface SearchPeopleOptions {
} }
export interface SearchOrderOptions { export interface SearchOrderOptions {
orderDirection?: 'ASC' | 'DESC'; orderDirection?: 'asc' | 'desc';
} }
export interface SearchPaginationOptions { export interface SearchPaginationOptions {
@ -148,20 +147,21 @@ export type SmartSearchOptions = SearchDateOptions &
export interface FaceEmbeddingSearch extends SearchEmbeddingOptions { export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
hasPerson?: boolean; hasPerson?: boolean;
numResults: number; numResults: number;
maxDistance?: number; maxDistance: number;
} }
export interface AssetDuplicateSearch { export interface AssetDuplicateSearch {
assetId: string; assetId: string;
embedding: number[]; embedding: number[];
maxDistance?: number; maxDistance: number;
type: AssetType; type: AssetType;
userIds: string[]; userIds: string[];
} }
export interface FaceSearchResult { export interface FaceSearchResult {
distance: number; distance: number;
face: AssetFaceEntity; id: string;
personId: string | null;
} }
export interface AssetDuplicateResult { export interface AssetDuplicateResult {

View file

@ -0,0 +1,25 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddTimeBucketIndices1734574016301 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE INDEX idx_local_date_time_month ON public.assets (date_trunc('MONTH', "localDateTime" at time zone 'UTC'))`,
);
await queryRunner.query(
`CREATE INDEX idx_local_date_time ON public.assets ((("localDateTime" at time zone 'UTC')::date))`,
);
await queryRunner.query(`DROP INDEX "IDX_day_of_month"`);
await queryRunner.query(`DROP INDEX "IDX_month"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX idx_local_date_time_month`);
await queryRunner.query(`DROP INDEX idx_local_date_time`);
await queryRunner.query(
`CREATE INDEX "IDX_day_of_month" ON assets (EXTRACT(DAY FROM "localDateTime" AT TIME ZONE 'UTC'))`,
);
await queryRunner.query(
`CREATE INDEX "IDX_month" ON assets (EXTRACT(MONTH FROM "localDateTime" AT TIME ZONE 'UTC'))`,
);
}
}

File diff suppressed because it is too large Load diff

View file

@ -1,641 +1,199 @@
-- NOTE: This file is auto generated by ./sql-generator -- NOTE: This file is auto generated by ./sql-generator
-- SearchRepository.searchMetadata -- SearchRepository.searchMetadata
SELECT DISTINCT select
"distinctAlias"."asset_id" AS "ids_asset_id", "assets".*
"distinctAlias"."asset_fileCreatedAt" from
FROM "assets"
( inner join "exif" on "assets"."id" = "exif"."assetId"
SELECT where
"asset"."id" AS "asset_id", "assets"."fileCreatedAt" >= $1
"asset"."deviceAssetId" AS "asset_deviceAssetId", and "exif"."lensModel" = $2
"asset"."ownerId" AS "asset_ownerId", and "assets"."ownerId" = any ($3::uuid [])
"asset"."libraryId" AS "asset_libraryId", and "assets"."isFavorite" = $4
"asset"."deviceId" AS "asset_deviceId", and "assets"."isArchived" = $5
"asset"."type" AS "asset_type", and "assets"."deletedAt" is null
"asset"."status" AS "asset_status", order by
"asset"."originalPath" AS "asset_originalPath", "assets"."fileCreatedAt" desc
"asset"."thumbhash" AS "asset_thumbhash", limit
"asset"."encodedVideoPath" AS "asset_encodedVideoPath", $6
"asset"."createdAt" AS "asset_createdAt", offset
"asset"."updatedAt" AS "asset_updatedAt", $7
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_fileCreatedAt" DESC,
"asset_id" ASC
LIMIT
101
-- SearchRepository.searchRandom
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" > $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
SELECT DISTINCT
"distinctAlias"."asset_id" AS "ids_asset_id",
"distinctAlias"."asset_id"
FROM
(
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."status" AS "asset_status",
"asset"."originalPath" AS "asset_originalPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND "asset"."ownerId" IN ($3)
AND 1 = 1
AND (
"asset"."isFavorite" = $4
AND "asset"."isArchived" = $5
)
AND "asset"."id" < $6
)
AND ("asset"."deletedAt" IS NULL)
) "distinctAlias"
ORDER BY
"distinctAlias"."asset_id" ASC,
"asset_id" ASC
LIMIT
100
-- SearchRepository.searchSmart -- SearchRepository.searchSmart
START TRANSACTION select
SET "assets".*
LOCAL vectors.hnsw_ef_search = 200; from
SELECT "assets"
"asset"."id" AS "asset_id", inner join "exif" on "assets"."id" = "exif"."assetId"
"asset"."deviceAssetId" AS "asset_deviceAssetId", inner join "smart_search" on "assets"."id" = "smart_search"."assetId"
"asset"."ownerId" AS "asset_ownerId", where
"asset"."libraryId" AS "asset_libraryId", "assets"."fileCreatedAt" >= $1
"asset"."deviceId" AS "asset_deviceId", and "exif"."lensModel" = $2
"asset"."type" AS "asset_type", and "assets"."ownerId" = any ($3::uuid [])
"asset"."status" AS "asset_status", and "assets"."isFavorite" = $4
"asset"."originalPath" AS "asset_originalPath", and "assets"."isArchived" = $5
"asset"."thumbhash" AS "asset_thumbhash", and "assets"."deletedAt" is null
"asset"."encodedVideoPath" AS "asset_encodedVideoPath", order by
"asset"."createdAt" AS "asset_createdAt", smart_search.embedding <= > $6::vector
"asset"."updatedAt" AS "asset_updatedAt", limit
"asset"."deletedAt" AS "asset_deletedAt", $7
"asset"."fileCreatedAt" AS "asset_fileCreatedAt", offset
"asset"."localDateTime" AS "asset_localDateTime", $8
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"stack"."id" AS "stack_id",
"stack"."ownerId" AS "stack_ownerId",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."status" AS "stackedAssets_status",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
WHERE
(
"asset"."fileCreatedAt" >= $1
AND "exifInfo"."lensModel" = $2
AND 1 = 1
AND 1 = 1
AND (
"asset"."isFavorite" = $3
AND "asset"."isArchived" = $4
)
AND "asset"."ownerId" IN ($5)
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
"search"."embedding" <= > $6 ASC
LIMIT
201
COMMIT
-- SearchRepository.searchDuplicates
WITH
"cte" AS (
SELECT
"asset"."duplicateId" AS "duplicateId",
"search"."assetId" AS "assetId",
"search"."embedding" <= > $1 AS "distance"
FROM
"assets" "asset"
INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
WHERE
(
"asset"."ownerId" IN ($2)
AND "asset"."id" != $3
AND "asset"."isVisible" = $4
AND "asset"."type" = $5
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
"search"."embedding" <= > $1 ASC
LIMIT
64
)
SELECT
res.*
FROM
"cte" "res"
WHERE
res.distance <= $6
-- SearchRepository.searchFaces -- SearchRepository.searchFaces
START TRANSACTION with
SET "cte" as (
LOCAL vectors.hnsw_ef_search = 100; select
WITH "asset_faces"."id",
"cte" AS ( "asset_faces"."personId",
SELECT face_search.embedding <= > $1::vector as "distance"
"faces"."id" AS "id", from
"faces"."assetId" AS "assetId", "asset_faces"
"faces"."personId" AS "personId", inner join "assets" on "assets"."id" = "asset_faces"."assetId"
"faces"."imageWidth" AS "imageWidth", inner join "face_search" on "face_search"."faceId" = "asset_faces"."id"
"faces"."imageHeight" AS "imageHeight", where
"faces"."boundingBoxX1" AS "boundingBoxX1", "assets"."ownerId" = any ($2::uuid [])
"faces"."boundingBoxY1" AS "boundingBoxY1", and "assets"."deletedAt" is null
"faces"."boundingBoxX2" AS "boundingBoxX2", order by
"faces"."boundingBoxY2" AS "boundingBoxY2", face_search.embedding <= > $3::vector
"faces"."sourceType" AS "sourceType", limit
"search"."embedding" <= > $1 AS "distance" $4
FROM
"asset_faces" "faces"
INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
AND ("asset"."deletedAt" IS NULL)
INNER JOIN "face_search" "search" ON "search"."faceId" = "faces"."id"
WHERE
"asset"."ownerId" IN ($2)
ORDER BY
"search"."embedding" <= > $1 ASC
LIMIT
64
) )
SELECT select
res.* *
FROM from
"cte" "res" "cte"
WHERE where
res.distance <= $3 "cte"."distance" <= $5
ORDER BY
res.distance ASC
COMMIT
-- SearchRepository.searchPlaces -- SearchRepository.searchPlaces
SELECT select
"geoplaces"."id" AS "geoplaces_id", *
"geoplaces"."name" AS "geoplaces_name", from
"geoplaces"."longitude" AS "geoplaces_longitude", "geodata_places"
"geoplaces"."latitude" AS "geoplaces_latitude", where
"geoplaces"."countryCode" AS "geoplaces_countryCode",
"geoplaces"."admin1Code" AS "geoplaces_admin1Code",
"geoplaces"."admin2Code" AS "geoplaces_admin2Code",
"geoplaces"."admin1Name" AS "geoplaces_admin1Name",
"geoplaces"."admin2Name" AS "geoplaces_admin2Name",
"geoplaces"."alternateNames" AS "geoplaces_alternateNames",
"geoplaces"."modificationDate" AS "geoplaces_modificationDate"
FROM
"geodata_places" "geoplaces"
WHERE
f_unaccent (name) %>> f_unaccent ($1) f_unaccent (name) %>> f_unaccent ($1)
OR f_unaccent ("admin2Name") %>> f_unaccent ($1) or f_unaccent ("admin2Name") %>> f_unaccent ($2)
OR f_unaccent ("admin1Name") %>> f_unaccent ($1) or f_unaccent ("admin1Name") %>> f_unaccent ($3)
OR f_unaccent ("alternateNames") %>> f_unaccent ($1) or f_unaccent ("alternateNames") %>> f_unaccent ($4)
ORDER BY order by
COALESCE(f_unaccent (name) <->>> f_unaccent ($1), 0.1) + COALESCE( coalesce(f_unaccent (name) <->>> f_unaccent ($5), 0.1) + coalesce(
f_unaccent ("admin2Name") <->>> f_unaccent ($1), f_unaccent ("admin2Name") <->>> f_unaccent ($6),
0.1 0.1
) + COALESCE( ) + coalesce(
f_unaccent ("admin1Name") <->>> f_unaccent ($1), f_unaccent ("admin1Name") <->>> f_unaccent ($7),
0.1 0.1
) + COALESCE( ) + coalesce(
f_unaccent ("alternateNames") <->>> f_unaccent ($1), f_unaccent ("alternateNames") <->>> f_unaccent ($8),
0.1 0.1
) ASC )
LIMIT limit
20 $9
-- SearchRepository.getAssetsByCity -- SearchRepository.getAssetsByCity
WITH RECURSIVE with recursive
cte AS ( "cte" as (
( (
SELECT select
city, "city",
"assetId" "assetId"
FROM from
exif "exif"
INNER JOIN assets ON exif."assetId" = assets.id inner join "assets" on "assets"."id" = "exif"."assetId"
WHERE where
"ownerId" = ANY ($1::uuid []) "assets"."ownerId" = any ($1::uuid [])
AND "isVisible" = $2 and "assets"."isVisible" = $2
AND "isArchived" = $3 and "assets"."isArchived" = $3
AND type = $4 and "assets"."type" = $4
ORDER BY and "assets"."deletedAt" is null
city order by
LIMIT "city"
1 limit
$5
)
union all
(
select
"l"."city",
"l"."assetId"
from
"cte"
inner join lateral (
select
"city",
"assetId"
from
"exif"
inner join "assets" on "assets"."id" = "exif"."assetId"
where
"assets"."ownerId" = any ($6::uuid [])
and "assets"."isVisible" = $7
and "assets"."isArchived" = $8
and "assets"."type" = $9
and "assets"."deletedAt" is null
and "exif"."city" > "cte"."city"
order by
"city"
limit
$10
) as "l" on true
) )
UNION ALL
SELECT
l.city,
l."assetId"
FROM
cte c,
LATERAL (
SELECT
city,
"assetId"
FROM
exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE
city > c.city
AND "ownerId" = ANY ($1::uuid [])
AND "isVisible" = $2
AND "isArchived" = $3
AND type = $4
ORDER BY
city
LIMIT
1
) l
) )
SELECT select
"asset"."id" AS "asset_id", "assets".*,
"asset"."deviceAssetId" AS "asset_deviceAssetId", to_jsonb("exif") as "exifInfo"
"asset"."ownerId" AS "asset_ownerId", from
"asset"."libraryId" AS "asset_libraryId", "assets"
"asset"."deviceId" AS "asset_deviceId", inner join "exif" on "assets"."id" = "exif"."assetId"
"asset"."type" AS "asset_type", inner join "cte" on "assets"."id" = "cte"."assetId"
"asset"."status" AS "asset_status", order by
"asset"."originalPath" AS "asset_originalPath", "exif"."city"
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"exif"."assetId" AS "exif_assetId",
"exif"."description" AS "exif_description",
"exif"."exifImageWidth" AS "exif_exifImageWidth",
"exif"."exifImageHeight" AS "exif_exifImageHeight",
"exif"."fileSizeInByte" AS "exif_fileSizeInByte",
"exif"."orientation" AS "exif_orientation",
"exif"."dateTimeOriginal" AS "exif_dateTimeOriginal",
"exif"."modifyDate" AS "exif_modifyDate",
"exif"."timeZone" AS "exif_timeZone",
"exif"."latitude" AS "exif_latitude",
"exif"."longitude" AS "exif_longitude",
"exif"."projectionType" AS "exif_projectionType",
"exif"."city" AS "exif_city",
"exif"."livePhotoCID" AS "exif_livePhotoCID",
"exif"."autoStackId" AS "exif_autoStackId",
"exif"."state" AS "exif_state",
"exif"."country" AS "exif_country",
"exif"."make" AS "exif_make",
"exif"."model" AS "exif_model",
"exif"."lensModel" AS "exif_lensModel",
"exif"."fNumber" AS "exif_fNumber",
"exif"."focalLength" AS "exif_focalLength",
"exif"."iso" AS "exif_iso",
"exif"."exposureTime" AS "exif_exposureTime",
"exif"."profileDescription" AS "exif_profileDescription",
"exif"."colorspace" AS "exif_colorspace",
"exif"."bitsPerSample" AS "exif_bitsPerSample",
"exif"."rating" AS "exif_rating",
"exif"."fps" AS "exif_fps"
FROM
"assets" "asset"
INNER JOIN "exif" "exif" ON "exif"."assetId" = "asset"."id"
INNER JOIN cte ON asset.id = cte."assetId"
ORDER BY
exif.city
-- SearchRepository.getCountries
SELECT DISTINCT
ON ("exif"."country") "exif"."country" AS "country"
FROM
"exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($1)
AND "exif"."country" != ''
AND "exif"."country" IS NOT NULL
-- SearchRepository.getStates -- SearchRepository.getStates
SELECT DISTINCT select distinct
ON ("exif"."state") "exif"."state" AS "state" on ("state") "state"
FROM from
"exif" "exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" inner join "assets" on "assets"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL) where
WHERE "ownerId" = any ($1::uuid [])
"asset"."ownerId" IN ($1) and "isVisible" = $2
AND "exif"."state" != '' and "deletedAt" is null
AND "exif"."state" IS NOT NULL and "state" is not null
-- SearchRepository.getCities -- SearchRepository.getCities
SELECT DISTINCT select distinct
ON ("exif"."city") "exif"."city" AS "city" on ("city") "city"
FROM from
"exif" "exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" inner join "assets" on "assets"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL) where
WHERE "ownerId" = any ($1::uuid [])
"asset"."ownerId" IN ($1) and "isVisible" = $2
AND "exif"."city" != '' and "deletedAt" is null
AND "exif"."city" IS NOT NULL and "city" is not null
-- SearchRepository.getCameraMakes -- SearchRepository.getCameraMakes
SELECT DISTINCT select distinct
ON ("exif"."make") "exif"."make" AS "make" on ("make") "make"
FROM from
"exif" "exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" inner join "assets" on "assets"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL) where
WHERE "ownerId" = any ($1::uuid [])
"asset"."ownerId" IN ($1) and "isVisible" = $2
AND "exif"."make" != '' and "deletedAt" is null
AND "exif"."make" IS NOT NULL and "make" is not null
-- SearchRepository.getCameraModels -- SearchRepository.getCameraModels
SELECT DISTINCT select distinct
ON ("exif"."model") "exif"."model" AS "model" on ("model") "model"
FROM from
"exif" "exif" "exif"
INNER JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" inner join "assets" on "assets"."id" = "exif"."assetId"
AND ("asset"."deletedAt" IS NULL) where
WHERE "ownerId" = any ($1::uuid [])
"asset"."ownerId" IN ($1) and "isVisible" = $2
AND "exif"."model" != '' and "deletedAt" is null
AND "exif"."model" IS NOT NULL and "model" is not null

View file

@ -1,79 +1,29 @@
-- NOTE: This file is auto generated by ./sql-generator -- NOTE: This file is auto generated by ./sql-generator
-- ViewRepository.getUniqueOriginalPaths
select distinct
substring("assets"."originalPath", $1) as "directoryPath"
from
"assets"
where
"ownerId" = $2::uuid
and "isVisible" = $3
and "isArchived" = $4
and "deletedAt" is null
-- ViewRepository.getAssetsByOriginalPath -- ViewRepository.getAssetsByOriginalPath
SELECT select
"asset"."id" AS "asset_id", "assets".*,
"asset"."deviceAssetId" AS "asset_deviceAssetId", to_jsonb("exif") as "exifInfo"
"asset"."ownerId" AS "asset_ownerId", from
"asset"."libraryId" AS "asset_libraryId", "assets"
"asset"."deviceId" AS "asset_deviceId", left join "exif" on "assets"."id" = "exif"."assetId"
"asset"."type" AS "asset_type", where
"asset"."status" AS "asset_status", "ownerId" = $1::uuid
"asset"."originalPath" AS "asset_originalPath", and "isVisible" = $2
"asset"."thumbhash" AS "asset_thumbhash", and "isArchived" = $3
"asset"."encodedVideoPath" AS "asset_encodedVideoPath", and "deletedAt" is null
"asset"."createdAt" AS "asset_createdAt", and "originalPath" like $4
"asset"."updatedAt" AS "asset_updatedAt", and "originalPath" not like $5
"asset"."deletedAt" AS "asset_deletedAt", order by
"asset"."fileCreatedAt" AS "asset_fileCreatedAt", regexp_replace("assets"."originalPath", $6, $7) asc
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"asset"."duplicateId" AS "asset_duplicateId",
"exifInfo"."assetId" AS "exifInfo_assetId",
"exifInfo"."description" AS "exifInfo_description",
"exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
"exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
"exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
"exifInfo"."orientation" AS "exifInfo_orientation",
"exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
"exifInfo"."modifyDate" AS "exifInfo_modifyDate",
"exifInfo"."timeZone" AS "exifInfo_timeZone",
"exifInfo"."latitude" AS "exifInfo_latitude",
"exifInfo"."longitude" AS "exifInfo_longitude",
"exifInfo"."projectionType" AS "exifInfo_projectionType",
"exifInfo"."city" AS "exifInfo_city",
"exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
"exifInfo"."autoStackId" AS "exifInfo_autoStackId",
"exifInfo"."state" AS "exifInfo_state",
"exifInfo"."country" AS "exifInfo_country",
"exifInfo"."make" AS "exifInfo_make",
"exifInfo"."model" AS "exifInfo_model",
"exifInfo"."lensModel" AS "exifInfo_lensModel",
"exifInfo"."fNumber" AS "exifInfo_fNumber",
"exifInfo"."focalLength" AS "exifInfo_focalLength",
"exifInfo"."iso" AS "exifInfo_iso",
"exifInfo"."exposureTime" AS "exifInfo_exposureTime",
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
"exifInfo"."colorspace" AS "exifInfo_colorspace",
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."rating" AS "exifInfo_rating",
"exifInfo"."fps" AS "exifInfo_fps"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
WHERE
(
(
"asset"."isVisible" = $1
AND "asset"."isArchived" = $2
AND "asset"."ownerId" = $3
)
AND (
"asset"."originalPath" LIKE $4
AND "asset"."originalPath" NOT LIKE $5
)
)
AND ("asset"."deletedAt" IS NULL)
ORDER BY
regexp_replace("asset"."originalPath", '.*/(.+)', '\1') ASC

File diff suppressed because it is too large Load diff

View file

@ -1,3 +1,4 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import { ImmichTelemetry } from 'src/enum'; import { ImmichTelemetry } from 'src/enum';
import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository'; import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository';
@ -79,14 +80,20 @@ describe('getEnv', () => {
it('should use defaults', () => { it('should use defaults', () => {
const { database } = getEnv(); const { database } = getEnv();
expect(database).toEqual({ expect(database).toEqual({
config: expect.objectContaining({ config: {
type: 'postgres', kysely: {
host: 'database', dialect: expect.any(PostgresJSDialect),
port: 5432, log: ['error'],
database: 'immich', },
username: 'postgres', typeorm: expect.objectContaining({
password: 'postgres', type: 'postgres',
}), host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
},
skipMigrations: false, skipMigrations: false,
vectorExtension: 'vectors', vectorExtension: 'vectors',
}); });

View file

@ -2,8 +2,10 @@ import { Inject, Injectable, Optional } from '@nestjs/common';
import { plainToInstance } from 'class-transformer'; import { plainToInstance } from 'class-transformer';
import { validateSync } from 'class-validator'; import { validateSync } from 'class-validator';
import { Request, Response } from 'express'; import { Request, Response } from 'express';
import { PostgresJSDialect } from 'kysely-postgres-js';
import { CLS_ID } from 'nestjs-cls'; import { CLS_ID } from 'nestjs-cls';
import { join, resolve } from 'node:path'; import { join, resolve } from 'node:path';
import postgres from 'postgres';
import { citiesFile, excludePaths, IWorker } from 'src/constants'; import { citiesFile, excludePaths, IWorker } from 'src/constants';
import { Telemetry } from 'src/decorators'; import { Telemetry } from 'src/decorators';
import { EnvDto } from 'src/dtos/env.dto'; import { EnvDto } from 'src/dtos/env.dto';
@ -96,6 +98,33 @@ const getEnv = (): EnvData => {
} }
} }
const driverOptions = {
max: 10,
types: {
date: {
to: 1184,
from: [1082, 1114, 1184],
serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x),
parse: (x: string) => new Date(x),
},
bigint: {
to: 20,
from: [20],
parse: (value: string) => Number.parseInt(value),
serialize: (value: number) => value.toString(),
},
},
};
const parts = {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
} as const;
return { return {
host: dto.IMMICH_HOST, host: dto.IMMICH_HOST,
port: dto.IMMICH_PORT || 2283, port: dto.IMMICH_PORT || 2283,
@ -150,24 +179,23 @@ const getEnv = (): EnvData => {
database: { database: {
config: { config: {
type: 'postgres', typeorm: {
entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'], type: 'postgres',
migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'], entities: [`${folders.dist}/entities` + '/*.entity.{js,ts}'],
subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'], migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
migrationsRun: false, subscribers: [`${folders.dist}/subscribers` + '/*.{js,ts}'],
synchronize: false, migrationsRun: false,
connectTimeoutMS: 10_000, // 10 seconds synchronize: false,
parseInt8: true, connectTimeoutMS: 10_000, // 10 seconds
...(databaseUrl parseInt8: true,
? { connectionType: 'url', url: databaseUrl } ...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts),
: { },
connectionType: 'parts', kysely: {
host: dto.DB_HOSTNAME || 'database', dialect: new PostgresJSDialect({
port: dto.DB_PORT || 5432, postgres: databaseUrl ? postgres(databaseUrl, driverOptions) : postgres({ ...parts, ...driverOptions }),
username: dto.DB_USERNAME || 'postgres', }),
password: dto.DB_PASSWORD || 'postgres', log: ['error'] as const,
database: dto.DB_DATABASE_NAME || 'immich', },
}),
}, },
skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false, skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,

View file

@ -21,7 +21,7 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm';
@Injectable() @Injectable()
export class DatabaseRepository implements IDatabaseRepository { export class DatabaseRepository implements IDatabaseRepository {
private vectorExtension: VectorExtension; private vectorExtension: VectorExtension;
readonly asyncLock = new AsyncLock(); private readonly asyncLock = new AsyncLock();
constructor( constructor(
@InjectDataSource() private dataSource: DataSource, @InjectDataSource() private dataSource: DataSource,

View file

@ -1,22 +1,16 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { Kysely, OrderByDirectionExpression, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators'; import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { AssetType, PaginationMode } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { import {
AssetDuplicateResult,
AssetDuplicateSearch, AssetDuplicateSearch,
AssetSearchOptions, AssetSearchOptions,
FaceEmbeddingSearch, FaceEmbeddingSearch,
FaceSearchResult,
GetCameraMakesOptions, GetCameraMakesOptions,
GetCameraModelsOptions, GetCameraModelsOptions,
GetCitiesOptions, GetCitiesOptions,
@ -25,40 +19,17 @@ import {
SearchPaginationOptions, SearchPaginationOptions,
SmartSearchOptions, SmartSearchOptions,
} from 'src/interfaces/search.interface'; } from 'src/interfaces/search.interface';
import { asVector, searchAssetBuilder } from 'src/utils/database'; import { anyUuid, asUuid, asVector } from 'src/utils/database';
import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination'; import { Paginated } from 'src/utils/pagination';
import { isValidInteger } from 'src/validation'; import { isValidInteger } from 'src/validation';
import { Repository } from 'typeorm';
@Injectable() @Injectable()
export class SearchRepository implements ISearchRepository { export class SearchRepository implements ISearchRepository {
private vectorExtension: VectorExtension;
private faceColumns: string[];
private assetsByCityQuery: string;
constructor( constructor(
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@Inject(ILoggerRepository) private logger: ILoggerRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(IConfigRepository) configRepository: IConfigRepository, @InjectKysely() private db: Kysely<DB>,
) { ) {
this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(SearchRepository.name); this.logger.setContext(SearchRepository.name);
this.faceColumns = this.assetFaceRepository.manager.connection
.getMetadata(AssetFaceEntity)
.ownColumns.map((column) => column.propertyName)
.filter((propertyName) => propertyName !== 'embedding');
this.assetsByCityQuery =
assetsByCityCte +
this.assetRepository
.createQueryBuilder('asset')
.innerJoinAndSelect('asset.exifInfo', 'exif')
.withDeleted()
.getQuery() +
' INNER JOIN cte ON asset.id = cte."assetId" ORDER BY exif.city';
} }
@GenerateSql({ @GenerateSql({
@ -74,14 +45,15 @@ export class SearchRepository implements ISearchRepository {
], ],
}) })
async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> { async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> {
let builder = this.assetRepository.createQueryBuilder('asset'); const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirectionExpression;
builder = searchAssetBuilder(builder, options).orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC'); const items = await searchAssetBuilder(this.db, options)
.orderBy('assets.fileCreatedAt', orderDirection)
return paginatedBuilder<AssetEntity>(builder, { .limit(pagination.size + 1)
mode: PaginationMode.SKIP_TAKE, .offset((pagination.page - 1) * pagination.size)
skip: (pagination.page - 1) * pagination.size, .execute();
take: pagination.size, const hasNextPage = items.length > pagination.size;
}); items.splice(pagination.size);
return { items: items as any as AssetEntity[], hasNextPage };
} }
@GenerateSql({ @GenerateSql({
@ -96,21 +68,15 @@ export class SearchRepository implements ISearchRepository {
}, },
], ],
}) })
async searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]> { searchRandom(size: number, options: AssetSearchOptions): Promise<AssetEntity[]> {
const builder1 = searchAssetBuilder(this.assetRepository.createQueryBuilder('asset'), options);
const builder2 = builder1.clone();
const uuid = randomUUID(); const uuid = randomUUID();
builder1.andWhere('asset.id > :uuid', { uuid }).orderBy('asset.id').take(size); const builder = searchAssetBuilder(this.db, options);
builder2.andWhere('asset.id < :uuid', { uuid }).orderBy('asset.id').take(size); return builder
.where('assets.id', '>', uuid)
const [assets1, assets2] = await Promise.all([builder1.getMany(), builder2.getMany()]); .orderBy('assets.id')
const missingCount = size - assets1.length; .limit(size)
for (let i = 0; i < missingCount && i < assets2.length; i++) { .unionAll(() => builder.where('assets.id', '<', uuid).orderBy('assets.id').limit(size))
assets1.push(assets2[i]); .execute() as any as Promise<AssetEntity[]>;
}
return assets1;
} }
@GenerateSql({ @GenerateSql({
@ -126,33 +92,21 @@ export class SearchRepository implements ISearchRepository {
}, },
], ],
}) })
async searchSmart( async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity> {
pagination: SearchPaginationOptions, if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) {
{ embedding, userIds, ...options }: SmartSearchOptions, throw new Error(`Invalid value for 'size': ${pagination.size}`);
): Paginated<AssetEntity> { }
let results: PaginationResult<AssetEntity> = { items: [], hasNextPage: false };
await this.assetRepository.manager.transaction(async (manager) => { const items = (await searchAssetBuilder(this.db, options)
let builder = manager.createQueryBuilder(AssetEntity, 'asset'); .innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
builder = searchAssetBuilder(builder, options); .orderBy(sql`smart_search.embedding <=> ${asVector(options.embedding)}`)
builder .limit(pagination.size + 1)
.innerJoin('asset.smartSearch', 'search') .offset((pagination.page - 1) * pagination.size)
.andWhere('asset.ownerId IN (:...userIds )') .execute()) as any as AssetEntity[];
.orderBy('search.embedding <=> :embedding')
.setParameters({ userIds, embedding: asVector(embedding) });
const runtimeConfig = this.getRuntimeConfig(pagination.size); const hasNextPage = items.length > pagination.size;
if (runtimeConfig) { items.splice(pagination.size);
await manager.query(runtimeConfig); return { items, hasNextPage };
}
results = await paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.LIMIT_OFFSET,
skip: (pagination.page - 1) * pagination.size,
take: pagination.size,
});
});
return results;
} }
@GenerateSql({ @GenerateSql({
@ -164,38 +118,30 @@ export class SearchRepository implements ISearchRepository {
}, },
], ],
}) })
searchDuplicates({ searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) {
assetId, const vector = asVector(embedding);
embedding, return this.db
maxDistance, .with('cte', (qb) =>
type, qb
userIds, .selectFrom('assets')
}: AssetDuplicateSearch): Promise<AssetDuplicateResult[]> { .select([
const cte = this.assetRepository.createQueryBuilder('asset'); 'assets.id as assetId',
cte 'assets.duplicateId',
.select('search.assetId', 'assetId') sql<number>`smart_search.embedding <=> ${vector}`.as('distance'),
.addSelect('asset.duplicateId', 'duplicateId') ])
.addSelect(`search.embedding <=> :embedding`, 'distance') .innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
.innerJoin('asset.smartSearch', 'search') .where('assets.ownerId', '=', anyUuid(userIds))
.where('asset.ownerId IN (:...userIds )') .where('assets.deletedAt', 'is', null)
.andWhere('asset.id != :assetId') .where('assets.isVisible', '=', true)
.andWhere('asset.isVisible = :isVisible') .where('assets.type', '=', type)
.andWhere('asset.type = :type') .where('assets.id', '!=', assetId)
.orderBy('search.embedding <=> :embedding') .orderBy(sql`smart_search.embedding <=> ${vector}`)
.limit(64) .limit(64),
.setParameters({ assetId, embedding: asVector(embedding), isVisible: true, type, userIds }); )
.selectFrom('cte')
const builder = this.assetRepository.manager .selectAll()
.createQueryBuilder() .where('cte.distance', '<=', maxDistance as number)
.addCommonTableExpression(cte, 'cte') .execute();
.from('cte', 'res')
.select('res.*');
if (maxDistance) {
builder.where('res.distance <= :maxDistance', { maxDistance });
}
return builder.getRawMany() as Promise<AssetDuplicateResult[]>;
} }
@GenerateSql({ @GenerateSql({
@ -208,120 +154,131 @@ export class SearchRepository implements ISearchRepository {
}, },
], ],
}) })
async searchFaces({ searchFaces({ userIds, embedding, numResults, maxDistance, hasPerson }: FaceEmbeddingSearch) {
userIds, if (!isValidInteger(numResults, { min: 1, max: 1000 })) {
embedding,
numResults,
maxDistance,
hasPerson,
}: FaceEmbeddingSearch): Promise<FaceSearchResult[]> {
if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`); throw new Error(`Invalid value for 'numResults': ${numResults}`);
} }
// setting this too low messes with prefilter recall const vector = asVector(embedding);
numResults = Math.max(numResults, 64); return this.db
.with('cte', (qb) =>
let results: Array<AssetFaceEntity & { distance: number }> = []; qb
await this.assetRepository.manager.transaction(async (manager) => { .selectFrom('asset_faces')
const cte = manager .select([
.createQueryBuilder(AssetFaceEntity, 'faces') 'asset_faces.id',
.select('search.embedding <=> :embedding', 'distance') 'asset_faces.personId',
.innerJoin('faces.asset', 'asset') sql<number>`face_search.embedding <=> ${vector}`.as('distance'),
.innerJoin('faces.faceSearch', 'search') ])
.where('asset.ownerId IN (:...userIds )') .innerJoin('assets', 'assets.id', 'asset_faces.assetId')
.orderBy('search.embedding <=> :embedding') .innerJoin('face_search', 'face_search.faceId', 'asset_faces.id')
.setParameters({ userIds, embedding: asVector(embedding) }); .where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.deletedAt', 'is', null)
cte.limit(numResults); .$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null))
.orderBy(sql`face_search.embedding <=> ${vector}`)
if (hasPerson) { .limit(numResults),
cte.andWhere('faces."personId" IS NOT NULL'); )
} .selectFrom('cte')
.selectAll()
for (const col of this.faceColumns) { .where('cte.distance', '<=', maxDistance)
cte.addSelect(`faces.${col}`, col); .execute();
}
const runtimeConfig = this.getRuntimeConfig(numResults);
if (runtimeConfig) {
await manager.query(runtimeConfig);
}
results = await manager
.createQueryBuilder()
.select('res.*')
.addCommonTableExpression(cte, 'cte')
.from('cte', 'res')
.where('res.distance <= :maxDistance', { maxDistance })
.orderBy('res.distance')
.getRawMany();
});
return results.map((row) => ({
face: this.assetFaceRepository.create(row),
distance: row.distance,
}));
} }
@GenerateSql({ params: [DummyValue.STRING] }) @GenerateSql({ params: [DummyValue.STRING] })
async searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]> { searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]> {
return await this.geodataPlacesRepository return this.db
.createQueryBuilder('geoplaces') .selectFrom('geodata_places')
.where(`f_unaccent(name) %>> f_unaccent(:placeName)`) .selectAll()
.orWhere(`f_unaccent("admin2Name") %>> f_unaccent(:placeName)`) .where(
.orWhere(`f_unaccent("admin1Name") %>> f_unaccent(:placeName)`) () =>
.orWhere(`f_unaccent("alternateNames") %>> f_unaccent(:placeName)`) // kysely doesn't support trigram %>> or <->>> operators
sql`
f_unaccent(name) %>> f_unaccent(${placeName}) or
f_unaccent("admin2Name") %>> f_unaccent(${placeName}) or
f_unaccent("admin1Name") %>> f_unaccent(${placeName}) or
f_unaccent("alternateNames") %>> f_unaccent(${placeName})
`,
)
.orderBy( .orderBy(
` sql`
COALESCE(f_unaccent(name) <->>> f_unaccent(:placeName), 0.1) + coalesce(f_unaccent(name) <->>> f_unaccent(${placeName}), 0.1) +
COALESCE(f_unaccent("admin2Name") <->>> f_unaccent(:placeName), 0.1) + coalesce(f_unaccent("admin2Name") <->>> f_unaccent(${placeName}), 0.1) +
COALESCE(f_unaccent("admin1Name") <->>> f_unaccent(:placeName), 0.1) + coalesce(f_unaccent("admin1Name") <->>> f_unaccent(${placeName}), 0.1) +
COALESCE(f_unaccent("alternateNames") <->>> f_unaccent(:placeName), 0.1) coalesce(f_unaccent("alternateNames") <->>> f_unaccent(${placeName}), 0.1)
`, `,
) )
.setParameters({ placeName })
.limit(20) .limit(20)
.getMany(); .execute() as Promise<GeodataPlacesEntity[]>;
} }
@GenerateSql({ params: [[DummyValue.UUID]] }) @GenerateSql({ params: [[DummyValue.UUID]] })
async getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> { getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> {
const parameters = [userIds, true, false, AssetType.IMAGE]; return this.db
const rawRes = await this.assetRepository.query(this.assetsByCityQuery, parameters); .withRecursive('cte', (qb) => {
const base = qb
.selectFrom('exif')
.select(['city', 'assetId'])
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', anyUuid(userIds))
.where('assets.isVisible', '=', true)
.where('assets.isArchived', '=', false)
.where('assets.type', '=', 'IMAGE')
.where('assets.deletedAt', 'is', null)
.orderBy('city')
.limit(1);
const items: AssetEntity[] = []; const recursive = qb
for (const res of rawRes) { .selectFrom('cte')
const item = { exifInfo: {} as Record<string, any> } as Record<string, any>; .select(['l.city', 'l.assetId'])
for (const [key, value] of Object.entries(res)) { .innerJoinLateral(
if (key.startsWith('exif_')) { (qb) =>
item.exifInfo[key.replace('exif_', '')] = value; qb
} else { .selectFrom('exif')
item[key.replace('asset_', '')] = value; .select(['city', 'assetId'])
} .innerJoin('assets', 'assets.id', 'exif.assetId')
} .where('assets.ownerId', '=', anyUuid(userIds))
items.push(item as AssetEntity); .where('assets.isVisible', '=', true)
} .where('assets.isArchived', '=', false)
.where('assets.type', '=', 'IMAGE')
.where('assets.deletedAt', 'is', null)
.whereRef('exif.city', '>', 'cte.city')
.orderBy('city')
.limit(1)
.as('l'),
(join) => join.onTrue(),
);
return items; return sql<{ city: string; assetId: string }>`(${base} union all ${recursive})`;
})
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.innerJoin('cte', 'assets.id', 'cte.assetId')
.selectAll('assets')
.select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo'))
.orderBy('exif.city')
.execute() as any as Promise<AssetEntity[]>;
} }
async upsert(assetId: string, embedding: number[]): Promise<void> { async upsert(assetId: string, embedding: number[]): Promise<void> {
await this.smartSearchRepository.upsert( const vector = asVector(embedding);
{ assetId, embedding: () => asVector(embedding, true) }, await this.db
{ conflictPaths: ['assetId'] }, .insertInto('smart_search')
); .values({ assetId: asUuid(assetId), embedding: vector } as any)
.onConflict((oc) => oc.column('assetId').doUpdateSet({ embedding: vector } as any))
.execute();
} }
async getDimensionSize(): Promise<number> { async getDimensionSize(): Promise<number> {
const res = await this.smartSearchRepository.manager.query(` const { rows } = await sql<{ dimsize: number }>`
SELECT atttypmod as dimsize select atttypmod as dimsize
FROM pg_attribute f from pg_attribute f
JOIN pg_class c ON c.oid = f.attrelid join pg_class c ON c.oid = f.attrelid
WHERE c.relkind = 'r'::char where c.relkind = 'r'::char
AND f.attnum > 0 and f.attnum > 0
AND c.relname = 'smart_search' and c.relname = 'smart_search'
AND f.attname = 'embedding'`); and f.attname = 'embedding'
`.execute(this.db);
const dimSize = res[0]['dimsize']; const dimSize = rows[0]['dimsize'];
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) { if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Could not retrieve CLIP dimension size`); throw new Error(`Could not retrieve CLIP dimension size`);
} }
@ -333,146 +290,71 @@ export class SearchRepository implements ISearchRepository {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`); throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
} }
return this.smartSearchRepository.manager.transaction(async (manager) => { return this.db.transaction().execute(async (trx) => {
await manager.clear(SmartSearchEntity); await sql`truncate ${sql.table('smart_search')}`.execute(trx);
await manager.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE vector(${dimSize})`); await trx.schema
await manager.query(`REINDEX INDEX clip_index`); .alterTable('smart_search')
.alterColumn('embedding', (col) => col.setDataType(sql.lit(`vector(${dimSize})`)))
.execute();
await sql`reindex index clip_index`.execute(trx);
}); });
} }
async deleteAllSearchEmbeddings(): Promise<void> { async deleteAllSearchEmbeddings(): Promise<void> {
return this.smartSearchRepository.clear(); await sql`truncate ${sql.table('smart_search')}`.execute(this.db);
} }
@GenerateSql({ params: [[DummyValue.UUID]] })
async getCountries(userIds: string[]): Promise<string[]> { async getCountries(userIds: string[]): Promise<string[]> {
const query = this.exifRepository const res = await this.getExifField('country', userIds).execute();
.createQueryBuilder('exif') return res.map((row) => row.country!);
.innerJoin('exif.asset', 'asset')
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.country != ''`)
.andWhere('exif.country IS NOT NULL')
.select('exif.country', 'country')
.distinctOn(['exif.country']);
const results = await query.getRawMany<{ country: string }>();
return results.map(({ country }) => country);
} }
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getStates(userIds: string[], { country }: GetStatesOptions): Promise<string[]> { async getStates(userIds: string[], { country }: GetStatesOptions): Promise<string[]> {
const query = this.exifRepository const res = await this.getExifField('state', userIds)
.createQueryBuilder('exif') .$if(!!country, (qb) => qb.where('country', '=', country!))
.innerJoin('exif.asset', 'asset') .execute();
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.state != ''`)
.andWhere('exif.state IS NOT NULL')
.select('exif.state', 'state')
.distinctOn(['exif.state']);
if (country) { return res.map((row) => row.state!);
query.andWhere('exif.country = :country', { country });
}
const result = await query.getRawMany<{ state: string }>();
return result.map(({ state }) => state);
} }
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING, DummyValue.STRING] }) @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING, DummyValue.STRING] })
async getCities(userIds: string[], { country, state }: GetCitiesOptions): Promise<string[]> { async getCities(userIds: string[], { country, state }: GetCitiesOptions): Promise<string[]> {
const query = this.exifRepository const res = await this.getExifField('city', userIds)
.createQueryBuilder('exif') .$if(!!country, (qb) => qb.where('country', '=', country!))
.innerJoin('exif.asset', 'asset') .$if(!!state, (qb) => qb.where('state', '=', state!))
.where('asset.ownerId IN (:...userIds )', { userIds }) .execute();
.andWhere(`exif.city != ''`)
.andWhere('exif.city IS NOT NULL')
.select('exif.city', 'city')
.distinctOn(['exif.city']);
if (country) { return res.map((row) => row.city!);
query.andWhere('exif.country = :country', { country });
}
if (state) {
query.andWhere('exif.state = :state', { state });
}
const results = await query.getRawMany<{ city: string }>();
return results.map(({ city }) => city);
} }
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getCameraMakes(userIds: string[], { model }: GetCameraMakesOptions): Promise<string[]> { async getCameraMakes(userIds: string[], { model }: GetCameraMakesOptions): Promise<string[]> {
const query = this.exifRepository const res = await this.getExifField('make', userIds)
.createQueryBuilder('exif') .$if(!!model, (qb) => qb.where('model', '=', model!))
.innerJoin('exif.asset', 'asset') .execute();
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.make != ''`)
.andWhere('exif.make IS NOT NULL')
.select('exif.make', 'make')
.distinctOn(['exif.make']);
if (model) { return res.map((row) => row.make!);
query.andWhere('exif.model = :model', { model });
}
const results = await query.getRawMany<{ make: string }>();
return results.map(({ make }) => make);
} }
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] }) @GenerateSql({ params: [[DummyValue.UUID], DummyValue.STRING] })
async getCameraModels(userIds: string[], { make }: GetCameraModelsOptions): Promise<string[]> { async getCameraModels(userIds: string[], { make }: GetCameraModelsOptions): Promise<string[]> {
const query = this.exifRepository const res = await this.getExifField('model', userIds)
.createQueryBuilder('exif') .$if(!!make, (qb) => qb.where('make', '=', make!))
.innerJoin('exif.asset', 'asset') .execute();
.where('asset.ownerId IN (:...userIds )', { userIds })
.andWhere(`exif.model != ''`)
.andWhere('exif.model IS NOT NULL')
.select('exif.model', 'model')
.distinctOn(['exif.model']);
if (make) { return res.map((row) => row.model!);
query.andWhere('exif.make = :make', { make });
}
const results = await query.getRawMany<{ model: string }>();
return results.map(({ model }) => model);
} }
private getRuntimeConfig(numResults?: number): string | undefined { private getExifField<K extends 'city' | 'state' | 'country' | 'make' | 'model'>(field: K, userIds: string[]) {
if (this.vectorExtension === DatabaseExtension.VECTOR) { return this.db
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall .selectFrom('exif')
} .select(field)
.distinctOn(field)
if (numResults && numResults !== 100) { .innerJoin('assets', 'assets.id', 'exif.assetId')
return `SET LOCAL vectors.hnsw_ef_search = ${Math.max(numResults, 100)};`; .where('ownerId', '=', anyUuid(userIds))
} .where('isVisible', '=', true)
.where('deletedAt', 'is', null)
.where(field, 'is not', null);
} }
} }
// the performance difference between this and the normal way is too huge to ignore, e.g. 3s vs 4ms
const assetsByCityCte = `
WITH RECURSIVE cte AS (
(
SELECT city, "assetId"
FROM exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
ORDER BY city
LIMIT 1
)
UNION ALL
SELECT l.city, l."assetId"
FROM cte c
, LATERAL (
SELECT city, "assetId"
FROM exif
INNER JOIN assets ON exif."assetId" = assets.id
WHERE city > c.city AND "ownerId" = ANY($1::uuid[]) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
ORDER BY city
LIMIT 1
) l
)
`;

View file

@ -1,48 +1,47 @@
import { InjectRepository } from '@nestjs/typeorm'; import { Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators'; import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity, withExif } from 'src/entities/asset.entity';
import { IViewRepository } from 'src/interfaces/view.interface'; import { IViewRepository } from 'src/interfaces/view.interface';
import { Brackets, Repository } from 'typeorm'; import { asUuid } from 'src/utils/database';
export class ViewRepository implements IViewRepository { export class ViewRepository implements IViewRepository {
constructor(@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>) {} constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
async getUniqueOriginalPaths(userId: string): Promise<string[]> { async getUniqueOriginalPaths(userId: string): Promise<string[]> {
const results = await this.assetRepository const results = await this.db
.createQueryBuilder('asset') .selectFrom('assets')
.where({ .select((eb) => eb.fn<string>('substring', ['assets.originalPath', eb.val('^(.*/)[^/]*$')]).as('directoryPath'))
isVisible: true, .distinct()
isArchived: false, .where('ownerId', '=', asUuid(userId))
ownerId: userId, .where('isVisible', '=', true)
}) .where('isArchived', '=', false)
.select("DISTINCT substring(asset.originalPath FROM '^(.*/)[^/]*$')", 'directoryPath') .where('deletedAt', 'is', null)
.getRawMany(); .execute();
return results.map((row: { directoryPath: string }) => row.directoryPath.replaceAll(/^\/|\/$/g, '')); return results.map((row) => row.directoryPath.replaceAll(/^\/|\/$/g, ''));
} }
@GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
async getAssetsByOriginalPath(userId: string, partialPath: string): Promise<AssetEntity[]> { async getAssetsByOriginalPath(userId: string, partialPath: string): Promise<AssetEntity[]> {
const normalizedPath = partialPath.replaceAll(/^\/|\/$/g, ''); const normalizedPath = partialPath.replaceAll(/^\/|\/$/g, '');
const assets = await this.assetRepository
.createQueryBuilder('asset')
.where({
isVisible: true,
isArchived: false,
ownerId: userId,
})
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.andWhere(
new Brackets((qb) => {
qb.where('asset.originalPath LIKE :likePath', { likePath: `%${normalizedPath}/%` }).andWhere(
'asset.originalPath NOT LIKE :notLikePath',
{ notLikePath: `%${normalizedPath}/%/%` },
);
}),
)
.orderBy(String.raw`regexp_replace(asset.originalPath, '.*/(.+)', '\1')`, 'ASC')
.getMany();
return assets; return this.db
.selectFrom('assets')
.selectAll('assets')
.$call(withExif)
.where('ownerId', '=', asUuid(userId))
.where('isVisible', '=', true)
.where('isArchived', '=', false)
.where('deletedAt', 'is', null)
.where('originalPath', 'like', `%${normalizedPath}/%`)
.where('originalPath', 'not like', `%${normalizedPath}/%/%`)
.orderBy(
(eb) => eb.fn('regexp_replace', ['assets.originalPath', eb.val('.*/(.+)'), eb.val(String.raw`\1`)]),
'asc',
)
.execute() as any as Promise<AssetEntity[]>;
} }
} }

View file

@ -23,7 +23,6 @@ import { fileStub } from 'test/fixtures/file.stub';
import { userStub } from 'test/fixtures/user.stub'; import { userStub } from 'test/fixtures/user.stub';
import { IAccessRepositoryMock } from 'test/repositories/access.repository.mock'; import { IAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newTestService } from 'test/utils'; import { newTestService } from 'test/utils';
import { QueryFailedError } from 'typeorm';
import { Mocked } from 'vitest'; import { Mocked } from 'vitest';
const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
@ -370,8 +369,8 @@ describe(AssetMediaService.name, () => {
originalName: 'asset_1.jpeg', originalName: 'asset_1.jpeg',
size: 0, size: 0,
}; };
const error = new QueryFailedError('', [], new Error('unique key violation')); const error = new Error('unique key violation');
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.create.mockRejectedValue(error); assetMock.create.mockRejectedValue(error);
assetMock.getUploadAssetIdByChecksum.mockResolvedValue(assetEntity.id); assetMock.getUploadAssetIdByChecksum.mockResolvedValue(assetEntity.id);
@ -397,8 +396,8 @@ describe(AssetMediaService.name, () => {
originalName: 'asset_1.jpeg', originalName: 'asset_1.jpeg',
size: 0, size: 0,
}; };
const error = new QueryFailedError('', [], new Error('unique key violation')); const error = new Error('unique key violation');
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.create.mockRejectedValue(error); assetMock.create.mockRejectedValue(error);
@ -480,7 +479,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset is not found', async () => { it('should throw an error if the asset is not found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(null);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(NotFoundException); await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(NotFoundException);
@ -512,7 +510,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset does not exist', async () => { it('should throw an error if the asset does not exist', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id]));
assetMock.getById.mockResolvedValue(null);
await expect( await expect(
sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.PREVIEW }), sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.PREVIEW }),
@ -618,7 +615,6 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset does not exist', async () => { it('should throw an error if the asset does not exist', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id]));
assetMock.getById.mockResolvedValue(null);
await expect(sut.playbackVideo(authStub.admin, assetStub.image.id)).rejects.toBeInstanceOf(NotFoundException); await expect(sut.playbackVideo(authStub.admin, assetStub.image.id)).rejects.toBeInstanceOf(NotFoundException);
}); });
@ -670,8 +666,6 @@ describe(AssetMediaService.name, () => {
describe('replaceAsset', () => { describe('replaceAsset', () => {
it('should error when update photo does not exist', async () => { it('should error when update photo does not exist', async () => {
assetMock.getById.mockResolvedValueOnce(null);
await expect(sut.replaceAsset(authStub.user1, 'id', replaceDto, fileStub.photo)).rejects.toThrow( await expect(sut.replaceAsset(authStub.user1, 'id', replaceDto, fileStub.photo)).rejects.toThrow(
'Not found or no asset.update access', 'Not found or no asset.update access',
); );
@ -785,8 +779,8 @@ describe(AssetMediaService.name, () => {
it('should handle a photo with sidecar to duplicate photo ', async () => { it('should handle a photo with sidecar to duplicate photo ', async () => {
const updatedFile = fileStub.photo; const updatedFile = fileStub.photo;
const error = new QueryFailedError('', [], new Error('unique key violation')); const error = new Error('unique key violation');
(error as any).constraint = ASSET_CHECKSUM_CONSTRAINT; (error as any).constraint_name = ASSET_CHECKSUM_CONSTRAINT;
assetMock.update.mockRejectedValue(error); assetMock.update.mockRejectedValue(error);
assetMock.getById.mockResolvedValueOnce(sidecarAsset); assetMock.getById.mockResolvedValueOnce(sidecarAsset);

View file

@ -30,7 +30,6 @@ import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util';
import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types'; import { mimeTypes } from 'src/utils/mime-types';
import { fromChecksum } from 'src/utils/request'; import { fromChecksum } from 'src/utils/request';
import { QueryFailedError } from 'typeorm';
export interface UploadRequest { export interface UploadRequest {
auth: AuthDto | null; auth: AuthDto | null;
fieldName: UploadFieldName; fieldName: UploadFieldName;
@ -302,7 +301,7 @@ export class AssetMediaService extends BaseService {
}); });
// handle duplicates with a success response // handle duplicates with a success response
if (error instanceof QueryFailedError && (error as any).constraint === ASSET_CHECKSUM_CONSTRAINT) { if (error.constraint_name === ASSET_CHECKSUM_CONSTRAINT) {
const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, file.checksum); const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, file.checksum);
if (!duplicateId) { if (!duplicateId) {
this.logger.error(`Error locating duplicate for checksum constraint`); this.logger.error(`Error locating duplicate for checksum constraint`);
@ -343,7 +342,7 @@ export class AssetMediaService extends BaseService {
localDateTime: dto.fileCreatedAt, localDateTime: dto.fileCreatedAt,
duration: dto.duration || null, duration: dto.duration || null,
livePhotoVideo: null, livePhotoVideoId: null,
sidecarPath: sidecarPath || null, sidecarPath: sidecarPath || null,
}); });

View file

@ -51,9 +51,7 @@ describe(AssetService.name, () => {
}); });
const mockGetById = (assets: AssetEntity[]) => { const mockGetById = (assets: AssetEntity[]) => {
assetMock.getById.mockImplementation((assetId) => assetMock.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId)));
Promise.resolve(assets.find((asset) => asset.id === assetId) ?? null),
);
}; };
beforeEach(() => { beforeEach(() => {
@ -250,27 +248,34 @@ describe(AssetService.name, () => {
it('should update the asset', async () => { it('should update the asset', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image); assetMock.getById.mockResolvedValue(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { isFavorite: true }); await sut.update(authStub.admin, 'asset-1', { isFavorite: true });
expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-1', isFavorite: true }); expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-1', isFavorite: true });
}); });
it('should update the exif description', async () => { it('should update the exif description', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image); assetMock.getById.mockResolvedValue(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { description: 'Test description' }); await sut.update(authStub.admin, 'asset-1', { description: 'Test description' });
expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' }); expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' });
}); });
it('should update the exif rating', async () => { it('should update the exif rating', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
assetMock.getById.mockResolvedValue(assetStub.image); assetMock.getById.mockResolvedValueOnce(assetStub.image);
assetMock.update.mockResolvedValueOnce(assetStub.image);
await sut.update(authStub.admin, 'asset-1', { rating: 3 }); await sut.update(authStub.admin, 'asset-1', { rating: 3 });
expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 }); expect(assetMock.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 });
}); });
it('should fail linking a live video if the motion part could not be found', async () => { it('should fail linking a live video if the motion part could not be found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValue(null);
await expect( await expect(
sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, {
@ -339,6 +344,7 @@ describe(AssetService.name, () => {
isVisible: true, isVisible: true,
}); });
assetMock.getById.mockResolvedValueOnce(assetStub.image); assetMock.getById.mockResolvedValueOnce(assetStub.image);
assetMock.update.mockResolvedValue(assetStub.image);
await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, {
livePhotoVideoId: assetStub.livePhotoMotionAsset.id, livePhotoVideoId: assetStub.livePhotoMotionAsset.id,
@ -366,7 +372,7 @@ describe(AssetService.name, () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoStillAsset); assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoStillAsset);
assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoMotionAsset); assetMock.getById.mockResolvedValueOnce(assetStub.livePhotoMotionAsset);
assetMock.getById.mockResolvedValueOnce(assetStub.image); assetMock.update.mockResolvedValueOnce(assetStub.image);
await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }); await sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null });
@ -383,15 +389,15 @@ describe(AssetService.name, () => {
it('should fail unlinking a live video if the asset could not be found', async () => { it('should fail unlinking a live video if the asset could not be found', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.livePhotoStillAsset.id]));
assetMock.getById.mockResolvedValue(null); // eslint-disable-next-line unicorn/no-useless-undefined
assetMock.getById.mockResolvedValueOnce(undefined);
await expect( await expect(
sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }), sut.update(authStub.admin, assetStub.livePhotoStillAsset.id, { livePhotoVideoId: null }),
).rejects.toBeInstanceOf(BadRequestException); ).rejects.toBeInstanceOf(BadRequestException);
expect(assetMock.update).not.toHaveBeenCalled(); expect(assetMock.update).not.toHaveBeenCalled();
expect(assetMock.update).not.toHaveBeenCalledWith(); expect(eventMock.emit).not.toHaveBeenCalled();
expect(eventMock.emit).not.toHaveBeenCalledWith();
}); });
}); });

View file

@ -74,29 +74,13 @@ export class AssetService extends BaseService {
async get(auth: AuthDto, id: string): Promise<AssetResponseDto | SanitizedAssetResponseDto> { async get(auth: AuthDto, id: string): Promise<AssetResponseDto | SanitizedAssetResponseDto> {
await this.requireAccess({ auth, permission: Permission.ASSET_READ, ids: [id] }); await this.requireAccess({ auth, permission: Permission.ASSET_READ, ids: [id] });
const asset = await this.assetRepository.getById( const asset = await this.assetRepository.getById(id, {
id, exifInfo: true,
{ owner: true,
exifInfo: true, faces: { person: true },
sharedLinks: true, stack: { assets: true },
tags: true, tags: true,
owner: true, });
faces: {
person: true,
},
stack: {
assets: {
exifInfo: true,
},
},
files: true,
},
{
faces: {
boundingBoxX1: 'ASC',
},
},
);
if (!asset) { if (!asset) {
throw new BadRequestException('Asset not found'); throw new BadRequestException('Asset not found');
@ -137,22 +121,12 @@ export class AssetService extends BaseService {
await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude, rating }); await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude, rating });
await this.assetRepository.update({ id, ...rest }); const asset = await this.assetRepository.update({ id, ...rest });
if (previousMotion) { if (previousMotion) {
await onAfterUnlink(repos, { userId: auth.user.id, livePhotoVideoId: previousMotion.id }); await onAfterUnlink(repos, { userId: auth.user.id, livePhotoVideoId: previousMotion.id });
} }
const asset = await this.assetRepository.getById(id, {
exifInfo: true,
owner: true,
tags: true,
faces: {
person: true,
},
files: true,
});
if (!asset) { if (!asset) {
throw new BadRequestException('Asset not found'); throw new BadRequestException('Asset not found');
} }
@ -202,9 +176,7 @@ export class AssetService extends BaseService {
const { id, deleteOnDisk } = job; const { id, deleteOnDisk } = job;
const asset = await this.assetRepository.getById(id, { const asset = await this.assetRepository.getById(id, {
faces: { faces: { person: true },
person: true,
},
library: true, library: true,
stack: { assets: true }, stack: { assets: true },
exifInfo: true, exifInfo: true,

View file

@ -71,10 +71,8 @@ export class BackupService extends BaseService {
@OnJob({ name: JobName.BACKUP_DATABASE, queue: QueueName.BACKUP_DATABASE }) @OnJob({ name: JobName.BACKUP_DATABASE, queue: QueueName.BACKUP_DATABASE })
async handleBackupDatabase(): Promise<JobStatus> { async handleBackupDatabase(): Promise<JobStatus> {
this.logger.debug(`Database Backup Started`); this.logger.debug(`Database Backup Started`);
const { database } = this.configRepository.getEnv();
const { const config = database.config.typeorm;
database: { config },
} = this.configRepository.getEnv();
const isUrlConnection = config.connectionType === 'url'; const isUrlConnection = config.connectionType === 'url';

View file

@ -1,3 +1,4 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import { IConfigRepository } from 'src/interfaces/config.interface'; import { IConfigRepository } from 'src/interfaces/config.interface';
import { import {
DatabaseExtension, DatabaseExtension,
@ -61,13 +62,19 @@ describe(DatabaseService.name, () => {
mockEnvData({ mockEnvData({
database: { database: {
config: { config: {
connectionType: 'parts', kysely: {
type: 'postgres', dialect: expect.any(PostgresJSDialect),
host: 'database', log: ['error'],
port: 5432, },
username: 'postgres', typeorm: {
password: 'postgres', connectionType: 'parts',
database: 'immich', type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
}, },
skipMigrations: false, skipMigrations: false,
vectorExtension: extension, vectorExtension: extension,
@ -291,13 +298,19 @@ describe(DatabaseService.name, () => {
mockEnvData({ mockEnvData({
database: { database: {
config: { config: {
connectionType: 'parts', kysely: {
type: 'postgres', dialect: expect.any(PostgresJSDialect),
host: 'database', log: ['error'],
port: 5432, },
username: 'postgres', typeorm: {
password: 'postgres', connectionType: 'parts',
database: 'immich', type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
}, },
skipMigrations: true, skipMigrations: true,
vectorExtension: DatabaseExtension.VECTORS, vectorExtension: DatabaseExtension.VECTORS,
@ -315,13 +328,19 @@ describe(DatabaseService.name, () => {
mockEnvData({ mockEnvData({
database: { database: {
config: { config: {
connectionType: 'parts', kysely: {
type: 'postgres', dialect: expect.any(PostgresJSDialect),
host: 'database', log: ['error'],
port: 5432, },
username: 'postgres', typeorm: {
password: 'postgres', connectionType: 'parts',
database: 'immich', type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
}, },
skipMigrations: true, skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR, vectorExtension: DatabaseExtension.VECTOR,

View file

@ -31,7 +31,12 @@ describe(SearchService.name, () => {
describe('getDuplicates', () => { describe('getDuplicates', () => {
it('should get duplicates', async () => { it('should get duplicates', async () => {
assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe, assetStub.hasDupe]); assetMock.getDuplicates.mockResolvedValue([
{
duplicateId: assetStub.hasDupe.duplicateId!,
assets: [assetStub.hasDupe, assetStub.hasDupe],
},
]);
await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([ await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([
{ {
duplicateId: assetStub.hasDupe.duplicateId, duplicateId: assetStub.hasDupe.duplicateId,
@ -42,12 +47,6 @@ describe(SearchService.name, () => {
}, },
]); ]);
}); });
it('should update assets with duplicateId', async () => {
assetMock.getDuplicates.mockResolvedValue([assetStub.hasDupe]);
await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([]);
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.hasDupe.id], { duplicateId: null });
});
}); });
describe('handleQueueSearchDuplicates', () => { describe('handleQueueSearchDuplicates', () => {

View file

@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
import { OnJob } from 'src/decorators'; import { OnJob } from 'src/decorators';
import { mapAsset } from 'src/dtos/asset-response.dto'; import { mapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto'; import { AuthDto } from 'src/dtos/auth.dto';
import { DuplicateResponseDto, mapDuplicateResponse } from 'src/dtos/duplicate.dto'; import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { WithoutProperty } from 'src/interfaces/asset.interface'; import { WithoutProperty } from 'src/interfaces/asset.interface';
import { JOBS_ASSET_PAGINATION_SIZE, JobName, JobOf, JobStatus, QueueName } from 'src/interfaces/job.interface'; import { JOBS_ASSET_PAGINATION_SIZE, JobName, JobOf, JobStatus, QueueName } from 'src/interfaces/job.interface';
@ -15,25 +15,11 @@ import { usePagination } from 'src/utils/pagination';
@Injectable() @Injectable()
export class DuplicateService extends BaseService { export class DuplicateService extends BaseService {
async getDuplicates(auth: AuthDto): Promise<DuplicateResponseDto[]> { async getDuplicates(auth: AuthDto): Promise<DuplicateResponseDto[]> {
const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] }); const duplicates = await this.assetRepository.getDuplicates(auth.user.id);
const uniqueAssetIds: string[] = []; return duplicates.map(({ duplicateId, assets }) => ({
const duplicates = mapDuplicateResponse(res.map((a) => mapAsset(a, { auth, withStack: true }))).filter( duplicateId,
(duplicate) => { assets: assets.map((asset) => mapAsset(asset, { auth })),
if (duplicate.assets.length === 1) { }));
uniqueAssetIds.push(duplicate.assets[0].id);
return false;
}
return true;
},
);
if (uniqueAssetIds.length > 0) {
try {
await this.assetRepository.updateAll(uniqueAssetIds, { duplicateId: null });
} catch (error: any) {
this.logger.error(`Failed to remove duplicateId from assets: ${error.message}`);
}
}
return duplicates;
} }
@OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION }) @OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION })

View file

@ -256,8 +256,6 @@ describe(LibraryService.name, () => {
exclusionPatterns: [], exclusionPatterns: [],
}; };
assetMock.getById.mockResolvedValue(null);
await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED); await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED);
expect(assetMock.remove).not.toHaveBeenCalled(); expect(assetMock.remove).not.toHaveBeenCalled();
@ -394,7 +392,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg', assetPath: '/data/user1/photo.jpg',
}; };
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image); assetMock.create.mockResolvedValue(assetStub.image);
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
@ -438,7 +435,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/video.mp4', assetPath: '/data/user1/video.mp4',
}; };
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.video); assetMock.create.mockResolvedValue(assetStub.video);
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
@ -482,7 +478,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg', assetPath: '/data/user1/photo.jpg',
}; };
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image); assetMock.create.mockResolvedValue(assetStub.image);
libraryMock.get.mockResolvedValue({ ...libraryStub.externalLibrary1, deletedAt: new Date() }); libraryMock.get.mockResolvedValue({ ...libraryStub.externalLibrary1, deletedAt: new Date() });
@ -548,7 +543,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg', assetPath: '/data/user1/photo.jpg',
}; };
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image); assetMock.create.mockResolvedValue(assetStub.image);
await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED); await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED);
@ -567,7 +561,6 @@ describe(LibraryService.name, () => {
assetPath: '/data/user1/photo.jpg', assetPath: '/data/user1/photo.jpg',
}; };
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.image); assetMock.create.mockResolvedValue(assetStub.image);
await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED); await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED);

View file

@ -2,6 +2,7 @@ import { BinaryField, ExifDateTime } from 'exiftool-vendored';
import { randomBytes } from 'node:crypto'; import { randomBytes } from 'node:crypto';
import { Stats } from 'node:fs'; import { Stats } from 'node:fs';
import { constants } from 'node:fs/promises'; import { constants } from 'node:fs/promises';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity'; import { ExifEntity } from 'src/entities/exif.entity';
import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum'; import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum';
import { IAlbumRepository } from 'src/interfaces/album.interface'; import { IAlbumRepository } from 'src/interfaces/album.interface';
@ -200,7 +201,6 @@ describe(MetadataService.name, () => {
exifInfo: { livePhotoCID: 'CID' } as ExifEntity, exifInfo: { livePhotoCID: 'CID' } as ExifEntity,
}, },
]); ]);
assetMock.findLivePhotoMatch.mockResolvedValue(null);
await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe( await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(
JobStatus.SKIPPED, JobStatus.SKIPPED,
@ -579,7 +579,6 @@ describe(MetadataService.name, () => {
EmbeddedVideoType: 'MotionPhoto_Data', EmbeddedVideoType: 'MotionPhoto_Data',
}); });
cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512); const video = randomBytes(512);
@ -624,7 +623,6 @@ describe(MetadataService.name, () => {
EmbeddedVideoType: 'MotionPhoto_Data', EmbeddedVideoType: 'MotionPhoto_Data',
}); });
cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512); const video = randomBytes(512);
@ -670,7 +668,6 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1, MicroVideoOffset: 1,
}); });
cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid); cryptoMock.randomUUID.mockReturnValue(fileStub.livePhotoMotion.uuid);
const video = randomBytes(512); const video = randomBytes(512);
@ -716,8 +713,9 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1, MicroVideoOffset: 1,
}); });
cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null); assetMock.create.mockImplementation(
assetMock.create.mockImplementation((asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset })); (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise<AssetEntity>,
);
const video = randomBytes(512); const video = randomBytes(512);
storageMock.readFile.mockResolvedValue(video); storageMock.readFile.mockResolvedValue(video);
@ -789,7 +787,6 @@ describe(MetadataService.name, () => {
MicroVideoOffset: 1, MicroVideoOffset: 1,
}); });
cryptoMock.hashSha1.mockReturnValue(randomBytes(512)); cryptoMock.hashSha1.mockReturnValue(randomBytes(512));
assetMock.getByChecksum.mockResolvedValue(null);
assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset); assetMock.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
const video = randomBytes(512); const video = randomBytes(512);
storageMock.readFile.mockResolvedValue(video); storageMock.readFile.mockResolvedValue(video);

View file

@ -1,16 +1,17 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored'; import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored';
import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime'; import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
import { Insertable } from 'kysely';
import _ from 'lodash'; import _ from 'lodash';
import { Duration } from 'luxon'; import { Duration } from 'luxon';
import { constants } from 'node:fs/promises'; import { constants } from 'node:fs/promises';
import path from 'node:path'; import path from 'node:path';
import { SystemConfig } from 'src/config'; import { SystemConfig } from 'src/config';
import { StorageCore } from 'src/cores/storage.core'; import { StorageCore } from 'src/cores/storage.core';
import { Exif } from 'src/db';
import { OnEvent, OnJob } from 'src/decorators'; import { OnEvent, OnJob } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { PersonEntity } from 'src/entities/person.entity'; import { PersonEntity } from 'src/entities/person.entity';
import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum'; import { AssetType, ExifOrientation, ImmichWorker, SourceType } from 'src/enum';
import { WithoutProperty } from 'src/interfaces/asset.interface'; import { WithoutProperty } from 'src/interfaces/asset.interface';
@ -166,7 +167,7 @@ export class MetadataService extends BaseService {
const { width, height } = this.getImageDimensions(exifTags); const { width, height } = this.getImageDimensions(exifTags);
const exifData: Partial<ExifEntity> = { const exifData: Insertable<Exif> = {
assetId: asset.id, assetId: asset.id,
// dates // dates

View file

@ -728,11 +728,13 @@ describe(PersonService.name, () => {
assetId: assetStub.image.id, assetId: assetStub.image.id,
facesRecognizedAt: expect.any(Date), facesRecognizedAt: expect.any(Date),
}); });
expect(assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt?.getTime()).toBeGreaterThan(start); const facesRecognizedAt = assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt as Date;
expect(facesRecognizedAt.getTime()).toBeGreaterThan(start);
}); });
it('should create a face with no person and queue recognition job', async () => { it('should create a face with no person and queue recognition job', async () => {
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock); machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
searchMock.searchFaces.mockResolvedValue([{ ...faceStub.face1, distance: 0.7 }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]); assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleDetectFaces({ id: assetStub.image.id }); await sut.handleDetectFaces({ id: assetStub.image.id });
@ -840,10 +842,10 @@ describe(PersonService.name, () => {
} }
const faces = [ const faces = [
{ face: faceStub.noPerson1, distance: 0 }, { ...faceStub.noPerson1, distance: 0 },
{ face: faceStub.primaryFace1, distance: 0.2 }, { ...faceStub.primaryFace1, distance: 0.2 },
{ face: faceStub.noPerson2, distance: 0.3 }, { ...faceStub.noPerson2, distance: 0.3 },
{ face: faceStub.face1, distance: 0.4 }, { ...faceStub.face1, distance: 0.4 },
] as FaceSearchResult[]; ] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
@ -867,8 +869,8 @@ describe(PersonService.name, () => {
it('should create a new person if the face is a core point with no person', async () => { it('should create a new person if the face is a core point with no person', async () => {
const faces = [ const faces = [
{ face: faceStub.noPerson1, distance: 0 }, { ...faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.3 }, { ...faceStub.noPerson2, distance: 0.3 },
] as FaceSearchResult[]; ] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
@ -889,7 +891,7 @@ describe(PersonService.name, () => {
}); });
it('should not queue face with no matches', async () => { it('should not queue face with no matches', async () => {
const faces = [{ face: faceStub.noPerson1, distance: 0 }] as FaceSearchResult[]; const faces = [{ ...faceStub.noPerson1, distance: 0 }] as FaceSearchResult[];
searchMock.searchFaces.mockResolvedValue(faces); searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1); personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
@ -905,8 +907,8 @@ describe(PersonService.name, () => {
it('should defer non-core faces to end of queue', async () => { it('should defer non-core faces to end of queue', async () => {
const faces = [ const faces = [
{ face: faceStub.noPerson1, distance: 0 }, { ...faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.4 }, { ...faceStub.noPerson2, distance: 0.4 },
] as FaceSearchResult[]; ] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });
@ -927,8 +929,8 @@ describe(PersonService.name, () => {
it('should not assign person to deferred non-core face with no matching person', async () => { it('should not assign person to deferred non-core face with no matching person', async () => {
const faces = [ const faces = [
{ face: faceStub.noPerson1, distance: 0 }, { ...faceStub.noPerson1, distance: 0 },
{ face: faceStub.noPerson2, distance: 0.4 }, { ...faceStub.noPerson2, distance: 0.4 },
] as FaceSearchResult[]; ] as FaceSearchResult[];
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } }); systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });

View file

@ -261,7 +261,7 @@ export class PersonService extends BaseService {
return force === false return force === false
? this.assetRepository.getWithout(pagination, WithoutProperty.FACES) ? this.assetRepository.getWithout(pagination, WithoutProperty.FACES)
: this.assetRepository.getAll(pagination, { : this.assetRepository.getAll(pagination, {
orderDirection: 'DESC', orderDirection: 'desc',
withFaces: true, withFaces: true,
withArchived: true, withArchived: true,
isVisible: true, isVisible: true,
@ -288,13 +288,7 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED; return JobStatus.SKIPPED;
} }
const relations = { const relations = { exifInfo: true, faces: { person: false }, files: true };
exifInfo: true,
faces: {
person: false,
},
files: true,
};
const [asset] = await this.assetRepository.getByIds([id], relations); const [asset] = await this.assetRepository.getByIds([id], relations);
const { previewFile } = getAssetFiles(asset.files); const { previewFile } = getAssetFiles(asset.files);
if (!asset || !previewFile) { if (!asset || !previewFile) {
@ -491,7 +485,7 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED; return JobStatus.SKIPPED;
} }
let personId = matches.find((match) => match.face.personId)?.face.personId; let personId = matches.find((match) => match.personId)?.personId;
if (!personId) { if (!personId) {
const matchWithPerson = await this.searchRepository.searchFaces({ const matchWithPerson = await this.searchRepository.searchFaces({
userIds: [face.asset.ownerId], userIds: [face.asset.ownerId],
@ -502,7 +496,7 @@ export class PersonService extends BaseService {
}); });
if (matchWithPerson.length > 0) { if (matchWithPerson.length > 0) {
personId = matchWithPerson[0].face.personId; personId = matchWithPerson[0].personId;
} }
} }

View file

@ -45,11 +45,11 @@ describe(SearchService.name, () => {
it('should get assets by city and tag', async () => { it('should get assets by city and tag', async () => {
assetMock.getAssetIdByCity.mockResolvedValue({ assetMock.getAssetIdByCity.mockResolvedValue({
fieldName: 'exifInfo.city', fieldName: 'exifInfo.city',
items: [{ value: 'Paris', data: assetStub.image.id }], items: [{ value: 'test-city', data: assetStub.withLocation.id }],
}); });
assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.image, assetStub.imageFrom2015]); assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]);
const expectedResponse = [ const expectedResponse = [
{ fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] }, { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] },
]; ];
const result = await sut.getExploreData(authStub.user1); const result = await sut.getExploreData(authStub.user1);

View file

@ -34,16 +34,10 @@ export class SearchService extends BaseService {
async getExploreData(auth: AuthDto): Promise<SearchExploreItem<AssetResponseDto>[]> { async getExploreData(auth: AuthDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
const options = { maxFields: 12, minAssetsPerField: 5 }; const options = { maxFields: 12, minAssetsPerField: 5 };
const result = await this.assetRepository.getAssetIdByCity(auth.user.id, options); const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options);
const results = [result]; const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data));
const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data))); const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) }));
const assets = await this.assetRepository.getByIdsWithAllRelations([...assetIds]); return [{ fieldName: cities.fieldName, items }];
const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
return results.map(({ fieldName, items }) => ({
fieldName,
items: items.map(({ value, data }) => ({ value, data: assetMap.get(data) as AssetResponseDto })),
}));
} }
async searchMetadata(auth: AuthDto, dto: MetadataSearchDto): Promise<SearchResponseDto> { async searchMetadata(auth: AuthDto, dto: MetadataSearchDto): Promise<SearchResponseDto> {
@ -57,14 +51,13 @@ export class SearchService extends BaseService {
const page = dto.page ?? 1; const page = dto.page ?? 1;
const size = dto.size || 250; const size = dto.size || 250;
const enumToOrder = { [AssetOrder.ASC]: 'ASC', [AssetOrder.DESC]: 'DESC' } as const;
const { hasNextPage, items } = await this.searchRepository.searchMetadata( const { hasNextPage, items } = await this.searchRepository.searchMetadata(
{ page, size }, { page, size },
{ {
...dto, ...dto,
checksum, checksum,
userIds, userIds,
orderDirection: dto.order ? enumToOrder[dto.order] : 'DESC', orderDirection: dto.order ?? AssetOrder.DESC,
}, },
); );

View file

@ -61,12 +61,15 @@ describe(TimelineService.name, () => {
userId: authStub.admin.user.id, userId: authStub.admin.user.id,
}), }),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })])); ).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', { expect(assetMock.getTimeBucket).toHaveBeenCalledWith(
size: TimeBucketSize.DAY, 'bucket',
timeBucket: 'bucket', expect.objectContaining({
isArchived: true, size: TimeBucketSize.DAY,
userIds: [authStub.admin.user.id], timeBucket: 'bucket',
}); isArchived: true,
userIds: [authStub.admin.user.id],
}),
);
}); });
it('should include partner shared assets', async () => { it('should include partner shared assets', async () => {
@ -143,11 +146,14 @@ describe(TimelineService.name, () => {
userId: authStub.admin.user.id, userId: authStub.admin.user.id,
}), }),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })])); ).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(assetMock.getTimeBucket).toHaveBeenCalledWith('bucket', { expect(assetMock.getTimeBucket).toHaveBeenCalledWith(
size: TimeBucketSize.DAY, 'bucket',
timeBucket: 'bucket', expect.objectContaining({
userIds: [authStub.admin.user.id], size: TimeBucketSize.DAY,
}); timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
}),
);
}); });
it('should throw an error if withParners is true and isArchived true or undefined', async () => { it('should throw an error if withParners is true and isArchived true or undefined', async () => {

View file

@ -1,8 +1,7 @@
import _ from 'lodash'; import { Expression, RawBuilder, sql, ValueExpression } from 'kysely';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { InsertObject } from 'node_modules/kysely/dist/cjs';
import { AssetEntity } from 'src/entities/asset.entity'; import { DB } from 'src/db';
import { AssetSearchBuilderOptions } from 'src/interfaces/search.interface'; import { Between, DataSource, LessThanOrEqual, MoreThanOrEqual } from 'typeorm';
import { Between, IsNull, LessThanOrEqual, MoreThanOrEqual, Not, SelectQueryBuilder } from 'typeorm';
/** /**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual * Allows optional values unlike the regular Between and uses MoreThanOrEqual
@ -18,131 +17,54 @@ export function OptionalBetween<T>(from?: T, to?: T) {
} }
} }
export const asVector = (embedding: number[], quote = false) => export const UPSERT_COLUMNS = {} as { [T in keyof DB]: { [K in keyof DB[T]]: RawBuilder<unknown> } };
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`; /** Any repository that upserts to a table using `mapUpsertColumns` should call this method in its constructor with that table. */
export const introspectTables = (dataSource: DataSource, ...tables: (keyof DB)[]) => {
export function searchAssetBuilder( for (const table of tables) {
builder: SelectQueryBuilder<AssetEntity>, if (table in UPSERT_COLUMNS) {
options: AssetSearchBuilderOptions, continue;
): SelectQueryBuilder<AssetEntity> {
builder.andWhere(
_.omitBy(
{
createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
},
_.isUndefined,
),
);
const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
const hasExifQuery = Object.keys(exifInfo).length > 0;
if (options.withExif && !hasExifQuery) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
}
if (hasExifQuery) {
if (options.withExif) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
} else {
builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
} }
for (const [key, value] of Object.entries(exifInfo)) { const metadata = dataSource.manager.connection.getMetadata(table);
if (value === null) { UPSERT_COLUMNS[table] = Object.fromEntries(
builder.andWhere(`exifInfo.${key} IS NULL`); metadata.ownColumns.map((column) => [column.propertyName, sql<string>`excluded.${sql.ref(column.propertyName)}`]),
} else { ) as any;
builder.andWhere(`exifInfo.${key} = :${key}`, { [key]: value }); }
} };
/** Generates the columns for an upsert statement, excluding the conflict keys.
* Assumes that all entries have the same keys. */
export const mapUpsertColumns = <T extends keyof DB>(
table: T,
entry: InsertObject<DB, T>,
conflictKeys: readonly (keyof DB[T])[],
) => {
const columns = UPSERT_COLUMNS[table] as { [K in keyof DB[T]]: RawBuilder<unknown> };
const upsertColumns: Partial<Record<keyof typeof entry, RawBuilder<unknown>>> = {};
for (const entryColumn in entry) {
if (!conflictKeys.includes(entryColumn as keyof DB[T])) {
upsertColumns[entryColumn as keyof typeof entry] = columns[entryColumn as keyof DB[T]];
} }
} }
const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId']); return upsertColumns as Expand<Record<keyof typeof entry, ValueExpression<DB, T, any>>>;
};
if (id.libraryId === null) { export const asUuid = (id: string | Expression<string>) => sql<string>`${id}::uuid`;
id.libraryId = IsNull() as unknown as string;
}
builder.andWhere(_.omitBy(id, _.isUndefined)); export const anyUuid = (ids: string[]) => sql<string>`any(${`{${ids}}`}::uuid[])`;
if (options.userIds) { export const asVector = (embedding: number[]) => sql<number[]>`${`[${embedding}]`}::vector`;
builder.andWhere(`${builder.alias}.ownerId IN (:...userIds)`, { userIds: options.userIds });
}
const path = _.pick(options, ['encodedVideoPath', 'originalPath']); /**
builder.andWhere(_.omitBy(path, _.isUndefined)); * Mainly for type debugging to make VS Code display a more useful tooltip.
* Source: https://stackoverflow.com/a/69288824
*/
export type Expand<T> = T extends infer O ? { [K in keyof O]: O[K] } : never;
if (options.originalFileName) { /** Recursive version of {@link Expand} from the same source. */
builder.andWhere(`f_unaccent(${builder.alias}.originalFileName) ILIKE f_unaccent(:originalFileName)`, { export type ExpandRecursively<T> = T extends object
originalFileName: `%${options.originalFileName}%`, ? T extends infer O
}); ? { [K in keyof O]: ExpandRecursively<O[K]> }
} : never
: T;
const status = _.pick(options, ['isFavorite', 'isVisible', 'type']);
const {
isArchived,
isEncoded,
isMotion,
withArchived,
isNotInAlbum,
withFaces,
withPeople,
personIds,
withStacked,
trashedAfter,
trashedBefore,
} = options;
builder.andWhere(
_.omitBy(
{
...status,
isArchived: isArchived ?? (withArchived ? undefined : false),
encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
},
_.isUndefined,
),
);
if (isNotInAlbum) {
builder
.leftJoin(`${builder.alias}.albums`, 'albums')
.andWhere('albums.id IS NULL')
.andWhere(`${builder.alias}.isVisible = true`);
}
if (withFaces || withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
}
if (withPeople) {
builder.leftJoinAndSelect('faces.person', 'person');
}
if (personIds && personIds.length > 0) {
const cte = builder
.createQueryBuilder()
.select('faces."assetId"')
.from(AssetFaceEntity, 'faces')
.where('faces."personId" IN (:...personIds)', { personIds })
.groupBy(`faces."assetId"`)
.having(`COUNT(DISTINCT faces."personId") = :personCount`, { personCount: personIds.length });
builder.addCommonTableExpression(cte, 'face_ids').innerJoin('face_ids', 'a', 'a."assetId" = asset.id');
builder.getQuery(); // typeorm mixes up parameters without this (੭ °ཀ°)੭
}
if (withStacked) {
builder.leftJoinAndSelect(`${builder.alias}.stack`, 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
}
const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
if (withDeleted) {
builder.withDeleted();
}
return builder;
}

View file

@ -33,7 +33,10 @@ export async function* usePagination<T>(
} }
} }
function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> { export function paginationHelper<Entity extends ObjectLiteral>(
items: Entity[],
take: number,
): PaginationResult<Entity> {
const hasNextPage = items.length > take; const hasNextPage = items.length > take;
items.splice(take); items.splice(take);

View file

@ -1,3 +1,5 @@
import { PostgresJSDialect } from 'kysely-postgres-js';
import postgres from 'postgres';
import { ImmichEnvironment, ImmichWorker } from 'src/enum'; import { ImmichEnvironment, ImmichWorker } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface';
@ -21,16 +23,24 @@ const envData: EnvData = {
database: { database: {
config: { config: {
connectionType: 'parts', kysely: {
database: 'immich', dialect: new PostgresJSDialect({
type: 'postgres', postgres: postgres({ database: 'immich', host: 'database', port: 5432 }),
host: 'database', }),
port: 5432, log: ['error'],
username: 'postgres', },
password: 'postgres', typeorm: {
name: 'immich', connectionType: 'parts',
synchronize: false, database: 'immich',
migrationsRun: true, type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
synchronize: false,
migrationsRun: true,
},
}, },
skipMigrations: false, skipMigrations: false,

View file

@ -19,7 +19,8 @@
"preserveWatchOutput": true, "preserveWatchOutput": true,
"baseUrl": "./", "baseUrl": "./",
"jsx": "react", "jsx": "react",
"types": ["vitest/globals"] "types": ["vitest/globals"],
"noErrorTruncation": true
}, },
"exclude": ["dist", "node_modules", "upload"] "exclude": ["dist", "node_modules", "upload"]
} }