mirror of
https://github.com/immich-app/immich.git
synced 2024-12-29 15:11:58 +00:00
feat: storage template file move hardening (#5917)
* fix: pgvecto.rs extension breaks typeorm schema:drop command * fix: parse postgres bigints to javascript number types when selecting data * feat: verify file size is the same as original asset after copying file for storage template job * feat: allow disabling of storage template job, defaults to disabled for new instances * fix: don't allow setting concurrency for storage template migration, can cause race conditions above 1 * feat: add checksum verification when file is copied for storage template job * fix: extract metadata for assets that aren't visible on timeline
This commit is contained in:
Zack Pollard
GitHub
parent
5f6bd4ae7e
commit
2e38fa73bf
36 changed files with 656 additions and 209 deletions
3
.github/workflows/test.yml
vendored
3
.github/workflows/test.yml
vendored
|
|
@ -269,6 +269,9 @@ jobs:
|
|||
- name: Run existing migrations
|
||||
run: npm run typeorm:migrations:run
|
||||
|
||||
- name: Test npm run schema:reset command works
|
||||
run: npm run typeorm:schema:reset
|
||||
|
||||
- name: Generate new migrations
|
||||
continue-on-error: true
|
||||
run: npm run typeorm:migrations:generate ./src/infra/migrations/TestMigration
|
||||
|
|
|
|||
18
cli/src/api/open-api/api.ts
generated
18
cli/src/api/open-api/api.ts
generated
|
|
@ -3780,12 +3780,6 @@ export interface SystemConfigJobDto {
|
|||
* @memberof SystemConfigJobDto
|
||||
*/
|
||||
'smartSearch': JobSettingsDto;
|
||||
/**
|
||||
*
|
||||
* @type {JobSettingsDto}
|
||||
* @memberof SystemConfigJobDto
|
||||
*/
|
||||
'storageTemplateMigration': JobSettingsDto;
|
||||
/**
|
||||
*
|
||||
* @type {JobSettingsDto}
|
||||
|
|
@ -4026,6 +4020,18 @@ export interface SystemConfigReverseGeocodingDto {
|
|||
* @interface SystemConfigStorageTemplateDto
|
||||
*/
|
||||
export interface SystemConfigStorageTemplateDto {
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
* @memberof SystemConfigStorageTemplateDto
|
||||
*/
|
||||
'enabled': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
* @memberof SystemConfigStorageTemplateDto
|
||||
*/
|
||||
'hashVerificationEnabled': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {string}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ x-server-build: &server-common
|
|||
volumes:
|
||||
- ../server:/usr/src/app
|
||||
- ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
|
||||
- ${UPLOAD_LOCATION}/photos/upload:/usr/src/app/upload/upload
|
||||
- /usr/src/app/node_modules
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
env_file:
|
||||
|
|
|
|||
BIN
mobile/openapi/doc/SystemConfigJobDto.md
generated
BIN
mobile/openapi/doc/SystemConfigJobDto.md
generated
Binary file not shown.
BIN
mobile/openapi/doc/SystemConfigStorageTemplateDto.md
generated
BIN
mobile/openapi/doc/SystemConfigStorageTemplateDto.md
generated
Binary file not shown.
BIN
mobile/openapi/lib/model/system_config_job_dto.dart
generated
BIN
mobile/openapi/lib/model/system_config_job_dto.dart
generated
Binary file not shown.
Binary file not shown.
BIN
mobile/openapi/test/system_config_job_dto_test.dart
generated
BIN
mobile/openapi/test/system_config_job_dto_test.dart
generated
Binary file not shown.
Binary file not shown.
|
|
@ -9171,9 +9171,6 @@
|
|||
"smartSearch": {
|
||||
"$ref": "#/components/schemas/JobSettingsDto"
|
||||
},
|
||||
"storageTemplateMigration": {
|
||||
"$ref": "#/components/schemas/JobSettingsDto"
|
||||
},
|
||||
"thumbnailGeneration": {
|
||||
"$ref": "#/components/schemas/JobSettingsDto"
|
||||
},
|
||||
|
|
@ -9186,7 +9183,6 @@
|
|||
"metadataExtraction",
|
||||
"videoConversion",
|
||||
"smartSearch",
|
||||
"storageTemplateMigration",
|
||||
"migration",
|
||||
"backgroundTask",
|
||||
"search",
|
||||
|
|
@ -9365,11 +9361,19 @@
|
|||
},
|
||||
"SystemConfigStorageTemplateDto": {
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"hashVerificationEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"template": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"enabled",
|
||||
"hashVerificationEnabled",
|
||||
"template"
|
||||
],
|
||||
"type": "object"
|
||||
|
|
|
|||
98
server/package-lock.json
generated
98
server/package-lock.json
generated
|
|
@ -41,7 +41,6 @@
|
|||
"joi": "^17.10.0",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.4.2",
|
||||
"mv": "^2.1.1",
|
||||
"nest-commander": "^3.11.1",
|
||||
"node-addon-api": "^7.0.0",
|
||||
"openid-client": "^5.4.3",
|
||||
|
|
@ -72,7 +71,6 @@
|
|||
"@types/lodash": "^4.14.197",
|
||||
"@types/mock-fs": "^4.13.1",
|
||||
"@types/multer": "^1.4.7",
|
||||
"@types/mv": "^2.1.2",
|
||||
"@types/node": "^20.5.7",
|
||||
"@types/sharp": "^0.31.1",
|
||||
"@types/supertest": "^2.0.12",
|
||||
|
|
@ -3548,12 +3546,6 @@
|
|||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/mv": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
|
||||
"integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
|
||||
|
|
@ -9293,45 +9285,6 @@
|
|||
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
|
||||
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
|
||||
},
|
||||
"node_modules/mv": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
|
||||
"integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
|
||||
"dependencies": {
|
||||
"mkdirp": "~0.5.1",
|
||||
"ncp": "~2.0.0",
|
||||
"rimraf": "~2.4.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mv/node_modules/glob": {
|
||||
"version": "6.0.4",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
|
||||
"integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
|
||||
"dependencies": {
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "2 || 3",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/mv/node_modules/rimraf": {
|
||||
"version": "2.4.5",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
|
||||
"integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
|
||||
"dependencies": {
|
||||
"glob": "^6.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"rimraf": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
|
|
@ -9355,14 +9308,6 @@
|
|||
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ncp": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
|
||||
"integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==",
|
||||
"bin": {
|
||||
"ncp": "bin/ncp"
|
||||
}
|
||||
},
|
||||
"node_modules/nearley": {
|
||||
"version": "2.20.1",
|
||||
"resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",
|
||||
|
|
@ -15519,12 +15464,6 @@
|
|||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"@types/mv": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
|
||||
"integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "20.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
|
||||
|
|
@ -19842,38 +19781,6 @@
|
|||
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
|
||||
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
|
||||
},
|
||||
"mv": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
|
||||
"integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
|
||||
"requires": {
|
||||
"mkdirp": "~0.5.1",
|
||||
"ncp": "~2.0.0",
|
||||
"rimraf": "~2.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"glob": {
|
||||
"version": "6.0.4",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
|
||||
"integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
|
||||
"requires": {
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "2 || 3",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.4.5",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
|
||||
"integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
|
||||
"requires": {
|
||||
"glob": "^6.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
|
|
@ -19897,11 +19804,6 @@
|
|||
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
|
||||
"dev": true
|
||||
},
|
||||
"ncp": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
|
||||
"integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA=="
|
||||
},
|
||||
"nearley": {
|
||||
"version": "2.20.1",
|
||||
"resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
"typeorm:migrations:generate": "typeorm migration:generate -d ./dist/infra/database.config.js",
|
||||
"typeorm:migrations:run": "typeorm migration:run -d ./dist/infra/database.config.js",
|
||||
"typeorm:migrations:revert": "typeorm migration:revert -d ./dist/infra/database.config.js",
|
||||
"typeorm:schema:drop": "typeorm schema:drop -d ./dist/infra/database.config.js",
|
||||
"typeorm:schema:drop": "typeorm query -d ./dist/infra/database.config.js 'DROP schema public cascade; CREATE schema public;'",
|
||||
"typeorm:schema:reset": "npm run typeorm:schema:drop && npm run typeorm:migrations:run",
|
||||
"api:typescript": "bash ./bin/generate-open-api.sh web",
|
||||
"api:dart": "bash ./bin/generate-open-api.sh mobile",
|
||||
|
|
@ -68,7 +68,6 @@
|
|||
"joi": "^17.10.0",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.4.2",
|
||||
"mv": "^2.1.1",
|
||||
"nest-commander": "^3.11.1",
|
||||
"node-addon-api": "^7.0.0",
|
||||
"openid-client": "^5.4.3",
|
||||
|
|
@ -99,7 +98,6 @@
|
|||
"@types/lodash": "^4.14.197",
|
||||
"@types/mock-fs": "^4.13.1",
|
||||
"@types/multer": "^1.4.7",
|
||||
"@types/mv": "^2.1.2",
|
||||
"@types/node": "^20.5.7",
|
||||
"@types/sharp": "^0.31.1",
|
||||
"@types/supertest": "^2.0.12",
|
||||
|
|
|
|||
|
|
@ -226,7 +226,6 @@ describe(JobService.name, () => {
|
|||
[QueueName.SEARCH]: { concurrency: 10 },
|
||||
[QueueName.SIDECAR]: { concurrency: 10 },
|
||||
[QueueName.LIBRARY]: { concurrency: 10 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
|
||||
[QueueName.MIGRATION]: { concurrency: 10 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
|
||||
|
|
@ -239,7 +238,6 @@ describe(JobService.name, () => {
|
|||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
|
||||
|
|
|
|||
|
|
@ -123,7 +123,11 @@ export class JobService {
|
|||
async registerHandlers(jobHandlers: Record<JobName, JobHandler>) {
|
||||
const config = await this.configCore.getConfig();
|
||||
for (const queueName of Object.values(QueueName)) {
|
||||
const concurrency = config.job[queueName].concurrency;
|
||||
let concurrency = 1;
|
||||
if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
|
||||
concurrency = config.job[queueName].concurrency;
|
||||
}
|
||||
|
||||
this.logger.debug(`Registering ${queueName} with a concurrency of ${concurrency}`);
|
||||
this.jobRepository.addHandler(queueName, concurrency, async (item: JobItem): Promise<void> => {
|
||||
const { name, data } = item;
|
||||
|
|
@ -143,7 +147,10 @@ export class JobService {
|
|||
this.configCore.config$.subscribe((config) => {
|
||||
this.logger.log(`Updating queue concurrency settings`);
|
||||
for (const queueName of Object.values(QueueName)) {
|
||||
const concurrency = config.job[queueName].concurrency;
|
||||
let concurrency = 1;
|
||||
if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
|
||||
concurrency = config.job[queueName].concurrency;
|
||||
}
|
||||
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
|
||||
this.jobRepository.setConcurrency(queueName, concurrency);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
assetStub,
|
||||
faceStub,
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newMediaRepositoryMock,
|
||||
newMoveRepositoryMock,
|
||||
|
|
@ -24,6 +25,7 @@ import {
|
|||
import { JobName } from '../job';
|
||||
import {
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IJobRepository,
|
||||
IMediaRepository,
|
||||
IMoveRepository,
|
||||
|
|
@ -43,6 +45,7 @@ describe(MediaService.name, () => {
|
|||
let moveMock: jest.Mocked<IMoveRepository>;
|
||||
let personMock: jest.Mocked<IPersonRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
|
||||
beforeEach(async () => {
|
||||
assetMock = newAssetRepositoryMock();
|
||||
|
|
@ -52,8 +55,9 @@ describe(MediaService.name, () => {
|
|||
moveMock = newMoveRepositoryMock();
|
||||
personMock = newPersonRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
|
||||
sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock);
|
||||
sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock, cryptoMock);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName }
|
|||
import {
|
||||
AudioStreamInfo,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IJobRepository,
|
||||
IMediaRepository,
|
||||
IMoveRepository,
|
||||
|
|
@ -52,9 +53,17 @@ export class MediaService {
|
|||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
|
||||
@Inject(IMoveRepository) moveRepository: IMoveRepository,
|
||||
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
||||
) {
|
||||
this.configCore = SystemConfigCore.create(configRepository);
|
||||
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
|
||||
this.storageCore = StorageCore.create(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
personRepository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
storageRepository,
|
||||
);
|
||||
}
|
||||
|
||||
async handleQueueGenerateThumbnails({ force }: IBaseJob) {
|
||||
|
|
|
|||
|
|
@ -239,15 +239,6 @@ describe(MetadataService.name, () => {
|
|||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle an asset with isVisible set to false', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, isVisible: false }]);
|
||||
|
||||
await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(false);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.upsertExif).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle a date in a sidecar file', async () => {
|
||||
const originalDate = new Date('2023-11-21T16:13:17.517Z');
|
||||
const sidecarDate = new Date('2022-01-01T00:00:00.000Z');
|
||||
|
|
|
|||
|
|
@ -114,7 +114,14 @@ export class MetadataService {
|
|||
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
|
||||
) {
|
||||
this.configCore = SystemConfigCore.create(configRepository);
|
||||
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
|
||||
this.storageCore = StorageCore.create(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
personRepository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
storageRepository,
|
||||
);
|
||||
}
|
||||
|
||||
async init() {
|
||||
|
|
@ -199,7 +206,7 @@ export class MetadataService {
|
|||
|
||||
async handleMetadataExtraction({ id }: IEntityJob) {
|
||||
const [asset] = await this.assetRepository.getByIds([id]);
|
||||
if (!asset || !asset.isVisible) {
|
||||
if (!asset) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
faceStub,
|
||||
newAccessRepositoryMock,
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newMachineLearningRepositoryMock,
|
||||
newMediaRepositoryMock,
|
||||
|
|
@ -22,6 +23,7 @@ import { CacheControl, ImmichFileResponse } from '../domain.util';
|
|||
import { JobName } from '../job';
|
||||
import {
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IJobRepository,
|
||||
IMachineLearningRepository,
|
||||
IMediaRepository,
|
||||
|
|
@ -73,6 +75,7 @@ describe(PersonService.name, () => {
|
|||
let personMock: jest.Mocked<IPersonRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
let sut: PersonService;
|
||||
|
||||
beforeEach(async () => {
|
||||
|
|
@ -86,6 +89,7 @@ describe(PersonService.name, () => {
|
|||
personMock = newPersonRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
smartInfoMock = newSmartInfoRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
sut = new PersonService(
|
||||
accessMock,
|
||||
assetMock,
|
||||
|
|
@ -97,6 +101,7 @@ describe(PersonService.name, () => {
|
|||
storageMock,
|
||||
jobMock,
|
||||
smartInfoMock,
|
||||
cryptoMock,
|
||||
);
|
||||
|
||||
mediaMock.crop.mockResolvedValue(croppedFace);
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
CropOptions,
|
||||
IAccessRepository,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IJobRepository,
|
||||
IMachineLearningRepository,
|
||||
IMediaRepository,
|
||||
|
|
@ -59,10 +60,18 @@ export class PersonService {
|
|||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
|
||||
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
||||
) {
|
||||
this.access = AccessCore.create(accessRepository);
|
||||
this.configCore = SystemConfigCore.create(configRepository);
|
||||
this.storageCore = StorageCore.create(assetRepository, moveRepository, repository, storageRepository);
|
||||
this.storageCore = StorageCore.create(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
repository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
storageRepository,
|
||||
);
|
||||
}
|
||||
|
||||
async getAll(auth: AuthDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
|
||||
|
|
|
|||
|
|
@ -30,11 +30,12 @@ export interface IStorageRepository {
|
|||
unlink(filepath: string): Promise<void>;
|
||||
unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
|
||||
removeEmptyDirs(folder: string, self?: boolean): Promise<void>;
|
||||
moveFile(source: string, target: string): Promise<void>;
|
||||
checkFileExists(filepath: string, mode?: number): Promise<boolean>;
|
||||
mkdirSync(filepath: string): void;
|
||||
checkDiskUsage(folder: string): Promise<DiskUsage>;
|
||||
readdir(folder: string): Promise<string[]>;
|
||||
stat(filepath: string): Promise<Stats>;
|
||||
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
|
||||
copyFile(source: string, target: string): Promise<void>;
|
||||
rename(source: string, target: string): Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,21 @@
|
|||
import { AssetPathType } from '@app/infra/entities';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
IUserRepository,
|
||||
StorageTemplateService,
|
||||
defaults,
|
||||
} from '@app/domain';
|
||||
import { AssetPathType, SystemConfigKey } from '@app/infra/entities';
|
||||
import {
|
||||
assetStub,
|
||||
newAlbumRepositoryMock,
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newMoveRepositoryMock,
|
||||
newPersonRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
|
|
@ -11,17 +24,7 @@ import {
|
|||
userStub,
|
||||
} from '@test';
|
||||
import { when } from 'jest-when';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
IUserRepository,
|
||||
} from '../repositories';
|
||||
import { defaults } from '../system-config/system-config.core';
|
||||
import { StorageTemplateService } from './storage-template.service';
|
||||
import { Stats } from 'node:fs';
|
||||
|
||||
describe(StorageTemplateService.name, () => {
|
||||
let sut: StorageTemplateService;
|
||||
|
|
@ -32,19 +35,21 @@ describe(StorageTemplateService.name, () => {
|
|||
let personMock: jest.Mocked<IPersonRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
let userMock: jest.Mocked<IUserRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
albumMock = newAlbumRepositoryMock();
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
moveMock = newMoveRepositoryMock();
|
||||
personMock = newPersonRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
userMock = newUserRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
|
||||
sut = new StorageTemplateService(
|
||||
albumMock,
|
||||
|
|
@ -55,27 +60,37 @@ describe(StorageTemplateService.name, () => {
|
|||
personMock,
|
||||
storageMock,
|
||||
userMock,
|
||||
cryptoMock,
|
||||
);
|
||||
|
||||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: true }]);
|
||||
});
|
||||
|
||||
describe('handleMigrationSingle', () => {
|
||||
it('should skip when storage template is disabled', async () => {
|
||||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: false }]);
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
expect(assetMock.getByIds).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
expect(moveMock.create).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).not.toHaveBeenCalled();
|
||||
expect(storageMock.stat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should migrate single moving picture', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const path = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}.jpg`;
|
||||
const newPath = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}+1.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoStillAsset.id)).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoStillAsset.id)).mockResolvedValue(false);
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(false);
|
||||
const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`;
|
||||
const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpeg`;
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newPath(assetStub.livePhotoStillAsset.id) })
|
||||
.calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newStillPicturePath })
|
||||
.mockResolvedValue(assetStub.livePhotoStillAsset);
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newPath(assetStub.livePhotoMotionAsset.id) })
|
||||
.calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newMotionPicturePath })
|
||||
.mockResolvedValue(assetStub.livePhotoMotionAsset);
|
||||
|
||||
when(assetMock.getByIds)
|
||||
|
|
@ -86,11 +101,265 @@ describe(StorageTemplateService.name, () => {
|
|||
.calledWith([assetStub.livePhotoMotionAsset.id])
|
||||
.mockResolvedValue([assetStub.livePhotoMotionAsset]);
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.livePhotoStillAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoStillAsset.originalPath,
|
||||
newPath: newStillPicturePath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.livePhotoStillAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoStillAsset.originalPath,
|
||||
newPath: newStillPicturePath,
|
||||
});
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.livePhotoMotionAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoMotionAsset.originalPath,
|
||||
newPath: newMotionPicturePath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '124',
|
||||
entityId: assetStub.livePhotoMotionAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoMotionAsset.originalPath,
|
||||
newPath: newMotionPicturePath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.livePhotoStillAsset.id,
|
||||
originalPath: newStillPicturePath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.livePhotoMotionAsset.id,
|
||||
originalPath: newMotionPicturePath,
|
||||
});
|
||||
});
|
||||
it('should migrate previously failed move from original path when it still exists', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(false);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(moveMock.update).toHaveBeenCalledWith({
|
||||
id: '123',
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
|
||||
when(storageMock.stat)
|
||||
.calledWith(previousFailedNewPath)
|
||||
.mockResolvedValue({ size: 5000 } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(assetStub.image.checksum);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).toHaveBeenCalledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
|
||||
it('should fail move if copying and hash of asset and the new file do not match', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.rename).calledWith(assetStub.image.originalPath, newPath).mockRejectedValue({ code: 'EXDEV' });
|
||||
when(storageMock.stat)
|
||||
.calledWith(newPath)
|
||||
.mockResolvedValue({ size: 5000 } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(Buffer.from('different-hash', 'utf-8'));
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(newPath);
|
||||
expect(moveMock.create).toHaveBeenCalledWith({
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: newPath,
|
||||
});
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledWith(newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each`
|
||||
failedPathChecksum | failedPathSize | reason
|
||||
${assetStub.image.checksum} | ${500} | ${'file size'}
|
||||
${Buffer.from('bad checksum', 'utf-8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'}
|
||||
`(
|
||||
'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',
|
||||
async ({ failedPathChecksum, failedPathSize }) => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
|
||||
when(storageMock.stat)
|
||||
.calledWith(previousFailedNewPath)
|
||||
.mockResolvedValue({ size: failedPathSize } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(failedPathChecksum);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('handle template migration', () => {
|
||||
|
|
@ -155,7 +424,8 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -175,7 +445,8 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -198,7 +469,7 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
|
|
@ -226,7 +497,7 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/label-1/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
|
|
@ -236,12 +507,84 @@ describe(StorageTemplateService.name, () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {
|
||||
const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg';
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
|
||||
userMock.getList.mockResolvedValue([userStub.user1]);
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
when(storageMock.stat)
|
||||
.calledWith(newPath)
|
||||
.mockResolvedValue({
|
||||
size: 5000,
|
||||
} as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(assetStub.image.checksum);
|
||||
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.rename).toHaveBeenCalledWith('/original/path.jpg', newPath);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledWith(assetStub.image.originalPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
|
||||
it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
|
||||
userMock.getList.mockResolvedValue([userStub.user1]);
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
});
|
||||
when(storageMock.stat)
|
||||
.calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg')
|
||||
.mockResolvedValue({
|
||||
size: 100,
|
||||
} as Stats);
|
||||
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not update the database if the move fails', async () => {
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
|
||||
storageMock.rename.mockRejectedValue(new Error('Read only system'));
|
||||
storageMock.copyFile.mockRejectedValue(new Error('Read only system'));
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: 'move-123',
|
||||
entityId: '123',
|
||||
|
|
@ -254,7 +597,7 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
|
|
@ -278,7 +621,8 @@ describe(StorageTemplateService.name, () => {
|
|||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
|
|||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
|
|
@ -61,6 +62,7 @@ export class StorageTemplateService {
|
|||
@Inject(IPersonRepository) personRepository: IPersonRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(IUserRepository) private userRepository: IUserRepository,
|
||||
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
||||
) {
|
||||
this.template = this.compile(config.storageTemplate.template);
|
||||
this.configCore = SystemConfigCore.create(configRepository);
|
||||
|
|
@ -70,10 +72,22 @@ export class StorageTemplateService {
|
|||
this.logger.debug(`Received config, compiling storage template: ${template}`);
|
||||
this.template = this.compile(template);
|
||||
});
|
||||
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
|
||||
this.storageCore = StorageCore.create(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
personRepository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
storageRepository,
|
||||
);
|
||||
}
|
||||
|
||||
async handleMigrationSingle({ id }: IEntityJob) {
|
||||
const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
|
||||
if (!storageTemplateEnabled) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const [asset] = await this.assetRepository.getByIds([id]);
|
||||
|
||||
const user = await this.userRepository.get(asset.ownerId, {});
|
||||
|
|
@ -93,6 +107,11 @@ export class StorageTemplateService {
|
|||
|
||||
async handleMigration() {
|
||||
this.logger.log('Starting storage template migration');
|
||||
const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
|
||||
if (!storageTemplateEnabled) {
|
||||
this.logger.log('Storage template migration disabled, skipping');
|
||||
return true;
|
||||
}
|
||||
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
||||
this.assetRepository.getAll(pagination),
|
||||
);
|
||||
|
|
@ -123,12 +142,23 @@ export class StorageTemplateService {
|
|||
return;
|
||||
}
|
||||
|
||||
const { id, sidecarPath, originalPath } = asset;
|
||||
const { id, sidecarPath, originalPath, exifInfo } = asset;
|
||||
const oldPath = originalPath;
|
||||
const newPath = await this.getTemplatePath(asset, metadata);
|
||||
|
||||
if (!exifInfo || !exifInfo.fileSizeInByte) {
|
||||
this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath });
|
||||
await this.storageCore.moveFile({
|
||||
entityId: id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath,
|
||||
newPath,
|
||||
assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum: asset.checksum },
|
||||
});
|
||||
if (sidecarPath) {
|
||||
await this.storageCore.moveFile({
|
||||
entityId: id,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,16 @@
|
|||
import { SystemConfigCore } from '@app/domain/system-config';
|
||||
import { AssetEntity, AssetPathType, PathType, PersonEntity, PersonPathType } from '@app/infra/entities';
|
||||
import { ImmichLogger } from '@app/infra/logger';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { APP_MEDIA_LOCATION } from '../domain.constant';
|
||||
import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories';
|
||||
import {
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
} from '../repositories';
|
||||
|
||||
export enum StorageFolder {
|
||||
ENCODED_VIDEO = 'encoded-video',
|
||||
|
|
@ -17,6 +25,10 @@ export interface MoveRequest {
|
|||
pathType: PathType;
|
||||
oldPath: string | null;
|
||||
newPath: string;
|
||||
assetInfo?: {
|
||||
sizeInBytes: number;
|
||||
checksum: Buffer;
|
||||
};
|
||||
}
|
||||
|
||||
type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO;
|
||||
|
|
@ -25,22 +37,35 @@ let instance: StorageCore | null;
|
|||
|
||||
export class StorageCore {
|
||||
private logger = new ImmichLogger(StorageCore.name);
|
||||
|
||||
private configCore;
|
||||
private constructor(
|
||||
private assetRepository: IAssetRepository,
|
||||
private moveRepository: IMoveRepository,
|
||||
private personRepository: IPersonRepository,
|
||||
private cryptoRepository: ICryptoRepository,
|
||||
private systemConfigRepository: ISystemConfigRepository,
|
||||
private repository: IStorageRepository,
|
||||
) {}
|
||||
) {
|
||||
this.configCore = SystemConfigCore.create(systemConfigRepository);
|
||||
}
|
||||
|
||||
static create(
|
||||
assetRepository: IAssetRepository,
|
||||
moveRepository: IMoveRepository,
|
||||
personRepository: IPersonRepository,
|
||||
cryptoRepository: ICryptoRepository,
|
||||
configRepository: ISystemConfigRepository,
|
||||
repository: IStorageRepository,
|
||||
) {
|
||||
if (!instance) {
|
||||
instance = new StorageCore(assetRepository, moveRepository, personRepository, repository);
|
||||
instance = new StorageCore(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
personRepository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
repository,
|
||||
);
|
||||
}
|
||||
|
||||
return instance;
|
||||
|
|
@ -131,7 +156,7 @@ export class StorageCore {
|
|||
}
|
||||
|
||||
async moveFile(request: MoveRequest) {
|
||||
const { entityId, pathType, oldPath, newPath } = request;
|
||||
const { entityId, pathType, oldPath, newPath, assetInfo } = request;
|
||||
if (!oldPath || oldPath === newPath) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -143,26 +168,94 @@ export class StorageCore {
|
|||
this.logger.log(`Attempting to finish incomplete move: ${move.oldPath} => ${move.newPath}`);
|
||||
const oldPathExists = await this.repository.checkFileExists(move.oldPath);
|
||||
const newPathExists = await this.repository.checkFileExists(move.newPath);
|
||||
const actualPath = newPathExists ? move.newPath : oldPathExists ? move.oldPath : null;
|
||||
const actualPath = oldPathExists ? move.oldPath : newPathExists ? move.newPath : null;
|
||||
if (!actualPath) {
|
||||
this.logger.warn('Unable to complete move. File does not exist at either location.');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(`Found file at ${actualPath === move.oldPath ? 'old' : 'new'} location`);
|
||||
const fileAtNewLocation = actualPath === move.newPath;
|
||||
this.logger.log(`Found file at ${fileAtNewLocation ? 'new' : 'old'} location`);
|
||||
|
||||
if (fileAtNewLocation) {
|
||||
if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, move.newPath, assetInfo))) {
|
||||
this.logger.fatal(
|
||||
`Skipping move as file verification failed, old file is missing and new file is different to what was expected`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
move = await this.moveRepository.update({ id: move.id, oldPath: actualPath, newPath });
|
||||
} else {
|
||||
move = await this.moveRepository.create({ entityId, pathType, oldPath, newPath });
|
||||
}
|
||||
|
||||
if (move.oldPath !== newPath) {
|
||||
await this.repository.moveFile(move.oldPath, newPath);
|
||||
if (pathType === AssetPathType.ORIGINAL && !assetInfo) {
|
||||
this.logger.warn(`Unable to complete move. Missing asset info for ${entityId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (move.oldPath !== newPath) {
|
||||
try {
|
||||
this.logger.debug(`Attempting to rename file: ${move.oldPath} => ${newPath}`);
|
||||
await this.repository.rename(move.oldPath, newPath);
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'EXDEV') {
|
||||
this.logger.warn(
|
||||
`Unable to complete move. Error renaming file with code ${err.code} and message: ${err.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.logger.debug(`Unable to rename file. Falling back to copy, verify and delete`);
|
||||
await this.repository.copyFile(move.oldPath, newPath);
|
||||
|
||||
if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, newPath, assetInfo))) {
|
||||
this.logger.warn(`Skipping move due to file size mismatch`);
|
||||
await this.repository.unlink(newPath);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.repository.unlink(move.oldPath);
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Unable to delete old file, it will now no longer be tracked by Immich: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.savePath(pathType, entityId, newPath);
|
||||
await this.moveRepository.delete(move);
|
||||
}
|
||||
|
||||
private async verifyNewPathContentsMatchesExpected(
|
||||
oldPath: string,
|
||||
newPath: string,
|
||||
assetInfo?: { sizeInBytes: number; checksum: Buffer },
|
||||
) {
|
||||
const oldPathSize = assetInfo ? assetInfo.sizeInBytes : (await this.repository.stat(oldPath)).size;
|
||||
const newPathSize = (await this.repository.stat(newPath)).size;
|
||||
this.logger.debug(`File size check: ${newPathSize} === ${oldPathSize}`);
|
||||
if (newPathSize !== oldPathSize) {
|
||||
this.logger.warn(`Unable to complete move. File size mismatch: ${newPathSize} !== ${oldPathSize}`);
|
||||
return false;
|
||||
}
|
||||
if (assetInfo && (await this.configCore.getConfig()).storageTemplate.hashVerificationEnabled) {
|
||||
const { checksum } = assetInfo;
|
||||
const newChecksum = await this.cryptoRepository.hashFile(newPath);
|
||||
if (!newChecksum.equals(checksum)) {
|
||||
this.logger.warn(
|
||||
`Unable to complete move. File checksum mismatch: ${newChecksum.toString('base64')} !== ${checksum.toString(
|
||||
'base64',
|
||||
)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
this.logger.debug(`File checksum check: ${newChecksum.toString('base64')} === ${checksum.toString('base64')}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
ensureFolders(input: string) {
|
||||
this.repository.mkdirSync(dirname(input));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@ export class JobSettingsDto {
|
|||
concurrency!: number;
|
||||
}
|
||||
|
||||
export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
|
||||
export class SystemConfigJobDto
|
||||
implements Record<Exclude<QueueName, QueueName.STORAGE_TEMPLATE_MIGRATION>, JobSettingsDto>
|
||||
{
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
|
|
@ -35,12 +37,6 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
|
|||
@Type(() => JobSettingsDto)
|
||||
[QueueName.SMART_SEARCH]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import { IsNotEmpty, IsString } from 'class-validator';
|
||||
import { IsBoolean, IsNotEmpty, IsString } from 'class-validator';
|
||||
|
||||
export class SystemConfigStorageTemplateDto {
|
||||
@IsBoolean()
|
||||
enabled!: boolean;
|
||||
@IsBoolean()
|
||||
hashVerificationEnabled!: boolean;
|
||||
@IsNotEmpty()
|
||||
@IsString()
|
||||
template!: string;
|
||||
|
|
|
|||
|
|
@ -53,7 +53,6 @@ export const defaults = Object.freeze<SystemConfig>({
|
|||
[QueueName.SEARCH]: { concurrency: 5 },
|
||||
[QueueName.SIDECAR]: { concurrency: 5 },
|
||||
[QueueName.LIBRARY]: { concurrency: 5 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
|
||||
|
|
@ -102,6 +101,8 @@ export const defaults = Object.freeze<SystemConfig>({
|
|||
enabled: true,
|
||||
},
|
||||
storageTemplate: {
|
||||
enabled: false,
|
||||
hashVerificationEnabled: true,
|
||||
template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
|
||||
},
|
||||
thumbnail: {
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
|||
[QueueName.SEARCH]: { concurrency: 5 },
|
||||
[QueueName.SIDECAR]: { concurrency: 5 },
|
||||
[QueueName.LIBRARY]: { concurrency: 5 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
|
||||
|
|
@ -102,6 +101,8 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
|||
enabled: true,
|
||||
},
|
||||
storageTemplate: {
|
||||
enabled: false,
|
||||
hashVerificationEnabled: true,
|
||||
template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
|
||||
},
|
||||
thumbnail: {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export const databaseConfig: PostgresConnectionOptions = {
|
|||
subscribers: [__dirname + '/subscribers/*.{js,ts}'],
|
||||
migrationsRun: false,
|
||||
connectTimeoutMS: 10000, // 10 seconds
|
||||
parseInt8: true,
|
||||
...urlOrParts,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,8 @@ export enum SystemConfigKey {
|
|||
|
||||
PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled',
|
||||
|
||||
STORAGE_TEMPLATE_ENABLED = 'storageTemplate.enabled',
|
||||
STORAGE_TEMPLATE_HASH_VERIFICATION_ENABLED = 'storageTemplate.hashVerificationEnabled',
|
||||
STORAGE_TEMPLATE = 'storageTemplate.template',
|
||||
|
||||
THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize',
|
||||
|
|
@ -171,7 +173,7 @@ export interface SystemConfig {
|
|||
accel: TranscodeHWAccel;
|
||||
tonemap: ToneMapping;
|
||||
};
|
||||
job: Record<QueueName, { concurrency: number }>;
|
||||
job: Record<Exclude<QueueName, QueueName.STORAGE_TEMPLATE_MIGRATION>, { concurrency: number }>;
|
||||
logging: {
|
||||
enabled: boolean;
|
||||
level: LogLevel;
|
||||
|
|
@ -216,6 +218,8 @@ export interface SystemConfig {
|
|||
enabled: boolean;
|
||||
};
|
||||
storageTemplate: {
|
||||
enabled: boolean;
|
||||
hashVerificationEnabled: boolean;
|
||||
template: string;
|
||||
};
|
||||
thumbnail: {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
import { MigrationInterface, QueryRunner } from "typeorm"
|
||||
|
||||
export class DefaultStorageTemplateOnForExistingInstallations1703288449127 implements MigrationInterface {
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
const adminCount = await queryRunner.query(`SELECT COUNT(*) FROM users WHERE "isAdmin" = true`)
|
||||
if(adminCount[0].count > 0) {
|
||||
await queryRunner.query(`INSERT INTO system_config (key, value) VALUES ('storageTemplate.enabled', 'true')`)
|
||||
}
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DELETE FROM system_config WHERE key = 'storageTemplate.enabled'`)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -9,14 +9,10 @@ import {
|
|||
import { ImmichLogger } from '@app/infra/logger';
|
||||
import archiver from 'archiver';
|
||||
import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
|
||||
import fs, { readdir, writeFile } from 'fs/promises';
|
||||
import fs, { copyFile, readdir, rename, writeFile } from 'fs/promises';
|
||||
import { glob } from 'glob';
|
||||
import mv from 'mv';
|
||||
import { promisify } from 'node:util';
|
||||
import path from 'path';
|
||||
|
||||
const moveFile = promisify<string, string, mv.Options>(mv);
|
||||
|
||||
export class FilesystemProvider implements IStorageRepository {
|
||||
private logger = new ImmichLogger(FilesystemProvider.name);
|
||||
|
||||
|
|
@ -54,15 +50,9 @@ export class FilesystemProvider implements IStorageRepository {
|
|||
|
||||
writeFile = writeFile;
|
||||
|
||||
async moveFile(source: string, destination: string): Promise<void> {
|
||||
this.logger.verbose(`Moving ${source} to ${destination}`);
|
||||
rename = rename;
|
||||
|
||||
if (await this.checkFileExists(destination)) {
|
||||
throw new Error(`Destination file already exists: ${destination}`);
|
||||
}
|
||||
|
||||
await moveFile(source, destination, { mkdirp: true, clobber: true });
|
||||
}
|
||||
copyFile = copyFile;
|
||||
|
||||
async checkFileExists(filepath: string, mode = constants.F_OK): Promise<boolean> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -13,12 +13,13 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepo
|
|||
unlink: jest.fn(),
|
||||
unlinkDir: jest.fn().mockResolvedValue(true),
|
||||
removeEmptyDirs: jest.fn(),
|
||||
moveFile: jest.fn(),
|
||||
checkFileExists: jest.fn(),
|
||||
mkdirSync: jest.fn(),
|
||||
checkDiskUsage: jest.fn(),
|
||||
readdir: jest.fn(),
|
||||
stat: jest.fn(),
|
||||
crawl: jest.fn(),
|
||||
rename: jest.fn(),
|
||||
copyFile: jest.fn(),
|
||||
};
|
||||
};
|
||||
|
|
|
|||
18
web/src/api/open-api/api.ts
generated
18
web/src/api/open-api/api.ts
generated
|
|
@ -3780,12 +3780,6 @@ export interface SystemConfigJobDto {
|
|||
* @memberof SystemConfigJobDto
|
||||
*/
|
||||
'smartSearch': JobSettingsDto;
|
||||
/**
|
||||
*
|
||||
* @type {JobSettingsDto}
|
||||
* @memberof SystemConfigJobDto
|
||||
*/
|
||||
'storageTemplateMigration': JobSettingsDto;
|
||||
/**
|
||||
*
|
||||
* @type {JobSettingsDto}
|
||||
|
|
@ -4026,6 +4020,18 @@ export interface SystemConfigReverseGeocodingDto {
|
|||
* @interface SystemConfigStorageTemplateDto
|
||||
*/
|
||||
export interface SystemConfigStorageTemplateDto {
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
* @memberof SystemConfigStorageTemplateDto
|
||||
*/
|
||||
'enabled': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
* @memberof SystemConfigStorageTemplateDto
|
||||
*/
|
||||
'hashVerificationEnabled': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {string}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@
|
|||
JobName.SmartSearch,
|
||||
JobName.RecognizeFaces,
|
||||
JobName.VideoConversion,
|
||||
JobName.StorageTemplateMigration,
|
||||
JobName.Migration,
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
|
||||
import { user } from '$lib/stores/user.store';
|
||||
import type { ResetOptions } from '$lib/utils/dipatch';
|
||||
import SettingSwitch from '$lib/components/admin-page/settings/setting-switch.svelte';
|
||||
|
||||
export let storageConfig: SystemConfigStorageTemplateDto;
|
||||
export let disabled = false;
|
||||
|
|
@ -145,7 +146,23 @@
|
|||
|
||||
<section class="dark:text-immich-dark-fg">
|
||||
{#await getConfigs() then}
|
||||
<div id="directory-path-builder" class="m-4">
|
||||
<div id="directory-path-builder" class="flex flex-col gap-4 m-4">
|
||||
<SettingSwitch
|
||||
title="ENABLED"
|
||||
{disabled}
|
||||
subtitle="Enable storage template engine"
|
||||
bind:checked={storageConfig.enabled}
|
||||
/>
|
||||
|
||||
<SettingSwitch
|
||||
title="HASH VERIFICATION ENABLED"
|
||||
{disabled}
|
||||
subtitle="Enables hash verification, don't disable this unless you're certain of the implications"
|
||||
bind:checked={storageConfig.hashVerificationEnabled}
|
||||
/>
|
||||
|
||||
<hr />
|
||||
|
||||
<h3 class="text-base font-medium text-immich-primary dark:text-immich-dark-primary">Variables</h3>
|
||||
|
||||
<section class="support-date">
|
||||
|
|
@ -191,8 +208,8 @@
|
|||
<div class="flex flex-col my-2">
|
||||
<label class="text-sm" for="preset-select">PRESET</label>
|
||||
<select
|
||||
class="p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
|
||||
{disabled}
|
||||
class="immich-form-input p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
|
||||
disabled={disabled || !storageConfig.enabled}
|
||||
name="presets"
|
||||
id="preset-select"
|
||||
bind:value={selectedPreset}
|
||||
|
|
@ -206,7 +223,7 @@
|
|||
<div class="flex gap-2 align-bottom">
|
||||
<SettingInputField
|
||||
label="TEMPLATE"
|
||||
{disabled}
|
||||
disabled={disabled || !storageConfig.enabled}
|
||||
required
|
||||
inputType={SettingInputFieldType.TEXT}
|
||||
bind:value={storageConfig.template}
|
||||
|
|
@ -239,7 +256,6 @@
|
|||
</p>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<SettingButtonsRow
|
||||
on:reset={({ detail }) => handleReset(detail)}
|
||||
on:save={saveSetting}
|
||||
|
|
|
|||
Loading…
Reference in a new issue