2023-03-30 21:38:55 +02:00
|
|
|
import { AssetEntity, SystemConfig } from '@app/infra/entities';
|
2023-02-25 15:12:03 +01:00
|
|
|
import { Inject, Injectable, Logger } from '@nestjs/common';
|
|
|
|
import { IAssetRepository } from '../asset/asset.repository';
|
2023-03-25 15:50:57 +01:00
|
|
|
import { APP_MEDIA_LOCATION } from '../domain.constant';
|
2023-05-22 20:05:06 +02:00
|
|
|
import { getLivePhotoMotionFilename, usePagination } from '../domain.util';
|
2023-05-26 21:43:24 +02:00
|
|
|
import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
|
2023-02-25 15:12:03 +01:00
|
|
|
import { IStorageRepository } from '../storage/storage.repository';
|
|
|
|
import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
|
2023-05-22 05:18:10 +02:00
|
|
|
import { IUserRepository } from '../user/user.repository';
|
2023-02-25 15:12:03 +01:00
|
|
|
import { StorageTemplateCore } from './storage-template.core';
|
|
|
|
|
2023-05-22 05:18:10 +02:00
|
|
|
export interface MoveAssetMetadata {
|
|
|
|
storageLabel: string | null;
|
|
|
|
filename: string;
|
|
|
|
}
|
|
|
|
|
2023-02-25 15:12:03 +01:00
|
|
|
@Injectable()
|
|
|
|
export class StorageTemplateService {
|
|
|
|
private logger = new Logger(StorageTemplateService.name);
|
|
|
|
private core: StorageTemplateCore;
|
|
|
|
|
|
|
|
constructor(
|
|
|
|
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
|
|
|
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
|
|
|
|
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
|
|
|
|
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
2023-05-22 05:18:10 +02:00
|
|
|
@Inject(IUserRepository) private userRepository: IUserRepository,
|
2023-02-25 15:12:03 +01:00
|
|
|
) {
|
|
|
|
this.core = new StorageTemplateCore(configRepository, config, storageRepository);
|
|
|
|
}
|
|
|
|
|
2023-05-26 21:43:24 +02:00
|
|
|
async handleMigrationSingle({ id }: IEntityJob) {
|
|
|
|
const [asset] = await this.assetRepository.getByIds([id]);
|
2023-03-28 22:04:11 +02:00
|
|
|
|
2023-05-26 21:43:24 +02:00
|
|
|
const user = await this.userRepository.get(asset.ownerId);
|
|
|
|
const storageLabel = user?.storageLabel || null;
|
|
|
|
const filename = asset.originalFileName || asset.id;
|
|
|
|
await this.moveAsset(asset, { storageLabel, filename });
|
|
|
|
|
|
|
|
// move motion part of live photo
|
|
|
|
if (asset.livePhotoVideoId) {
|
|
|
|
const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
|
|
|
|
const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
|
|
|
|
await this.moveAsset(livePhotoVideo, { storageLabel, filename: motionFilename });
|
2023-03-28 22:04:11 +02:00
|
|
|
}
|
2023-05-26 21:43:24 +02:00
|
|
|
|
|
|
|
return true;
|
2023-03-28 22:04:11 +02:00
|
|
|
}
|
|
|
|
|
2023-05-26 14:52:52 +02:00
|
|
|
async handleMigration() {
|
2023-02-25 15:12:03 +01:00
|
|
|
try {
|
|
|
|
console.time('migrating-time');
|
|
|
|
|
2023-05-22 20:05:06 +02:00
|
|
|
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
|
|
|
this.assetRepository.getAll(pagination),
|
|
|
|
);
|
|
|
|
const users = await this.userRepository.getList();
|
2023-02-25 15:12:03 +01:00
|
|
|
|
2023-05-22 20:05:06 +02:00
|
|
|
for await (const assets of assetPagination) {
|
|
|
|
for (const asset of assets) {
|
|
|
|
const user = users.find((user) => user.id === asset.ownerId);
|
|
|
|
const storageLabel = user?.storageLabel || null;
|
|
|
|
const filename = asset.originalFileName || asset.id;
|
|
|
|
await this.moveAsset(asset, { storageLabel, filename });
|
2023-02-25 15:12:03 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
this.logger.debug('Cleaning up empty directories...');
|
2023-03-25 15:50:57 +01:00
|
|
|
await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION);
|
2023-02-25 15:12:03 +01:00
|
|
|
} finally {
|
|
|
|
console.timeEnd('migrating-time');
|
|
|
|
}
|
2023-05-26 21:43:24 +02:00
|
|
|
|
|
|
|
return true;
|
2023-02-25 15:12:03 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: use asset core (once in domain)
|
2023-05-22 05:18:10 +02:00
|
|
|
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-22 04:33:20 +02:00
|
|
|
if (asset.isReadOnly) {
|
|
|
|
this.logger.verbose(`Not moving read-only asset: ${asset.originalPath}`);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2023-05-22 05:18:10 +02:00
|
|
|
const destination = await this.core.getTemplatePath(asset, metadata);
|
2023-02-25 15:12:03 +01:00
|
|
|
if (asset.originalPath !== destination) {
|
|
|
|
const source = asset.originalPath;
|
|
|
|
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
let sidecarMoved = false;
|
2023-02-25 15:12:03 +01:00
|
|
|
try {
|
|
|
|
await this.storageRepository.moveFile(asset.originalPath, destination);
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
|
|
|
|
let sidecarDestination;
|
2023-02-25 15:12:03 +01:00
|
|
|
try {
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
if (asset.sidecarPath) {
|
|
|
|
sidecarDestination = `${destination}.xmp`;
|
|
|
|
await this.storageRepository.moveFile(asset.sidecarPath, sidecarDestination);
|
|
|
|
sidecarMoved = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
await this.assetRepository.save({ id: asset.id, originalPath: destination, sidecarPath: sidecarDestination });
|
2023-02-25 15:12:03 +01:00
|
|
|
asset.originalPath = destination;
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
asset.sidecarPath = sidecarDestination || null;
|
2023-02-25 15:12:03 +01:00
|
|
|
} catch (error: any) {
|
2023-06-20 23:24:47 +02:00
|
|
|
this.logger.warn(
|
|
|
|
`Unable to save new originalPath to database, undoing move for path ${asset.originalPath} - filename ${asset.originalFileName} - id ${asset.id}`,
|
|
|
|
error?.stack,
|
|
|
|
);
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
|
|
|
|
// Either sidecar move failed or the save failed. Eithr way, move media back
|
2023-02-25 15:12:03 +01:00
|
|
|
await this.storageRepository.moveFile(destination, source);
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-25 03:59:30 +02:00
|
|
|
|
|
|
|
if (asset.sidecarPath && sidecarDestination && sidecarMoved) {
|
|
|
|
// If the sidecar was moved, that means the saved failed. So move both the sidecar and the
|
|
|
|
// media back into their original positions
|
|
|
|
await this.storageRepository.moveFile(sidecarDestination, asset.sidecarPath);
|
|
|
|
}
|
2023-02-25 15:12:03 +01:00
|
|
|
}
|
|
|
|
} catch (error: any) {
|
|
|
|
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return asset;
|
|
|
|
}
|
|
|
|
}
|