From cf9e04c8ec276b7513e7fb199e466771d1b33c5b Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey <10599762+fyfrey@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:28:50 +0200 Subject: [PATCH] feat(server): asset entity audit (#3824) * feat(server): audit log * feedback * Insert to database * migration * test * controller/repository/service * test * module * feat(server): implement audit endpoint * directly return changed assets * add daily cleanup of audit table * fix tests * review feedback * ci * refactor(server): audit implementation * chore: open api --------- Co-authored-by: Alex Tran Co-authored-by: Fynn Petersen-Frey Co-authored-by: Jason Rasmussen --- cli/src/api/open-api/api.ts | 215 +++++++++++++++++- mobile/openapi/.openapi-generator/FILES | 9 + mobile/openapi/README.md | Bin 18666 -> 18861 bytes mobile/openapi/doc/AssetApi.md | Bin 58716 -> 58856 bytes mobile/openapi/doc/AuditApi.md | Bin 0 -> 2481 bytes mobile/openapi/doc/AuditDeletesResponseDto.md | Bin 0 -> 478 bytes mobile/openapi/doc/EntityType.md | Bin 0 -> 376 bytes mobile/openapi/lib/api.dart | Bin 6037 -> 6141 bytes mobile/openapi/lib/api/asset_api.dart | Bin 53647 -> 53913 bytes mobile/openapi/lib/api/audit_api.dart | Bin 0 -> 2478 bytes mobile/openapi/lib/api_client.dart | Bin 18898 -> 19083 bytes mobile/openapi/lib/api_helper.dart | Bin 4566 -> 4664 bytes .../lib/model/audit_deletes_response_dto.dart | Bin 0 -> 3250 bytes mobile/openapi/lib/model/entity_type.dart | Bin 0 -> 2544 bytes mobile/openapi/test/asset_api_test.dart | Bin 5475 -> 5498 bytes mobile/openapi/test/audit_api_test.dart | Bin 0 -> 637 bytes .../test/audit_deletes_response_dto_test.dart | Bin 0 -> 724 bytes mobile/openapi/test/entity_type_test.dart | Bin 0 -> 417 bytes server/immich-openapi-specs.json | 93 ++++++++ server/src/domain/asset/asset.service.spec.ts | 2 +- server/src/domain/asset/asset.service.ts | 15 +- .../asset/response-dto/asset-response.dto.ts | 5 + .../response-dto/memory-lane-response.dto.ts | 6 - server/src/domain/audit/audi.service.spec.ts | 61 +++++ server/src/domain/audit/audit.dto.ts | 24 ++ server/src/domain/audit/audit.repository.ts | 14 ++ server/src/domain/audit/audit.service.ts | 43 ++++ server/src/domain/audit/index.ts | 3 + server/src/domain/domain.constant.ts | 3 + server/src/domain/domain.module.ts | 2 + server/src/domain/index.ts | 1 + server/src/domain/job/job.constants.ts | 2 + server/src/domain/job/job.repository.ts | 3 + server/src/domain/job/job.service.spec.ts | 1 + server/src/domain/job/job.service.ts | 1 + .../immich/api-v1/asset/asset-repository.ts | 3 +- .../api-v1/asset/dto/asset-search.dto.ts | 9 +- server/src/immich/app.module.ts | 2 + .../immich/controllers/asset.controller.ts | 4 +- .../immich/controllers/audit.controller.ts | 18 ++ server/src/immich/controllers/index.ts | 1 + server/src/infra/database.config.ts | 1 + server/src/infra/entities/audit.entity.ts | 34 +++ server/src/infra/entities/index.ts | 3 + server/src/infra/infra.module.ts | 3 + .../migrations/1692804658140-AddAuditTable.ts | 16 ++ .../infra/repositories/audit.repository.ts | 26 +++ server/src/infra/repositories/index.ts | 1 + .../src/infra/subscribers/audit.subscriber.ts | 38 ++++ server/src/microservices/app.service.ts | 3 + .../metadata-extraction.processor.ts | 6 +- server/test/fixtures/audit.stub.ts | 29 +++ server/test/fixtures/index.ts | 1 + .../repositories/access.repository.mock.ts | 4 +- .../repositories/audit.repository.mock.ts | 8 + server/test/repositories/index.ts | 1 + web/src/api/open-api/api.ts | 215 +++++++++++++++++- 57 files changed, 899 insertions(+), 30 deletions(-) create mode 100644 mobile/openapi/doc/AuditApi.md create mode 100644 mobile/openapi/doc/AuditDeletesResponseDto.md create mode 100644 mobile/openapi/doc/EntityType.md create mode 100644 mobile/openapi/lib/api/audit_api.dart create mode 100644 mobile/openapi/lib/model/audit_deletes_response_dto.dart create mode 100644 mobile/openapi/lib/model/entity_type.dart create mode 100644 mobile/openapi/test/audit_api_test.dart create mode 100644 mobile/openapi/test/audit_deletes_response_dto_test.dart create mode 100644 mobile/openapi/test/entity_type_test.dart delete mode 100644 server/src/domain/asset/response-dto/memory-lane-response.dto.ts create mode 100644 server/src/domain/audit/audi.service.spec.ts create mode 100644 server/src/domain/audit/audit.dto.ts create mode 100644 server/src/domain/audit/audit.repository.ts create mode 100644 server/src/domain/audit/audit.service.ts create mode 100644 server/src/domain/audit/index.ts create mode 100644 server/src/immich/controllers/audit.controller.ts create mode 100644 server/src/infra/entities/audit.entity.ts create mode 100644 server/src/infra/migrations/1692804658140-AddAuditTable.ts create mode 100644 server/src/infra/repositories/audit.repository.ts create mode 100644 server/src/infra/subscribers/audit.subscriber.ts create mode 100644 server/test/fixtures/audit.stub.ts create mode 100644 server/test/repositories/audit.repository.mock.ts diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index 16ba34df5e..f73b6bcdf7 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -752,6 +752,25 @@ export const AudioCodec = { export type AudioCodec = typeof AudioCodec[keyof typeof AudioCodec]; +/** + * + * @export + * @interface AuditDeletesResponseDto + */ +export interface AuditDeletesResponseDto { + /** + * + * @type {Array} + * @memberof AuditDeletesResponseDto + */ + 'ids': Array; + /** + * + * @type {boolean} + * @memberof AuditDeletesResponseDto + */ + 'needsFullSync': boolean; +} /** * * @export @@ -1243,6 +1262,20 @@ export interface DownloadResponseDto { */ 'totalSize': number; } +/** + * + * @export + * @enum {string} + */ + +export const EntityType = { + Asset: 'ASSET', + Album: 'ALBUM' +} as const; + +export type EntityType = typeof EntityType[keyof typeof EntityType]; + + /** * * @export @@ -5120,11 +5153,12 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration * @param {boolean} [isArchived] * @param {boolean} [withoutThumbs] Include assets without thumbnails * @param {number} [skip] + * @param {string} [updatedAfter] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { + getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { const localVarPath = `/asset`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -5166,6 +5200,12 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration localVarQueryParameter['skip'] = skip; } + if (updatedAfter !== undefined) { + localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ? + (updatedAfter as any).toISOString() : + updatedAfter; + } + if (ifNoneMatch != null) { localVarHeaderParameter['if-none-match'] = String(ifNoneMatch); } @@ -6274,12 +6314,13 @@ export const AssetApiFp = function(configuration?: Configuration) { * @param {boolean} [isArchived] * @param {boolean} [withoutThumbs] Include assets without thumbnails * @param {number} [skip] + * @param {string} [updatedAfter] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, withoutThumbs, skip, ifNoneMatch, options); + async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, withoutThumbs, skip, updatedAfter, ifNoneMatch, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, /** @@ -6576,7 +6617,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath * @throws {RequiredError} */ getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise> { - return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); + return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); }, /** * Get a single asset\'s information @@ -6895,6 +6936,13 @@ export interface AssetApiGetAllAssetsRequest { */ readonly skip?: number + /** + * + * @type {string} + * @memberof AssetApiGetAllAssets + */ + readonly updatedAfter?: string + /** * ETag of data already cached on the client * @type {string} @@ -7460,7 +7508,7 @@ export class AssetApi extends BaseAPI { * @memberof AssetApi */ public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) { - return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); + return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); } /** @@ -7671,6 +7719,163 @@ export class AssetApi extends BaseAPI { } +/** + * AuditApi - axios parameter creator + * @export + */ +export const AuditApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * + * @param {EntityType} entityType + * @param {string} after + * @param {string} [userId] + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditDeletes: async (entityType: EntityType, after: string, userId?: string, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'entityType' is not null or undefined + assertParamExists('getAuditDeletes', 'entityType', entityType) + // verify required parameter 'after' is not null or undefined + assertParamExists('getAuditDeletes', 'after', after) + const localVarPath = `/audit/deletes`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + if (entityType !== undefined) { + localVarQueryParameter['entityType'] = entityType; + } + + if (userId !== undefined) { + localVarQueryParameter['userId'] = userId; + } + + if (after !== undefined) { + localVarQueryParameter['after'] = (after as any instanceof Date) ? + (after as any).toISOString() : + after; + } + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * AuditApi - functional programming interface + * @export + */ +export const AuditApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = AuditApiAxiosParamCreator(configuration) + return { + /** + * + * @param {EntityType} entityType + * @param {string} after + * @param {string} [userId] + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getAuditDeletes(entityType: EntityType, after: string, userId?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAuditDeletes(entityType, after, userId, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + } +}; + +/** + * AuditApi - factory interface + * @export + */ +export const AuditApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = AuditApiFp(configuration) + return { + /** + * + * @param {AuditApiGetAuditDeletesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditDeletes(requestParameters: AuditApiGetAuditDeletesRequest, options?: AxiosRequestConfig): AxiosPromise { + return localVarFp.getAuditDeletes(requestParameters.entityType, requestParameters.after, requestParameters.userId, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * Request parameters for getAuditDeletes operation in AuditApi. + * @export + * @interface AuditApiGetAuditDeletesRequest + */ +export interface AuditApiGetAuditDeletesRequest { + /** + * + * @type {EntityType} + * @memberof AuditApiGetAuditDeletes + */ + readonly entityType: EntityType + + /** + * + * @type {string} + * @memberof AuditApiGetAuditDeletes + */ + readonly after: string + + /** + * + * @type {string} + * @memberof AuditApiGetAuditDeletes + */ + readonly userId?: string +} + +/** + * AuditApi - object-oriented interface + * @export + * @class AuditApi + * @extends {BaseAPI} + */ +export class AuditApi extends BaseAPI { + /** + * + * @param {AuditApiGetAuditDeletesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApi + */ + public getAuditDeletes(requestParameters: AuditApiGetAuditDeletesRequest, options?: AxiosRequestConfig) { + return AuditApiFp(this.configuration).getAuditDeletes(requestParameters.entityType, requestParameters.after, requestParameters.userId, options).then((request) => request(this.axios, this.basePath)); + } +} + + /** * AuthenticationApi - axios parameter creator * @export diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 93b7049766..506dfbf6a7 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -29,6 +29,8 @@ doc/AssetResponseDto.md doc/AssetStatsResponseDto.md doc/AssetTypeEnum.md doc/AudioCodec.md +doc/AuditApi.md +doc/AuditDeletesResponseDto.md doc/AuthDeviceResponseDto.md doc/AuthenticationApi.md doc/BulkIdResponseDto.md @@ -50,6 +52,7 @@ doc/DeleteAssetStatus.md doc/DownloadArchiveInfo.md doc/DownloadInfoDto.md doc/DownloadResponseDto.md +doc/EntityType.md doc/ExifResponseDto.md doc/ImportAssetDto.md doc/JobApi.md @@ -134,6 +137,7 @@ lib/api.dart lib/api/album_api.dart lib/api/api_key_api.dart lib/api/asset_api.dart +lib/api/audit_api.dart lib/api/authentication_api.dart lib/api/job_api.dart lib/api/o_auth_api.dart @@ -176,6 +180,7 @@ lib/model/asset_response_dto.dart lib/model/asset_stats_response_dto.dart lib/model/asset_type_enum.dart lib/model/audio_codec.dart +lib/model/audit_deletes_response_dto.dart lib/model/auth_device_response_dto.dart lib/model/bulk_id_response_dto.dart lib/model/bulk_ids_dto.dart @@ -196,6 +201,7 @@ lib/model/delete_asset_status.dart lib/model/download_archive_info.dart lib/model/download_info_dto.dart lib/model/download_response_dto.dart +lib/model/entity_type.dart lib/model/exif_response_dto.dart lib/model/import_asset_dto.dart lib/model/job_command.dart @@ -292,6 +298,8 @@ test/asset_response_dto_test.dart test/asset_stats_response_dto_test.dart test/asset_type_enum_test.dart test/audio_codec_test.dart +test/audit_api_test.dart +test/audit_deletes_response_dto_test.dart test/auth_device_response_dto_test.dart test/authentication_api_test.dart test/bulk_id_response_dto_test.dart @@ -313,6 +321,7 @@ test/delete_asset_status_test.dart test/download_archive_info_test.dart test/download_info_dto_test.dart test/download_response_dto_test.dart +test/entity_type_test.dart test/exif_response_dto_test.dart test/import_asset_dto_test.dart test/job_api_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index fb3873a3270dba56b5ee2d91e938942257549af2..188c09f5f7c70370d729b34e0015f91c4ef91205 100644 GIT binary patch delta 174 zcmaDgk#X&0#tqyO5hB6UeXt-< znO<&+GEhw-NKFbvjV4g1mX^C~h?bUuK3GB@EHjx;T7I*lM7b;%!o0~B4TZT7=1)$v f5Zye@G@L~$uOzdiGNiH~6>6U=TwwEKOLl$$sR}(6 delta 28 kcmZ2Gneo*`#tqyOlPjdfHaAI>%WgI|3uW27!Rjv`0G~Jt&j0`b diff --git a/mobile/openapi/doc/AssetApi.md b/mobile/openapi/doc/AssetApi.md index 2d76e5f9a5e4ae32dab6712868f8c08f7f97d589..140f509b0b05fa67eb0a9054bf04d13042aef001 100644 GIT binary patch delta 83 zcmca}iuuK9<_#LwJf#IGi6yBij%g*SMVr;E-C3D|B9jxlY$oSA972`Vb$q~xA~N~> QL$S#okN7w9I{PmI0DgNRr2qf` delta 28 kcmaEHn)%Kt<_#Lwo6W3UStsvuIJCLT@gC#mna=);0Kqv84FCWD diff --git a/mobile/openapi/doc/AuditApi.md b/mobile/openapi/doc/AuditApi.md new file mode 100644 index 0000000000000000000000000000000000000000..63a1c97a3bb5088631cca0c12aaa2a58e65b6260 GIT binary patch literal 2481 zcmd5;U2oe)5PavaSU?3tu_i^yL6OL<0mV`h!|?}X`5{mZ`$Vp!t0^8!9vdsM|9j_n zlqIW3;k11y0Ym2P-rnxa-VPh2GEPM%1M}A}W2yU@beu+mJ}MEDUNH49jwut-otU$8 zdwUy;Tsv}^sa#XDUhmaW+9rx|XdW0>$LS{tU5$Y=B%qy2Ar_w!t<`(b!oveZon)| zrLi@)5W@4`567NI?aI)uEigF10@lAobPj3zh!Gc=N(&yKv>jiUQ!Vvsyvlfw#7uL( zM`gkJaoj?1Al;zhy;x?+ccI&E)A)RFPDd)8$$1HM4NZ5vB2fa-Tz&!6D(z9E)SYCf zjhbenOboq8i9}98l_v;BdQs*4Gi)ck>Lb9R+q4^-Ez=oTrC+G|>HKPRgL7_9hM&MN zawyYCEtd#EQ%=-p!)(F2%qYuwChy7RU>?w02K{-9W-4E&xkqwFDRa#6zt8Q$9)=9< zNjA5a;e+aK0NX!EqTc!cL})j;7eQ3rK?~t-8%-+;4dq9oVCn>Pg`Z2E%XCg-b;s$; z5d5HZW>2Ww%mxOC=$n2NaZyy!+`8G0j)s?)H{kL<_%&WNXt&7UnTiq6Yr636C@jou$+ee}fXC_mTkYP!tg%)YV zxM1rc<-)@{Y-R_{2ZVx#Pt(Lqr#g(8DsbqAoqfL(`rUBcdDH8Lz5VdVu+s~}YUcpR zjOCIy`|KFP#{Ji?Li6~qqgm*8I@8#ncJ~>vrf+`YS3C>DxQbE3Y38ENSM=2(JT{4( z3yc>8+b+ru8O+*x6#vaIJg8!2XejO4SU};CFq*)u%b;n?p*g&dxKamD&ndi;#B|~s zLH}@P3is#-{xvl%^k|jI%-su-izO5yFC6C#{{a~@rtz~SKZMsa1sVB19m1|>I^{q` zHf%y_9tcXcVbh2d#HsaQ&NI%HR8)SRx2DeXp3IiJ+HPs7trJxNBa&;dWXc?2BIhY- zl;#+D{DyEn$z;vdmZq7+Y%FtYpVS4>tV$sDpgkJ)2d6_Luy(PvN0WLC?bz#8<6y7< zj@>TSCqDZySg?pIy2uAa5I+q}eM6e(^$bk}nUg6*rE)L>CDXJ@ZkE&uN?~^lh;n;#A=);)Ow$ O#`oE73+B%8K;%ym8YX}M literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/AuditDeletesResponseDto.md b/mobile/openapi/doc/AuditDeletesResponseDto.md new file mode 100644 index 0000000000000000000000000000000000000000..c7c9594f1bb291121d9aa962122ab1d20d6575fc GIT binary patch literal 478 zcma)2L2AP=5WMRZ3qGVV*txf*km3{yNeD@A#$eUjq@uD`Nb5tOkFVsqH0`C)Ma*bs zRy!GxLo1ywds1l1<)AcIXpk}NF*vdTD<&bb4AvwS@EgJ?1t+_Ac|y^)t&*OF?j3oC z*~H&Y!HX201hcy&9g}uGK6=&|#8&nQAMkQ^>8dbvUOU!gBid5@g(xmgBri}={D(!J zYZdZ5>hgT81K%ImYkPVasmRs}k8&^!Owh$;8S18qE{mwb(_qZuY`e+-N@Tu$Le%wR zEFk$dsW~t)y=iXBayLdbBLlLdB7MF;*Am@Z(os?(^!)PoR6FY2haKBsWmQog)sOk z)Lv5`Yqg delta 22 ecmeyXKUIH&8S~~qW+v9njU4{Wo8NQu@c;l_-3L7Y diff --git a/mobile/openapi/lib/api/asset_api.dart b/mobile/openapi/lib/api/asset_api.dart index 0f8b69a1b6a902ebc8fea7a6474756e7f94906b9..a2f6b7b3a92a919136eced5a45fd6b81e4b643cf 100644 GIT binary patch delta 209 zcmeBQ%sg``^M?6)lNC;LF_#vkOkPzgHTggw=j0W72gFZ4${cF)OAn{*<54P b#Dv9C_sM>yHMlK4D1~gM6{?Z#W*oBt>jhEx delta 64 zcmbQal(~N~^M?6)lO60hC!g0lFnNcW%4B{gyUDL4ttT63^lZLh@Rf11sc{7pRE7Iw Q6|)+ks{NZMo3qUZ0ADy7V*mgE diff --git a/mobile/openapi/lib/api/audit_api.dart b/mobile/openapi/lib/api/audit_api.dart new file mode 100644 index 0000000000000000000000000000000000000000..4eabd17c9c0fd681e82a5132a10f47fe027f60fc GIT binary patch literal 2478 zcmd5;-*ehV5PsKRvB^V#j0&kI4;>Pl5`z%QInmcbwz3}F0WG5_K9^n7|bUQ8!&HhVjbVKRsF`2ya}C$qC( zAF-O#{5chdj*t339!2gbZ zV(@b*4Vx>m;qO8i{v|Y)of~_<`AREGLQx`94k$B;#+%!FgXL18O>(VnSdG>~B|kk& zR%>Bw6md8Loq;Y;i9+`I^CF6(RW7WBahXaQ7ZPqGI2^iHEIG|32Oa^dhsR|PhRq3% zThF5rq?m~^cdmjqhXBZ=5;+`XjfH}Rw566=GVXBiu*XMTOYFSgCTGCFkFYRh!T(?KLp79Ck(VaQ}&t^N!l&KicG4NJMdXQF{j;`7#hMJ)c4AN@#+IOFK z50&;*rwAv<;8^Otm-MeiddceyEj5EcYooW0=BHm*SU8?$N)5E!a>NPvbR8A3U&j@% zc1-UF-S20EiR|MVnemzLH<}QbSHCa?*X*w}T(b+d^_LS0&`Q)VJOEOJV%UA5C;O|5h|(=gcd~EU65sEJ&Sv TQBQ=^6~>)pYp~fsbCCi7dUG9Z delta 19 bcmeC4%6MrqH<%l@i|o delta 12 Tcmdm?a!q-|LH5nc9HLwRBNqfq diff --git a/mobile/openapi/lib/model/audit_deletes_response_dto.dart b/mobile/openapi/lib/model/audit_deletes_response_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..a0bc0dd4d929e9b41615ef036517283fc0d0f748 GIT binary patch literal 3250 zcmbVOZExC05dO}um`GJIs%!50=|l=`bR?&~1u3B?oe;7%_8M=`czwI;B9!vqZ)Vpv zc8Zgzl}cbeug~+$c*o&fCDKmW&{QhP@KgA}Oiie3*VIndOk3*jEEzgZ#{}$UqDs$32c{4AKlzlJM~!IeJk9?4uAmYYz-m{25wE9^HhAG$`u*dJ^cdiuThkn)s5m3xX1=FdwxckN<#KojsjbKW_ z22wMqOn;;C4$YpbXK8D!Yv6)K}c2leQ~yJeJ#!9wlMoTG*XSL(yKDa9WHrmdVF4s!3kyBr?r4o zZ+pWwD#Qr2qsGo~7b0ry$fO!-DqwN!l-Tlgc-Odh(679Vr0%>LxCzwXY!o}!6*cX3 z)Q294<0L_K#SSF2YOkxlX!ZJPbAeSnN`>WY>Zs})H{4X!#bpA`9V5ioJl@b#M%Y%)@n62DHE#W#Ua5vh*hbf8)UbQ} zNsH8iCEW{tCrpY*G-7lK>fz+IvqtA?RpSc@z35U%L2caNA%=gd}5Pc1hEs^=nTpuUgZ zEK}ZSqq%{i3^m>*7;;~tZ01s0xeNCyA-LCR19H}fA0*b zn-w<=5L=hwT+W<1N*5dc zPbmx@S1iZRf*JgF+*ovO?D^w~R@O1)Vv%A%kqK`6aeFqH6pGteYPCl*cVH_1<)=6~ zFk?p}4Cg>+pbM^8AtL<$FdB^#$*hGH`gYa2DmZ)}0Zg&>+YneV$AN+3N8qZ=;V9ez z+<`-3^fM-v!seODl5ZnC=dN{wtol$_@~ zZpdXO6qE4iOl#@geAdt07>ti5^-HJ2#rO*5pMq0w+P?n7YBhX`;5w1ii7y`?S8JG` z7GN&@<4a5*;MDx?*QYOM%;_w#xdLZPh{JIOGh)KpFlCYYIXC00 zDCLPx`MB4~+BcQgPiX-XpWFN~G8mIHK(2Ub-@OC6kA%I~Zj~3VYP_L8ZD1TIgfR(jGVR_s5d#HJI^W`i2Yv4; zcnBY?FXo_1Df`XgYX zgVaSF!UW^AJ4l!Uq!M*%3wSO33@4n1t4}U)x}KVe_sA5Eap>?C8eXK#G1#%>8(H^H zN3TRMc?c=i)ZU#bW+SKn9LcPr_Uch33EP>e&yFj5~e<73Xz=E`mO_ z1AFbMZc+6%4HmxR+YK3S2@cjPi&P5R5{LJGD_5^imVPAVE_>N}oSo^s?hFo&lR{H~82#c)ECF$nFa>+RUJ@Bhgl#3pt< zDO`48{qb~Vj5eXj1CEJ0g@R;$kPzls0q&)=f+JOf9+!XK0tvp|V0Eo};PtR*@tif&`k0~!s=vZKPH#(doLnws1w zRoh7vBkE%-RHQ;=2(MJY;)YBP;cZ_V5@tCx%Vbvfua`&URXk`MNk(Jux5-(1dehmb zJY%SuGc2z8qa-id68g!Ig`<|i((Ft6E#q4`zI;;O6gDz!;ebEVJ+N{HKTyeG6E-p4 rP&HCV$`55L5cm;0Be_G(>dDDqbdgQ9Xi=M9?XQ9LqSKELB#-|Bm4QL& literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/asset_api_test.dart b/mobile/openapi/test/asset_api_test.dart index ef8ae195a7d39404bb2f55f285578d1aa33b27e5..076521b0e8e8ba98e90b561dc17ee7b7d9ad3d0e 100644 GIT binary patch delta 34 ocmaE?^-F6*2D_+BVo7R9W^SrNX+a8*nc|pMl3KJmjr}h#0Pz$JmH+?% delta 12 TcmeyR^;l~|2K(lE_TRh!C;tVK diff --git a/mobile/openapi/test/audit_api_test.dart b/mobile/openapi/test/audit_api_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..68ffede19c056ff71bb12146dffb13c14309f26c GIT binary patch literal 637 zcmZvZPjA~W48`yM6x=BZkk(yJL;s-x>T-jC3-^;akKJq=KX_BU}elClP@A)o&T$gzUo9&N$30VQVqJ*DCw%xpZ5iF7qwf20sy8OCI z;*nZoJ*vif)li4mFqk1=t+2x$4KGh;>Yep`wnBHm^*$&!u=4mmke70!ds&_KvNQT1 z;y#=&udE5I4J)dpAvJ`4EDvX!b!0H8&YD&*e$_^OyHeGw_90288DZvh^`=|AwhoPU zl*)4%UC(YlA&i03xSW6(N(%8y00{-wLruJPl>D+(L$Arq7jQ@bglfp>4rm;|Lb&@2 z&lfj|Ag#B9o6kOr&6e(U5 IEpEr;5B!MJ*#H0l literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/audit_deletes_response_dto_test.dart b/mobile/openapi/test/audit_deletes_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..45dbccc28d7430d301a0380f015d78f827ea7dd9 GIT binary patch literal 724 zcma)(Pfy!G5XJBQ6w^}@Dd57X;!o8`P7o5s5`+UnXxZ9hXT)ZAyE{u2ReX2Gn{ucK zf`iwd)%(3SGcL=bEMa!DtoP35m-DOHa$dpW{9-d4>`B%30(3JtE&^7OT)wReGc659OHo2YzZ74UbWAh^|QIlT_@ zW(-lxL)?uw&PLXj6}1vb9bw3L+!J_lS-@)olb_~Nt+~p zE-w=9?k8doc?j`$`2%=`;DanA@CF`)e%QL}xUo$hVPIs8WnI;of8O?)FMl~_z#_JB Vk&L+(;(V- literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/entity_type_test.dart b/mobile/openapi/test/entity_type_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..81f023308c2edf844a6eeb6591ab1e24a28e4ee6 GIT binary patch literal 417 zcmZvY(M!WH5XRs0SDa7Xpf0*6*$^DGlYy=v^}(lbwAU`!CNW8sBKzM>l!CB6}Ugn8XVPvI_gRJbkedn+WJ`sc3(I;VFoWOeGJkDyTU$HgEsu0tP9Ppo3~e5 zIgiQ { + let sut: AuditService; + let accessMock: IAccessRepositoryMock; + let auditMock: jest.Mocked; + + beforeEach(async () => { + accessMock = newAccessRepositoryMock(); + auditMock = newAuditRepositoryMock(); + sut = new AuditService(accessMock, auditMock); + }); + + it('should work', () => { + expect(sut).toBeDefined(); + }); + + describe('handleCleanup', () => { + it('should delete old audit entries', async () => { + await expect(sut.handleCleanup()).resolves.toBe(true); + expect(auditMock.removeBefore).toBeCalledWith(expect.any(Date)); + }); + }); + + describe('getDeletes', () => { + it('should require full sync if the request is older than 100 days', async () => { + auditMock.getAfter.mockResolvedValue([]); + + const date = new Date(2022, 0, 1); + await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ + needsFullSync: true, + ids: [], + }); + + expect(auditMock.getAfter).toHaveBeenCalledWith(date, { + action: DatabaseAction.DELETE, + ownerId: authStub.admin.id, + entityType: EntityType.ASSET, + }); + }); + + it('should get any new or updated assets and deleted ids', async () => { + auditMock.getAfter.mockResolvedValue([auditStub.delete]); + + const date = new Date(); + await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ + needsFullSync: false, + ids: ['asset-deleted'], + }); + + expect(auditMock.getAfter).toHaveBeenCalledWith(date, { + action: DatabaseAction.DELETE, + ownerId: authStub.admin.id, + entityType: EntityType.ASSET, + }); + }); + }); +}); diff --git a/server/src/domain/audit/audit.dto.ts b/server/src/domain/audit/audit.dto.ts new file mode 100644 index 0000000000..2494883e04 --- /dev/null +++ b/server/src/domain/audit/audit.dto.ts @@ -0,0 +1,24 @@ +import { EntityType } from '@app/infra/entities'; +import { ApiProperty } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { IsDate, IsEnum, IsOptional, IsUUID } from 'class-validator'; + +export class AuditDeletesDto { + @IsDate() + @Type(() => Date) + after!: Date; + + @ApiProperty({ enum: EntityType, enumName: 'EntityType' }) + @IsEnum(EntityType) + entityType!: EntityType; + + @IsOptional() + @IsUUID('4') + @ApiProperty({ format: 'uuid' }) + userId?: string; +} + +export class AuditDeletesResponseDto { + needsFullSync!: boolean; + ids!: string[]; +} diff --git a/server/src/domain/audit/audit.repository.ts b/server/src/domain/audit/audit.repository.ts new file mode 100644 index 0000000000..774ab1e422 --- /dev/null +++ b/server/src/domain/audit/audit.repository.ts @@ -0,0 +1,14 @@ +import { AuditEntity, DatabaseAction, EntityType } from '@app/infra/entities'; + +export const IAuditRepository = 'IAuditRepository'; + +export interface AuditSearch { + action?: DatabaseAction; + entityType?: EntityType; + ownerId?: string; +} + +export interface IAuditRepository { + getAfter(since: Date, options: AuditSearch): Promise; + removeBefore(before: Date): Promise; +} diff --git a/server/src/domain/audit/audit.service.ts b/server/src/domain/audit/audit.service.ts new file mode 100644 index 0000000000..47d98e6886 --- /dev/null +++ b/server/src/domain/audit/audit.service.ts @@ -0,0 +1,43 @@ +import { DatabaseAction } from '@app/infra/entities'; +import { Inject, Injectable } from '@nestjs/common'; +import { DateTime } from 'luxon'; +import { AccessCore, IAccessRepository, Permission } from '../access'; +import { AuthUserDto } from '../auth'; +import { AUDIT_LOG_MAX_DURATION } from '../domain.constant'; +import { AuditDeletesDto, AuditDeletesResponseDto } from './audit.dto'; +import { IAuditRepository } from './audit.repository'; + +@Injectable() +export class AuditService { + private access: AccessCore; + + constructor( + @Inject(IAccessRepository) accessRepository: IAccessRepository, + @Inject(IAuditRepository) private repository: IAuditRepository, + ) { + this.access = new AccessCore(accessRepository); + } + + async handleCleanup(): Promise { + await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate()); + return true; + } + + async getDeletes(authUser: AuthUserDto, dto: AuditDeletesDto): Promise { + const userId = dto.userId || authUser.id; + await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId); + + const audits = await this.repository.getAfter(dto.after, { + ownerId: userId, + entityType: dto.entityType, + action: DatabaseAction.DELETE, + }); + + const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after)); + + return { + needsFullSync: duration > AUDIT_LOG_MAX_DURATION, + ids: audits.map(({ entityId }) => entityId), + }; + } +} diff --git a/server/src/domain/audit/index.ts b/server/src/domain/audit/index.ts new file mode 100644 index 0000000000..2074b86f38 --- /dev/null +++ b/server/src/domain/audit/index.ts @@ -0,0 +1,3 @@ +export * from './audit.dto'; +export * from './audit.repository'; +export * from './audit.service'; diff --git a/server/src/domain/domain.constant.ts b/server/src/domain/domain.constant.ts index 7b60b796aa..04b8309760 100644 --- a/server/src/domain/domain.constant.ts +++ b/server/src/domain/domain.constant.ts @@ -1,8 +1,11 @@ import { AssetType } from '@app/infra/entities'; import { BadRequestException } from '@nestjs/common'; +import { Duration } from 'luxon'; import { extname } from 'node:path'; import pkg from 'src/../../package.json'; +export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 }); + const [major, minor, patch] = pkg.version.split('.'); export interface IServerVersion { diff --git a/server/src/domain/domain.module.ts b/server/src/domain/domain.module.ts index 72d5300836..a2efd8796a 100644 --- a/server/src/domain/domain.module.ts +++ b/server/src/domain/domain.module.ts @@ -2,6 +2,7 @@ import { DynamicModule, Global, Module, ModuleMetadata, OnApplicationShutdown, P import { AlbumService } from './album'; import { APIKeyService } from './api-key'; import { AssetService } from './asset'; +import { AuditService } from './audit'; import { AuthService } from './auth'; import { FacialRecognitionService } from './facial-recognition'; import { JobService } from './job'; @@ -23,6 +24,7 @@ const providers: Provider[] = [ AlbumService, APIKeyService, AssetService, + AuditService, AuthService, FacialRecognitionService, JobService, diff --git a/server/src/domain/index.ts b/server/src/domain/index.ts index 201e31ff31..c66c4eadc5 100644 --- a/server/src/domain/index.ts +++ b/server/src/domain/index.ts @@ -2,6 +2,7 @@ export * from './access'; export * from './album'; export * from './api-key'; export * from './asset'; +export * from './audit'; export * from './auth'; export * from './communication'; export * from './crypto'; diff --git a/server/src/domain/job/job.constants.ts b/server/src/domain/job/job.constants.ts index 02fa588c9b..7062ab86b6 100644 --- a/server/src/domain/job/job.constants.ts +++ b/server/src/domain/job/job.constants.ts @@ -55,6 +55,7 @@ export enum JobName { // cleanup DELETE_FILES = 'delete-files', + CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs', // search SEARCH_INDEX_ASSETS = 'search-index-assets', @@ -84,6 +85,7 @@ export const JOBS_TO_QUEUE: Record = { [JobName.USER_DELETE_CHECK]: QueueName.BACKGROUND_TASK, [JobName.USER_DELETION]: QueueName.BACKGROUND_TASK, [JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK, + [JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK, [JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK, // conversion diff --git a/server/src/domain/job/job.repository.ts b/server/src/domain/job/job.repository.ts index f605bef4b4..a452ad4f9b 100644 --- a/server/src/domain/job/job.repository.ts +++ b/server/src/domain/job/job.repository.ts @@ -68,6 +68,9 @@ export type JobItem = // Filesystem | { name: JobName.DELETE_FILES; data: IDeleteFilesJob } + // Audit log cleanup + | { name: JobName.CLEAN_OLD_AUDIT_LOGS; data?: IBaseJob } + // Asset Deletion | { name: JobName.PERSON_CLEANUP; data?: IBaseJob } diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index 503440a5cf..f8a323bbba 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -51,6 +51,7 @@ describe(JobService.name, () => { [{ name: JobName.USER_DELETE_CHECK }], [{ name: JobName.PERSON_CLEANUP }], [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }], + [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }], ]); }); }); diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index 1c82908911..cc90e4ccd3 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -136,6 +136,7 @@ export class JobService { await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK }); await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }); + await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS }); } /** diff --git a/server/src/immich/api-v1/asset/asset-repository.ts b/server/src/immich/api-v1/asset/asset-repository.ts index ce85bd369b..b19b33632f 100644 --- a/server/src/immich/api-v1/asset/asset-repository.ts +++ b/server/src/immich/api-v1/asset/asset-repository.ts @@ -1,7 +1,7 @@ import { AssetEntity, ExifEntity } from '@app/infra/entities'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { IsNull, Not } from 'typeorm'; +import { IsNull, MoreThan, Not } from 'typeorm'; import { In } from 'typeorm/find-options/operator/In'; import { Repository } from 'typeorm/repository/Repository'; import { AssetSearchDto } from './dto/asset-search.dto'; @@ -131,6 +131,7 @@ export class AssetRepository implements IAssetRepository { isVisible: true, isFavorite: dto.isFavorite, isArchived: dto.isArchived, + updatedAt: dto.updatedAfter ? MoreThan(dto.updatedAfter) : undefined, }, relations: { exifInfo: true, diff --git a/server/src/immich/api-v1/asset/dto/asset-search.dto.ts b/server/src/immich/api-v1/asset/dto/asset-search.dto.ts index a629c915c8..72d16edbed 100644 --- a/server/src/immich/api-v1/asset/dto/asset-search.dto.ts +++ b/server/src/immich/api-v1/asset/dto/asset-search.dto.ts @@ -1,7 +1,7 @@ import { toBoolean } from '@app/domain'; import { ApiProperty } from '@nestjs/swagger'; -import { Transform } from 'class-transformer'; -import { IsBoolean, IsNotEmpty, IsNumber, IsOptional, IsUUID } from 'class-validator'; +import { Transform, Type } from 'class-transformer'; +import { IsBoolean, IsDate, IsNotEmpty, IsNumber, IsOptional, IsUUID } from 'class-validator'; export class AssetSearchDto { @IsOptional() @@ -32,4 +32,9 @@ export class AssetSearchDto { @IsUUID('4') @ApiProperty({ format: 'uuid' }) userId?: string; + + @IsOptional() + @IsDate() + @Type(() => Date) + updatedAfter?: Date; } diff --git a/server/src/immich/app.module.ts b/server/src/immich/app.module.ts index 1067485ac0..6b08228bd8 100644 --- a/server/src/immich/app.module.ts +++ b/server/src/immich/app.module.ts @@ -16,6 +16,7 @@ import { APIKeyController, AppController, AssetController, + AuditController, AuthController, JobController, OAuthController, @@ -42,6 +43,7 @@ import { AppController, AlbumController, APIKeyController, + AuditController, AuthController, JobController, OAuthController, diff --git a/server/src/immich/controllers/asset.controller.ts b/server/src/immich/controllers/asset.controller.ts index 652238a1ca..7a69c34e85 100644 --- a/server/src/immich/controllers/asset.controller.ts +++ b/server/src/immich/controllers/asset.controller.ts @@ -9,14 +9,14 @@ import { AuthUserDto, DownloadInfoDto, DownloadResponseDto, + MapMarkerDto, MapMarkerResponseDto, MemoryLaneDto, + MemoryLaneResponseDto, TimeBucketAssetDto, TimeBucketDto, TimeBucketResponseDto, } from '@app/domain'; -import { MapMarkerDto } from '@app/domain/asset/dto/map-marker.dto'; -import { MemoryLaneResponseDto } from '@app/domain/asset/response-dto/memory-lane-response.dto'; import { Body, Controller, Get, HttpCode, HttpStatus, Param, Post, Put, Query, StreamableFile } from '@nestjs/common'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { Authenticated, AuthUser, SharedLinkRoute } from '../app.guard'; diff --git a/server/src/immich/controllers/audit.controller.ts b/server/src/immich/controllers/audit.controller.ts new file mode 100644 index 0000000000..8b28f6e9f3 --- /dev/null +++ b/server/src/immich/controllers/audit.controller.ts @@ -0,0 +1,18 @@ +import { AuditDeletesDto, AuditDeletesResponseDto, AuditService, AuthUserDto } from '@app/domain'; +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { Authenticated, AuthUser } from '../app.guard'; +import { UseValidation } from '../app.utils'; + +@ApiTags('Audit') +@Controller('audit') +@Authenticated() +@UseValidation() +export class AuditController { + constructor(private service: AuditService) {} + + @Get('deletes') + getAuditDeletes(@AuthUser() authUser: AuthUserDto, @Query() dto: AuditDeletesDto): Promise { + return this.service.getDeletes(authUser, dto); + } +} diff --git a/server/src/immich/controllers/index.ts b/server/src/immich/controllers/index.ts index e257c5a9ad..b28e82ecbf 100644 --- a/server/src/immich/controllers/index.ts +++ b/server/src/immich/controllers/index.ts @@ -2,6 +2,7 @@ export * from './album.controller'; export * from './api-key.controller'; export * from './app.controller'; export * from './asset.controller'; +export * from './audit.controller'; export * from './auth.controller'; export * from './job.controller'; export * from './oauth.controller'; diff --git a/server/src/infra/database.config.ts b/server/src/infra/database.config.ts index 8df8777051..089fd6878c 100644 --- a/server/src/infra/database.config.ts +++ b/server/src/infra/database.config.ts @@ -17,6 +17,7 @@ export const databaseConfig: PostgresConnectionOptions = { entities: [__dirname + '/entities/*.entity.{js,ts}'], synchronize: false, migrations: [__dirname + '/migrations/*.{js,ts}'], + subscribers: [__dirname + '/subscribers/*.{js,ts}'], migrationsRun: true, connectTimeoutMS: 10000, // 10 seconds ...urlOrParts, diff --git a/server/src/infra/entities/audit.entity.ts b/server/src/infra/entities/audit.entity.ts new file mode 100644 index 0000000000..be5e14891c --- /dev/null +++ b/server/src/infra/entities/audit.entity.ts @@ -0,0 +1,34 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; + +export enum DatabaseAction { + CREATE = 'CREATE', + UPDATE = 'UPDATE', + DELETE = 'DELETE', +} + +export enum EntityType { + ASSET = 'ASSET', + ALBUM = 'ALBUM', +} + +@Entity('audit') +@Index('IDX_ownerId_createdAt', ['ownerId', 'createdAt']) +export class AuditEntity { + @PrimaryGeneratedColumn('increment') + id!: number; + + @Column() + entityType!: EntityType; + + @Column({ type: 'uuid' }) + entityId!: string; + + @Column() + action!: DatabaseAction; + + @Column({ type: 'uuid' }) + ownerId!: string; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; +} diff --git a/server/src/infra/entities/index.ts b/server/src/infra/entities/index.ts index 6864a3f739..632a8d6b44 100644 --- a/server/src/infra/entities/index.ts +++ b/server/src/infra/entities/index.ts @@ -2,6 +2,7 @@ import { AlbumEntity } from './album.entity'; import { APIKeyEntity } from './api-key.entity'; import { AssetFaceEntity } from './asset-face.entity'; import { AssetEntity } from './asset.entity'; +import { AuditEntity } from './audit.entity'; import { PartnerEntity } from './partner.entity'; import { PersonEntity } from './person.entity'; import { SharedLinkEntity } from './shared-link.entity'; @@ -15,6 +16,7 @@ export * from './album.entity'; export * from './api-key.entity'; export * from './asset-face.entity'; export * from './asset.entity'; +export * from './audit.entity'; export * from './exif.entity'; export * from './partner.entity'; export * from './person.entity'; @@ -30,6 +32,7 @@ export const databaseEntities = [ APIKeyEntity, AssetEntity, AssetFaceEntity, + AuditEntity, PartnerEntity, PersonEntity, SharedLinkEntity, diff --git a/server/src/infra/infra.module.ts b/server/src/infra/infra.module.ts index 060c64ae35..98d4387ebe 100644 --- a/server/src/infra/infra.module.ts +++ b/server/src/infra/infra.module.ts @@ -2,6 +2,7 @@ import { IAccessRepository, IAlbumRepository, IAssetRepository, + IAuditRepository, ICommunicationRepository, ICryptoRepository, IFaceRepository, @@ -35,6 +36,7 @@ import { AlbumRepository, APIKeyRepository, AssetRepository, + AuditRepository, CommunicationRepository, CryptoRepository, FaceRepository, @@ -58,6 +60,7 @@ const providers: Provider[] = [ { provide: IAccessRepository, useClass: AccessRepository }, { provide: IAlbumRepository, useClass: AlbumRepository }, { provide: IAssetRepository, useClass: AssetRepository }, + { provide: IAuditRepository, useClass: AuditRepository }, { provide: ICommunicationRepository, useClass: CommunicationRepository }, { provide: ICryptoRepository, useClass: CryptoRepository }, { provide: IFaceRepository, useClass: FaceRepository }, diff --git a/server/src/infra/migrations/1692804658140-AddAuditTable.ts b/server/src/infra/migrations/1692804658140-AddAuditTable.ts new file mode 100644 index 0000000000..71b8c7b2c6 --- /dev/null +++ b/server/src/infra/migrations/1692804658140-AddAuditTable.ts @@ -0,0 +1,16 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddAuditTable1692804658140 implements MigrationInterface { + name = 'AddAuditTable1692804658140' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "audit" ("id" SERIAL NOT NULL, "entityType" character varying NOT NULL, "entityId" uuid NOT NULL, "action" character varying NOT NULL, "ownerId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_1d3d120ddaf7bc9b1ed68ed463a" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX "IDX_ownerId_createdAt" ON "audit" ("ownerId", "createdAt") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_ownerId_createdAt"`); + await queryRunner.query(`DROP TABLE "audit"`); + } + +} diff --git a/server/src/infra/repositories/audit.repository.ts b/server/src/infra/repositories/audit.repository.ts new file mode 100644 index 0000000000..b19d385777 --- /dev/null +++ b/server/src/infra/repositories/audit.repository.ts @@ -0,0 +1,26 @@ +import { AuditSearch, IAuditRepository } from '@app/domain'; +import { InjectRepository } from '@nestjs/typeorm'; +import { LessThan, MoreThan, Repository } from 'typeorm'; +import { AuditEntity } from '../entities'; + +export class AuditRepository implements IAuditRepository { + constructor(@InjectRepository(AuditEntity) private repository: Repository) {} + + getAfter(since: Date, options: AuditSearch): Promise { + return this.repository + .createQueryBuilder('audit') + .where({ + createdAt: MoreThan(since), + action: options.action, + entityType: options.entityType, + ownerId: options.ownerId, + }) + .distinctOn(['audit.entityId', 'audit.entityType']) + .orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC') + .getMany(); + } + + async removeBefore(before: Date): Promise { + await this.repository.delete({ createdAt: LessThan(before) }); + } +} diff --git a/server/src/infra/repositories/index.ts b/server/src/infra/repositories/index.ts index 5c7261b2d4..c52c350fbc 100644 --- a/server/src/infra/repositories/index.ts +++ b/server/src/infra/repositories/index.ts @@ -2,6 +2,7 @@ export * from './access.repository'; export * from './album.repository'; export * from './api-key.repository'; export * from './asset.repository'; +export * from './audit.repository'; export * from './communication.repository'; export * from './crypto.repository'; export * from './face.repository'; diff --git a/server/src/infra/subscribers/audit.subscriber.ts b/server/src/infra/subscribers/audit.subscriber.ts new file mode 100644 index 0000000000..c0e8313077 --- /dev/null +++ b/server/src/infra/subscribers/audit.subscriber.ts @@ -0,0 +1,38 @@ +import { EntitySubscriberInterface, EventSubscriber, RemoveEvent } from 'typeorm'; +import { AlbumEntity, AssetEntity, AuditEntity, DatabaseAction, EntityType } from '../entities'; + +@EventSubscriber() +export class AuditSubscriber implements EntitySubscriberInterface { + async afterRemove(event: RemoveEvent): Promise { + await this.onEvent(DatabaseAction.DELETE, event); + } + + private async onEvent(action: DatabaseAction, event: RemoveEvent): Promise { + const audit = this.getAudit(event.metadata.name, { ...event.entity, id: event.entityId }); + if (audit && audit.entityId && audit.ownerId) { + await event.manager.getRepository(AuditEntity).save({ ...audit, action }); + } + } + + private getAudit(entityName: string, entity: any): Partial | null { + switch (entityName) { + case AssetEntity.name: + const asset = entity as AssetEntity; + return { + entityType: EntityType.ASSET, + entityId: asset.id, + ownerId: asset.ownerId, + }; + + case AlbumEntity.name: + const album = entity as AlbumEntity; + return { + entityType: EntityType.ALBUM, + entityId: album.id, + ownerId: album.ownerId, + }; + } + + return null; + } +} diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 1204a6ebdd..32d3c17b46 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -1,4 +1,5 @@ import { + AuditService, FacialRecognitionService, IDeleteFilesJob, JobName, @@ -35,11 +36,13 @@ export class AppService { private storageService: StorageService, private systemConfigService: SystemConfigService, private userService: UserService, + private auditService: AuditService, ) {} async init() { await this.jobService.registerHandlers({ [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data), + [JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(), [JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(), [JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data), [JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data), diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts index 7c58f7102e..5d421ebfd2 100644 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ b/server/src/microservices/processors/metadata-extraction.processor.ts @@ -408,7 +408,11 @@ export class MetadataExtractionProcessor { } await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); - await this.assetRepository.save({ id: asset.id, fileCreatedAt: fileCreatedAt || undefined }); + await this.assetRepository.save({ + id: asset.id, + fileCreatedAt: fileCreatedAt || undefined, + updatedAt: new Date(), + }); return true; } diff --git a/server/test/fixtures/audit.stub.ts b/server/test/fixtures/audit.stub.ts new file mode 100644 index 0000000000..c915ed8214 --- /dev/null +++ b/server/test/fixtures/audit.stub.ts @@ -0,0 +1,29 @@ +import { AuditEntity, DatabaseAction, EntityType } from '@app/infra/entities'; +import { authStub } from './auth.stub'; + +export const auditStub = { + create: Object.freeze({ + id: 1, + entityId: 'asset-created', + action: DatabaseAction.CREATE, + entityType: EntityType.ASSET, + ownerId: authStub.admin.id, + createdAt: new Date(), + }), + update: Object.freeze({ + id: 2, + entityId: 'asset-updated', + action: DatabaseAction.UPDATE, + entityType: EntityType.ASSET, + ownerId: authStub.admin.id, + createdAt: new Date(), + }), + delete: Object.freeze({ + id: 3, + entityId: 'asset-deleted', + action: DatabaseAction.DELETE, + entityType: EntityType.ASSET, + ownerId: authStub.admin.id, + createdAt: new Date(), + }), +}; diff --git a/server/test/fixtures/index.ts b/server/test/fixtures/index.ts index c0e8aed3ce..624cc0758e 100644 --- a/server/test/fixtures/index.ts +++ b/server/test/fixtures/index.ts @@ -1,6 +1,7 @@ export * from './album.stub'; export * from './api-key.stub'; export * from './asset.stub'; +export * from './audit.stub'; export * from './auth.stub'; export * from './device.stub'; export * from './error.stub'; diff --git a/server/test/repositories/access.repository.mock.ts b/server/test/repositories/access.repository.mock.ts index 1c8d09c507..ad2f68ca50 100644 --- a/server/test/repositories/access.repository.mock.ts +++ b/server/test/repositories/access.repository.mock.ts @@ -1,10 +1,10 @@ import { IAccessRepository } from '@app/domain'; -export type IAccessRepositoryMock = { +export interface IAccessRepositoryMock { asset: jest.Mocked; album: jest.Mocked; library: jest.Mocked; -}; +} export const newAccessRepositoryMock = (): IAccessRepositoryMock => { return { diff --git a/server/test/repositories/audit.repository.mock.ts b/server/test/repositories/audit.repository.mock.ts new file mode 100644 index 0000000000..bd1a4b815a --- /dev/null +++ b/server/test/repositories/audit.repository.mock.ts @@ -0,0 +1,8 @@ +import { IAuditRepository } from '@app/domain'; + +export const newAuditRepositoryMock = (): jest.Mocked => { + return { + getAfter: jest.fn(), + removeBefore: jest.fn(), + }; +}; diff --git a/server/test/repositories/index.ts b/server/test/repositories/index.ts index aa62a5f01a..2b2c190262 100644 --- a/server/test/repositories/index.ts +++ b/server/test/repositories/index.ts @@ -2,6 +2,7 @@ export * from './access.repository.mock'; export * from './album.repository.mock'; export * from './api-key.repository.mock'; export * from './asset.repository.mock'; +export * from './audit.repository.mock'; export * from './communication.repository.mock'; export * from './crypto.repository.mock'; export * from './face.repository.mock'; diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 16ba34df5e..f73b6bcdf7 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -752,6 +752,25 @@ export const AudioCodec = { export type AudioCodec = typeof AudioCodec[keyof typeof AudioCodec]; +/** + * + * @export + * @interface AuditDeletesResponseDto + */ +export interface AuditDeletesResponseDto { + /** + * + * @type {Array} + * @memberof AuditDeletesResponseDto + */ + 'ids': Array; + /** + * + * @type {boolean} + * @memberof AuditDeletesResponseDto + */ + 'needsFullSync': boolean; +} /** * * @export @@ -1243,6 +1262,20 @@ export interface DownloadResponseDto { */ 'totalSize': number; } +/** + * + * @export + * @enum {string} + */ + +export const EntityType = { + Asset: 'ASSET', + Album: 'ALBUM' +} as const; + +export type EntityType = typeof EntityType[keyof typeof EntityType]; + + /** * * @export @@ -5120,11 +5153,12 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration * @param {boolean} [isArchived] * @param {boolean} [withoutThumbs] Include assets without thumbnails * @param {number} [skip] + * @param {string} [updatedAfter] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { + getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { const localVarPath = `/asset`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -5166,6 +5200,12 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration localVarQueryParameter['skip'] = skip; } + if (updatedAfter !== undefined) { + localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ? + (updatedAfter as any).toISOString() : + updatedAfter; + } + if (ifNoneMatch != null) { localVarHeaderParameter['if-none-match'] = String(ifNoneMatch); } @@ -6274,12 +6314,13 @@ export const AssetApiFp = function(configuration?: Configuration) { * @param {boolean} [isArchived] * @param {boolean} [withoutThumbs] Include assets without thumbnails * @param {number} [skip] + * @param {string} [updatedAfter] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, withoutThumbs, skip, ifNoneMatch, options); + async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, withoutThumbs?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, withoutThumbs, skip, updatedAfter, ifNoneMatch, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, /** @@ -6576,7 +6617,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath * @throws {RequiredError} */ getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise> { - return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); + return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); }, /** * Get a single asset\'s information @@ -6895,6 +6936,13 @@ export interface AssetApiGetAllAssetsRequest { */ readonly skip?: number + /** + * + * @type {string} + * @memberof AssetApiGetAllAssets + */ + readonly updatedAfter?: string + /** * ETag of data already cached on the client * @type {string} @@ -7460,7 +7508,7 @@ export class AssetApi extends BaseAPI { * @memberof AssetApi */ public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) { - return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); + return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.withoutThumbs, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); } /** @@ -7671,6 +7719,163 @@ export class AssetApi extends BaseAPI { } +/** + * AuditApi - axios parameter creator + * @export + */ +export const AuditApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * + * @param {EntityType} entityType + * @param {string} after + * @param {string} [userId] + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditDeletes: async (entityType: EntityType, after: string, userId?: string, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'entityType' is not null or undefined + assertParamExists('getAuditDeletes', 'entityType', entityType) + // verify required parameter 'after' is not null or undefined + assertParamExists('getAuditDeletes', 'after', after) + const localVarPath = `/audit/deletes`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + if (entityType !== undefined) { + localVarQueryParameter['entityType'] = entityType; + } + + if (userId !== undefined) { + localVarQueryParameter['userId'] = userId; + } + + if (after !== undefined) { + localVarQueryParameter['after'] = (after as any instanceof Date) ? + (after as any).toISOString() : + after; + } + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * AuditApi - functional programming interface + * @export + */ +export const AuditApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = AuditApiAxiosParamCreator(configuration) + return { + /** + * + * @param {EntityType} entityType + * @param {string} after + * @param {string} [userId] + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getAuditDeletes(entityType: EntityType, after: string, userId?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAuditDeletes(entityType, after, userId, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + } +}; + +/** + * AuditApi - factory interface + * @export + */ +export const AuditApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = AuditApiFp(configuration) + return { + /** + * + * @param {AuditApiGetAuditDeletesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditDeletes(requestParameters: AuditApiGetAuditDeletesRequest, options?: AxiosRequestConfig): AxiosPromise { + return localVarFp.getAuditDeletes(requestParameters.entityType, requestParameters.after, requestParameters.userId, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * Request parameters for getAuditDeletes operation in AuditApi. + * @export + * @interface AuditApiGetAuditDeletesRequest + */ +export interface AuditApiGetAuditDeletesRequest { + /** + * + * @type {EntityType} + * @memberof AuditApiGetAuditDeletes + */ + readonly entityType: EntityType + + /** + * + * @type {string} + * @memberof AuditApiGetAuditDeletes + */ + readonly after: string + + /** + * + * @type {string} + * @memberof AuditApiGetAuditDeletes + */ + readonly userId?: string +} + +/** + * AuditApi - object-oriented interface + * @export + * @class AuditApi + * @extends {BaseAPI} + */ +export class AuditApi extends BaseAPI { + /** + * + * @param {AuditApiGetAuditDeletesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApi + */ + public getAuditDeletes(requestParameters: AuditApiGetAuditDeletesRequest, options?: AxiosRequestConfig) { + return AuditApiFp(this.configuration).getAuditDeletes(requestParameters.entityType, requestParameters.after, requestParameters.userId, options).then((request) => request(this.axios, this.basePath)); + } +} + + /** * AuthenticationApi - axios parameter creator * @export