sort filenames, vacuum after migration, clean folder table

update asset mock

update nightly test

exclude archived assets

update sql

remove vacuuming logic

keep varchar type for filename

set not null

server/src/interfaces/asset.interface.ts

@@ -149,6 +149,7 @@ export const IAssetRepository = 'IAssetRepository';
 export interface IAssetRepository {
   getAssetsByOriginalPath(userId: string, partialPath: string): Promise<AssetEntity[]>;
   getUniqueOriginalPaths(userId: string): Promise<string[]>;
+  removeEmptyFolders(): Promise<void>;
   create(asset: AssetCreate): Promise<AssetEntity>;
   getByIds(
     ids: string[],

server/src/interfaces/job.interface.ts

@@ -85,6 +85,7 @@ export enum JobName {
   DELETE_FILES = 'delete-files',
   CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
   CLEAN_OLD_SESSION_TOKENS = 'clean-old-session-tokens',
+  CLEAN_FOLDER_TABLE = 'clean-folder-table',
 
   // smart search
   QUEUE_SMART_SEARCH = 'queue-smart-search',
@@ -260,6 +261,7 @@ export type JobItem =
   // Cleanup
   | { name: JobName.CLEAN_OLD_AUDIT_LOGS; data?: IBaseJob }
   | { name: JobName.CLEAN_OLD_SESSION_TOKENS; data?: IBaseJob }
+  | { name: JobName.CLEAN_FOLDER_TABLE; data?: IBaseJob }
 
   // Asset Deletion
   | { name: JobName.PERSON_CLEANUP; data?: IBaseJob }

server/src/migrations/1724802318088-AddAssetFolderRelation.ts

@@ -21,7 +21,11 @@ export class AddAssetFolderRelation1724802318088 implements MigrationInterface {
             path text not null collate numeric
         )`);
 
-    // so postgres chooses the right plan
+    await queryRunner.query(`drop index "IDX_4d66e76dada1ca180f67a205dc"`); // unused index on "originalFileName"
+
+    await queryRunner.query(`alter table assets alter column "originalFileName" set data type varchar collate numeric`);
+
+    // to make sure postgres chooses the right plan
     await queryRunner.query(`alter table asset_folders alter column path set statistics 500`);
 
     await queryRunner.query(`
@@ -42,9 +46,11 @@ export class AddAssetFolderRelation1724802318088 implements MigrationInterface {
         from inserted
         where file_parent("originalPath") = inserted.path`);
 
+    await queryRunner.query(`alter table assets alter column "folderId" set not null`);
+
     await queryRunner.query(`create unique index idx_asset_folders_path on asset_folders (path collate numeric)`);
 
-    await queryRunner.query(`create index idx_assets_folder_id on assets ("folderId")`);
+    await queryRunner.query(`create index idx_assets_folder_id_originalfilename on assets ("folderId", "originalFileName" collate numeric)`);
   }
 
   public async down(queryRunner: QueryRunner): Promise<void> {

server/src/queries/asset.repository.sql

@@ -1132,6 +1132,30 @@ WHERE
   AND "asset"."ownerId" IN ($1)
   AND "asset"."updatedAt" > $2
 
+-- AssetRepository.getUniqueOriginalPaths
+SELECT
+  path
+FROM
+  "asset_folders" "folder"
+WHERE
+  EXISTS (
+    SELECT
+      1
+    FROM
+      "assets" "AssetEntity"
+    WHERE
+      (
+        "ownerId" = $1
+        AND "isVisible" = true
+        AND "isArchived" = false
+        AND "deletedAt" is null
+        AND "folderId" = "folder"."id"
+      )
+      AND ("AssetEntity"."deletedAt" IS NULL)
+  )
+ORDER BY
+  path ASC
+
 -- AssetRepository.getAssetsByOriginalPath
 SELECT
   "asset"."id" AS "asset_id",
@@ -1189,51 +1213,37 @@ SELECT
   "exifInfo"."colorspace" AS "exifInfo_colorspace",
   "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
   "exifInfo"."rating" AS "exifInfo_rating",
-  "exifInfo"."fps" AS "exifInfo_fps",
-  "stack"."id" AS "stack_id",
-  "stack"."ownerId" AS "stack_ownerId",
-  "stack"."primaryAssetId" AS "stack_primaryAssetId",
-  "stackedAssets"."id" AS "stackedAssets_id",
-  "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
-  "stackedAssets"."ownerId" AS "stackedAssets_ownerId",
-  "stackedAssets"."libraryId" AS "stackedAssets_libraryId",
-  "stackedAssets"."deviceId" AS "stackedAssets_deviceId",
-  "stackedAssets"."type" AS "stackedAssets_type",
-  "stackedAssets"."originalPath" AS "stackedAssets_originalPath",
-  "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
-  "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
-  "stackedAssets"."createdAt" AS "stackedAssets_createdAt",
-  "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
-  "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
-  "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
-  "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
-  "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
-  "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
-  "stackedAssets"."isArchived" AS "stackedAssets_isArchived",
-  "stackedAssets"."isExternal" AS "stackedAssets_isExternal",
-  "stackedAssets"."isOffline" AS "stackedAssets_isOffline",
-  "stackedAssets"."checksum" AS "stackedAssets_checksum",
-  "stackedAssets"."duration" AS "stackedAssets_duration",
-  "stackedAssets"."isVisible" AS "stackedAssets_isVisible",
-  "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
-  "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
-  "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId",
-  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
+  "exifInfo"."fps" AS "exifInfo_fps"
 FROM
   "assets" "asset"
-  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
-  LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
-  LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
-  AND ("stackedAssets"."deletedAt" IS NULL)
-WHERE
-  "asset"."ownerId" = $1
+  INNER JOIN "asset_folders" "folder" ON "folder"."id" = "asset"."folderId"
   AND (
-    "asset"."originalPath" LIKE $2
-    AND "asset"."originalPath" NOT LIKE $3
+    asset."folderId" = "folder"."id"
+    and "folder"."path" = $1
   )
+  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
+WHERE
+  (
+    "asset"."isVisible" = true
+    AND "asset"."isArchived" = false
+    AND "asset"."deletedAt" is null
+    AND "asset"."ownerId" = $2
+  )
+  AND ("asset"."deletedAt" IS NULL)
 ORDER BY
-  regexp_replace("asset"."originalPath", '.*/(.+)', '\1') ASC
+  "asset"."originalFileName" ASC
+
+-- AssetRepository.removeEmptyFolders
+delete from asset_folders
+where
+  not exists (
+    select
+      1
+    from
+      assets
+    where
+      "folderId" = asset_folders.id
+  )
 
 -- AssetRepository.upsertFile
 INSERT INTO

server/src/repositories/asset.repository.ts

@@ -870,19 +870,22 @@ export class AssetRepository implements IAssetRepository {
     return builder.getMany();
   }
 
+  @GenerateSql({ params: [DummyValue.UUID] })
   async getUniqueOriginalPaths(userId: string): Promise<string[]> {
-    const folders: { path: string }[] = await this.repository
-      .createQueryBuilder('asset')
+    const folders: { path: string }[] = await this.folderRepository
+      .createQueryBuilder('folder')
       .select('path')
       .whereExists(
         this.repository
           .createQueryBuilder()
           .select('1')
-          .where('ownerId = :userId', { userId })
-          .andWhere('isVisible = true')
-          .andWhere('deletedAt is null')
-          .andWhere('folderId = asset_folders.id'),
+          .where('"ownerId" = :userId', { userId })
+          .andWhere('"isVisible" = true')
+          .andWhere('"isArchived" = false')
+          .andWhere('"deletedAt" is null')
+          .andWhere('"folderId" = folder.id'),
       )
+      .orderBy('path')
       .getRawMany();
 
     return folders.map((row) => row.path);
@@ -895,13 +898,26 @@ export class AssetRepository implements IAssetRepository {
       .innerJoin('asset.folder', 'folder', 'asset."folderId" = folder.id and folder.path = :path', { path })
       .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
       .andWhere('asset.isVisible = true')
+      .andWhere('asset.isArchived = false')
       .andWhere('asset.deletedAt is null')
       .andWhere('asset.ownerId = :userId', { userId })
+      .orderBy('asset.originalFileName')
       .getMany();
 
     return assets;
   }
 
+  @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
+  async removeEmptyFolders(): Promise<void> {
+    await this.repository.manager.query(`
+      delete from asset_folders
+      where not exists(
+        select 1
+        from assets
+        where "folderId" = asset_folders.id
+      )`);
+  }
+
   @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] })
   async upsertFile({ assetId, type, path }: { assetId: string; type: AssetFileType; path: string }): Promise<void> {
     await this.fileRepository.upsert({ assetId, type, path }, { conflictPaths: ['assetId', 'type'] });

server/src/repositories/job.repository.ts

@@ -29,6 +29,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.CLEAN_OLD_SESSION_TOKENS]: QueueName.BACKGROUND_TASK,
   [JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
   [JobName.USER_SYNC_USAGE]: QueueName.BACKGROUND_TASK,
+  [JobName.CLEAN_FOLDER_TABLE]: QueueName.BACKGROUND_TASK,
 
   // conversion
   [JobName.QUEUE_VIDEO_CONVERSION]: QueueName.VIDEO_CONVERSION,

server/src/services/asset.service.ts

@@ -318,6 +318,11 @@ export class AssetService {
     await this.jobRepository.queueAll(jobs);
   }
 
+  async handleCleanupFolders() {
+    await this.assetRepository.removeEmptyFolders();
+    return JobStatus.SUCCESS;
+  }
+
   private async updateMetadata(dto: ISidecarWriteJob) {
     const { id, description, dateTimeOriginal, latitude, longitude, rating } = dto;
     const writes = _.omitBy({ description, dateTimeOriginal, latitude, longitude, rating }, _.isUndefined);

server/src/services/job.service.spec.ts

@@ -73,6 +73,7 @@ describe(JobService.name, () => {
         { name: JobName.USER_SYNC_USAGE },
         { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
         { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+        { name: JobName.CLEAN_FOLDER_TABLE },
       ]);
     });
   });

server/src/services/job.service.ts

@@ -212,6 +212,7 @@ export class JobService {
       { name: JobName.USER_SYNC_USAGE },
       { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
       { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+      { name: JobName.CLEAN_FOLDER_TABLE },
     ]);
   }
 

server/src/services/microservices.service.ts

@@ -51,6 +51,7 @@ export class MicroservicesService {
       [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
       [JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(),
       [JobName.CLEAN_OLD_SESSION_TOKENS]: () => this.sessionService.handleCleanup(),
+      [JobName.CLEAN_FOLDER_TABLE]: () => this.assetService.handleCleanupFolders(),
       [JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
       [JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
       [JobName.USER_SYNC_USAGE]: () => this.userService.handleUserSyncUsage(),

server/test/repositories/asset.repository.mock.ts

@@ -45,5 +45,6 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     upsertFile: vitest.fn(),
     getAssetsByOriginalPath: vitest.fn(),
     getUniqueOriginalPaths: vitest.fn(),
+    removeEmptyFolders: vitest.fn(),
   };
 };