From e6d30d72faa98ead4f9b87be049d6d40cca2d1c6 Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Mon, 4 Jul 2022 20:20:43 +0100 Subject: [PATCH] Fix typeorm migrations (#297) * fix: remove config parameter from typeorm cli and update config the config parameter is no longer supported since version 0.3 the config now needs to export a DataSource object to work with the 0.3 cli * fix: update all typeorm entities and migrations to be aligned with database structure * Fixed test-util import databaseConfig * Fixed column mismatch in raw query with new migration * Remove dist build directory when starting dev server Co-authored-by: Alex Tran --- Makefile | 6 +-- .../immich/src/api-v1/asset/asset.service.ts | 2 +- server/apps/immich/test/test-utils.ts | 2 +- .../database/src/config/database.config.ts | 4 +- .../src/entities/asset-album.entity.ts | 5 +- .../database/src/entities/asset.entity.ts | 4 +- .../libs/database/src/entities/exif.entity.ts | 15 ++++++ .../libs/database/src/entities/user.entity.ts | 10 ++-- ...656888591977-RenameAssetAlbumIdSequence.ts | 15 ++++++ ...6888918620-DropExifTextSearchableColumn.ts | 34 ++++++++++++++ ...1566-MatchMigrationsWithTypeORMEntities.ts | 46 +++++++++++++++++++ server/package.json | 2 +- 12 files changed, 128 insertions(+), 17 deletions(-) create mode 100644 server/libs/database/src/migrations/1656888591977-RenameAssetAlbumIdSequence.ts create mode 100644 server/libs/database/src/migrations/1656888918620-DropExifTextSearchableColumn.ts create mode 100644 server/libs/database/src/migrations/1656889061566-MatchMigrationsWithTypeORMEntities.ts diff --git a/Makefile b/Makefile index 4fdea22137..52be5d8055 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ dev: - docker-compose -f ./docker/docker-compose.dev.yml up --remove-orphans + rm -rf ./server/dist && docker-compose -f ./docker/docker-compose.dev.yml up --remove-orphans dev-update: - docker-compose -f ./docker/docker-compose.dev.yml up --build -V --remove-orphans + rm -rf ./server/dist && docker-compose -f ./docker/docker-compose.dev.yml up --build -V --remove-orphans dev-scale: - docker-compose -f ./docker/docker-compose.dev.yml up --build -V --scale immich-server=3 --remove-orphans + rm -rf ./server/dist && docker-compose -f ./docker/docker-compose.dev.yml up --build -V --scale immich-server=3 --remove-orphans stage: docker-compose -f ./docker/docker-compose.staging.yml up --build -V --remove-orphans diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index 9d12206b20..a1fb560efa 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -405,7 +405,7 @@ export class AssetService { ( TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR - e.exif_text_searchable_column @@ PLAINTO_TSQUERY('english', $2) + e."exifTextSearchableColumn" @@ PLAINTO_TSQUERY('english', $2) ); `; diff --git a/server/apps/immich/test/test-utils.ts b/server/apps/immich/test/test-utils.ts index 5a3ff47f81..f3ed1a72da 100644 --- a/server/apps/immich/test/test-utils.ts +++ b/server/apps/immich/test/test-utils.ts @@ -3,7 +3,7 @@ import { CanActivate, ExecutionContext } from '@nestjs/common'; import { TestingModuleBuilder } from '@nestjs/testing'; import { AuthUserDto } from '../src/decorators/auth-user.decorator'; import { JwtAuthGuard } from '../src/modules/immich-jwt/guards/jwt-auth.guard'; -import databaseConfig from '@app/database/config/database.config'; +import { databaseConfig } from '@app/database/config/database.config'; type CustomAuthCallback = () => AuthUserDto; diff --git a/server/libs/database/src/config/database.config.ts b/server/libs/database/src/config/database.config.ts index 7eb3e154e7..87e9b88b47 100644 --- a/server/libs/database/src/config/database.config.ts +++ b/server/libs/database/src/config/database.config.ts @@ -1,5 +1,5 @@ -import { TypeOrmModuleOptions } from '@nestjs/typeorm'; import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; +import {DataSource} from "typeorm"; export const databaseConfig: PostgresConnectionOptions = { type: 'postgres', @@ -14,4 +14,4 @@ export const databaseConfig: PostgresConnectionOptions = { migrationsRun: true, }; -export default databaseConfig; +export const dataSource = new DataSource(databaseConfig); diff --git a/server/libs/database/src/entities/asset-album.entity.ts b/server/libs/database/src/entities/asset-album.entity.ts index b803b418ff..f4dc19e113 100644 --- a/server/libs/database/src/entities/asset-album.entity.ts +++ b/server/libs/database/src/entities/asset-album.entity.ts @@ -1,9 +1,9 @@ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import {Column, Entity, JoinColumn, ManyToOne, OneToOne, PrimaryGeneratedColumn, Unique} from 'typeorm'; import { AlbumEntity } from './album.entity'; import { AssetEntity } from './asset.entity'; @Entity('asset_album') -@Unique('PK_unique_asset_in_album', ['albumId', 'assetId']) +@Unique('UQ_unique_asset_in_album', ['albumId', 'assetId']) export class AssetAlbumEntity { @PrimaryGeneratedColumn() id!: string; @@ -12,6 +12,7 @@ export class AssetAlbumEntity { albumId!: string; @Column() + @OneToOne(() => AssetEntity, (entity) => entity.id) assetId!: string; @ManyToOne(() => AlbumEntity, (album) => album.assets, { diff --git a/server/libs/database/src/entities/asset.entity.ts b/server/libs/database/src/entities/asset.entity.ts index 618befcdb6..ed5e4ee216 100644 --- a/server/libs/database/src/entities/asset.entity.ts +++ b/server/libs/database/src/entities/asset.entity.ts @@ -26,10 +26,10 @@ export class AssetEntity { @Column({ type: 'varchar', nullable: true }) resizePath!: string | null; - @Column({ type: 'varchar', nullable: true }) + @Column({ type: 'varchar', nullable: true, default: '' }) webpPath!: string | null; - @Column({ type: 'varchar', nullable: true }) + @Column({ type: 'varchar', nullable: true, default: '' }) encodedVideoPath!: string; @Column() diff --git a/server/libs/database/src/entities/exif.entity.ts b/server/libs/database/src/entities/exif.entity.ts index fa9d8f8000..26cabf8e14 100644 --- a/server/libs/database/src/entities/exif.entity.ts +++ b/server/libs/database/src/entities/exif.entity.ts @@ -73,4 +73,19 @@ export class ExifEntity { @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true }) @JoinColumn({ name: 'assetId', referencedColumnName: 'id' }) asset?: ExifEntity; + + @Index("exif_text_searchable", { synchronize: false }) + @Column({ + type: 'tsvector', + generatedType: 'STORED', + asExpression: `TO_TSVECTOR('english', + COALESCE(make, '') || ' ' || + COALESCE(model, '') || ' ' || + COALESCE(orientation, '') || ' ' || + COALESCE("lensModel", '') || ' ' || + COALESCE("city", '') || ' ' || + COALESCE("state", '') || ' ' || + COALESCE("country", ''))` + }) + exifTextSearchableColumn!: string } diff --git a/server/libs/database/src/entities/user.entity.ts b/server/libs/database/src/entities/user.entity.ts index 40f5273471..4ba3e4afc7 100644 --- a/server/libs/database/src/entities/user.entity.ts +++ b/server/libs/database/src/entities/user.entity.ts @@ -5,13 +5,13 @@ export class UserEntity { @PrimaryGeneratedColumn('uuid') id!: string; - @Column() + @Column({ default: '' }) firstName!: string; - @Column() + @Column({ default: '' }) lastName!: string; - @Column() + @Column({ default: false }) isAdmin!: boolean; @Column() @@ -23,10 +23,10 @@ export class UserEntity { @Column({ select: false }) salt?: string; - @Column() + @Column({ default: '' }) profileImagePath!: string; - @Column() + @Column({ default: true }) shouldChangePassword!: boolean; @CreateDateColumn() diff --git a/server/libs/database/src/migrations/1656888591977-RenameAssetAlbumIdSequence.ts b/server/libs/database/src/migrations/1656888591977-RenameAssetAlbumIdSequence.ts new file mode 100644 index 0000000000..82646234a7 --- /dev/null +++ b/server/libs/database/src/migrations/1656888591977-RenameAssetAlbumIdSequence.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class RenameAssetAlbumIdSequence1656888591977 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`alter sequence asset_shared_album_id_seq rename to asset_album_id_seq;`); + await queryRunner.query(`alter table asset_album alter column id set default nextval('asset_album_id_seq'::regclass);`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`alter sequence asset_album_id_seq rename to asset_shared_album_id_seq;`); + await queryRunner.query(`alter table asset_album alter column id set default nextval('asset_shared_album_id_seq'::regclass);`); + } + +} diff --git a/server/libs/database/src/migrations/1656888918620-DropExifTextSearchableColumn.ts b/server/libs/database/src/migrations/1656888918620-DropExifTextSearchableColumn.ts new file mode 100644 index 0000000000..cbe2339395 --- /dev/null +++ b/server/libs/database/src/migrations/1656888918620-DropExifTextSearchableColumn.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class DropExifTextSearchableColumns1656888918620 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "exif_text_searchable_column"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE exif + DROP COLUMN IF EXISTS exif_text_searchable_column; + + ALTER TABLE exif + ADD COLUMN IF NOT EXISTS exif_text_searchable_column tsvector + GENERATED ALWAYS AS ( + TO_TSVECTOR('english', + COALESCE(make, '') || ' ' || + COALESCE(model, '') || ' ' || + COALESCE(orientation, '') || ' ' || + COALESCE("lensModel", '') || ' ' || + COALESCE("city", '') || ' ' || + COALESCE("state", '') || ' ' || + COALESCE("country", '') + ) + ) STORED; + + CREATE INDEX exif_text_searchable_idx + ON exif + USING GIN (exif_text_searchable_column); + `); + } + +} diff --git a/server/libs/database/src/migrations/1656889061566-MatchMigrationsWithTypeORMEntities.ts b/server/libs/database/src/migrations/1656889061566-MatchMigrationsWithTypeORMEntities.ts new file mode 100644 index 0000000000..8e7074a8de --- /dev/null +++ b/server/libs/database/src/migrations/1656889061566-MatchMigrationsWithTypeORMEntities.ts @@ -0,0 +1,46 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class MatchMigrationsWithTypeORMEntities1656889061566 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "exif" ADD "exifTextSearchableColumn" tsvector GENERATED ALWAYS AS (TO_TSVECTOR('english', + COALESCE(make, '') || ' ' || + COALESCE(model, '') || ' ' || + COALESCE(orientation, '') || ' ' || + COALESCE("lensModel", '') || ' ' || + COALESCE("city", '') || ' ' || + COALESCE("state", '') || ' ' || + COALESCE("country", ''))) STORED`); + await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" SET NOT NULL`); + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","exifTextSearchableColumn","postgres","public","exif"]); + await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["postgres","public","exif","GENERATED_COLUMN","exifTextSearchableColumn","TO_TSVECTOR('english',\n COALESCE(make, '') || ' ' ||\n COALESCE(model, '') || ' ' ||\n COALESCE(orientation, '') || ' ' ||\n COALESCE(\"lensModel\", '') || ' ' ||\n COALESCE(\"city\", '') || ' ' ||\n COALESCE(\"state\", '') || ' ' ||\n COALESCE(\"country\", ''))"]); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "firstName" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "lastName" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "isAdmin" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "profileImagePath" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "shouldChangePassword" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "FK_a8b79a84996cef6ba6a3662825d"`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "FK_64f2e7d68d1d1d8417acc844a4a"`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "UQ_a1e2734a1ce361e7a26f6b28288"`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "UQ_unique_asset_in_album" UNIQUE ("albumId", "assetId")`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "FK_256a30a03a4a0aff0394051397d" FOREIGN KEY ("albumId") REFERENCES "albums"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "FK_7ae4e03729895bf87e056d7b598" FOREIGN KEY ("assetId") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "shouldChangePassword" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "profileImagePath" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "isAdmin" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "lastName" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "firstName" DROP NOT NULL`); + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","exifTextSearchableColumn","immich","public","exif"]); + await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "exifTextSearchableColumn"`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "FK_7ae4e03729895bf87e056d7b598"`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "FK_256a30a03a4a0aff0394051397d"`); + await queryRunner.query(`ALTER TABLE "asset_album" DROP CONSTRAINT "UQ_unique_asset_in_album"`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "UQ_a1e2734a1ce361e7a26f6b28288" UNIQUE ("albumId", "assetId")`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "FK_64f2e7d68d1d1d8417acc844a4a" FOREIGN KEY ("assetId") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "asset_album" ADD CONSTRAINT "FK_a8b79a84996cef6ba6a3662825d" FOREIGN KEY ("albumId") REFERENCES "albums"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + +} \ No newline at end of file diff --git a/server/package.json b/server/package.json index 9e96ce5f67..d43a7b49bd 100644 --- a/server/package.json +++ b/server/package.json @@ -22,7 +22,7 @@ "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./apps/immich/test/jest-e2e.json", - "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js --config libs/database/src/config/database.config.ts" + "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js" }, "dependencies": { "@mapbox/mapbox-sdk": "^0.13.3",