diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index eaedabc7bd..1f6669794f 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto { */ 'existingIds': Array; } -/** - * - * @export - * @enum {string} - */ - -export const CitiesFile = { - Cities15000: 'cities15000', - Cities5000: 'cities5000', - Cities1000: 'cities1000', - Cities500: 'cities500' -} as const; - -export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile]; - - /** * * @export @@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto { * @interface SystemConfigReverseGeocodingDto */ export interface SystemConfigReverseGeocodingDto { - /** - * - * @type {CitiesFile} - * @memberof SystemConfigReverseGeocodingDto - */ - 'citiesFileOverride': CitiesFile; /** * * @type {boolean} @@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto { */ 'enabled': boolean; } - - /** * * @export diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 10f10fb01a..f54b788a4e 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -46,7 +46,6 @@ doc/CQMode.md doc/ChangePasswordDto.md doc/CheckExistingAssetsDto.md doc/CheckExistingAssetsResponseDto.md -doc/CitiesFile.md doc/ClassificationConfig.md doc/Colorspace.md doc/CreateAlbumDto.md @@ -231,7 +230,6 @@ lib/model/bulk_ids_dto.dart lib/model/change_password_dto.dart lib/model/check_existing_assets_dto.dart lib/model/check_existing_assets_response_dto.dart -lib/model/cities_file.dart lib/model/classification_config.dart lib/model/clip_config.dart lib/model/clip_mode.dart @@ -388,7 +386,6 @@ test/bulk_ids_dto_test.dart test/change_password_dto_test.dart test/check_existing_assets_dto_test.dart test/check_existing_assets_response_dto_test.dart -test/cities_file_test.dart test/classification_config_test.dart test/clip_config_test.dart test/clip_mode_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 38aefc4452..903919c050 100644 Binary files a/mobile/openapi/README.md and b/mobile/openapi/README.md differ diff --git a/mobile/openapi/doc/CitiesFile.md b/mobile/openapi/doc/CitiesFile.md deleted file mode 100644 index 9acca959c7..0000000000 Binary files a/mobile/openapi/doc/CitiesFile.md and /dev/null differ diff --git a/mobile/openapi/doc/SystemConfigReverseGeocodingDto.md b/mobile/openapi/doc/SystemConfigReverseGeocodingDto.md index 36eab47477..9fca6c2094 100644 Binary files a/mobile/openapi/doc/SystemConfigReverseGeocodingDto.md and b/mobile/openapi/doc/SystemConfigReverseGeocodingDto.md differ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3052d5d8b2..8941626933 100644 Binary files a/mobile/openapi/lib/api.dart and b/mobile/openapi/lib/api.dart differ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 77a9997010..42a0e5cbb3 100644 Binary files a/mobile/openapi/lib/api_client.dart and b/mobile/openapi/lib/api_client.dart differ diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index d3f7971e3e..728a4ed833 100644 Binary files a/mobile/openapi/lib/api_helper.dart and b/mobile/openapi/lib/api_helper.dart differ diff --git a/mobile/openapi/lib/model/cities_file.dart b/mobile/openapi/lib/model/cities_file.dart deleted file mode 100644 index 96f5d8e573..0000000000 Binary files a/mobile/openapi/lib/model/cities_file.dart and /dev/null differ diff --git a/mobile/openapi/lib/model/system_config_reverse_geocoding_dto.dart b/mobile/openapi/lib/model/system_config_reverse_geocoding_dto.dart index 727e5534fd..d995d96673 100644 Binary files a/mobile/openapi/lib/model/system_config_reverse_geocoding_dto.dart and b/mobile/openapi/lib/model/system_config_reverse_geocoding_dto.dart differ diff --git a/mobile/openapi/test/cities_file_test.dart b/mobile/openapi/test/cities_file_test.dart deleted file mode 100644 index cfe63b7548..0000000000 Binary files a/mobile/openapi/test/cities_file_test.dart and /dev/null differ diff --git a/mobile/openapi/test/system_config_reverse_geocoding_dto_test.dart b/mobile/openapi/test/system_config_reverse_geocoding_dto_test.dart index 12f7655ead..b4aa477df3 100644 Binary files a/mobile/openapi/test/system_config_reverse_geocoding_dto_test.dart and b/mobile/openapi/test/system_config_reverse_geocoding_dto_test.dart differ diff --git a/server/Dockerfile b/server/Dockerfile index be8a5ec999..5f2b796340 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -31,7 +31,7 @@ COPY --from=prod /usr/src/app/node_modules ./node_modules COPY --from=prod /usr/src/app/dist ./dist COPY --from=prod /usr/src/app/bin ./bin COPY --from=web /usr/src/app/build ./www -COPY server/assets assets +COPY server/resources resources COPY server/package.json server/package-lock.json ./ COPY server/start*.sh ./ RUN npm link && npm cache clean --force diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index bd62f44f61..e3ed6402aa 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -6989,15 +6989,6 @@ ], "type": "object" }, - "CitiesFile": { - "enum": [ - "cities15000", - "cities5000", - "cities1000", - "cities500" - ], - "type": "string" - }, "ClassificationConfig": { "properties": { "enabled": { @@ -9112,15 +9103,11 @@ }, "SystemConfigReverseGeocodingDto": { "properties": { - "citiesFileOverride": { - "$ref": "#/components/schemas/CitiesFile" - }, "enabled": { "type": "boolean" } }, "required": [ - "citiesFileOverride", "enabled" ], "type": "object" diff --git a/server/package-lock.json b/server/package-lock.json index 6ae9ae2596..917e643740 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -38,7 +38,6 @@ "i18n-iso-countries": "^7.6.0", "ioredis": "^5.3.2", "joi": "^17.10.0", - "local-reverse-geocoder": "0.16.5", "lodash": "^4.17.21", "luxon": "^3.4.2", "mv": "^2.1.1", @@ -4132,18 +4131,6 @@ "node": ">=0.6" } }, - "node_modules/binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "dependencies": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - }, - "engines": { - "node": "*" - } - }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -4329,14 +4316,6 @@ "node": ">=4" } }, - "node_modules/buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", - "engines": { - "node": ">=0.2.0" - } - }, "node_modules/buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -4500,17 +4479,6 @@ } ] }, - "node_modules/chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "dependencies": { - "traverse": ">=0.3.0 <0.4" - }, - "engines": { - "node": "*" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -5161,19 +5129,6 @@ "node": ">= 8" } }, - "node_modules/csv-parse": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz", - "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw==" - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" - } - }, "node_modules/date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -6300,28 +6255,6 @@ "bser": "2.1.1" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -6575,17 +6508,6 @@ "node": ">= 6" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -8323,11 +8245,6 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/kdt": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz", - "integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg==" - }, "node_modules/keyv": { "version": "4.5.3", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz", @@ -8425,41 +8342,6 @@ "node": ">=6.11.5" } }, - "node_modules/local-reverse-geocoder": { - "version": "0.16.5", - "resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz", - "integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==", - "hasInstallScript": true, - "dependencies": { - "async": "^3.2.4", - "csv-parse": "^5.5.0", - "debug": "^4.3.4", - "kdt": "^0.1.0", - "node-fetch": "^3.3.2", - "unzip-stream": "^0.3.1" - }, - "engines": { - "node": ">=11.0.0", - "npm": ">=6.4.1" - } - }, - "node_modules/local-reverse-geocoder/node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -9077,24 +8959,6 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==" }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -11717,14 +11581,6 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", - "engines": { - "node": "*" - } - }, "node_modules/tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -12314,15 +12170,6 @@ "node": ">=8" } }, - "node_modules/unzip-stream": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz", - "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==", - "dependencies": { - "binary": "^0.3.0", - "mkdirp": "^0.5.1" - } - }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -12480,14 +12327,6 @@ "defaults": "^1.0.3" } }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -15937,15 +15776,6 @@ "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==", "dev": true }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -16077,11 +15907,6 @@ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" - }, "buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -16194,14 +16019,6 @@ "integrity": "sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA==", "dev": true }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -16681,16 +16498,6 @@ "which": "^2.0.1" } }, - "csv-parse": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz", - "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw==" - }, - "data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" - }, "date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -17523,15 +17330,6 @@ "bser": "2.1.1" } }, - "fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "requires": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - } - }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -17717,14 +17515,6 @@ "mime-types": "^2.1.12" } }, - "formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "requires": { - "fetch-blob": "^3.1.2" - } - }, "formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -19005,11 +18795,6 @@ "universalify": "^2.0.0" } }, - "kdt": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz", - "integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg==" - }, "keyv": { "version": "4.5.3", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz", @@ -19094,31 +18879,6 @@ "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", "dev": true }, - "local-reverse-geocoder": { - "version": "0.16.5", - "resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz", - "integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==", - "requires": { - "async": "^3.2.4", - "csv-parse": "^5.5.0", - "debug": "^4.3.4", - "kdt": "^0.1.0", - "node-fetch": "^3.3.2", - "unzip-stream": "^0.3.1" - }, - "dependencies": { - "node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "requires": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - } - } - } - }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -19599,11 +19359,6 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==" }, - "node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" - }, "node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -21569,11 +21324,6 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==" - }, "tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -21900,15 +21650,6 @@ "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", "dev": true }, - "unzip-stream": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz", - "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==", - "requires": { - "binary": "^0.3.0", - "mkdirp": "^0.5.1" - } - }, "update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -22028,11 +21769,6 @@ "defaults": "^1.0.3" } }, - "web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" - }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/server/package.json b/server/package.json index 556740764d..21e19ee59c 100644 --- a/server/package.json +++ b/server/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "@babel/runtime": "^7.22.11", + "@immich/cli": "^2.0.3", "@nestjs/bullmq": "^10.0.1", "@nestjs/common": "^10.2.2", "@nestjs/config": "^3.0.0", @@ -65,10 +66,8 @@ "glob": "^10.3.3", "handlebars": "^4.7.8", "i18n-iso-countries": "^7.6.0", - "@immich/cli": "^2.0.3", "ioredis": "^5.3.2", "joi": "^17.10.0", - "local-reverse-geocoder": "0.16.5", "lodash": "^4.17.21", "luxon": "^3.4.2", "mv": "^2.1.1", diff --git a/server/assets/style-dark.json b/server/resources/style-dark.json similarity index 100% rename from server/assets/style-dark.json rename to server/resources/style-dark.json diff --git a/server/assets/style-light.json b/server/resources/style-light.json similarity index 100% rename from server/assets/style-light.json rename to server/resources/style-light.json diff --git a/server/src/domain/metadata/metadata.service.spec.ts b/server/src/domain/metadata/metadata.service.spec.ts index bb2d706224..7ce7db054a 100644 --- a/server/src/domain/metadata/metadata.service.spec.ts +++ b/server/src/domain/metadata/metadata.service.spec.ts @@ -1,4 +1,4 @@ -import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities'; +import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities'; import { assetStub, newAlbumRepositoryMock, @@ -15,7 +15,7 @@ import { randomBytes } from 'crypto'; import { Stats } from 'fs'; import { constants } from 'fs/promises'; import { when } from 'jest-when'; -import { JobName, QueueName } from '../job'; +import { JobName } from '../job'; import { IAlbumRepository, IAssetRepository, @@ -78,10 +78,7 @@ describe(MetadataService.name, () => { describe('init', () => { beforeEach(async () => { - configMock.load.mockResolvedValue([ - { key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }, - { key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 }, - ]); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]); await sut.init(); }); @@ -90,42 +87,10 @@ describe(MetadataService.name, () => { configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]); await sut.init(); - expect(metadataMock.deleteCache).not.toHaveBeenCalled(); expect(jobMock.pause).toHaveBeenCalledTimes(1); expect(metadataMock.init).toHaveBeenCalledTimes(1); expect(jobMock.resume).toHaveBeenCalledTimes(1); }); - - it('should return if deleteCache is false and the cities precision has not changed', async () => { - await sut.init(); - - expect(metadataMock.deleteCache).not.toHaveBeenCalled(); - expect(jobMock.pause).toHaveBeenCalledTimes(1); - expect(metadataMock.init).toHaveBeenCalledTimes(1); - expect(jobMock.resume).toHaveBeenCalledTimes(1); - }); - - it('should re-init if deleteCache is false but the cities precision has changed', async () => { - configMock.load.mockResolvedValue([ - { key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 }, - ]); - - await sut.init(); - - expect(metadataMock.deleteCache).not.toHaveBeenCalled(); - expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION); - expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 }); - expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION); - }); - - it('should re-init and delete cache if deleteCache is true', async () => { - await sut.init(true); - - expect(metadataMock.deleteCache).toHaveBeenCalled(); - expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION); - expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 }); - expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION); - }); }); describe('handleLivePhotoLinking', () => { diff --git a/server/src/domain/metadata/metadata.service.ts b/server/src/domain/metadata/metadata.service.ts index f600f75a9b..b3a19dac20 100644 --- a/server/src/domain/metadata/metadata.service.ts +++ b/server/src/domain/metadata/metadata.service.ts @@ -97,31 +97,24 @@ export class MetadataService { this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository); } - async init(deleteCache = false) { + async init() { if (!this.subscription) { this.subscription = this.configCore.config$.subscribe(() => this.init()); } const { reverseGeocoding } = await this.configCore.getConfig(); - const { citiesFileOverride } = reverseGeocoding; + const { enabled } = reverseGeocoding; - if (!reverseGeocoding.enabled) { + if (!enabled) { return; } try { - if (deleteCache) { - await this.repository.deleteCache(); - } else if (this.oldCities && this.oldCities === citiesFileOverride) { - return; - } - await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); - await this.repository.init({ citiesFileOverride }); + await this.repository.init(); await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); - this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`); - this.oldCities = citiesFileOverride; + this.logger.log(`Initialized local reverse geocoder`); } catch (error: Error | any) { this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack); } @@ -258,8 +251,9 @@ export class MetadataService { } try { - const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude }); - Object.assign(exifData, { city, state, country }); + const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude }); + if (!reverseGeocode) return; + Object.assign(exifData, reverseGeocode); } catch (error: Error | any) { this.logger.warn( `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`, diff --git a/server/src/domain/repositories/index.ts b/server/src/domain/repositories/index.ts index ff098d8dbb..f812e6ee59 100644 --- a/server/src/domain/repositories/index.ts +++ b/server/src/domain/repositories/index.ts @@ -20,6 +20,7 @@ export * from './shared-link.repository'; export * from './smart-info.repository'; export * from './storage.repository'; export * from './system-config.repository'; +export * from './system-metadata.repository'; export * from './tag.repository'; export * from './user-token.repository'; export * from './user.repository'; diff --git a/server/src/domain/repositories/metadata.repository.ts b/server/src/domain/repositories/metadata.repository.ts index 0c3b78462b..c0a0fef46a 100644 --- a/server/src/domain/repositories/metadata.repository.ts +++ b/server/src/domain/repositories/metadata.repository.ts @@ -1,5 +1,4 @@ import { Tags } from 'exiftool-vendored'; -import { InitOptions } from 'local-reverse-geocoder'; export const IMetadataRepository = 'IMetadataRepository'; @@ -31,9 +30,8 @@ export interface ImmichTags extends Omit { } export interface IMetadataRepository { - init(options: Partial): Promise; + init(): Promise; teardown(): Promise; - reverseGeocode(point: GeoPoint): Promise; - deleteCache(): Promise; + reverseGeocode(point: GeoPoint): Promise; getExifTags(path: string): Promise; } diff --git a/server/src/domain/repositories/system-metadata.repository.ts b/server/src/domain/repositories/system-metadata.repository.ts new file mode 100644 index 0000000000..4d571953bc --- /dev/null +++ b/server/src/domain/repositories/system-metadata.repository.ts @@ -0,0 +1,8 @@ +import { SystemMetadata } from '@app/infra/entities'; + +export const ISystemMetadataRepository = 'ISystemMetadataRepository'; + +export interface ISystemMetadataRepository { + get(key: T): Promise; + set(key: T, value: SystemMetadata[T]): Promise; +} diff --git a/server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts b/server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts index be20a02c79..aa224ccc6c 100644 --- a/server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts +++ b/server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts @@ -1,12 +1,6 @@ -import { CitiesFile } from '@app/infra/entities'; -import { ApiProperty } from '@nestjs/swagger'; -import { IsBoolean, IsEnum } from 'class-validator'; +import { IsBoolean } from 'class-validator'; export class SystemConfigReverseGeocodingDto { @IsBoolean() enabled!: boolean; - - @IsEnum(CitiesFile) - @ApiProperty({ enum: CitiesFile, enumName: 'CitiesFile' }) - citiesFileOverride!: CitiesFile; } diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index b3a030487a..bfab4bb4fc 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -1,6 +1,5 @@ import { AudioCodec, - CitiesFile, Colorspace, CQMode, SystemConfig, @@ -85,7 +84,6 @@ export const defaults = Object.freeze({ }, reverseGeocoding: { enabled: true, - citiesFileOverride: CitiesFile.CITIES_500, }, oauth: { enabled: false, diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index cdeb552b09..6ff4ac5c45 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -1,6 +1,5 @@ import { AudioCodec, - CitiesFile, Colorspace, CQMode, SystemConfig, @@ -85,7 +84,6 @@ const updatedConfig = Object.freeze({ }, reverseGeocoding: { enabled: true, - citiesFileOverride: CitiesFile.CITIES_500, }, oauth: { autoLaunch: true, diff --git a/server/src/domain/system-config/system-config.service.ts b/server/src/domain/system-config/system-config.service.ts index 5e9743ba5a..c81c462e89 100644 --- a/server/src/domain/system-config/system-config.service.ts +++ b/server/src/domain/system-config/system-config.service.ts @@ -79,7 +79,7 @@ export class SystemConfigService { return this.repository.fetchStyle(styleUrl); } - return JSON.parse(await this.repository.readFile(`./assets/style-${theme}.json`)); + return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`)); } async getCustomCss(): Promise { diff --git a/server/src/infra/entities/geodata-admin1.entity.ts b/server/src/infra/entities/geodata-admin1.entity.ts new file mode 100644 index 0000000000..36cf0a805e --- /dev/null +++ b/server/src/infra/entities/geodata-admin1.entity.ts @@ -0,0 +1,10 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity('geodata_admin1') +export class GeodataAdmin1Entity { + @PrimaryColumn({ type: 'varchar' }) + key!: string; + + @Column({ type: 'varchar' }) + name!: string; +} diff --git a/server/src/infra/entities/geodata-admin2.entity.ts b/server/src/infra/entities/geodata-admin2.entity.ts new file mode 100644 index 0000000000..bd03e83776 --- /dev/null +++ b/server/src/infra/entities/geodata-admin2.entity.ts @@ -0,0 +1,10 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity('geodata_admin2') +export class GeodataAdmin2Entity { + @PrimaryColumn({ type: 'varchar' }) + key!: string; + + @Column({ type: 'varchar' }) + name!: string; +} diff --git a/server/src/infra/entities/geodata-places.entity.ts b/server/src/infra/entities/geodata-places.entity.ts new file mode 100644 index 0000000000..244e4261b0 --- /dev/null +++ b/server/src/infra/entities/geodata-places.entity.ts @@ -0,0 +1,59 @@ +import { GeodataAdmin1Entity } from '@app/infra/entities/geodata-admin1.entity'; +import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity'; +import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; + +@Entity('geodata_places', { synchronize: false }) +export class GeodataPlacesEntity { + @PrimaryColumn({ type: 'integer' }) + id!: number; + + @Column({ type: 'varchar', length: 200 }) + name!: string; + + @Column({ type: 'float' }) + longitude!: number; + + @Column({ type: 'float' }) + latitude!: number; + + // @Column({ + // generatedType: 'STORED', + // asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)', + // type: 'earth', + // }) + earthCoord!: unknown; + + @Column({ type: 'char', length: 2 }) + countryCode!: string; + + @Column({ type: 'varchar', length: 20, nullable: true }) + admin1Code!: string; + + @Column({ type: 'varchar', length: 80, nullable: true }) + admin2Code!: string; + + @Column({ + type: 'varchar', + generatedType: 'STORED', + asExpression: `"countryCode" || '.' || "admin1Code"`, + nullable: true, + }) + admin1Key!: string; + + @ManyToOne(() => GeodataAdmin1Entity, { eager: true, nullable: true, createForeignKeyConstraints: false }) + admin1!: GeodataAdmin1Entity; + + @Column({ + type: 'varchar', + generatedType: 'STORED', + asExpression: `"countryCode" || '.' || "admin1Code" || '.' || "admin2Code"`, + nullable: true, + }) + admin2Key!: string; + + @ManyToOne(() => GeodataAdmin2Entity, { eager: true, nullable: true, createForeignKeyConstraints: false }) + admin2!: GeodataAdmin2Entity; + + @Column({ type: 'date' }) + modificationDate!: Date; +} diff --git a/server/src/infra/entities/index.ts b/server/src/infra/entities/index.ts index e4b5c38b4d..6c662a20ad 100644 --- a/server/src/infra/entities/index.ts +++ b/server/src/infra/entities/index.ts @@ -1,3 +1,4 @@ +import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity'; import { ActivityEntity } from './activity.entity'; import { AlbumEntity } from './album.entity'; import { APIKeyEntity } from './api-key.entity'; @@ -6,6 +7,8 @@ import { AssetJobStatusEntity } from './asset-job-status.entity'; import { AssetEntity } from './asset.entity'; import { AuditEntity } from './audit.entity'; import { ExifEntity } from './exif.entity'; +import { GeodataAdmin1Entity } from './geodata-admin1.entity'; +import { GeodataPlacesEntity } from './geodata-places.entity'; import { LibraryEntity } from './library.entity'; import { MoveEntity } from './move.entity'; import { PartnerEntity } from './partner.entity'; @@ -13,6 +16,7 @@ import { PersonEntity } from './person.entity'; import { SharedLinkEntity } from './shared-link.entity'; import { SmartInfoEntity } from './smart-info.entity'; import { SystemConfigEntity } from './system-config.entity'; +import { SystemMetadataEntity } from './system-metadata.entity'; import { TagEntity } from './tag.entity'; import { UserTokenEntity } from './user-token.entity'; import { UserEntity } from './user.entity'; @@ -25,6 +29,9 @@ export * from './asset-job-status.entity'; export * from './asset.entity'; export * from './audit.entity'; export * from './exif.entity'; +export * from './geodata-admin1.entity'; +export * from './geodata-admin2.entity'; +export * from './geodata-places.entity'; export * from './library.entity'; export * from './move.entity'; export * from './partner.entity'; @@ -32,6 +39,7 @@ export * from './person.entity'; export * from './shared-link.entity'; export * from './smart-info.entity'; export * from './system-config.entity'; +export * from './system-metadata.entity'; export * from './tag.entity'; export * from './user-token.entity'; export * from './user.entity'; @@ -45,12 +53,16 @@ export const databaseEntities = [ AssetJobStatusEntity, AuditEntity, ExifEntity, + GeodataPlacesEntity, + GeodataAdmin1Entity, + GeodataAdmin2Entity, MoveEntity, PartnerEntity, PersonEntity, SharedLinkEntity, SmartInfoEntity, SystemConfigEntity, + SystemMetadataEntity, TagEntity, UserEntity, UserTokenEntity, diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 84e72e6380..f6c14e1a7d 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -66,7 +66,6 @@ export enum SystemConfigKey { MAP_DARK_STYLE = 'map.darkStyle', REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled', - REVERSE_GEOCODING_CITIES_FILE_OVERRIDE = 'reverseGeocoding.citiesFileOverride', NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled', @@ -145,13 +144,6 @@ export enum Colorspace { P3 = 'p3', } -export enum CitiesFile { - CITIES_15000 = 'cities15000', - CITIES_5000 = 'cities5000', - CITIES_1000 = 'cities1000', - CITIES_500 = 'cities500', -} - export interface SystemConfig { ffmpeg: { crf: number; @@ -200,7 +192,6 @@ export interface SystemConfig { }; reverseGeocoding: { enabled: boolean; - citiesFileOverride: CitiesFile; }; oauth: { enabled: boolean; diff --git a/server/src/infra/entities/system-metadata.entity.ts b/server/src/infra/entities/system-metadata.entity.ts new file mode 100644 index 0000000000..623806db79 --- /dev/null +++ b/server/src/infra/entities/system-metadata.entity.ts @@ -0,0 +1,18 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity('system_metadata') +export class SystemMetadataEntity { + @PrimaryColumn() + key!: string; + + @Column({ type: 'jsonb', default: '{}', transformer: { to: JSON.stringify, from: JSON.parse } }) + value!: { [key: string]: unknown }; +} + +export enum SystemMetadataKey { + REVERSE_GEOCODING_STATE = 'reverse-geocoding-state', +} + +export interface SystemMetadata extends Record { + [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string }; +} diff --git a/server/src/infra/infra.config.ts b/server/src/infra/infra.config.ts index 90477d8ca3..7f24230326 100644 --- a/server/src/infra/infra.config.ts +++ b/server/src/infra/infra.config.ts @@ -74,6 +74,3 @@ function parseTypeSenseConfig(): ConfigurationOptions { } export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig(); - -export const REVERSE_GEOCODING_DUMP_DIRECTORY = - process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/'; diff --git a/server/src/infra/infra.module.ts b/server/src/infra/infra.module.ts index 276058c0b3..e0d5711d63 100644 --- a/server/src/infra/infra.module.ts +++ b/server/src/infra/infra.module.ts @@ -21,6 +21,7 @@ import { ISmartInfoRepository, IStorageRepository, ISystemConfigRepository, + ISystemMetadataRepository, ITagRepository, IUserRepository, IUserTokenRepository, @@ -56,6 +57,7 @@ import { SharedLinkRepository, SmartInfoRepository, SystemConfigRepository, + SystemMetadataRepository, TagRepository, TypesenseRepository, UserRepository, @@ -84,6 +86,7 @@ const providers: Provider[] = [ { provide: ISmartInfoRepository, useClass: SmartInfoRepository }, { provide: IStorageRepository, useClass: FilesystemProvider }, { provide: ISystemConfigRepository, useClass: SystemConfigRepository }, + { provide: ISystemMetadataRepository, useClass: SystemMetadataRepository }, { provide: ITagRepository, useClass: TagRepository }, { provide: IMediaRepository, useClass: MediaRepository }, { provide: IUserRepository, useClass: UserRepository }, diff --git a/server/src/infra/migrations/1700345818045-SystemMetadata.ts b/server/src/infra/migrations/1700345818045-SystemMetadata.ts new file mode 100644 index 0000000000..0bd9162db7 --- /dev/null +++ b/server/src/infra/migrations/1700345818045-SystemMetadata.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class SystemMetadata1700345818045 implements MigrationInterface { + name = 'SystemMetadata1700345818045' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL DEFAULT '{}', CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key"))`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "system_metadata"`); + } + +} diff --git a/server/src/infra/migrations/1700362016675-Geodata.ts b/server/src/infra/migrations/1700362016675-Geodata.ts new file mode 100644 index 0000000000..1ef562ff7e --- /dev/null +++ b/server/src/infra/migrations/1700362016675-Geodata.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class Geodata1700362016675 implements MigrationInterface { + name = 'Geodata1700362016675' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS cube`) + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS earthdistance`) + await queryRunner.query(`CREATE TABLE "geodata_admin2" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_1e3886455dbb684d6f6b4756726" PRIMARY KEY ("key"))`); + await queryRunner.query(`CREATE TABLE "geodata_admin1" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_3fe3a89c5aac789d365871cb172" PRIMARY KEY ("key"))`); + await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin1Key","\"countryCode\" || '.' || \"admin1Code\""]); + await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin2Key","\"countryCode\" || '.' || \"admin1Code\" || '.' || \"admin2Code\""]); + await queryRunner.query(`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "admin1Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code") STORED, "admin2Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code" || '.' || "admin2Code") STORED, "modificationDate" date NOT NULL, CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id"))`); + await queryRunner.query(`ALTER TABLE "geodata_places" ADD "earthCoord" earth GENERATED ALWAYS AS (ll_to_earth(latitude, longitude)) STORED`) + await queryRunner.query(`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" USING gist ("earthCoord");`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord"`); + await queryRunner.query(`DROP TABLE "geodata_places"`); + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin2Key","immich","public","geodata_places"]); + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin1Key","immich","public","geodata_places"]); + await queryRunner.query(`DROP TABLE "geodata_admin1"`); + await queryRunner.query(`DROP TABLE "geodata_admin2"`); + await queryRunner.query(`DROP EXTENSION cube`); + await queryRunner.query(`DROP EXTENSION earthdistance`); + } + +} diff --git a/server/src/infra/repositories/index.ts b/server/src/infra/repositories/index.ts index 81ea7dd81f..0324fef43c 100644 --- a/server/src/infra/repositories/index.ts +++ b/server/src/infra/repositories/index.ts @@ -19,6 +19,7 @@ export * from './server-info.repository'; export * from './shared-link.repository'; export * from './smart-info.repository'; export * from './system-config.repository'; +export * from './system-metadata.repository'; export * from './tag.repository'; export * from './typesense.repository'; export * from './user-token.repository'; diff --git a/server/src/infra/repositories/metadata.repository.ts b/server/src/infra/repositories/metadata.repository.ts index 63bc29dcba..8f8d068e56 100644 --- a/server/src/infra/repositories/metadata.repository.ts +++ b/server/src/infra/repositories/metadata.repository.ts @@ -1,77 +1,182 @@ -import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from '@app/domain'; -import { REVERSE_GEOCODING_DUMP_DIRECTORY } from '@app/infra'; -import { Injectable, Logger } from '@nestjs/common'; +import { + GeoPoint, + IMetadataRepository, + ImmichTags, + ISystemMetadataRepository, + ReverseGeocodeResult, +} from '@app/domain'; +import { GeodataAdmin1Entity, GeodataAdmin2Entity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities'; +import { DatabaseLock } from '@app/infra/utils/database-locks'; +import { Inject, Logger } from '@nestjs/common'; +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { DefaultReadTaskOptions, exiftool } from 'exiftool-vendored'; -import { readdir, rm } from 'fs/promises'; +import { createReadStream, existsSync } from 'fs'; +import { readFile } from 'fs/promises'; import * as geotz from 'geo-tz'; import { getName } from 'i18n-iso-countries'; -import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder'; -import path from 'path'; -import { promisify } from 'util'; +import * as readLine from 'readline'; +import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm'; -export interface AdminCode { - name: string; - asciiName: string; - geoNameId: string; -} +type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity; +type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity; -export type GeoData = AddressObject & { - admin1Code?: AdminCode | string; - admin2Code?: AdminCode | string; -}; +const CITIES_FILE = 'cities500.txt'; -const lookup = promisify(geocoder.lookUp).bind(geocoder); - -@Injectable() export class MetadataRepository implements IMetadataRepository { + constructor( + @InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository, + @InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository, + @InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository, + @Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository, + @InjectDataSource() private dataSource: DataSource, + ) {} + private logger = new Logger(MetadataRepository.name); - async init(options: Partial): Promise { - return new Promise((resolve) => { - geocoder.init( - { - load: { - admin1: true, - admin2: true, - admin3And4: false, - alternateNames: false, - }, - countries: [], - dumpDirectory: REVERSE_GEOCODING_DUMP_DIRECTORY, - ...options, - }, - resolve, - ); + async init(): Promise { + this.logger.log('Initializing metadata repository'); + const geodataDate = await readFile('/usr/src/resources/geodata-date.txt', 'utf8'); + + await this.geodataPlacesRepository.query('SELECT pg_advisory_lock($1)', [DatabaseLock.GeodataImport]); + + const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE); + + if (geocodingMetadata?.lastUpdate === geodataDate) { + await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]); + return; + } + + this.logger.log('Importing geodata to database from file'); + + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.connect(); + + try { + await queryRunner.startTransaction(); + + await this.loadCities500(queryRunner); + await this.loadAdmin1(queryRunner); + await this.loadAdmin2(queryRunner); + + await queryRunner.commitTransaction(); + } catch (e) { + this.logger.fatal('Error importing geodata', e); + await queryRunner.rollbackTransaction(); + throw e; + } finally { + await queryRunner.release(); + } + + await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, { + lastUpdate: geodataDate, + lastImportFileName: CITIES_FILE, }); + + await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]); + this.logger.log('Geodata import completed'); + } + + private async loadGeodataToTableFromFile( + queryRunner: QueryRunner, + lineToEntityMapper: (lineSplit: string[]) => T, + filePath: string, + entity: GeoEntityClass, + ) { + if (!existsSync(filePath)) { + this.logger.error(`Geodata file ${filePath} not found`); + throw new Error(`Geodata file ${filePath} not found`); + } + await queryRunner.manager.clear(entity); + + const input = createReadStream(filePath); + let buffer: DeepPartial[] = []; + const lineReader = readLine.createInterface({ input: input }); + + for await (const line of lineReader) { + const lineSplit = line.split('\t'); + buffer.push(lineToEntityMapper(lineSplit)); + if (buffer.length > 1000) { + await queryRunner.manager.save(buffer); + buffer = []; + } + } + await queryRunner.manager.save(buffer); + } + + private async loadCities500(queryRunner: QueryRunner) { + await this.loadGeodataToTableFromFile( + queryRunner, + (lineSplit: string[]) => + this.geodataPlacesRepository.create({ + id: parseInt(lineSplit[0]), + name: lineSplit[1], + latitude: parseFloat(lineSplit[4]), + longitude: parseFloat(lineSplit[5]), + countryCode: lineSplit[8], + admin1Code: lineSplit[10], + admin2Code: lineSplit[11], + modificationDate: lineSplit[18], + }), + `/usr/src/resources/${CITIES_FILE}`, + GeodataPlacesEntity, + ); + } + + private async loadAdmin1(queryRunner: QueryRunner) { + await this.loadGeodataToTableFromFile( + queryRunner, + (lineSplit: string[]) => + this.geodataAdmin1Repository.create({ + key: lineSplit[0], + name: lineSplit[1], + }), + '/usr/src/resources/admin1CodesASCII.txt', + GeodataAdmin1Entity, + ); + } + + private async loadAdmin2(queryRunner: QueryRunner) { + await this.loadGeodataToTableFromFile( + queryRunner, + (lineSplit: string[]) => + this.geodataAdmin2Repository.create({ + key: lineSplit[0], + name: lineSplit[1], + }), + '/usr/src/resources/admin2Codes.txt', + GeodataAdmin2Entity, + ); } async teardown() { await exiftool.end(); } - async deleteCache() { - const dumpDirectory = REVERSE_GEOCODING_DUMP_DIRECTORY; - if (dumpDirectory) { - // delete contents - const items = await readdir(dumpDirectory, { withFileTypes: true }); - const folders = items.filter((item) => item.isDirectory()); - for (const { name } of folders) { - await rm(path.join(dumpDirectory, name), { recursive: true, force: true }); - } - } - } - - async reverseGeocode(point: GeoPoint): Promise { + async reverseGeocode(point: GeoPoint): Promise { this.logger.debug(`Request: ${point.latitude},${point.longitude}`); - const [address] = await lookup([point], 1); - this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`); + const response = await this.geodataPlacesRepository + .createQueryBuilder('geoplaces') + .leftJoinAndSelect('geoplaces.admin1', 'admin1') + .leftJoinAndSelect('geoplaces.admin2', 'admin2') + .where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point) + .orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")') + .limit(1) + .getOne(); - const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData; + if (!response) { + this.logger.warn( + `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`, + ); + return null; + } + + this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`); + + const { countryCode, name: city, admin1, admin2 } = response; const country = getName(countryCode, 'en') ?? null; - const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name); + const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name); const state = stateParts.length > 0 ? stateParts.join(', ') : null; - this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`); return { country, state, city }; } diff --git a/server/src/infra/repositories/system-metadata.repository.ts b/server/src/infra/repositories/system-metadata.repository.ts new file mode 100644 index 0000000000..a4f3eeff02 --- /dev/null +++ b/server/src/infra/repositories/system-metadata.repository.ts @@ -0,0 +1,20 @@ +import { ISystemMetadataRepository } from '@app/domain/repositories/system-metadata.repository'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { SystemMetadata, SystemMetadataEntity } from '../entities'; + +export class SystemMetadataRepository implements ISystemMetadataRepository { + constructor( + @InjectRepository(SystemMetadataEntity) + private repository: Repository, + ) {} + async get(key: T): Promise { + const metadata = await this.repository.findOne({ where: { key } }); + if (!metadata) return null; + return metadata.value as SystemMetadata[T]; + } + + async set(key: T, value: SystemMetadata[T]): Promise { + await this.repository.upsert({ key, value }, { conflictPaths: { key: true } }); + } +} diff --git a/server/src/infra/utils/database-locks.ts b/server/src/infra/utils/database-locks.ts new file mode 100644 index 0000000000..756437743b --- /dev/null +++ b/server/src/infra/utils/database-locks.ts @@ -0,0 +1,3 @@ +export enum DatabaseLock { + GeodataImport = 100, +} diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 67d995e331..554519114e 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -92,16 +92,6 @@ export class AppService { [JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(), }); - process.on('uncaughtException', async (error: Error | any) => { - const isCsvError = error.code === 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH'; - if (!isCsvError) { - throw error; - } - - this.logger.warn('Geocoding csv parse error, trying again without cache...'); - await this.metadataService.init(true); - }); - await this.metadataService.init(); await this.searchService.init(); } diff --git a/server/test/repositories/metadata.repository.mock.ts b/server/test/repositories/metadata.repository.mock.ts index 76c6f777a5..c602c54d56 100644 --- a/server/test/repositories/metadata.repository.mock.ts +++ b/server/test/repositories/metadata.repository.mock.ts @@ -2,7 +2,6 @@ import { IMetadataRepository } from '@app/domain'; export const newMetadataRepositoryMock = (): jest.Mocked => { return { - deleteCache: jest.fn(), getExifTags: jest.fn(), init: jest.fn(), teardown: jest.fn(), diff --git a/server/test/test-utils.ts b/server/test/test-utils.ts index 2cbd4f19a6..dc7c1b6988 100644 --- a/server/test/test-utils.ts +++ b/server/test/test-utils.ts @@ -25,7 +25,9 @@ export const db = { const tableNames = entities.length > 0 ? entities.map((entity) => em.getRepository(entity).metadata.tableName) - : dataSource.entityMetadatas.map((entity) => entity.tableName); + : dataSource.entityMetadatas + .map((entity) => entity.tableName) + .filter((tableName) => !tableName.startsWith('geodata')); let deleteUsers = false; for (const tableName of tableNames) { diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index eaedabc7bd..1f6669794f 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto { */ 'existingIds': Array; } -/** - * - * @export - * @enum {string} - */ - -export const CitiesFile = { - Cities15000: 'cities15000', - Cities5000: 'cities5000', - Cities1000: 'cities1000', - Cities500: 'cities500' -} as const; - -export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile]; - - /** * * @export @@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto { * @interface SystemConfigReverseGeocodingDto */ export interface SystemConfigReverseGeocodingDto { - /** - * - * @type {CitiesFile} - * @memberof SystemConfigReverseGeocodingDto - */ - 'citiesFileOverride': CitiesFile; /** * * @type {boolean} @@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto { */ 'enabled': boolean; } - - /** * * @export diff --git a/web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte b/web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte index 7093a0eeb0..fe2f879690 100644 --- a/web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte +++ b/web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte @@ -4,13 +4,12 @@ NotificationType, } from '$lib/components/shared-components/notification/notification'; import { handleError } from '$lib/utils/handle-error'; - import { api, CitiesFile, SystemConfigDto } from '@api'; + import { api, SystemConfigDto } from '@api'; import { cloneDeep, isEqual } from 'lodash-es'; import { fade } from 'svelte/transition'; import SettingAccordion from '../setting-accordion.svelte'; import SettingButtonsRow from '../setting-buttons-row.svelte'; import SettingSwitch from '../setting-switch.svelte'; - import SettingSelect from '../setting-select.svelte'; import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte'; export let config: SystemConfigDto; // this is the config that is being edited @@ -39,7 +38,6 @@ }, reverseGeocoding: { enabled: config.reverseGeocoding.enabled, - citiesFileOverride: config.reverseGeocoding.citiesFileOverride, }, }, }); @@ -131,24 +129,6 @@ subtitle="Enable reverse geocoding" bind:checked={config.reverseGeocoding.enabled} /> - -
- -