refactor(server): reverse geocoding (#2167)

* refactor(server): reverse geocoding

* fix: nullable results

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -4,8 +4,8 @@ import {
   IAssetRepository,
   IAssetUploadedJob,
   IBaseJob,
+  IGeocodingRepository,
   IJobRepository,
-  IReverseGeocodingJob,
   JobName,
   QueueName,
   WithoutProperty,
@@ -15,12 +15,10 @@ import { Process, Processor } from '@nestjs/bull';
 import { Inject, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { InjectRepository } from '@nestjs/typeorm';
+import tz_lookup from '@photostructure/tz-lookup';
 import { Job } from 'bull';
 import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
-import tz_lookup from '@photostructure/tz-lookup';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
-import { getName } from 'i18n-iso-countries';
-import geocoder, { InitOptions } from 'local-reverse-geocoder';
 import { Duration } from 'luxon';
 import fs from 'node:fs';
 import path from 'path';
@@ -34,125 +32,44 @@ interface ImmichTags extends Tags {
   ContentIdentifier?: string;
 }
 
-function geocoderInit(init: InitOptions) {
-  return new Promise<void>(function (resolve) {
-    geocoder.init(init, () => {
-      resolve();
-    });
-  });
-}
-
-function geocoderLookup(points: { latitude: number; longitude: number }[]) {
-  return new Promise<GeoData>(function (resolve) {
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-ignore
-    geocoder.lookUp(points, 1, (err, addresses) => {
-      resolve(addresses[0][0] as GeoData);
-    });
-  });
-}
-
-const geocodingPrecisionLevels = ['cities15000', 'cities5000', 'cities1000', 'cities500'];
-
-export type AdminCode = {
-  name: string;
-  asciiName: string;
-  geoNameId: string;
-};
-
-export type GeoData = {
-  geoNameId: string;
-  name: string;
-  asciiName: string;
-  alternateNames: string;
-  latitude: string;
-  longitude: string;
-  featureClass: string;
-  featureCode: string;
-  countryCode: string;
-  cc2?: any;
-  admin1Code?: AdminCode | string;
-  admin2Code?: AdminCode | string;
-  admin3Code?: any;
-  admin4Code?: any;
-  population: string;
-  elevation: string;
-  dem: string;
-  timezone: string;
-  modificationDate: string;
-  distance: number;
-};
-
 @Processor(QueueName.METADATA_EXTRACTION)
 export class MetadataExtractionProcessor {
   private logger = new Logger(MetadataExtractionProcessor.name);
-  private isGeocodeInitialized = false;
   private assetCore: AssetCore;
+  private reverseGeocodingEnabled: boolean;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
-
+    @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository,
-    @InjectRepository(ExifEntity)
+    @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
-    private exifRepository: Repository<ExifEntity>,
 
     configService: ConfigService,
   ) {
     this.assetCore = new AssetCore(assetRepository, jobRepository);
+    this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING');
+    this.init();
+  }
 
-    if (!configService.get('DISABLE_REVERSE_GEOCODING')) {
+  private async init() {
+    this.logger.warn(`Reverse geocoding is ${this.reverseGeocodingEnabled ? 'enabled' : 'disabled'}`);
+    if (!this.reverseGeocodingEnabled) {
+      return;
+    }
+
+    try {
       this.logger.log('Initializing Reverse Geocoding');
-      geocoderInit({
+
-        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
-        // @ts-ignore
+      await this.geocodingRepository.init();
-        citiesFileOverride: geocodingPrecisionLevels[configService.get('REVERSE_GEOCODING_PRECISION')],
+      await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
-        load: {
+
-          admin1: true,
+      this.logger.log('Reverse Geocoding Initialized');
-          admin2: true,
+    } catch (error: any) {
-          admin3And4: false,
+      this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
-          alternateNames: false,
-        },
-        countries: [],
-        dumpDirectory:
-          configService.get('REVERSE_GEOCODING_DUMP_DIRECTORY') || process.cwd() + '/.reverse-geocoding-dump/',
-      }).then(() => {
-        this.isGeocodeInitialized = true;
-        this.logger.log('Reverse Geocoding Initialised');
-      });
     }
   }
 
-  private async reverseGeocodeExif(
-    latitude: number,
-    longitude: number,
-  ): Promise<{ country: string; state: string; city: string }> {
-    const geoCodeInfo = await geocoderLookup([{ latitude, longitude }]);
-
-    const country = getName(geoCodeInfo.countryCode, 'en');
-    const city = geoCodeInfo.name;
-
-    let state = '';
-
-    if (geoCodeInfo.admin2Code) {
-      const adminCode2 = geoCodeInfo.admin2Code as AdminCode;
-      state += adminCode2.name;
-    }
-
-    if (geoCodeInfo.admin1Code) {
-      const adminCode1 = geoCodeInfo.admin1Code as AdminCode;
-
-      if (geoCodeInfo.admin2Code) {
-        const adminCode2 = geoCodeInfo.admin2Code as AdminCode;
-        if (adminCode2.name) {
-          state += ', ';
-        }
-      }
-      state += adminCode1.name;
-    }
-
-    return { country, state, city };
-  }
-
   @Process(JobName.QUEUE_METADATA_EXTRACTION)
   async handleQueueMetadataExtraction(job: Job<IBaseJob>) {
     try {
@@ -241,18 +158,7 @@ export class MetadataExtractionProcessor {
         }
       }
 
-      /**
+      await this.applyReverseGeocoding(newExif);
-       * Reverse Geocoding
-       *
-       * Get the city, state or region name of the asset
-       * based on lat/lon GPS coordinates.
-       */
-      if (this.isGeocodeInitialized && newExif.latitude && newExif.longitude) {
-        const { country, state, city } = await this.reverseGeocodeExif(newExif.latitude, newExif.longitude);
-        newExif.country = country;
-        newExif.state = state;
-        newExif.city = city;
-      }
 
       /**
        * IF the EXIF doesn't contain the width and height of the image,
@@ -282,15 +188,6 @@ export class MetadataExtractionProcessor {
     }
   }
 
-  @Process({ name: JobName.REVERSE_GEOCODING })
-  async reverseGeocoding(job: Job<IReverseGeocodingJob>) {
-    if (this.isGeocodeInitialized) {
-      const { latitude, longitude } = job.data;
-      const { country, state, city } = await this.reverseGeocodeExif(latitude, longitude);
-      await this.exifRepository.update({ assetId: job.data.assetId }, { city, state, country });
-    }
-  }
-
   @Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 })
   async extractVideoMetadata(job: Job<IAssetUploadedJob>) {
     let asset = job.data.asset;
@@ -377,13 +274,7 @@ export class MetadataExtractionProcessor {
         }
       }
 
-      // Reverse GeoCoding
+      await this.applyReverseGeocoding(newExif);
-      if (this.isGeocodeInitialized && newExif.longitude && newExif.latitude) {
-        const { country, state, city } = await this.reverseGeocodeExif(newExif.latitude, newExif.longitude);
-        newExif.country = country;
-        newExif.state = state;
-        newExif.city = city;
-      }
 
       for (const stream of data.streams) {
         if (stream.codec_type === 'video') {
@@ -418,6 +309,20 @@ export class MetadataExtractionProcessor {
     }
   }
 
+  private async applyReverseGeocoding(newExif: ExifEntity) {
+    const { assetId, latitude, longitude } = newExif;
+    if (this.reverseGeocodingEnabled && longitude && latitude) {
+      try {
+        const { country, state, city } = await this.geocodingRepository.reverseGeocode({ latitude, longitude });
+        newExif.country = country;
+        newExif.state = state;
+        newExif.city = city;
+      } catch (error: any) {
+        this.logger.warn(`Unable to run reverse geocoding for asset: ${assetId}, due to ${error}`, error?.stack);
+      }
+    }
+  }
+
   private extractDuration(duration: number | string | null) {
     const videoDurationInSecond = Number(duration);
     if (!videoDurationInSecond) {

server/libs/domain/src/index.ts

@@ -11,6 +11,7 @@ export * from './domain.module';
 export * from './domain.util';
 export * from './job';
 export * from './media';
+export * from './metadata';
 export * from './oauth';
 export * from './search';
 export * from './server-info';

server/libs/domain/src/job/job.constants.ts

@@ -33,7 +33,6 @@ export enum JobName {
   QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
   EXIF_EXTRACTION = 'exif-extraction',
   EXTRACT_VIDEO_METADATA = 'extract-video-metadata',
-  REVERSE_GEOCODING = 'reverse-geocoding',
 
   // user deletion
   USER_DELETION = 'user-deletion',

server/libs/domain/src/job/job.interface.ts

@@ -28,11 +28,3 @@ export interface IDeleteFilesJob extends IBaseJob {
 export interface IUserDeletionJob extends IBaseJob {
   user: UserEntity;
 }
-
-export interface IReverseGeocodingJob extends IBaseJob {
-  assetId: string;
-  latitude: number;
-  longitude: number;
-}
-
-export type IMetadataExtractionJob = IAssetUploadedJob | IReverseGeocodingJob;

server/libs/domain/src/job/job.repository.ts

@@ -5,7 +5,6 @@ import {
   IBaseJob,
   IBulkEntityJob,
   IDeleteFilesJob,
-  IReverseGeocodingJob,
   IUserDeletionJob,
 } from './job.interface';
 
@@ -49,7 +48,6 @@ export type JobItem =
   | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
   | { name: JobName.EXIF_EXTRACTION; data: IAssetUploadedJob }
   | { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetUploadedJob }
-  | { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingJob }
 
   // Object Tagging
   | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob }

server/libs/domain/src/metadata/geocoding.repository.ts

@@ -0,0 +1,17 @@
+export const IGeocodingRepository = 'IGeocodingRepository';
+
+export interface GeoPoint {
+  latitude: number;
+  longitude: number;
+}
+
+export interface ReverseGeocodeResult {
+  country: string | null;
+  state: string | null;
+  city: string | null;
+}
+
+export interface IGeocodingRepository {
+  init(): Promise<void>;
+  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
+}

server/libs/domain/src/metadata/index.ts

@@ -0,0 +1 @@
+export * from './geocoding.repository';

server/libs/infra/src/infra.config.ts

@@ -1,6 +1,7 @@
 import { QueueName } from '@app/domain';
 import { BullModuleOptions } from '@nestjs/bull';
 import { RedisOptions } from 'ioredis';
+import { InitOptions } from 'local-reverse-geocoder';
 import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
 
 function parseRedisConfig(): RedisOptions {
@@ -69,3 +70,21 @@ function parseTypeSenseConfig(): ConfigurationOptions {
 }
 
 export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();
+
+function parseLocalGeocodingConfig(): InitOptions {
+  const precision = Number(process.env.REVERSE_GEOCODING_PRECISION);
+
+  return {
+    citiesFileOverride: precision ? ['cities15000', 'cities5000', 'cities1000', 'cities500'][precision] : undefined,
+    load: {
+      admin1: true,
+      admin2: true,
+      admin3And4: false,
+      alternateNames: false,
+    },
+    countries: [],
+    dumpDirectory: process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/',
+  };
+}
+
+export const localGeocodingConfig: InitOptions = parseLocalGeocodingConfig();

server/libs/infra/src/infra.module.ts

@@ -4,6 +4,7 @@ import {
   ICommunicationRepository,
   ICryptoRepository,
   IDeviceInfoRepository,
+  IGeocodingRepository,
   IJobRepository,
   IKeyRepository,
   IMachineLearningRepository,
@@ -33,6 +34,7 @@ import {
   CryptoRepository,
   DeviceInfoRepository,
   FilesystemProvider,
+  GeocodingRepository,
   JobRepository,
   MachineLearningRepository,
   MediaRepository,
@@ -50,8 +52,9 @@ const providers: Provider[] = [
   { provide: ICommunicationRepository, useClass: CommunicationRepository },
   { provide: ICryptoRepository, useClass: CryptoRepository },
   { provide: IDeviceInfoRepository, useClass: DeviceInfoRepository },
-  { provide: IKeyRepository, useClass: APIKeyRepository },
+  { provide: IGeocodingRepository, useClass: GeocodingRepository },
   { provide: IJobRepository, useClass: JobRepository },
+  { provide: IKeyRepository, useClass: APIKeyRepository },
   { provide: IMachineLearningRepository, useClass: MachineLearningRepository },
   { provide: IMediaRepository, useClass: MediaRepository },
   { provide: ISearchRepository, useClass: TypesenseRepository },

server/libs/infra/src/repositories/geocoding.repository.ts

@@ -0,0 +1,44 @@
+import { GeoPoint, ReverseGeocodeResult } from '@app/domain';
+import { localGeocodingConfig } from '@app/infra';
+import { Injectable, Logger } from '@nestjs/common';
+import { getName } from 'i18n-iso-countries';
+import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
+import { promisify } from 'util';
+
+export interface AdminCode {
+  name: string;
+  asciiName: string;
+  geoNameId: string;
+}
+
+export type GeoData = AddressObject & {
+  admin1Code?: AdminCode | string;
+  admin2Code?: AdminCode | string;
+};
+
+const init = (options: InitOptions): Promise<void> => new Promise<void>((resolve) => geocoder.init(options, resolve));
+const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
+
+@Injectable()
+export class GeocodingRepository {
+  private logger = new Logger(GeocodingRepository.name);
+
+  async init(): Promise<void> {
+    await init(localGeocodingConfig);
+  }
+
+  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
+    this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
+
+    const [address] = await lookup([point], 1);
+    this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`);
+
+    const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData;
+    const country = getName(countryCode, 'en');
+    const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name);
+    const state = stateParts.length > 0 ? stateParts.join(', ') : null;
+    this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`);
+
+    return { country, state, city };
+  }
+}

server/libs/infra/src/repositories/index.ts

@@ -5,6 +5,7 @@ export * from './communication.repository';
 export * from './crypto.repository';
 export * from './device-info.repository';
 export * from './filesystem.provider';
+export * from './geocoding.repository';
 export * from './job.repository';
 export * from './machine-learning.repository';
 export * from './media.repository';

server/libs/infra/src/repositories/job.repository.ts

@@ -1,8 +1,8 @@
 import {
   IAssetJob,
+  IAssetUploadedJob,
   IBaseJob,
   IJobRepository,
-  IMetadataExtractionJob,
   JobCounts,
   JobItem,
   JobName,
@@ -30,7 +30,7 @@ export class JobRepository implements IJobRepository {
     @InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue,
     @InjectQueue(QueueName.OBJECT_TAGGING) private objectTagging: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.CLIP_ENCODING) private clipEmbedding: Queue<IAssetJob | IBaseJob>,
-    @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IMetadataExtractionJob | IBaseJob>,
+    @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IAssetUploadedJob | IBaseJob>,
     @InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue,
     @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue,
     @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IAssetJob | IBaseJob>,
@@ -88,7 +88,6 @@ export class JobRepository implements IJobRepository {
       case JobName.QUEUE_METADATA_EXTRACTION:
       case JobName.EXIF_EXTRACTION:
       case JobName.EXTRACT_VIDEO_METADATA:
-      case JobName.REVERSE_GEOCODING:
         await this.metadataExtraction.add(item.name, item.data);
         break;