From c383e115af197038389c97b2e505699068cea940 Mon Sep 17 00:00:00 2001 From: Zack Pollard <zackpollard@ymail.com> Date: Tue, 5 Nov 2024 16:20:56 +0000 Subject: [PATCH] chore: better observability for geodata import (#13931) --- server/src/repositories/map.repository.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index 7ad94016e8..f87ba6d0ac 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -249,6 +249,7 @@ export class MapRepository implements IMapRepository { const input = createReadStream(filePath); let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = []; const lineReader = readLine.createInterface({ input }); + let count = 0; for await (const line of lineReader) { const lineSplit = line.split('\t'); @@ -257,8 +258,12 @@ export class MapRepository implements IMapRepository { } const geoData = lineToEntityMapper(lineSplit); bufferGeodata.push(geoData); - if (bufferGeodata.length > 1000) { + if (bufferGeodata.length >= 1000) { await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']); + count += bufferGeodata.length; + if (count % 10_000 === 0) { + this.logger.log(`${count} geodata records imported`); + } bufferGeodata = []; } }