From aa023378477fb1a65d24f4614de0940f6a769794 Mon Sep 17 00:00:00 2001 From: Rufus Uttley Date: Mon, 28 Sep 2026 12:06:30 +0100 Subject: [PATCH] fix(server): use GeoNames for reverse-geocoded country names (#30199) Co-authored-by: Daniel Dietzler --- e2e/src/specs/server/api/map.e2e-spec.ts | 8 +-- e2e/src/specs/server/api/search.e2e-spec.ts | 12 ++-- pnpm-lock.yaml | 16 ----- server/package.json | 1 - server/src/repositories/config.repository.ts | 2 + server/src/repositories/map.repository.ts | 64 +++++++++++++++++-- .../1790587508209-RenameGeoNamesCountries.ts | 56 ++++++++++++++++ server/src/schema/migrations/ORDER | 1 + server/test/factories/asset-exif.factory.ts | 2 +- .../repositories/config.repository.mock.ts | 1 + 10 files changed, 129 insertions(+), 34 deletions(-) create mode 100644 server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts diff --git a/e2e/src/specs/server/api/map.e2e-spec.ts b/e2e/src/specs/server/api/map.e2e-spec.ts index cf6a2abdb..d70b047ea 100644 --- a/e2e/src/specs/server/api/map.e2e-spec.ts +++ b/e2e/src/specs/server/api/map.e2e-spec.ts @@ -65,7 +65,7 @@ describe('/map', () => { expect(body).toEqual([ { city: 'Palisade', - country: 'United States of America', + country: 'United States', id: expect.any(String), lat: expect.closeTo(39.115), lon: expect.closeTo(-108.400968), @@ -73,7 +73,7 @@ describe('/map', () => { }, { city: 'Ralston', - country: 'United States of America', + country: 'United States', id: expect.any(String), lat: expect.closeTo(41.2203), lon: expect.closeTo(-96.071625), @@ -113,7 +113,7 @@ describe('/map', () => { expect(body).toEqual([ { city: 'Palisade', - country: 'United States of America', + country: 'United States', id: expect.any(String), lat: expect.closeTo(39.115), lon: expect.closeTo(-108.400968), @@ -121,7 +121,7 @@ describe('/map', () => { }, { city: 'Ralston', - country: 'United States of America', + country: 'United States', id: expect.any(String), lat: expect.closeTo(41.2203), lon: expect.closeTo(-96.071625), diff --git a/e2e/src/specs/server/api/search.e2e-spec.ts b/e2e/src/specs/server/api/search.e2e-spec.ts index 82ff06138..e4616bd03 100644 --- a/e2e/src/specs/server/api/search.e2e-spec.ts +++ b/e2e/src/specs/server/api/search.e2e-spec.ts @@ -491,6 +491,7 @@ describe('/search', () => { .get('/search/suggestions?type=country&includeNull=true') .set('Authorization', `Bearer ${admin.accessToken}`); expect(body).toEqual([ + 'China', 'Cuba', 'France', 'Georgia', @@ -498,12 +499,11 @@ describe('/search', () => { 'Ghana', 'Japan', 'Morocco', - "People's Republic of China", - 'Russian Federation', + 'Russia', 'Singapore', 'Spain', 'Switzerland', - 'United States of America', + 'United States', null, ]); expect(status).toBe(200); @@ -514,6 +514,7 @@ describe('/search', () => { .get('/search/suggestions?type=country') .set('Authorization', `Bearer ${admin.accessToken}`); expect(body).toEqual([ + 'China', 'Cuba', 'France', 'Georgia', @@ -521,12 +522,11 @@ describe('/search', () => { 'Ghana', 'Japan', 'Morocco', - "People's Republic of China", - 'Russian Federation', + 'Russia', 'Singapore', 'Spain', 'Switzerland', - 'United States of America', + 'United States', ]); expect(status).toBe(200); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 640c52e74..a4cf14391 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -529,9 +529,6 @@ importers: helmet: specifier: ^8.1.0 version: 8.3.0 - i18n-iso-countries: - specifier: ^7.6.0 - version: 7.14.0 ioredis: specifier: ^5.8.2 version: 5.11.1(supports-color@8.1.1) @@ -7436,9 +7433,6 @@ packages: dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} - diacritics@1.3.0: - resolution: {integrity: sha512-wlwEkqcsaxvPJML+rDh/2iS824jbREk6DUMUKkEaSlxdYHeS43cClJtsWglvw2RfeXGm6ohKDqsXteJ5sP5enA==} - didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} @@ -8651,10 +8645,6 @@ packages: resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} engines: {node: '>=10.18'} - i18n-iso-countries@7.14.0: - resolution: {integrity: sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg==} - engines: {node: '>= 12'} - iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -20614,8 +20604,6 @@ snapshots: asap: 2.0.6 wrappy: 1.0.2 - diacritics@1.3.0: {} - didyoumean@1.2.2: {} diff-sequences@29.6.3: {} @@ -22341,10 +22329,6 @@ snapshots: hyperdyperid@1.2.0: {} - i18n-iso-countries@7.14.0: - dependencies: - diacritics: 1.3.0 - iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2 diff --git a/server/package.json b/server/package.json index 3c2334037..f857ea667 100644 --- a/server/package.json +++ b/server/package.json @@ -82,7 +82,6 @@ "geo-tz": "^8.0.0", "handlebars": "^4.7.8", "helmet": "^8.1.0", - "i18n-iso-countries": "^7.6.0", "ioredis": "^5.8.2", "jose": "^6.0.0", "js-yaml": "^5.0.0", diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 7c8c7b70e..85621179b 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -90,6 +90,7 @@ export interface EnvData { admin1: string; admin2: string; cities500: string; + countryInfo: string; naturalEarthCountriesPath: string; }; web: { @@ -338,6 +339,7 @@ const getEnv = (): EnvData => { admin1: join(folders.geodata, 'admin1CodesASCII.txt'), admin2: join(folders.geodata, 'admin2Codes.txt'), cities500: join(folders.geodata, citiesFile), + countryInfo: join(folders.geodata, 'countryInfo.txt'), naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'), }, web: { diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index d7234faf0..d797c8c16 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; -import isoCountries from 'i18n-iso-countries'; -import { type Expression, type Insertable, type Kysely, type NotNull, type SqlBool, sql } from 'kysely'; +import { Expression, Insertable, Kysely, NotNull, SqlBool, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { createReadStream, existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; @@ -38,8 +37,15 @@ interface MapDB extends DB { naturalearth_countries_tmp: NaturalEarthCountriesTable; } +interface CountryNames { + byAlpha2: Map; + byAlpha3: Map; +} + @Injectable() export class MapRepository { + private countryNames?: Promise; + constructor( private configRepository: ConfigRepository, private metadataRepository: SystemMetadataRepository, @@ -148,6 +154,8 @@ export class MapRepository { async reverseGeocode(point: GeoPoint): Promise { this.logger.debug(`Request: ${point.latitude},${point.longitude}`); + const countryNames = await this.loadCountryNames(); + const response = await this.db .selectFrom('geodata_places') .selectAll() @@ -166,8 +174,7 @@ export class MapRepository { this.logger.verboseFn(() => `Raw: ${JSON.stringify(response, null, 2)}`); const { countryCode, name: city, admin1Name } = response; - // eslint-disable-next-line import-x/no-named-as-default-member - const country = isoCountries.getName(countryCode, 'en') ?? null; + const country = countryNames.byAlpha2.get(countryCode) ?? null; const state = admin1Name; return { country, state, city }; @@ -195,14 +202,59 @@ export class MapRepository { this.logger.verboseFn(() => `Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`); const { admin_a3 } = ne_response; - // eslint-disable-next-line import-x/no-named-as-default-member - const country = isoCountries.getName(admin_a3, 'en') ?? null; + const country = countryNames.byAlpha3.get(admin_a3) ?? null; const state = null; const city = null; return { country, state, city }; } + // init() skips the geodata import when it is up to date, so load the names on demand + private async loadCountryNames(): Promise { + this.countryNames ??= this.readCountryNames(); + try { + return await this.countryNames; + } catch (error) { + this.countryNames = undefined; + throw error; + } + } + + private async readCountryNames(): Promise { + const byAlpha2 = new Map(); + const byAlpha3 = new Map(); + + const { resourcePaths } = this.configRepository.getEnv(); + const filePath = resourcePaths.geodata.countryInfo; + if (!existsSync(filePath)) { + throw new Error(`Geodata file ${filePath} not found`); + } + + const lineReader = readLine.createInterface({ input: createReadStream(filePath) }); + for await (const line of lineReader) { + if (line.startsWith('#')) { + continue; + } + // Columns: ISO alpha-2, ISO alpha-3, ISO numeric, fips, Country, ... + const fields = line.split('\t', 5); + const alpha2 = fields[0]; + const alpha3 = fields[1]; + const name = fields[4]?.trim(); + if (!name) { + continue; + } + if (alpha2) { + byAlpha2.set(alpha2, name); + } + if (alpha3) { + byAlpha3.set(alpha3, name); + } + } + + this.logger.log(`Loaded ${byAlpha2.size} country names from ${filePath}`); + return { byAlpha2, byAlpha3 }; + } + private async importNaturalEarthCountries() { const { resourcePaths } = this.configRepository.getEnv(); const geoJSONData = JSON.parse(await readFile(resourcePaths.geodata.naturalEarthCountriesPath, 'utf8')); diff --git a/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts new file mode 100644 index 000000000..b1e016fb6 --- /dev/null +++ b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts @@ -0,0 +1,56 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + // (#30199) Reverse geocoding now uses the GeoNames names, so rewrite the stored ones + await sql` + WITH renames ("old", "new") AS ( + VALUES + ('Åland Islands', 'Aland Islands'), + ('Bonaire, Sint Eustatius and Saba', 'Bonaire, Saint Eustatius and Saba'), + ('Brunei Darussalam', 'Brunei'), + ('Cape Verde', 'Cabo Verde'), + ('Cocos (Keeling) Islands', 'Cocos Islands'), + ('Cote d''Ivoire', 'Ivory Coast'), + ('Cote D''Ivoire', 'Ivory Coast'), + ('Curaçao', 'Curacao'), + ('Czech Republic', 'Czechia'), + ('Falkland Islands (Malvinas)', 'Falkland Islands'), + ('Holy See (Vatican City State)', 'Vatican'), + ('Islamic Republic of Iran', 'Iran'), + ('Lao People''s Democratic Republic', 'Laos'), + ('Micronesia, Federated States of', 'Micronesia'), + ('Moldova, Republic of', 'Moldova'), + ('Netherlands', 'The Netherlands'), + ('People''s Republic of China', 'China'), + ('Republic of The Gambia', 'Gambia'), + ('Russian Federation', 'Russia'), + ('Saint Barthélemy', 'Saint Barthelemy'), + ('Saint Martin (French part)', 'Saint Martin'), + ('Sint Maarten (Dutch part)', 'Sint Maarten'), + ('State of Palestine', 'Palestinian Territory'), + ('Syrian Arab Republic', 'Syria'), + ('Taiwan, Province of China', 'Taiwan'), + ('The Republic of North Macedonia', 'North Macedonia'), + ('Timor-Leste', 'Timor Leste'), + ('Türkiye', 'Turkey'), + ('United Republic of Tanzania', 'Tanzania'), + ('United States of America', 'United States'), + ('Virgin Islands, British', 'British Virgin Islands'), + ('Virgin Islands, U.S.', 'U.S. Virgin Islands') + ), + exif AS ( + UPDATE "asset_exif" + SET "country" = renames."new" + FROM renames + WHERE "country" = renames."old" + ) + UPDATE "workflow_step" + SET "config" = jsonb_set("config", '{region,country}', to_jsonb(renames."new")) + FROM renames + WHERE "config"->'region'->>'country' = renames."old" + `.execute(db); +} + +export async function down(): Promise { + // Not implemented: the old names came from a removed dependency +} diff --git a/server/src/schema/migrations/ORDER b/server/src/schema/migrations/ORDER index f4bd7df46..2db4ae0ca 100644 --- a/server/src/schema/migrations/ORDER +++ b/server/src/schema/migrations/ORDER @@ -95,3 +95,4 @@ 1787148183729-ClusterGroups 1787148183730-DeleteMismatchedMemoryAssets 1789419229196-ConvertUserOAuthIdEmptyStringToNull +1790587508209-RenameGeoNamesCountries diff --git a/server/test/factories/asset-exif.factory.ts b/server/test/factories/asset-exif.factory.ts index f6b0e7d48..c01515156 100644 --- a/server/test/factories/asset-exif.factory.ts +++ b/server/test/factories/asset-exif.factory.ts @@ -19,7 +19,7 @@ export class AssetExifFactory { bitsPerSample: null, city: 'Austin', colorspace: null, - country: 'United States of America', + country: 'United States', dateTimeOriginal: factory.date(), description: '', exifImageHeight: 420, diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 4ccf89e17..2b4aa2a19 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -70,6 +70,7 @@ export const envData: EnvData = { admin1: '/build/geodata/admin1CodesASCII.txt', admin2: '/build/geodata/admin2Codes.txt', cities500: '/build/geodata/cities500.txt', + countryInfo: '/build/geodata/countryInfo.txt', naturalEarthCountriesPath: 'build/ne_10m_admin_0_countries.geojson', }, web: {