Skip to content

Commit

Permalink
chore: deburred and refreshed data
Browse files Browse the repository at this point in the history
chore: integrated xcarpentier#475 from kanimetov
  • Loading branch information
mrflip committed Sep 22, 2022
1 parent 66a9696 commit 4127a79
Show file tree
Hide file tree
Showing 8 changed files with 20,543 additions and 2,047 deletions.
2 changes: 1 addition & 1 deletion data/countries-emoji.json

Large diffs are not rendered by default.

7,305 changes: 7,305 additions & 0 deletions data/countries-more-translations.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion data/countries.json

Large diffs are not rendered by default.

90 changes: 90 additions & 0 deletions scripts/deburr.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// From
// https://github.com/lodash/lodash/blob/master/deburr.js

/** Used to match Latin Unicode letters (excluding mathematical operators). */
const reLatin = /[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g

/** Used to compose unicode character classes. */
const rsComboMarksRange = '\\u0300-\\u036f'
const reComboHalfMarksRange = '\\ufe20-\\ufe2f'
const rsComboSymbolsRange = '\\u20d0-\\u20ff'
const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff'
const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff'
const rsComboRange = rsComboMarksRange + reComboHalfMarksRange + rsComboSymbolsRange + rsComboMarksExtendedRange + rsComboMarksSupplementRange

/** Used to compose unicode capture groups. */
const rsCombo = `[${rsComboRange}]`

/**
* Used to match [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks) and
* [combining diacritical marks for symbols](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks_for_Symbols).
*/
const reComboMark = RegExp(rsCombo, 'g')

/** Used to map Latin Unicode letters to basic Latin letters. */
const deburredLetters = {
// Latin-1 Supplement block.
'\xc0': 'A', '\xc1': 'A', '\xc2': 'A', '\xc3': 'A', '\xc4': 'A', '\xc5': 'A',
'\xe0': 'a', '\xe1': 'a', '\xe2': 'a', '\xe3': 'a', '\xe4': 'a', '\xe5': 'a',
'\xc7': 'C', '\xe7': 'c',
'\xd0': 'D', '\xf0': 'd',
'\xc8': 'E', '\xc9': 'E', '\xca': 'E', '\xcb': 'E',
'\xe8': 'e', '\xe9': 'e', '\xea': 'e', '\xeb': 'e',
'\xcc': 'I', '\xcd': 'I', '\xce': 'I', '\xcf': 'I',
'\xec': 'i', '\xed': 'i', '\xee': 'i', '\xef': 'i',
'\xd1': 'N', '\xf1': 'n',
'\xd2': 'O', '\xd3': 'O', '\xd4': 'O', '\xd5': 'O', '\xd6': 'O', '\xd8': 'O',
'\xf2': 'o', '\xf3': 'o', '\xf4': 'o', '\xf5': 'o', '\xf6': 'o', '\xf8': 'o',
'\xd9': 'U', '\xda': 'U', '\xdb': 'U', '\xdc': 'U',
'\xf9': 'u', '\xfa': 'u', '\xfb': 'u', '\xfc': 'u',
'\xdd': 'Y', '\xfd': 'y', '\xff': 'y',
'\xc6': 'Ae', '\xe6': 'ae',
'\xde': 'Th', '\xfe': 'th',
'\xdf': 'ss',
// Latin Extended-A block.
'\u0100': 'A', '\u0102': 'A', '\u0104': 'A',
'\u0101': 'a', '\u0103': 'a', '\u0105': 'a',
'\u0106': 'C', '\u0108': 'C', '\u010a': 'C', '\u010c': 'C',
'\u0107': 'c', '\u0109': 'c', '\u010b': 'c', '\u010d': 'c',
'\u010e': 'D', '\u0110': 'D', '\u010f': 'd', '\u0111': 'd',
'\u0112': 'E', '\u0114': 'E', '\u0116': 'E', '\u0118': 'E', '\u011a': 'E',
'\u0113': 'e', '\u0115': 'e', '\u0117': 'e', '\u0119': 'e', '\u011b': 'e',
'\u011c': 'G', '\u011e': 'G', '\u0120': 'G', '\u0122': 'G',
'\u011d': 'g', '\u011f': 'g', '\u0121': 'g', '\u0123': 'g',
'\u0124': 'H', '\u0126': 'H', '\u0125': 'h', '\u0127': 'h',
'\u0128': 'I', '\u012a': 'I', '\u012c': 'I', '\u012e': 'I', '\u0130': 'I',
'\u0129': 'i', '\u012b': 'i', '\u012d': 'i', '\u012f': 'i', '\u0131': 'i',
'\u0134': 'J', '\u0135': 'j',
'\u0136': 'K', '\u0137': 'k', '\u0138': 'k',
'\u0139': 'L', '\u013b': 'L', '\u013d': 'L', '\u013f': 'L', '\u0141': 'L',
'\u013a': 'l', '\u013c': 'l', '\u013e': 'l', '\u0140': 'l', '\u0142': 'l',
'\u0143': 'N', '\u0145': 'N', '\u0147': 'N', '\u014a': 'N',
'\u0144': 'n', '\u0146': 'n', '\u0148': 'n', '\u014b': 'n',
'\u014c': 'O', '\u014e': 'O', '\u0150': 'O',
'\u014d': 'o', '\u014f': 'o', '\u0151': 'o',
'\u0154': 'R', '\u0156': 'R', '\u0158': 'R',
'\u0155': 'r', '\u0157': 'r', '\u0159': 'r',
'\u015a': 'S', '\u015c': 'S', '\u015e': 'S', '\u0160': 'S',
'\u015b': 's', '\u015d': 's', '\u015f': 's', '\u0161': 's',
'\u0162': 'T', '\u0164': 'T', '\u0166': 'T',
'\u0163': 't', '\u0165': 't', '\u0167': 't',
'\u0168': 'U', '\u016a': 'U', '\u016c': 'U', '\u016e': 'U', '\u0170': 'U', '\u0172': 'U',
'\u0169': 'u', '\u016b': 'u', '\u016d': 'u', '\u016f': 'u', '\u0171': 'u', '\u0173': 'u',
'\u0174': 'W', '\u0175': 'w',
'\u0176': 'Y', '\u0177': 'y', '\u0178': 'Y',
'\u0179': 'Z', '\u017b': 'Z', '\u017d': 'Z',
'\u017a': 'z', '\u017c': 'z', '\u017e': 'z',
'\u0132': 'IJ', '\u0133': 'ij',
'\u0152': 'Oe', '\u0153': 'oe',
'\u0149': "'n", '\u017f': 's'
}

const deburrLetter = function(str) {
return deburredLetters[str] || str
}

function deburr(str) {
return str && str.replace(reLatin, deburrLetter).replace(reComboMark, '')
}

module.exports = deburr
77 changes: 61 additions & 16 deletions scripts/transform-world-countries.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

const countries = require('world-countries')
const flags = require('./countryFlags')
const deburr = require('./deburr')
const moreTranslations = require('../data/countries-more-translations.json')

const isEmoji = process.argv.includes('--emoji')
const isCca2 = process.argv.includes('--cca2')
Expand All @@ -10,7 +12,7 @@ const isSubRegion = process.argv.includes('--subregions')

const getCountryNameAsyncs = (common, translations) =>
Object.keys(translations)
.filter(k => k !== 'common')
.filter(k => (k !== 'common'))
.map(key => ({ [key]: translations[key].common }))
.reduce(
(prev, cur) => ({
Expand All @@ -20,26 +22,65 @@ const getCountryNameAsyncs = (common, translations) =>
{}
)

function uniq(arr) { return [...new Set(arr)].sort() }

const TranslationLanguageCodes = {}

function bagsort(bag) {
const pairs = Object.entries(bag).sort(([kk1], [kk2]) => (kk1.localeCompare(kk2)))
return Object.fromEntries(pairs)
}

const newcountries = countries
.map(
({
cca2,
currency,
callingCode,
region,
subregion,
name: { common },
translations
}) => ({
[cca2]: {
currency,
callingCode,
(info) => {
const {
cca2,
idd,
region,
subregion,
flag: isEmoji ? `flag-${cca2.toLowerCase()}` : flags[cca2],
name: { common, ...getCountryNameAsyncs(common, translations) }
name: { common, official, native },
translations,
currencies,
altSpellings,
latlng,
flag,
..._rest
} = info
const callingCodeRoot = idd.root.replace(/^\+/, '')
const callingCodes = (cca2 === 'US') ? ['1'] : idd.suffixes.map((suff) => (callingCodeRoot + suff)).sort()
const natives = {}
Object.keys(native).forEach((lang) => { natives[lang] = native[lang].common })
const deburred = deburr(common)
const translatedNames = {
...moreTranslations[cca2].name, ...natives, ...getCountryNameAsyncs(common, translations),
}
})
const names = { common, deburred, ...bagsort(translatedNames) }
if (names.per && (! names.fas)) { names.fas = names.per }
if (! names.eng) { names.eng = names.common }
const altsearch = uniq([...altSpellings, ...altSpellings.map(deburr), ...Object.values(names).map(deburr)])
.filter((str) => /\w\w\w/.test(str))
Object.keys(names).forEach((lang) => { TranslationLanguageCodes[lang] = 1 + (TranslationLanguageCodes[lang] || 0) })
// if (/^T/.test(cca2)) { console.warn(info, natives, native, names) }
return {
[cca2]: {
code: cca2,
common: names.common,
official,
native,
latlng,
flagchar: flag,
currency: Object.keys(currencies || []).sort(),
callingCode: callingCodes,
region,
subregion,
flag: isEmoji ? `flag-${cca2.toLowerCase()}` : flags[cca2],
altSpellings,
name: names,
altsearch,
}
}
}
)
.sort((a, b) => {
if (a[Object.keys(a)[0]].name.common === b[Object.keys(b)[0]].name.common) {
Expand All @@ -59,6 +100,10 @@ const newcountries = countries
{}
)

// console.warn(TranslationLanguageCodes)
const WellTranslated = Object.fromEntries(Object.entries(TranslationLanguageCodes).filter(([lang, ct]) => (ct > 50)))
console.warn('export const TranslationLanguageCodeList = ', Object.keys(WellTranslated), 'as const')

if (isCca2) {
console.log(JSON.stringify(Object.keys(newcountries)))
} else if (isRegion) {
Expand Down
Loading

0 comments on commit 4127a79

Please sign in to comment.