Normalize Latin-1 to Latin-9

livingdata-co · Sep 17, 2024 · fc5fd47 · fc5fd47
1 parent 795a08e
commit fc5fd47
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 13 deletions.
diff --git a/lib/detect-encoding.js b/lib/detect-encoding.js
@@ -6,16 +6,14 @@ const ENCODING_TO_DETECT = new Set([
   'windows-1252'
 ])
 
-const DEFAULT_ENCODING = 'UTF-8'
-
 export function selectEncoding(result) {
   const candidate = result.find(r => ENCODING_TO_DETECT.has(r.name))
 
-  if (candidate) {
-    return candidate.name
+  if (!candidate || candidate.name === 'UTF-8') {
+    return 'UTF-8'
   }
 
-  return DEFAULT_ENCODING
+  return 'ISO-8859-15'
 }
 
 export function detectEncoding(buffer) {

diff --git a/test/auto-decode-stream.js b/test/auto-decode-stream.js
@@ -19,7 +19,7 @@ test('createAutoDecodeStream / ISO-8859-1', async t => {
       .pipe(autoDecodeStream)
   )
 
-  t.is(encoding, 'ISO-8859-1')
+  t.is(encoding, 'ISO-8859-15')
   t.is(decodedString, 'éléphant')
 })
 

diff --git a/test/csv.js b/test/csv.js
@@ -31,7 +31,7 @@ test('detecting CSV/ISO-8859-1', async t => {
   t.deepEqual(result, {
     format: 'csv',
     formatOptions: {
-      encoding: 'ISO-8859-1',
+      encoding: 'ISO-8859-15',
       delimiter: ',',
       linebreak: '\n',
       quoteChar: '"'
@@ -50,7 +50,7 @@ test('parsing invalid CSV', async t => {
   t.deepEqual(result, {
     format: 'csv',
     formatOptions: {
-      encoding: 'ISO-8859-1',
+      encoding: 'ISO-8859-15',
       delimiter: ',',
       linebreak: '\n',
       quoteChar: '"'

diff --git a/test/detect-encoding.js b/test/detect-encoding.js
@@ -7,20 +7,20 @@ test('selectEncoding empty', t => {
   t.is(selectEncoding([]), 'UTF-8')
 })
 
-test('selectEncoding one allowed', t => {
-  t.is(selectEncoding([{name: 'ISO-8859-1'}]), 'ISO-8859-1')
+test('selectEncoding fallback Latin-9', t => {
+  t.is(selectEncoding([{name: 'ISO-8859-1'}]), 'ISO-8859-15')
 })
 
 test('selectEncoding one rejected', t => {
   t.is(selectEncoding([{name: 'ISO-8859-9'}]), 'UTF-8')
 })
 
-test('selectEncoding multiple allowed', t => {
+test('selectEncoding multiple allowed fallback', t => {
   t.is(selectEncoding([
     {name: 'ISO-8859-9'},
     {name: 'ISO-8859-1'},
     {name: 'ISO-8859-6'}
-  ]), 'ISO-8859-1')
+  ]), 'ISO-8859-15')
 })
 
 test('selectEncoding multiple rejected', t => {
@@ -37,7 +37,7 @@ test('detectEncoding / UTF-8', t => {
 
 test('detectEncoding / ISO-8859-1', t => {
   const text = iconv.encode('éléphant', 'ISO-8859-1')
-  t.is(detectEncoding(text), 'ISO-8859-1')
+  t.is(detectEncoding(text), 'ISO-8859-15')
 })
 
 test('detectEncoding / fallback UTF-8 when unknown', t => {