fix: not found fields

lib/schema/index.js

@@ -26,6 +26,7 @@ exports.fields = {
     required: true,
     trim: true,
     aliases: ['cle_intero', 'cle_interro'],
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     parse(v, {addError, setAdditionnalValues}) {
       if (v.toLowerCase() !== v) {
         addError('casse_invalide')
@@ -82,10 +83,12 @@ exports.fields = {
 
   uid_adresse: {
     trim: true,
+    formats: ['1.1', '1.2', '1.3'],
     aliases: ['uid_adress']
   },
 
   id_ban_commune: {
+    formats: ['1.4'],
     trim: true,
     parse(v, {addError}) {
       if (!isUuid(v)) {
@@ -97,6 +100,7 @@ exports.fields = {
   },
 
   id_ban_toponyme: {
+    formats: ['1.4'],
     trim: true,
     parse(v, {addError}) {
       if (!isUuid(v)) {
@@ -108,6 +112,7 @@ exports.fields = {
   },
 
   id_ban_adresse: {
+    formats: ['1.4'],
     trim: true,
     parse(v, {addError}) {
       if (!isUuid(v)) {
@@ -120,6 +125,7 @@ exports.fields = {
 
   voie_nom: {
     required: true,
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     allowLocales: true,
     parse(v, {addError}) {
@@ -149,14 +155,15 @@ exports.fields = {
   },
 
   lieudit_complement_nom: {
-    version: '1.2',
+    formats: ['1.2', '1.3', '1.4'],
     trim: true,
     allowLocales: true,
     aliases: ['lieudit_co']
   },
 
   numero: {
     required: true,
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     aliases: ['nulmero'],
     parse(v, {addError}) {
@@ -179,6 +186,7 @@ exports.fields = {
   },
 
   suffixe: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     parse(v, {addError}) {
       if (!/^[\da-z]/i.test(v)) {
@@ -194,8 +202,8 @@ exports.fields = {
   },
 
   commune_insee: {
-    version: '1.2',
     required: true,
+    formats: ['1.2', '1.3', '1.4'],
     trim: true,
     aliases: ['commune_in'],
     parse(v, {addError}) {
@@ -216,13 +224,14 @@ exports.fields = {
 
   commune_nom: {
     required: true,
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     allowLocales: true,
     aliases: ['commune_no']
   },
 
   commune_deleguee_insee: {
-    version: '1.2',
+    formats: ['1.2', '1.3', '1.4'],
     trim: true,
     parse(v, {addError}) {
       const code = v.toUpperCase()
@@ -245,12 +254,13 @@ exports.fields = {
   },
 
   commune_deleguee_nom: {
-    version: '1.2',
+    formats: ['1.2', '1.3', '1.4'],
     trim: true,
     allowLocales: true
   },
 
   position: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     enum: [
       'délivrance postale',
@@ -265,6 +275,7 @@ exports.fields = {
   },
 
   x: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     aliases: ['x_l93'],
     trim: true,
     parse(v, {addError}) {
@@ -282,6 +293,7 @@ exports.fields = {
   },
 
   y: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     aliases: ['y_l93'],
     trim: true,
     parse(v, {addError}) {
@@ -299,6 +311,7 @@ exports.fields = {
   },
 
   long: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     aliases: ['long_wgs84', 'lon'],
     trim: true,
     parse(v, {addError}) {
@@ -316,6 +329,7 @@ exports.fields = {
   },
 
   lat: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     aliases: ['lat_wgs84'],
     trim: true,
     parse(v, {addError}) {
@@ -333,7 +347,7 @@ exports.fields = {
   },
 
   cad_parcelles: {
-    version: '1.2',
+    formats: ['1.2', '1.3', '1.4'],
     aliases: ['cad_parcel'],
     trim: true,
 
@@ -361,12 +375,14 @@ exports.fields = {
   },
 
   source: {
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     required: true,
     trim: true
   },
 
   date_der_maj: {
     required: true,
+    formats: ['1.1', '1.2', '1.3', '1.4'],
     trim: true,
     aliases: ['date_der_m', 'dmaj', 'date_maj'],
     parse(v, {addError}) {
@@ -392,7 +408,7 @@ exports.fields = {
   },
 
   certification_commune: {
-    version: '1.3',
+    formats: ['1.3', '1.4'],
     required: false,
     trim: true,
     aliases: ['certification_adresse'],

lib/schema/profiles/1.1-strict.js

@@ -73,5 +73,6 @@ module.exports = {
   isUsed: false,
   relax: false,
   errors,
-  warnings
+  warnings,
+  format: '1.1'
 }

lib/schema/profiles/1.2-strict.js

@@ -87,5 +87,6 @@ module.exports = {
   isUsed: false,
   relax: false,
   errors,
-  warnings
+  warnings,
+  format: '1.2'
 }

lib/schema/profiles/1.3-relax.js

@@ -106,6 +106,7 @@ module.exports = {
   relax: true,
   errors,
   warnings,
-  infos
+  infos,
+  format: '1.3'
 }
 

lib/schema/profiles/1.3-strict.js

@@ -90,5 +90,6 @@ module.exports = {
   isUsed: true,
   relax: false,
   errors,
-  warnings
+  warnings,
+  format: '1.3'
 }

lib/schema/profiles/1.3.js

@@ -113,6 +113,7 @@ module.exports = {
   relax: false,
   errors,
   warnings,
-  infos
+  infos,
+  format: '1.3'
 }
 

lib/schema/profiles/1.4-strict.js

@@ -124,6 +124,7 @@ module.exports = {
   relax: false,
   errors,
   warnings,
-  infos
+  infos,
+  format: '1.4'
 }
 

lib/schema/profiles/1.4.js

@@ -113,9 +113,10 @@ module.exports = {
   code: '1.4',
   name: 'BAL 1.4 (beta)',
   isUsed: false,
-  relax: true,
+  relax: false,
   errors,
   warnings,
-  infos
+  infos,
+  format: '1.4'
 }
 

lib/validate/__tests__/1.4/index.js

@@ -41,7 +41,8 @@ test('Valid file 1.4 with relaxFieldsDetection', async t => {
 
 test('Valid file 1.4 with profile relax', async t => {
   const buffer = await readAsBuffer('1.4-valid-relax.csv')
-  const report = await validate(buffer, {profile: '1.4'})
+  const report = await validate(buffer, {profile: '1.4', relaxFieldsDetection: true})
+
   t.is(report.encoding, 'utf-8')
   t.is(report.parseOk, true)
   t.is(report.profilesValidation['1.4-strict'].isValid, true)

lib/validate/__tests__/index.js

@@ -59,7 +59,7 @@ test('validate a file with aliases / relaxFieldsDetection true', async t => {
     t.true(fields.some(f => f.schemaName === field))
   }
 
-  t.true(notFoundFields.length === 8)
+  t.true(notFoundFields.length === 5)
 
   for (const field of [
     'lieudit_complement_nom',
@@ -113,7 +113,7 @@ test('validate a file with aliases with 1.3-relax', async t => {
     t.true(fields.some(f => f.schemaName === field))
   }
 
-  t.true(notFoundFields.length === 8)
+  t.true(notFoundFields.length === 5)
 
   for (const field of [
     'lieudit_complement_nom',
@@ -148,7 +148,7 @@ test('validate a file with aliases / profile relax and relaxFieldsDetection fals
 
   t.is(aliasedFields.length, 0)
   t.is(knownFields.length, 5)
-  t.is(notFoundFields.length, 17)
+  t.is(notFoundFields.length, 14)
   t.is(unknownFields.length, 9)
 })
 
@@ -172,7 +172,7 @@ test('validate a file with aliases / relaxFieldsDetection false', async t => {
 
   t.is(aliasedFields.length, 0)
   t.is(knownFields.length, 5)
-  t.is(notFoundFields.length, 17)
+  t.is(notFoundFields.length, 14)
   t.is(unknownFields.length, 9)
 })
 
@@ -184,7 +184,7 @@ test('validate a binary file', async t => {
 test('validate an arbitrary CSV file', async t => {
   const buffer = await readAsBuffer('junk.ascii.csv')
   const {notFoundFields} = await validate(buffer)
-  t.is(notFoundFields.length, 22)
+  t.is(notFoundFields.length, 19)
 })
 
 test('validation avec locales', async t => {

lib/validate/fields.js

@@ -5,7 +5,7 @@ function getSchemaVersion(schemaName) {
   return schema.fields[schemaName].version || '1.1'
 }
 
-function computeFields(originalFields, {relaxFieldsDetection, globalErrors}) {
+function computeFields(originalFields, format, {relaxFieldsDetection, globalErrors}) {
   const fields = originalFields.map(field => ({name: field}))
   const foundFields = new Set()
   const notFoundFields = new Set()
@@ -18,7 +18,6 @@ function computeFields(originalFields, {relaxFieldsDetection, globalErrors}) {
         if (candidate) {
           const exactMatch = schemaName === field
           candidate.schemaName = schemaName
-          candidate.version = getSchemaVersion(schemaName)
           foundFields.add(schemaName)
 
           if (!exactMatch) {
@@ -39,7 +38,7 @@ function computeFields(originalFields, {relaxFieldsDetection, globalErrors}) {
     }
 
     // Si le champ n'est pas trouvé on créé des erreurs
-    if (!foundFields.has(schemaName)) {
+    if (!foundFields.has(schemaName) && schema.fields[schemaName].formats.includes(format)) {
       notFoundFields.add(schemaName)
       globalErrors.add(`field.${schemaName}.missing`)
     }

lib/validate/index.js

@@ -104,7 +104,7 @@ function validateRows(parsedRows, {globalErrors}) {
   checkUseBanIdsEveryRow(parsedRows, {globalErrors})
 }
 
-async function prevalidate(file, relaxFieldsDetection) {
+async function prevalidate(file, format, relaxFieldsDetection) {
   const globalErrors = new Set()
   const rowsErrors = new Set()
 
@@ -114,7 +114,7 @@ async function prevalidate(file, relaxFieldsDetection) {
     return {encoding, linebreak, delimiter, originalFields, parseOk, parseErrors, parsedRows}
   }
 
-  const {fields, notFoundFields} = computeFields(originalFields, {globalErrors, relaxFieldsDetection})
+  const {fields, notFoundFields} = computeFields(originalFields, format, {globalErrors, relaxFieldsDetection})
   const {rows} = await computeRows(parsedRows, {fields, rowsErrors})
   const fileValidation = validateFile({linebreak, encoding, delimiter}, {globalErrors})
   validateRows(parsedRows, {globalErrors})
@@ -200,7 +200,8 @@ async function validate(file, options = {}) {
     relaxFieldsDetection = profiles[profile].relax
   }
 
-  const prevalidateResult = await prevalidate(file, relaxFieldsDetection)
+  const {format} = profiles[profile]
+  const prevalidateResult = await prevalidate(file, format, relaxFieldsDetection)
   return validateProfile(prevalidateResult, profile)
 }
 
@@ -220,4 +221,4 @@ function humanizeLinebreak(linebreak) {
   return 'Inconnu'
 }
 
-module.exports = {validate, validateProfile, prevalidate}
+module.exports = {validate, validateProfile}