diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index f1890630..c6bb0bd4 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -1,12 +1,58 @@ import {Transaction} from 'sequelize' -import {sequelize, District, CommonToponym, Address} from '../../util/sequelize.js' +import {createFantoirCommune} from '@ban-team/fantoir' +import {findCodePostal} from 'codes-postaux/full.js' import mongo from '../../util/mongo.cjs' +import {sequelize, District, CommonToponym, Address} from '../../util/sequelize.js' +import {derivePositionProps} from '../../util/geo.cjs' + +// Seetings + +// The number of records to process per page +const PAGE_SIZE = 100 + +// The path to the fantoir sqlite database +const FANTOIR_PATH = process.env.FANTOIR_PATH || 'data/fantoir.sqlite' + +// The min and max zoom levels to use for the tiles +const COMMON_TOPONYM_TILES_MIN_ZOOM = 10 +const COMMON_TOPONYM_TILES_MAX_ZOOM = 14 +const ADDRESS_TILES_MIN_ZOOM = 12 +const ADDRESS_TILES_MAX_ZOOM = 14 + +// Collections names +const DISTRICT_COLLECTION = 'districts' +const COMMON_TOPONYM_COLLECTION = 'common_toponyms' +const ADDRESS_COLLECTION = 'addresses' + +// The priority of each position type +const POSITION_TYPES_PRIORITY = { + entrance: 1, + building: 2, + 'staircase identifier': 3, + 'unit identifier': 4, + 'utility service': 5, + 'postal delivery': 6, + parcel: 7, + segment: 8, + other: 9 +} -const DistrictCollection = 'districts' -const CommonToponymCollection = 'common_toponyms' -const AddressCollection = 'addresses' - -const pageSize = 100 +const commonToponymPageQuery = ` + SELECT + CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT."createdAt", CT."updatedAt", + ST_Centroid(ST_Collect(ST_GeomFromGeoJSON((A.positions[1])->'geometry'))) AS centroid + FROM + "CommonToponyms" AS CT + LEFT JOIN + "Addresses" AS A + ON + CT.id = A."mainCommonToponymID" + WHERE CT."districtID" = :districtID + GROUP BY CT.id + ORDER BY CT.id ASC + OFFSET :offset + LIMIT :limit +` export default async function exportToExploitationDB({data}) { const {districtID} = data @@ -34,51 +80,157 @@ export default async function exportToExploitationDB({data}) { await deleteAllDataRelatedToDistrict(districtID) // Insert the district - await mongo.db.collection(DistrictCollection).insertOne(district) + await mongo.db.collection(DISTRICT_COLLECTION).insertOne(district) // CommonToponym + // Prepare fantoir finder from cog and fantoir sqlite database + const {meta: {insee: {cog}}} = district + const fantoirFinder = await createFantoirCommune(cog, {FANTOIR_PATH}) + + // Map to store the fantoir code for each common toponym + const commonToponymIDFantoirCodeMap = new Map() + + // Count the total number of common toponyms and pages to process const totalCommonToponymRecords = await CommonToponym.count({ where: {districtID}, transaction, }) - const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / pageSize) - - const fetchAndExportDataFromPage = async (model, collection, pageNumber) => { - const offset = (pageNumber - 1) * pageSize - const pageData = await model.findAll({ - where: {districtID}, - order: [['id', 'ASC']], - offset, - limit: pageSize, - transaction, - raw: true, - }) - // Insert the common toponyms from the related page - await mongo.db.collection(collection).insertMany(pageData, {ordered: false}) + const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE) + + // Helpers + // Helpers to calculate the fantoir code + const calculateCommonToponymFantoirCode = commonToponym => { + // Find the label in 'fra' if possible, otherwise take the first one + const labelValue = commonToponym?.labels?.find(({isoCode}) => isoCode === 'fra')?.value || commonToponym?.labels[0]?.value + const fantoirCode = fantoirFinder.findVoie(labelValue, cog)?.codeFantoir + // Store the fantoir code for the common toponym to be able to calculate the postal codes later + commonToponymIDFantoirCodeMap.set(commonToponym.id, fantoirCode) + return fantoirCode + } + + // Helpers to calculate the tiles + const calculateCommonToponymTiles = commonToponym => { + const {centroid} = commonToponym + if (!centroid) { + return {} + } + + const {crs, ...position} = centroid + const {tiles, x, y} = derivePositionProps(position, COMMON_TOPONYM_TILES_MIN_ZOOM, COMMON_TOPONYM_TILES_MAX_ZOOM) + return {tiles, x, y} + } + + const calculateAddressTiles = address => { + const {positions} = address + // Find the position with the highest priority + const positionPrioritized = positions.reduce((max, item) => (POSITION_TYPES_PRIORITY[item.type] < POSITION_TYPES_PRIORITY[max.type] ? item : max), positions[0]) + // Calculate the tiles for the position with the highest priority + const {tiles, x, y} = derivePositionProps(positionPrioritized?.geometry, ADDRESS_TILES_MIN_ZOOM, ADDRESS_TILES_MAX_ZOOM) + return {tiles, x, y} + } + + // Helpers to calculate the postal code + const calculateCommonToponymPostalCode = commonToponym => { + const fantoirCode = commonToponymIDFantoirCodeMap.get(commonToponym.id) + const {codePostal} = findCodePostal(cog, fantoirCode) + return codePostal + } + + const calculateAddressPostalCode = address => { + const fantoirCode = commonToponymIDFantoirCodeMap.get(address.mainCommonToponymID) + const {number, suffix} = address + const {codePostal} = findCodePostal(cog, fantoirCode, number, suffix) + return codePostal + } + + const formatPageData = (pageData, type) => { + if (type === 'commonToponym') { + return pageData.map(commonToponym => { + // Calculate the fantoir code for each common toponym + const fantoirCode = calculateCommonToponymFantoirCode(commonToponym) + // Calculate the tiles for each common toponym + const {tiles, x, y} = calculateCommonToponymTiles(commonToponym) + // Calculate the postal code for each common toponym + const postalCode = calculateCommonToponymPostalCode(commonToponym) + // Remove the centroid data from the common toponym + const {centroid, ...commonToponymCleaned} = commonToponym + return {...commonToponymCleaned, + meta: { + ...commonToponym.meta, + ...(fantoirCode ? {dgfip: {...commonToponym.meta?.dgfip, fantoir: fantoirCode}} : {}), + ...(tiles && x && y ? {geography: {...commonToponym.meta?.geography, tiles, x, y}} : {}), + ...(postalCode ? {laposte: {...commonToponym.meta?.laposte, codePostal: postalCode}} : {}) + }} + }) + } + + if (type === 'adresse') { + return pageData.map(address => { + // Calculate the tiles for each address + const {tiles, x, y} = calculateAddressTiles(address) + // Calculate the postal code for each address + const postalCode = calculateAddressPostalCode(address) + return {...address, + meta: { + ...(tiles && x && y ? {geography: {...address.meta?.geography, tiles, x, y}} : {}), + ...(postalCode ? {laposte: {...address.meta?.laposte, codePostal: postalCode}} : {}) + } + } + }) + } + } + + const fetchAndExportDataFromPage = async (type, model, collection, pageNumber) => { + const offset = (pageNumber - 1) * PAGE_SIZE + let pageData = [] + // Export the data from the page + if (type === 'commonToponym') { + [pageData] = await sequelize.query(commonToponymPageQuery, { + replacements: {districtID, offset, limit: PAGE_SIZE}, + transaction, + raw: true, + }) + } else if (type === 'adresse') { + pageData = await model.findAll({ + where: {districtID}, + order: [['id', 'ASC']], + offset, + limit: PAGE_SIZE, + transaction, + raw: true, + }) + } + + // Format the data and calculate the fantoir code, tiles and postal code + const formatedPageData = formatPageData(pageData, type) + + // Insert the data in the collection + await mongo.db.collection(collection).insertMany(formatedPageData, {ordered: false}) } const commonToponymsExportPromises = [] for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) { commonToponymsExportPromises.push( - fetchAndExportDataFromPage(CommonToponym, CommonToponymCollection, pageNumber) + fetchAndExportDataFromPage('commonToponym', CommonToponym, COMMON_TOPONYM_COLLECTION, pageNumber) ) } await Promise.all(commonToponymsExportPromises) // Address + // Count the total number of addresses and pages to process const totalAddressRecords = await Address.count({ where: {districtID}, transaction, }) - const totalAddressPages = Math.ceil(totalAddressRecords / pageSize) + const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE) const addressesExportPromises = [] for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) { addressesExportPromises.push( - fetchAndExportDataFromPage(Address, AddressCollection, pageNumber) + fetchAndExportDataFromPage('adresse', Address, ADDRESS_COLLECTION, pageNumber) ) } @@ -94,8 +246,8 @@ export default async function exportToExploitationDB({data}) { const deleteAllDataRelatedToDistrict = async districtID => { await Promise.all([ - mongo.db.collection(DistrictCollection).deleteOne({id: districtID}), - mongo.db.collection(CommonToponymCollection).deleteMany({districtID}), - mongo.db.collection(AddressCollection).deleteMany({districtID}) + mongo.db.collection(DISTRICT_COLLECTION).deleteOne({id: districtID}), + mongo.db.collection(COMMON_TOPONYM_COLLECTION).deleteMany({districtID}), + mongo.db.collection(ADDRESS_COLLECTION).deleteMany({districtID}) ]) }