diff --git a/src/moa/index.js b/src/moa/index.js index 8c1543f5..a22c3cc4 100644 --- a/src/moa/index.js +++ b/src/moa/index.js @@ -544,6 +544,18 @@ const parseRelevance = (moaRecord) => { return relevance; }; +const removeRecords = async (conn, records) => { + if (records.length && records.length > 0) { + for (const record of records) { + try{ + await conn.deleteRecord('Statement', record['@rid']); + logger.info(`Removing Statement ${record['@rid']} that are out of date`); + } catch (err) { + logger.warn(`${err}`); + } + } + } +}; const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) => { const source = await conn.addSource(SOURCE_DEFN); @@ -559,6 +571,12 @@ const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) => const existing = {}; + const newIds = records.map(record => record.assertion_id); + let toRemove = existingRecords.filter( + record => !newIds.includes(parseInt(record.sourceId, 10)), + ); + await removeRecords(conn, toRemove); + for (const record of existingRecords) { const key = record.sourceId; @@ -573,41 +591,31 @@ const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) => logger.info(`loading: ${rawRecord.assertion_id} / ${records.length}`); const record = fixStringNulls(rawRecord); + if (record.assertion_id === 258){ + logger.info(`loading: ${rawRecord.assertion_id}`); + } // handle empty space in url if (record.sources[0].url && record.sources[0].url.includes(' ')) { record.sources[0].url = record.sources[0].url.replace(/\s/g, ''); } checkSpec(validateMoaRecord, record); const key = `${record.assertion_id}`; - const lastUpdate = new Date(record.last_updated).getTime(); const relevance = parseRelevance(record); - // do we have the expected number of GraphKB records for this MOA assertion - if (existing[key] && existing[key].length === relevance.length) { - // check the last update date of the assertion against the timestamp in GraphKB - if (existing[key].every(r => lastUpdate <= r.updatedAt)) { - logger.debug('Skip. Current record exists and does not need updating'); - counts.skipped++; - continue; - } - } const updatedRecords = (await loadRecord(conn, record, source, relevance)).map(r => r['@rid']); if (existing[key]) { - const toRemove = existing[key].map(r => r['@rid']).filter(r => !updatedRecords.includes(r)); - - if (toRemove.length) { - logger.warn(`Removing ${toRemove.length} records that are out of date`); - - for (const recordId of toRemove) { - await conn.deleteRecord('Statement', recordId); - } - } + toRemove = existing[key].filter(r => !updatedRecords.includes(r['@rid'])); + await removeRecords(conn, toRemove); } counts.success++; } catch (err) { logger.warn(`${err}`); counts.error++; + toRemove = existingRecords.filter( + record => rawRecord.assertion_id === parseInt(record.sourceId, 10), + ); + await removeRecords(conn, toRemove); if (err.toString().includes('Cannot read property')) { throw err;