From f5841f0d6f5889fb59283d56a96362da0ca6f264 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Thu, 17 Oct 2024 17:41:42 +0200 Subject: [PATCH] feat: use OpenAlex for all unsupported DOI registry agents --- data-generation/real-data.js | 1 + .../mention/ImportMentions/ImportReportBody.tsx | 2 -- .../mention/ImportMentions/apiImportMentions.tsx | 14 ++++++-------- .../components/mention/ImportMentions/index.tsx | 2 +- frontend/utils/getCrossref.ts | 3 ++- frontend/utils/getDOI.ts | 12 ++++++------ frontend/utils/getDataCite.ts | 1 + .../rsd/scraper/doi/MainMentions.java | 12 ++++++++---- 8 files changed, 25 insertions(+), 22 deletions(-) diff --git a/data-generation/real-data.js b/data-generation/real-data.js index 40ad7944a..00001968a 100644 --- a/data-generation/real-data.js +++ b/data-generation/real-data.js @@ -61,6 +61,7 @@ const dois = [ '10.2777/28598', '10.2788/50967', '10.2788/52504', + '10.3968/10006', '10.4233/uuid:4bb38399-9267-428f-b10a-80b86e101f23', '10.5194/egusphere-egu21-4805', '10.5194/ems2022-105', diff --git a/frontend/components/mention/ImportMentions/ImportReportBody.tsx b/frontend/components/mention/ImportMentions/ImportReportBody.tsx index 5f3284283..051975afb 100644 --- a/frontend/components/mention/ImportMentions/ImportReportBody.tsx +++ b/frontend/components/mention/ImportMentions/ImportReportBody.tsx @@ -82,8 +82,6 @@ export default function ImportReportBody({initialResults,onCancel,onImport}: Bul return 'Not a valid DOI' case 'doiNotFound': return 'DOI not found' - case 'unsupportedRA': - return 'Registration agent (RA) is not supported' case 'alreadyImported': return 'This publication is already imported' default: diff --git a/frontend/components/mention/ImportMentions/apiImportMentions.tsx b/frontend/components/mention/ImportMentions/apiImportMentions.tsx index cf3e9b0be..8edee62de 100644 --- a/frontend/components/mention/ImportMentions/apiImportMentions.tsx +++ b/frontend/components/mention/ImportMentions/apiImportMentions.tsx @@ -68,15 +68,15 @@ export async function validateInputList(doiList: string[], mentions: MentionItem return false } }) - // extract DOI string from serch info + // extract DOI string from search info .map(search => search.term.toLowerCase()) - // if no valid doi's left return report + // if no valid DOIs left return report if (validDois.length === 0) { return mentionResultPerDoi } - // FIND DOI's already in RSD + // FIND DOIs already in RSD const existingMentionsResponse = await getMentionsByDoiFromRsd({dois: validDois, token}) if (existingMentionsResponse.status === 200) { const existingMentions = existingMentionsResponse.message as MentionItemProps[] @@ -91,7 +91,7 @@ export async function validateInputList(doiList: string[], mentions: MentionItem } // DOI NOT IN RSD - // valid dois not present in mentionResultPerDoi map at this point are not in RSD + // valid DOIs not present in mentionResultPerDoi map at this point are not in RSD const doisNotInDatabase: string[] = validDois.filter(entry => !mentionResultPerDoi.has(entry)) if (doisNotInDatabase.length > 0) { @@ -104,17 +104,15 @@ export async function validateInputList(doiList: string[], mentions: MentionItem const openalexDois: string[] = [] doiRas.forEach(doiRa => { const doi = doiRa.DOI.toLowerCase() - if (typeof doiRa?.RA === 'undefined') { + if (doiRa?.RA === undefined || doiRa.RA === 'invalid doi' || doiRa.RA === 'doi does not exist' || doiRa.RA === 'unknown') { // Invalid DOI -> RA not found mentionResultPerDoi.set(doi, {doi, status: 'doiNotFound', include: false}) } else if (doiRa.RA === 'Crossref') { crossrefDois.push(doi) } else if (doiRa.RA === 'DataCite') { dataciteDois.push(doi) - } else if (doiRa.RA === 'OP') { - openalexDois.push(doi) } else { - mentionResultPerDoi.set(doi, {doi, status: 'unsupportedRA', include: false}) + openalexDois.push(doi) } }) diff --git a/frontend/components/mention/ImportMentions/index.tsx b/frontend/components/mention/ImportMentions/index.tsx index d11a41d3f..75d4149af 100644 --- a/frontend/components/mention/ImportMentions/index.tsx +++ b/frontend/components/mention/ImportMentions/index.tsx @@ -23,7 +23,7 @@ import {DoiBulkImportReport, addMentions, linkMentionToEntity} from './apiImport export type SearchResult = { doi: string - status: 'valid' | 'invalidDoi' | 'doiNotFound' |'unsupportedRA' | 'alreadyImported' | 'unknown', + status: 'valid' | 'invalidDoi' | 'doiNotFound' | 'alreadyImported' | 'unknown', include: boolean source?: 'RSD' | 'Crossref' | 'DataCite' | 'OpenAlex', mention?: MentionItemProps diff --git a/frontend/utils/getCrossref.ts b/frontend/utils/getCrossref.ts index 5daea2add..605b2c1cc 100644 --- a/frontend/utils/getCrossref.ts +++ b/frontend/utils/getCrossref.ts @@ -75,7 +75,8 @@ export function crossrefItemToMentionItem(item: CrossrefSelectItem) { image_url: null, mention_type: crossrefToRsdType(item.type), source: 'Crossref', - note: null + note: null, + openalex_id: null } // debugger return mention diff --git a/frontend/utils/getDOI.ts b/frontend/utils/getDOI.ts index 497296c22..52d917c31 100644 --- a/frontend/utils/getDOI.ts +++ b/frontend/utils/getDOI.ts @@ -188,18 +188,18 @@ export async function getMentionByDoi(doi: string) { if (doiRA && doiRA.RA) { switch (doiRA.RA.toLowerCase()) { case 'crossref': - // get from crossref return getItemFromCrossref(doi) case 'datacite': - // get from datacite return getItemFromDatacite(doi) - case 'op': - return getItemFromOpenalex(doi) - default: + case 'invalid doi': + case 'doi does not exist': + case 'unknown': return { status: 400, - message: `${doiRA.RA} not supported. RSD supports Crossref and DataCite api` + message: 'Invalid or unknown DOI' } + default: + return getItemFromOpenalex(doi) } } return { diff --git a/frontend/utils/getDataCite.ts b/frontend/utils/getDataCite.ts index 4ec2136ec..c097d1f3c 100644 --- a/frontend/utils/getDataCite.ts +++ b/frontend/utils/getDataCite.ts @@ -186,6 +186,7 @@ export function dataCiteGraphQLItemToMentionItem(item: WorkResponse) { mention_type: dataciteToRsdType(item), source: 'DataCite', note: null, + openalex_id: null } return mention } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java index 0a136be81..4661e2703 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -75,9 +76,10 @@ public static void main(String[] args) { Instant now = Instant.now(); // DATACITE + final String dataciteDoiRaKey = "DataCite"; Collection dataciteDois = doiToSource.entrySet() .stream() - .filter(doiSourceEntry -> doiSourceEntry.getValue().equals("DataCite")) + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(dataciteDoiRaKey)) .map(Map.Entry::getKey) .map(Doi::fromString) .toList(); @@ -104,9 +106,10 @@ public static void main(String[] args) { // END DATACITE // CROSSREF + final String crossrefDoiRaKey = "Crossref"; Collection crossrefDois = doiToSource.entrySet() .stream() - .filter(doiSourceEntry -> doiSourceEntry.getValue().equals("Crossref")) + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(crossrefDoiRaKey)) .map(Map.Entry::getKey) .map(Doi::fromString) .toList(); @@ -132,13 +135,14 @@ public static void main(String[] args) { } // END CROSSREF - // OPENALEX (for European Publication Office DOIs) + // OPENALEX (other DOI registry agents) String email = Config.crossrefContactEmail().orElse(null); Collection scrapedOpenalexMentions = new ArrayList<>(); OpenAlexConnector openAlexConnector = new OpenAlexConnector(); + Collection invalidDoiRas = Set.of(dataciteDoiRaKey, crossrefDoiRaKey, "Invalid DOI", "DOI does not exist", "Unknown"); Collection europeanPublicationsOfficeDois = doiToSource.entrySet() .stream() - .filter(doiSourceEntry -> doiSourceEntry.getValue().equals("OP")) + .filter(doiSourceEntry -> !invalidDoiRas.contains(doiSourceEntry.getValue())) .map(Map.Entry::getKey) .map(Doi::fromString) .toList();