Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: use OpenAlex for all unsupported DOI registry agents #1323

Merged
merged 1 commit into from
Oct 21, 2024
Merged

feat: use OpenAlex for all unsupported DOI registry agents #1323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions data-generation/real-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ const dois = [
'10.2777/28598',
'10.2788/50967',
'10.2788/52504',
'10.3968/10006',
'10.4233/uuid:4bb38399-9267-428f-b10a-80b86e101f23',
'10.5194/egusphere-egu21-4805',
'10.5194/ems2022-105',
Expand Down
2 changes: 0 additions & 2 deletions frontend/components/mention/ImportMentions/ImportReportBody.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ export default function ImportReportBody({initialResults,onCancel,onImport}: Bul
return 'Not a valid DOI'
case 'doiNotFound':
return 'DOI not found'
case 'unsupportedRA':
return 'Registration agent (RA) is not supported'
case 'alreadyImported':
return 'This publication is already imported'
default:
Expand Down
14 changes: 6 additions & 8 deletions frontend/components/mention/ImportMentions/apiImportMentions.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,15 @@ export async function validateInputList(doiList: string[], mentions: MentionItem
return false
}
})
// extract DOI string from serch info
// extract DOI string from search info
.map(search => search.term.toLowerCase())

// if no valid doi's left return report
// if no valid DOIs left return report
if (validDois.length === 0) {
return mentionResultPerDoi
}

// FIND DOI's already in RSD
// FIND DOIs already in RSD
const existingMentionsResponse = await getMentionsByDoiFromRsd({dois: validDois, token})
if (existingMentionsResponse.status === 200) {
const existingMentions = existingMentionsResponse.message as MentionItemProps[]
Expand All @@ -91,7 +91,7 @@ export async function validateInputList(doiList: string[], mentions: MentionItem
}

// DOI NOT IN RSD
// valid dois not present in mentionResultPerDoi map at this point are not in RSD
// valid DOIs not present in mentionResultPerDoi map at this point are not in RSD
const doisNotInDatabase: string[] = validDois.filter(entry => !mentionResultPerDoi.has(entry))

if (doisNotInDatabase.length > 0) {
Expand All @@ -104,17 +104,15 @@ export async function validateInputList(doiList: string[], mentions: MentionItem
const openalexDois: string[] = []
doiRas.forEach(doiRa => {
const doi = doiRa.DOI.toLowerCase()
if (typeof doiRa?.RA === 'undefined') {
if (doiRa?.RA === undefined || doiRa.RA === 'invalid doi' || doiRa.RA === 'doi does not exist' || doiRa.RA === 'unknown') {
// Invalid DOI -> RA not found
mentionResultPerDoi.set(doi, {doi, status: 'doiNotFound', include: false})
} else if (doiRa.RA === 'Crossref') {
crossrefDois.push(doi)
} else if (doiRa.RA === 'DataCite') {
dataciteDois.push(doi)
} else if (doiRa.RA === 'OP') {
openalexDois.push(doi)
} else {
mentionResultPerDoi.set(doi, {doi, status: 'unsupportedRA', include: false})
openalexDois.push(doi)
}
})

Expand Down
2 changes: 1 addition & 1 deletion frontend/components/mention/ImportMentions/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import {DoiBulkImportReport, addMentions, linkMentionToEntity} from './apiImport

export type SearchResult = {
doi: string
status: 'valid' | 'invalidDoi' | 'doiNotFound' |'unsupportedRA' | 'alreadyImported' | 'unknown',
status: 'valid' | 'invalidDoi' | 'doiNotFound' | 'alreadyImported' | 'unknown',
include: boolean
source?: 'RSD' | 'Crossref' | 'DataCite' | 'OpenAlex',
mention?: MentionItemProps
Expand Down
3 changes: 2 additions & 1 deletion frontend/utils/getCrossref.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ export function crossrefItemToMentionItem(item: CrossrefSelectItem) {
image_url: null,
mention_type: crossrefToRsdType(item.type),
source: 'Crossref',
note: null
note: null,
openalex_id: null
}
// debugger
return mention
Expand Down
12 changes: 6 additions & 6 deletions frontend/utils/getDOI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,18 +188,18 @@ export async function getMentionByDoi(doi: string) {
if (doiRA && doiRA.RA) {
switch (doiRA.RA.toLowerCase()) {
case 'crossref':
// get from crossref
return getItemFromCrossref(doi)
case 'datacite':
// get from datacite
return getItemFromDatacite(doi)
case 'op':
return getItemFromOpenalex(doi)
default:
case 'invalid doi':
case 'doi does not exist':
case 'unknown':
return {
status: 400,
message: `${doiRA.RA} not supported. RSD supports Crossref and DataCite api`
message: 'Invalid or unknown DOI'
}
default:
return getItemFromOpenalex(doi)
}
}
return {
Expand Down
1 change: 1 addition & 0 deletions frontend/utils/getDataCite.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ export function dataCiteGraphQLItemToMentionItem(item: WorkResponse) {
mention_type: dataciteToRsdType(item),
source: 'DataCite',
note: null,
openalex_id: null
}
return mention
}
Expand Down
12 changes: 8 additions & 4 deletions scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -75,9 +76,10 @@ public static void main(String[] args) {
Instant now = Instant.now();

// DATACITE
final String dataciteDoiRaKey = "DataCite";
Collection<Doi> dataciteDois = doiToSource.entrySet()
.stream()
.filter(doiSourceEntry -> doiSourceEntry.getValue().equals("DataCite"))
.filter(doiSourceEntry -> doiSourceEntry.getValue().equals(dataciteDoiRaKey))
.map(Map.Entry::getKey)
.map(Doi::fromString)
.toList();
Expand All @@ -104,9 +106,10 @@ public static void main(String[] args) {
// END DATACITE

// CROSSREF
final String crossrefDoiRaKey = "Crossref";
Collection<Doi> crossrefDois = doiToSource.entrySet()
.stream()
.filter(doiSourceEntry -> doiSourceEntry.getValue().equals("Crossref"))
.filter(doiSourceEntry -> doiSourceEntry.getValue().equals(crossrefDoiRaKey))
.map(Map.Entry::getKey)
.map(Doi::fromString)
.toList();
Expand All @@ -132,13 +135,14 @@ public static void main(String[] args) {
}
// END CROSSREF

// OPENALEX (for European Publication Office DOIs)
// OPENALEX (other DOI registry agents)
String email = Config.crossrefContactEmail().orElse(null);
Collection<ExternalMentionRecord> scrapedOpenalexMentions = new ArrayList<>();
OpenAlexConnector openAlexConnector = new OpenAlexConnector();
Collection<String> invalidDoiRas = Set.of(dataciteDoiRaKey, crossrefDoiRaKey, "Invalid DOI", "DOI does not exist", "Unknown");
Collection<Doi> europeanPublicationsOfficeDois = doiToSource.entrySet()
.stream()
.filter(doiSourceEntry -> doiSourceEntry.getValue().equals("OP"))
.filter(doiSourceEntry -> !invalidDoiRas.contains(doiSourceEntry.getValue()))
.map(Map.Entry::getKey)
.map(Doi::fromString)
.toList();
Expand Down