Skip to content

Commit

Permalink
Merge branch 'main' of github.com:pkiraly/qa-catalogue
Browse files Browse the repository at this point in the history
  • Loading branch information
@pkiraly
pkiraly committed Dec 18, 2023
2 parents 7cb1a8a + 011b467 commit 7fc279a
Show file tree
Hide file tree
Showing 3 changed files with 9,733 additions and 11,650 deletions.
14 changes: 7 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,32 +108,32 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.13.0</version>
<version>3.14.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.15.3</version>
<version>2.16.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.3</version>
<version>2.16.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.15.3</version>
<version>2.16.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.21.1</version>
<version>2.22.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.21.1</version>
<version>2.22.0</version>
</dependency>
<dependency>
<artifactId>solr-core</artifactId>
Expand Down Expand Up @@ -182,7 +182,7 @@
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>5.8</version>
<version>5.9</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
<dependency>
Expand Down
70 changes: 40 additions & 30 deletions src/main/java/de/gwdg/metadataqa/marc/utils/marcreader/schema/AvramMarc21SchemaReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,17 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import java.util.logging.Logger;

public class AvramMarc21SchemaReader {
private static final Logger logger = Logger.getLogger(AvramMarc21SchemaReader.class.getCanonicalName());
private static final Map<String, Integer> knownFieldProperties = Map.of(
"label", 1, "repeatable", 1, "indicators", 1, "subfields", 1, "positions", 1);
"label", 1, "repeatable", 1, "indicator1", 1, "indicator2", 1, "subfields", 1, "positions", 1);
private static final Map<String, Integer> knownSubfieldProperties = Map.of(
"label", 1, "repeatable", 1, "static", 1, "codes", 1);
"label", 1, "repeatable", 1, "codes", 1);
private static final Map<String, Integer> knownSubfieldCodeProperties = Map.of(
"label", 1, "value",1);
"label", 1, "code",1);
private static final Map<String, Integer> knownIndicatorProperties = Map.of(
"label", 1, "codes",1);
private JSONParser parser = new JSONParser(JSONParser.MODE_RFC4627);
Expand Down Expand Up @@ -75,18 +76,24 @@ private void process(JSONObject jsonObject) throws IOException, ParseException,
(boolean) field.get("repeatable"),
!field.containsKey("subfields")
);
if (field.containsKey("indicators"))
processIndicators((JSONObject) field.get("indicators"), (Marc21DataFieldDefinition) tag);
Marc21DataFieldDefinition tagDef = (Marc21DataFieldDefinition)tag;
if (field.containsKey("indicator1") && !field.get("indicator1").equals(null))
tagDef.setInd1(processIndicator(field.get("indicator1"), tag.getTag()));
if (field.containsKey("indicator2") && !field.get("indicator2").equals(null))
tagDef.setInd2(processIndicator(field.get("indicator2"), tag.getTag()));
if (field.containsKey("subfields"))
processSubfields((JSONObject) field.get("subfields"), (Marc21DataFieldDefinition) tag);
if (field.containsKey("positions"))
processPositions((JSONArray) field.get("positions"), (Marc21DataFieldDefinition) tag);
processPositions((JSONObject) field.get("positions"), (Marc21DataFieldDefinition) tag);
for (String property : field.keySet()) {
if (!knownFieldProperties.containsKey(property))
logger.warning("unhandled field property: " + property);
}
if (id.equals("008") || id.equals("leader"))
if (id.equals("008") || id.equals("LDR")) {
tag = new DefaultControlFieldDefinition((Marc21DataFieldDefinition) tag);
if (id.equals("LDR"))
id = "leader";
}

if (!map.containsKey(id))
map.put(id, tag);
Expand All @@ -95,20 +102,36 @@ private void process(JSONObject jsonObject) throws IOException, ParseException,
}
}

private void processPositions(JSONArray positions, Marc21DataFieldDefinition tag) {
List<ControlfieldPositionDefinition> positionDefinitions = new LinkedList<>();
for (Object position : positions) {
positionDefinitions.add(processPosition(position, tag));
private void processPositions(JSONObject positions, Marc21DataFieldDefinition tag) {
TreeSet<ControlfieldPositionDefinition> sortedPositions = new TreeSet<>((a,b) -> a.getPositionStart() - b.getPositionStart());

for (String key : positions.keySet()) {
sortedPositions.add(processPosition(key, positions.get(key), tag));
}
tag.setPositions(positionDefinitions);

tag.setPositions(new LinkedList<ControlfieldPositionDefinition>(sortedPositions));
}

private ControlfieldPositionDefinition processPosition(Object positionNode, Marc21DataFieldDefinition tag) {
private ControlfieldPositionDefinition processPosition(String positionKey, Object positionNode, Marc21DataFieldDefinition tag) {
if (positionNode instanceof JSONObject) {
JSONObject position = (JSONObject) positionNode;
String label = position.getAsString("label");
int start = (int) position.getAsNumber("start");
int end = (int) position.getAsNumber("stop") + 1;

String[] charPos = positionKey.split("-");
int start = Integer.parseInt(charPos[0]);
int end = charPos.length > 1 ? Integer.parseInt(charPos[1]) : start;
if (position.containsKey("start")) {
int startKey = (int) position.getAsNumber("start");
if (startKey != start)
logger.warning("position start " + start + "does not match character positions: " + positionKey);
}
if (position.containsKey("end")) {
int endKey = (int) position.getAsNumber("end");
if (endKey != end)
logger.warning("position end " + end + "does not match character positions: " + positionKey);
}
end++;

ControlfieldPositionDefinition definition = new ControlfieldPositionDefinition(label, start, end);
if (position.containsKey("codes")) {
definition.setCodes(extractEncodedValues(position));
Expand All @@ -128,19 +151,6 @@ private ControlfieldPositionDefinition processPosition(Object positionNode, Marc
return null;
}

private void processIndicators(JSONObject indicators, Marc21DataFieldDefinition tag) {
for (Map.Entry<String, Object> indicator : indicators.entrySet()) {
String key = indicator.getKey();
Indicator indicatorDef = processIndicator(indicator.getValue(), tag.getTag());
if (key.equals("1"))
tag.setInd1(indicatorDef);
else if (key.equals("2"))
tag.setInd2(indicatorDef);
else
logger.warning("Wrong indicator number: " + key);
}
}

private Indicator processIndicator(Object indicatorNode, String tag) {
if (indicatorNode instanceof JSONObject) {
JSONObject indicator = (JSONObject) indicatorNode;
Expand Down Expand Up @@ -211,8 +221,8 @@ private List<EncodedValue> processStaticValues(Object staticValuesObject) {
String code = codeEntry.getKey();
if (codeEntry.getValue() instanceof JSONObject) {
JSONObject property = (JSONObject) codeEntry.getValue();
if (!code.equals(property.get("value")))
logger.warning(String.format("code is different to code value: %s vs %s", code, (String)property.get("value")));
if (!code.equals(property.get("code")))
logger.warning(String.format("code is different to code value: %s vs %s", code, (String)property.get("coden")));
else
list.add(new EncodedValue(code, (String) property.get("label")));
for (String key : property.keySet()) {
Expand Down
Loading

0 comments on commit 7fc279a

Please sign in to comment.