DD-314: Map languages to ISO 639-2(#12)

src/main/assembly/dist/cfg/languages.csv

@@ -0,0 +1,152 @@
+"found","iso 639-2 (B)"
+"Afrikaans","afr"
+"South African","afr"
+"Arabic","ara"
+"Aramaic","arc"
+"Official Aramaic; Imperial Aramaic (arc)","arc"
+"Berber","ber"
+"Bosnisch","bos"
+"Chinese","chi"
+"Chinese Script, Modern Mandarin, Hakka, Taiwanese, Bunun, Tsou, Ami","chi"
+"Hakka","chi"
+"Modern Mandarin Chinese","chi"
+"漢字, 現代華語, 客家話, 臺灣話, 布農話, 鄒語, 阿美話","chi"
+"Czech","cze"
+"(and Dutch questionnaire)","dut"
+"Brabants dialect","dut"
+"Codebook in Dutch","dut"
+"Data: Nederlands","dut"
+"dut/nld","dut"
+"Dutch","dut"
+"dutch","dut"
+"Dutch (Questionnaires presented to participants only)","dut"
+"Dutch, heavy Texel accent","dut"
+"Dutch/Nederlands","dut"
+"Gronings","dut"
+"Gronings dialect","dut"
+"In het Nederland","dut"
+"In het Nederlands","dut"
+"in het Nederlands","dut"
+"In het Nederlands.","dut"
+"in het Nederlands.","dut"
+"In Nederlands","dut"
+"in Nederlands","dut"
+"Interview: Dutch","dut"
+"Nderlands","dut"
+"ned","dut"
+"Nederland","dut"
+"Nederlands","dut"
+"nederlands","dut"
+"Nederlandse","dut"
+"nedl","dut"
+"partly Dutch","dut"
+"Subtitles: Dutch","dut"
+"Subtitling: Dutch","dut"
+"Survey instruments in Dutch","dut"
+"Transcripts in Dutch","dut"
+"Vlaams","dut"
+"Artikelen: Nederlands en Engels","dut,eng"
+"Data files with Greek characters in their names are also included in the data set. However, language to describe the content of this data set is primarily English and occasionally Dutch.","dut,eng"
+"Dutch and English","dut,eng"
+"Dutch, English","dut,eng"
+"English and Dutch","dut,eng"
+"English with summary in Dutch","dut,eng"
+"In het Nederlands, samenvatting in het Engels.","dut,eng"
+"Nederlands en Engels","dut,eng"
+"Description in English","eng"
+"Documentation in English","eng"
+"eng","eng"
+"Engels","eng"
+"English","eng"
+"English (Summary)","eng"
+"English (summary)","eng"
+"English summary","eng"
+"Interview: English","eng"
+"partially English","eng"
+"Subtitles: English","eng"
+"Subtitling: English","eng"
+"summary and conclusion in English","eng"
+"Summary in English","eng"
+"summary in English","eng"
+"English and Portuguese","eng,por"
+"Ewe","ewe"
+"Fante","fat"
+"Filipino","fil"
+"Frans","fre"
+"fre/fra","fre"
+"French","fre"
+"North Frisian","frr"
+"een paar woorden Fries","fry"
+"Fries","fry"
+"Frisian","fry"
+"Deutsch (Zusammenfassung)","ger"
+"Deutsche Zusammenfassung","ger"
+"Duits","ger"
+"ger/deu","ger"
+"German","ger"
+"German (appendix 2)","ger"
+"German (phonetic notation)","ger"
+"German (summary)","ger"
+"German summary","ger"
+"Duits/Nederlands","ger,dut"
+"German, Dutch and English","ger,eng,dut"
+"Ancient Greek","grc"
+"greek (new testament)","grc"
+"Greek","gre"
+"Ancient Hebrew (hbo)","heb"
+"Biblical Hebrew","heb"
+"Classical Hebrew","heb"
+"Hebrew","heb"
+"Hebrew (some interviews)","heb"
+"Interview: Hebrew","heb"
+"Hindi","hin"
+"Croatian","hrv"
+"Hungarian","hun"
+"Hungarian, German","hun,ger"
+"Bahasa","ind"
+"Bahasa Indonesia","ind"
+"Bahasa indonesia","ind"
+"Bugis","ind"
+"Indonesian","ind"
+"Indonesisch","ind"
+"Italian","ita"
+"Includes Dutch, Italian and Swedish (filmtitles)","ita,dut,swe"
+"Javaans","jav"
+"Javanese","jav"
+"Japanese","jpn"
+"Japans","jpn"
+"Kinyarwanda","kin"
+"Latin","lat"
+"Names of Regions, Cities, Districts are in original languge - in Latvian","lav"
+"Limburg","lim"
+"Limburgs","lim"
+"Limburgs/Nederlands","lim,dut"
+"limburgs/nederlands","lim,dut"
+"Maa","luo"
+"Ambon-Maleis","may"
+"Maleis","may"
+"Maleis doorspekt met Nederlandse woorden en zinsdelen","may"
+"Moluks-Maleis","msa"
+"Nepali","nep"
+"Papiamento","pap"
+"Papiamentu","pap"
+"Polish","pol"
+"One documentary available with English and Polish subtitles","pol,eng"
+"one documentary available with English and Polish subtitles","pol,eng"
+"Brazilian Portuguese","por"
+"Portuguese","por"
+"Interview: Russian","rus"
+"Russian","rus"
+"Russian (some interviews)","rus"
+"Russisch","rus"
+"dse","sgn"
+"Sign Language of the Netherlands","sgn"
+"Spanish","spa"
+"Sranan Tongo","srn"
+"Biblical Aramaic","syc"
+"Syriac","syr"
+"a little bit of Jiddish","yid"
+"Jiddisch","yid"
+"Yiddish","yid"
+"Interview: English and some Yiddish","yid,eng"
+"Interview: Russian and Yiddish","yid,rus"

src/main/scala/nl.knaw.dans.easy.bag2deposit/EasyConvertBagToDepositApp.scala

@@ -19,6 +19,8 @@ import better.files.File
 import better.files.File.CopyOptions
 import nl.knaw.dans.bag.v0.DansV0Bag
 import nl.knaw.dans.easy.bag2deposit.Command.FeedBackMessage
+import nl.knaw.dans.easy.bag2deposit.ddm.LanguageRewriteRule
+import nl.knaw.dans.easy.bag2deposit.ddm.LanguageRewriteRule.logNotMapped
 import nl.knaw.dans.lib.logging.DebugEnhancedLogging
 
 import java.io.{ FileNotFoundException, IOException }
@@ -75,6 +77,7 @@ class EasyConvertBagToDepositApp(configuration: Configuration) extends DebugEnha
       _ = formatDiff(ddmIn, ddmOut).foreach(s => logger.info(s))
       _ = ddmFile.writeText(ddmOut.serialize)
       props <- depositPropertiesFactory.create(bagInfo, ddmOut)
+      _ = logNotMapped(ddmOut, props.getString("identifier.fedora", ""))
       _ = props.save((bagParentDir / "deposit.properties").toJava)
       _ = bagInfoKeysToRemove.foreach(mutableBagMetadata.remove)
       _ <- BagFacade.updateMetadata(bag)

src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/DdmTransformer.scala

@@ -31,6 +31,7 @@ case class DdmTransformer(cfgDir: File) extends DebugEnhancedLogging {
     reportRewriteRule,
     AbrRewriteRule(cfgDir / "ABR-period.csv", "temporal", "ddm:temporal"),
     AbrRewriteRule(cfgDir / "ABR-complex.csv", "subject", "ddm:subject"),
+    LanguageRewriteRule(cfgDir / "languages.csv"),
   )
 
   private case class DdmRewriteRule(reportNumberFromFirstTitle: NodeSeq) extends RewriteRule {

src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/LanguageRewriteRule.scala

@@ -0,0 +1,66 @@
+/**
+ * Copyright (C) 2020 DANS - Data Archiving and Networked Services ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.knaw.dans.easy.bag2deposit.ddm
+
+import better.files.File
+import nl.knaw.dans.easy.bag2deposit.{ parseCsv, printer }
+import nl.knaw.dans.lib.logging.DebugEnhancedLogging
+
+import scala.xml.{ Node, NodeSeq }
+import scala.xml.transform.RewriteRule
+
+case class LanguageRewriteRule(cfgFile: File) extends RewriteRule {
+
+  private val freeLanguageToCodes = parseCsv(cfgFile, 0).toSeq
+    .map(r => r.get(0) -> r.get(1).split(",").toSeq)
+    .toMap
+
+  private val codeToElem = Map(
+    "nld" -> elem("Dutch")("dut"),
+    "dut" -> elem("Dutch")("dut"),
+    "eng" -> elem("English")("eng"),
+    "fre" -> elem("French")("fre"),
+    "fra" -> elem("French")("fre"),
+    "deu" -> elem("German")("ger"),
+    "ger" -> elem("German")("ger"),
+  )
+
+  override def transform(node: Node): Seq[Node] = {
+    def mapFreeLanguageValue = {
+      freeLanguageToCodes
+        .get(node.text)
+        .map(_.map(elem(node.text.trim)))
+        .getOrElse(node)
+    }
+
+    if (node.label != "language") node
+    else if (node.attributes.mkString.matches("(.*:)?ISO639-[23]."))
+           codeToElem.getOrElse(node.text.trim.toLowerCase, mapFreeLanguageValue)
+         else mapFreeLanguageValue
+  }
+
+  private def elem(value: String)(code: String) =
+    <ddm:language encodingScheme='ISO639-2' code={ code.toLowerCase }>{ value.trim }</ddm:language>
+}
+object LanguageRewriteRule extends DebugEnhancedLogging {
+  def logNotMapped(ddm: Node, datasetId: String): Unit = {
+    (ddm \\ "language").filter(
+      !_.attributes.mkString.contains("encodingScheme")
+    ).foreach(n =>
+      logger.info(s"NOT MAPPED $datasetId ${printer.format(n)}")
+    )
+  }
+}

src/test/resources/debug-config/languages.csv

@@ -0,0 +1,132 @@
+"Afrikaans","afr"
+"Arabic","ara"
+"Aramaic","arc"
+"Official Aramaic; Imperial Aramaic (arc)","arc"
+"Berber","ber"
+"Bosnisch","bos"
+"Modern Mandarin Chinese","chi"
+"Hakka","chi"
+"Chinese Script, Modern Mandarin, Hakka, Taiwanese, Bunun, Tsou, Ami","chi"
+"Chinese","chi"
+"Czech","cze"
+"(and Dutch questionnaire)","dut"
+"Codebook in Dutch","dut"
+"Data: Nederlands","dut"
+"dut/nld","dut"
+"Dutch","dut"
+"dutch","dut"
+"Dutch (Questionnaires presented to participants only)","dut"
+"Dutch, heavy Texel accent","dut"
+"Dutch/Nederlands","dut"
+"In het Nederland","dut"
+"In het Nederlands","dut"
+"in het Nederlands","dut"
+"In het Nederlands.","dut"
+"in het Nederlands.","dut"
+"In Nederlands","dut"
+"in Nederlands","dut"
+"Interview: Dutch","dut"
+"Nderlands","dut"
+"ned","dut"
+"Nederland","dut"
+"Nederlands","dut"
+"nederlands","dut"
+"Nederlandse","dut"
+"nedl","dut"
+"partly Dutch","dut"
+"Subtitles: Dutch","dut"
+"Subtitling: Dutch","dut"
+"Survey instruments in Dutch","dut"
+"Transcripts in Dutch","dut"
+"Vlaams","dut"
+"Artikelen: Nederlands en Engels","dut,eng"
+"Data files with Greek characters in their names are also included in the data set. However, language to describe the content of this data set is primarily English and occasionally Dutch.","dut,eng"
+"Dutch and English","dut,eng"
+"Dutch, English","dut,eng"
+"English and Dutch","dut,eng"
+"Description in English","eng"
+"Documentation in English","eng"
+"eng","eng"
+"Engels","eng"
+"English","eng"
+"English (Summary)","eng"
+"English (summary)","eng"
+"English summary","eng"
+"Interview: English","eng"
+"partially English","eng"
+"Subtitles: English","eng"
+"Subtitling: English","eng"
+"summary and conclusion in English","eng"
+"Summary in English","eng"
+"summary in English","eng"
+"English and Portuguese","eng,por"
+"Ewe","ewe"
+"Fante","fat"
+"Filipino","fil"
+"Frans","fre"
+"fre/fra","fre"
+"French","fre"
+"North Frisian","frr"
+"een paar woorden Fries","fry"
+"Fries","fry"
+"Frisian","fry"
+"Deutsch (Zusammenfassung)","ger"
+"Deutsche Zusammenfassung","ger"
+"Duits","ger"
+"ger/deu","ger"
+"German","ger"
+"German (appendix 2)","ger"
+"German (phonetic notation)","ger"
+"German (summary)","ger"
+"German summary","ger"
+"Duits/Nederlands","ger,dut"
+"greek (new testament)","grc"
+"Ancient Greek","grc"
+"Greek","gre"
+"Hebrew","heb"
+"Hebrew (some interviews)","heb"
+"Interview: Hebrew","heb"
+"Hindi","hin"
+"Croatian","hrv"
+"Hungarian","hun"
+"Hungarian, German","hun,ger"
+"Bahasa","ind"
+"Bahasa Indonesia","ind"
+"Bahasa indonesia","ind"
+"Indonesian","ind"
+"Indonesisch","ind"
+"Italian","ita"
+"Includes Dutch, Italian and Swedish (filmtitles)","ita,dut,swe"
+"Javaans","jav"
+"Javanese","jav"
+"Japanese","jpn"
+"Japans","jpn"
+"Kinyarwanda","kin"
+"Latin","lat"
+"Names of Regions, Cities, Districts are in original languge - in Latvian","lav"
+"Limburg","lim"
+"Limburgs","lim"
+"Ambon-Maleis","may"
+"Maleis","may"
+"Maleis doorspekt met Nederlandse woorden en zinsdelen","may"
+"Nepali","nep"
+"Papiamento","pap"
+"Papiamentu","pap"
+"Polish","pol"
+"One documentary available with English and Polish subtitles","pol,eng"
+"one documentary available with English and Polish subtitles","pol,eng"
+"Portuguese","por"
+"Interview: Russian","rus"
+"Russian","rus"
+"Russian (some interviews)","rus"
+"Russisch","rus"
+"dse","sgn"
+"Sign Language of the Netherlands","sgn"
+"Spanish","spa"
+"Sranan Tongo","srn"
+"Syriac","syr"
+"a little bit of Jiddish","yid"
+"Jiddisch","yid"
+"Yiddish","yid"
+"Interview: English and some Yiddish","yid,eng"
+"Interview: Russian and Yiddish","yid,rus"