From 46d341794f2d8c7c2fda997cabb268ee3c05962c Mon Sep 17 00:00:00 2001 From: Jo Pol Date: Wed, 27 Jan 2021 15:07:25 +0100 Subject: [PATCH] DD-314: Map languages to ISO 639-2(#12) --- src/main/assembly/dist/cfg/languages.csv | 152 ++++++++++++++++++ .../EasyConvertBagToDepositApp.scala | 3 + .../easy/bag2deposit/ddm/DdmTransformer.scala | 1 + .../bag2deposit/ddm/LanguageRewriteRule.scala | 66 ++++++++ src/test/resources/debug-config/languages.csv | 132 +++++++++++++++ .../RewriteSpec.scala | 53 +++++- 6 files changed, 402 insertions(+), 5 deletions(-) create mode 100644 src/main/assembly/dist/cfg/languages.csv create mode 100644 src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/LanguageRewriteRule.scala create mode 100644 src/test/resources/debug-config/languages.csv diff --git a/src/main/assembly/dist/cfg/languages.csv b/src/main/assembly/dist/cfg/languages.csv new file mode 100644 index 00000000..ce22376c --- /dev/null +++ b/src/main/assembly/dist/cfg/languages.csv @@ -0,0 +1,152 @@ +"found","iso 639-2 (B)" +"Afrikaans","afr" +"South African","afr" +"Arabic","ara" +"Aramaic","arc" +"Official Aramaic; Imperial Aramaic (arc)","arc" +"Berber","ber" +"Bosnisch","bos" +"Chinese","chi" +"Chinese Script, Modern Mandarin, Hakka, Taiwanese, Bunun, Tsou, Ami","chi" +"Hakka","chi" +"Modern Mandarin Chinese","chi" +"漢字, 現代華語, 客家話, 臺灣話, 布農話, 鄒語, 阿美話","chi" +"Czech","cze" +"(and Dutch questionnaire)","dut" +"Brabants dialect","dut" +"Codebook in Dutch","dut" +"Data: Nederlands","dut" +"dut/nld","dut" +"Dutch","dut" +"dutch","dut" +"Dutch (Questionnaires presented to participants only)","dut" +"Dutch, heavy Texel accent","dut" +"Dutch/Nederlands","dut" +"Gronings","dut" +"Gronings dialect","dut" +"In het Nederland","dut" +"In het Nederlands","dut" +"in het Nederlands","dut" +"In het Nederlands.","dut" +"in het Nederlands.","dut" +"In Nederlands","dut" +"in Nederlands","dut" +"Interview: Dutch","dut" +"Nderlands","dut" +"ned","dut" +"Nederland","dut" +"Nederlands","dut" +"nederlands","dut" +"Nederlandse","dut" +"nedl","dut" +"partly Dutch","dut" +"Subtitles: Dutch","dut" +"Subtitling: Dutch","dut" +"Survey instruments in Dutch","dut" +"Transcripts in Dutch","dut" +"Vlaams","dut" +"Artikelen: Nederlands en Engels","dut,eng" +"Data files with Greek characters in their names are also included in the data set. However, language to describe the content of this data set is primarily English and occasionally Dutch.","dut,eng" +"Dutch and English","dut,eng" +"Dutch, English","dut,eng" +"English and Dutch","dut,eng" +"English with summary in Dutch","dut,eng" +"In het Nederlands, samenvatting in het Engels.","dut,eng" +"Nederlands en Engels","dut,eng" +"Description in English","eng" +"Documentation in English","eng" +"eng","eng" +"Engels","eng" +"English","eng" +"English (Summary)","eng" +"English (summary)","eng" +"English summary","eng" +"Interview: English","eng" +"partially English","eng" +"Subtitles: English","eng" +"Subtitling: English","eng" +"summary and conclusion in English","eng" +"Summary in English","eng" +"summary in English","eng" +"English and Portuguese","eng,por" +"Ewe","ewe" +"Fante","fat" +"Filipino","fil" +"Frans","fre" +"fre/fra","fre" +"French","fre" +"North Frisian","frr" +"een paar woorden Fries","fry" +"Fries","fry" +"Frisian","fry" +"Deutsch (Zusammenfassung)","ger" +"Deutsche Zusammenfassung","ger" +"Duits","ger" +"ger/deu","ger" +"German","ger" +"German (appendix 2)","ger" +"German (phonetic notation)","ger" +"German (summary)","ger" +"German summary","ger" +"Duits/Nederlands","ger,dut" +"German, Dutch and English","ger,eng,dut" +"Ancient Greek","grc" +"greek (new testament)","grc" +"Greek","gre" +"Ancient Hebrew (hbo)","heb" +"Biblical Hebrew","heb" +"Classical Hebrew","heb" +"Hebrew","heb" +"Hebrew (some interviews)","heb" +"Interview: Hebrew","heb" +"Hindi","hin" +"Croatian","hrv" +"Hungarian","hun" +"Hungarian, German","hun,ger" +"Bahasa","ind" +"Bahasa Indonesia","ind" +"Bahasa indonesia","ind" +"Bugis","ind" +"Indonesian","ind" +"Indonesisch","ind" +"Italian","ita" +"Includes Dutch, Italian and Swedish (filmtitles)","ita,dut,swe" +"Javaans","jav" +"Javanese","jav" +"Japanese","jpn" +"Japans","jpn" +"Kinyarwanda","kin" +"Latin","lat" +"Names of Regions, Cities, Districts are in original languge - in Latvian","lav" +"Limburg","lim" +"Limburgs","lim" +"Limburgs/Nederlands","lim,dut" +"limburgs/nederlands","lim,dut" +"Maa","luo" +"Ambon-Maleis","may" +"Maleis","may" +"Maleis doorspekt met Nederlandse woorden en zinsdelen","may" +"Moluks-Maleis","msa" +"Nepali","nep" +"Papiamento","pap" +"Papiamentu","pap" +"Polish","pol" +"One documentary available with English and Polish subtitles","pol,eng" +"one documentary available with English and Polish subtitles","pol,eng" +"Brazilian Portuguese","por" +"Portuguese","por" +"Interview: Russian","rus" +"Russian","rus" +"Russian (some interviews)","rus" +"Russisch","rus" +"dse","sgn" +"Sign Language of the Netherlands","sgn" +"Spanish","spa" +"Sranan Tongo","srn" +"Biblical Aramaic","syc" +"Syriac","syr" +"a little bit of Jiddish","yid" +"Jiddisch","yid" +"Yiddish","yid" +"Interview: English and some Yiddish","yid,eng" +"Interview: Russian and Yiddish","yid,rus" \ No newline at end of file diff --git a/src/main/scala/nl.knaw.dans.easy.bag2deposit/EasyConvertBagToDepositApp.scala b/src/main/scala/nl.knaw.dans.easy.bag2deposit/EasyConvertBagToDepositApp.scala index 778a6660..3c50f21f 100644 --- a/src/main/scala/nl.knaw.dans.easy.bag2deposit/EasyConvertBagToDepositApp.scala +++ b/src/main/scala/nl.knaw.dans.easy.bag2deposit/EasyConvertBagToDepositApp.scala @@ -19,6 +19,8 @@ import better.files.File import better.files.File.CopyOptions import nl.knaw.dans.bag.v0.DansV0Bag import nl.knaw.dans.easy.bag2deposit.Command.FeedBackMessage +import nl.knaw.dans.easy.bag2deposit.ddm.LanguageRewriteRule +import nl.knaw.dans.easy.bag2deposit.ddm.LanguageRewriteRule.logNotMapped import nl.knaw.dans.lib.logging.DebugEnhancedLogging import java.io.{ FileNotFoundException, IOException } @@ -75,6 +77,7 @@ class EasyConvertBagToDepositApp(configuration: Configuration) extends DebugEnha _ = formatDiff(ddmIn, ddmOut).foreach(s => logger.info(s)) _ = ddmFile.writeText(ddmOut.serialize) props <- depositPropertiesFactory.create(bagInfo, ddmOut) + _ = logNotMapped(ddmOut, props.getString("identifier.fedora", "")) _ = props.save((bagParentDir / "deposit.properties").toJava) _ = bagInfoKeysToRemove.foreach(mutableBagMetadata.remove) _ <- BagFacade.updateMetadata(bag) diff --git a/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/DdmTransformer.scala b/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/DdmTransformer.scala index 734083f5..e858ab28 100644 --- a/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/DdmTransformer.scala +++ b/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/DdmTransformer.scala @@ -31,6 +31,7 @@ case class DdmTransformer(cfgDir: File) extends DebugEnhancedLogging { reportRewriteRule, AbrRewriteRule(cfgDir / "ABR-period.csv", "temporal", "ddm:temporal"), AbrRewriteRule(cfgDir / "ABR-complex.csv", "subject", "ddm:subject"), + LanguageRewriteRule(cfgDir / "languages.csv"), ) private case class DdmRewriteRule(reportNumberFromFirstTitle: NodeSeq) extends RewriteRule { diff --git a/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/LanguageRewriteRule.scala b/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/LanguageRewriteRule.scala new file mode 100644 index 00000000..bcbd1dfe --- /dev/null +++ b/src/main/scala/nl/knaw/dans/easy/bag2deposit/ddm/LanguageRewriteRule.scala @@ -0,0 +1,66 @@ +/** + * Copyright (C) 2020 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.easy.bag2deposit.ddm + +import better.files.File +import nl.knaw.dans.easy.bag2deposit.{ parseCsv, printer } +import nl.knaw.dans.lib.logging.DebugEnhancedLogging + +import scala.xml.{ Node, NodeSeq } +import scala.xml.transform.RewriteRule + +case class LanguageRewriteRule(cfgFile: File) extends RewriteRule { + + private val freeLanguageToCodes = parseCsv(cfgFile, 0).toSeq + .map(r => r.get(0) -> r.get(1).split(",").toSeq) + .toMap + + private val codeToElem = Map( + "nld" -> elem("Dutch")("dut"), + "dut" -> elem("Dutch")("dut"), + "eng" -> elem("English")("eng"), + "fre" -> elem("French")("fre"), + "fra" -> elem("French")("fre"), + "deu" -> elem("German")("ger"), + "ger" -> elem("German")("ger"), + ) + + override def transform(node: Node): Seq[Node] = { + def mapFreeLanguageValue = { + freeLanguageToCodes + .get(node.text) + .map(_.map(elem(node.text.trim))) + .getOrElse(node) + } + + if (node.label != "language") node + else if (node.attributes.mkString.matches("(.*:)?ISO639-[23].")) + codeToElem.getOrElse(node.text.trim.toLowerCase, mapFreeLanguageValue) + else mapFreeLanguageValue + } + + private def elem(value: String)(code: String) = + { value.trim } +} +object LanguageRewriteRule extends DebugEnhancedLogging { + def logNotMapped(ddm: Node, datasetId: String): Unit = { + (ddm \\ "language").filter( + !_.attributes.mkString.contains("encodingScheme") + ).foreach(n => + logger.info(s"NOT MAPPED $datasetId ${printer.format(n)}") + ) + } +} diff --git a/src/test/resources/debug-config/languages.csv b/src/test/resources/debug-config/languages.csv new file mode 100644 index 00000000..f2a9f75b --- /dev/null +++ b/src/test/resources/debug-config/languages.csv @@ -0,0 +1,132 @@ +"Afrikaans","afr" +"Arabic","ara" +"Aramaic","arc" +"Official Aramaic; Imperial Aramaic (arc)","arc" +"Berber","ber" +"Bosnisch","bos" +"Modern Mandarin Chinese","chi" +"Hakka","chi" +"Chinese Script, Modern Mandarin, Hakka, Taiwanese, Bunun, Tsou, Ami","chi" +"Chinese","chi" +"Czech","cze" +"(and Dutch questionnaire)","dut" +"Codebook in Dutch","dut" +"Data: Nederlands","dut" +"dut/nld","dut" +"Dutch","dut" +"dutch","dut" +"Dutch (Questionnaires presented to participants only)","dut" +"Dutch, heavy Texel accent","dut" +"Dutch/Nederlands","dut" +"In het Nederland","dut" +"In het Nederlands","dut" +"in het Nederlands","dut" +"In het Nederlands.","dut" +"in het Nederlands.","dut" +"In Nederlands","dut" +"in Nederlands","dut" +"Interview: Dutch","dut" +"Nderlands","dut" +"ned","dut" +"Nederland","dut" +"Nederlands","dut" +"nederlands","dut" +"Nederlandse","dut" +"nedl","dut" +"partly Dutch","dut" +"Subtitles: Dutch","dut" +"Subtitling: Dutch","dut" +"Survey instruments in Dutch","dut" +"Transcripts in Dutch","dut" +"Vlaams","dut" +"Artikelen: Nederlands en Engels","dut,eng" +"Data files with Greek characters in their names are also included in the data set. However, language to describe the content of this data set is primarily English and occasionally Dutch.","dut,eng" +"Dutch and English","dut,eng" +"Dutch, English","dut,eng" +"English and Dutch","dut,eng" +"Description in English","eng" +"Documentation in English","eng" +"eng","eng" +"Engels","eng" +"English","eng" +"English (Summary)","eng" +"English (summary)","eng" +"English summary","eng" +"Interview: English","eng" +"partially English","eng" +"Subtitles: English","eng" +"Subtitling: English","eng" +"summary and conclusion in English","eng" +"Summary in English","eng" +"summary in English","eng" +"English and Portuguese","eng,por" +"Ewe","ewe" +"Fante","fat" +"Filipino","fil" +"Frans","fre" +"fre/fra","fre" +"French","fre" +"North Frisian","frr" +"een paar woorden Fries","fry" +"Fries","fry" +"Frisian","fry" +"Deutsch (Zusammenfassung)","ger" +"Deutsche Zusammenfassung","ger" +"Duits","ger" +"ger/deu","ger" +"German","ger" +"German (appendix 2)","ger" +"German (phonetic notation)","ger" +"German (summary)","ger" +"German summary","ger" +"Duits/Nederlands","ger,dut" +"greek (new testament)","grc" +"Ancient Greek","grc" +"Greek","gre" +"Hebrew","heb" +"Hebrew (some interviews)","heb" +"Interview: Hebrew","heb" +"Hindi","hin" +"Croatian","hrv" +"Hungarian","hun" +"Hungarian, German","hun,ger" +"Bahasa","ind" +"Bahasa Indonesia","ind" +"Bahasa indonesia","ind" +"Indonesian","ind" +"Indonesisch","ind" +"Italian","ita" +"Includes Dutch, Italian and Swedish (filmtitles)","ita,dut,swe" +"Javaans","jav" +"Javanese","jav" +"Japanese","jpn" +"Japans","jpn" +"Kinyarwanda","kin" +"Latin","lat" +"Names of Regions, Cities, Districts are in original languge - in Latvian","lav" +"Limburg","lim" +"Limburgs","lim" +"Ambon-Maleis","may" +"Maleis","may" +"Maleis doorspekt met Nederlandse woorden en zinsdelen","may" +"Nepali","nep" +"Papiamento","pap" +"Papiamentu","pap" +"Polish","pol" +"One documentary available with English and Polish subtitles","pol,eng" +"one documentary available with English and Polish subtitles","pol,eng" +"Portuguese","por" +"Interview: Russian","rus" +"Russian","rus" +"Russian (some interviews)","rus" +"Russisch","rus" +"dse","sgn" +"Sign Language of the Netherlands","sgn" +"Spanish","spa" +"Sranan Tongo","srn" +"Syriac","syr" +"a little bit of Jiddish","yid" +"Jiddisch","yid" +"Yiddish","yid" +"Interview: English and some Yiddish","yid,eng" +"Interview: Russian and Yiddish","yid,rus" \ No newline at end of file diff --git a/src/test/scala/nl.knaw.dans.easy.bag2deposit/RewriteSpec.scala b/src/test/scala/nl.knaw.dans.easy.bag2deposit/RewriteSpec.scala index c8bbec08..e2dcc4bf 100644 --- a/src/test/scala/nl.knaw.dans.easy.bag2deposit/RewriteSpec.scala +++ b/src/test/scala/nl.knaw.dans.easy.bag2deposit/RewriteSpec.scala @@ -17,7 +17,7 @@ package nl.knaw.dans.easy.bag2deposit import better.files.File import nl.knaw.dans.easy.bag2deposit.Fixture.SchemaSupport -import nl.knaw.dans.easy.bag2deposit.ddm.AbrRewriteRule +import nl.knaw.dans.easy.bag2deposit.ddm.{ AbrRewriteRule, LanguageRewriteRule } import org.apache.commons.csv.CSVRecord import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -29,7 +29,7 @@ import scala.xml.NodeBuffer class RewriteSpec extends AnyFlatSpec with SchemaSupport with Matchers { private val cfgDir: File = File("src/main/assembly/dist/cfg") private val cfg = Configuration(cfgDir.parent) - override val schema = "https://raw.githubusercontent.com/DANS-KNAW/easy-schema/9c2759d913cde537a2b49cbc0300532da56898c7/lib/src/main/resources/md/ddm/ddm.xsd" + override val schema = "https://raw.githubusercontent.com/DANS-KNAW/easy-schema/a7b54e1/lib/src/main/resources/md/ddm/ddm.xsd" private val mandatoryInProfile = YYY @@ -59,7 +59,7 @@ class RewriteSpec extends AnyFlatSpec with SchemaSupport with Matchers { private def getDuplicates(records: Iterable[CSVRecord]) = records.groupBy(_.get(0)).filter(_._2.size > 1) - "transform" should "convert" in { + "ABR rules" should "convert" in { val ddmIn = ddm( Rapport 123 @@ -154,8 +154,51 @@ class RewriteSpec extends AnyFlatSpec with SchemaSupport with Matchers { |""".stripMargin ) - cfg.ddmTransformer.transform(ddmIn).map(normalized) shouldBe - Success(normalized(expectedDDM)) + cfg.ddmTransformer.transform(ddmIn).map(normalized) + .getOrElse(fail("no DDM returned")) shouldBe normalized(expectedDDM) + + assume(schemaIsAvailable) + validate(expectedDDM) shouldBe Success(()) + } + + "LanguageRewriteRule" should "convert" in { + val ddmIn = ddm( + + language test + { mandatoryInProfile } + + + in het Nederlands + Engels + ratjetoe + huh + nld + ENG + deu + fra + + ) + val expectedDDM = ddm( + + language test + { mandatoryInProfile } + + + in het Nederlands + Engels + ratjetoe + huh + Dutch + English + German + French + + ) + cfg.ddmTransformer.transform(ddmIn).map(normalized) + .getOrElse(fail("no DDM returned")) shouldBe normalized(expectedDDM) + + // TODO manually check logging of not mapped language fields + LanguageRewriteRule.logNotMapped(expectedDDM,"eas-dataset:123") assume(schemaIsAvailable) validate(expectedDDM) shouldBe Success(())