Skip to content

Commit

Permalink
Release Data Preparation Tool 3.4.0.
Browse files Browse the repository at this point in the history
  • Loading branch information
olivergoetze committed Feb 25, 2021
1 parent ee12eb1 commit 9436fea
Show file tree
Hide file tree
Showing 50 changed files with 58,555 additions and 47,288 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,4 @@ data_output/
utils/validation_results/
utils/xml_enriched_with_uuids/
utils/ddb_id_lists
modules/provider_specific/.provider_script_sets/.deleted_sets
4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
![Data Preparation Tool](https://github.com/Deutsche-Digitale-Bibliothek/ddblabs-datapreparationtool/raw/master/dpt_screenshot.png "Data Preparation Tool")

![Data Preparation Tool](https://github.com/Deutsche-Digitale-Bibliothek/ddblabs-datapreparationtool/raw/master/dpt_screenshot_02.png "Data Preparation Tool")

![PyPI - Python Version](https://img.shields.io/pypi/pyversions/validify)

Das DDB Data Preparation Tool ist eine Open-Source-Anwendung zur Aufbereitung von Daten im [EAD(DDB)-Format](https://wiki.deutsche-digitale-bibliothek.de/pages/viewpage.action?pageId=19010180) für den Ingest in die Deutsche Digitale Bibliothek und das Archivportal-D. Es wird vornehmlich zur Datenanalyse und -anpassung in der Fachstelle Archiv verwendet, soll aber auch der Validierung von Exportdateien durch Datengeber, Schnittstellenentwickler und Softwarehersteller dienen.
Expand All @@ -28,6 +26,6 @@ python3 main_gui.py


### Weiterführende Informationen ...
... finden Sie in der [Dokumentation zum Lieferformat EAD(DDB)](https://wiki.deutsche-digitale-bibliothek.de/display/DFD/DDB+Data+Preparation+Tool).
... finden Sie [hier](https://wiki.deutsche-digitale-bibliothek.de/display/DFD/DDB+Data+Preparation+Tool).

Bei Fragen können Sie sich gerne an die [Fachstelle Archiv der Deutschen Digitalen Bibliothek](https://pro.deutsche-digitale-bibliothek.de/fachstelle-archiv) wenden
11 changes: 6 additions & 5 deletions build_standalone_dist.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@
# Variablen für den Build-Prozess (64 bit Python-Umgebung):
include_providerspecific_modules = False
qt_lib_path = "C:\\Users\\OGoetze\\venv\\build\\ddbmappings_build\\Lib\\site-packages\\PyQt5\\Qt\\bin"
msvc_path = "C:\\Program Files (x86)\\Windows Kits\\10\\Redist\\ucrt\\DLLs\\x64"
msvc_path = "C:\\Program Files (x86)\\Windows Kits\\10\\Redist\\10.0.19041.0\\ucrt\\DLLs\\x64"
icon_path = "gui_components/ui_templates/resources/datapreparationtool.ico"
data_files = []

# Angepasste Variablen für 32 bit Python-Umgebung:
python_arch = struct.calcsize("P") * 8
if python_arch == 32: # 32 bit Python
qt_lib_path = "C:\\Users\\OGoetze\\venv\\build\\ddbmappings_build_32bit\\Lib\\site-packages\\PyQt5\\Qt\\bin"
msvc_path = "C:\\Program Files (x86)\\Windows Kits\\10\\Redist\\ucrt\\DLLs\\x86"
msvc_path = "C:\\Program Files (x86)\\Windows Kits\\10\\Redist\\10.0.19041.0\\ucrt\\DLLs\\x86"
logger.info("Baue stand-alone Distribution für Windows ({architecture} bit).", architecture=str(python_arch))

# Aufräumen:
Expand All @@ -41,7 +41,6 @@
logger.info("Erstellen von Unterzeichnissen ...")
os.makedirs("dist/gui_session/templates")
os.makedirs("dist/utils/xml_enriched_with_uuids")
os.makedirs("dist/modules/xsl_transform")
os.makedirs("dist/modules/ead2mets")
os.makedirs("dist/modules/common/provider_metadata")
os.makedirs("dist/modules/analysis/enrichment")
Expand All @@ -57,6 +56,7 @@
logger.info("Kopieren der UI-Ressourcen ...")
copytree("gui_components/ui_templates/resources/html", "dist/gui_components/ui_templates/resources/html")
copyfile("gui_components/ui_templates/resources/list.png", "dist/gui_components/ui_templates/resources/list.png")
copytree("gui_components/ui_templates/resources/i18n", "dist/gui_components/ui_templates/resources/i18n")

logger.info("Kopieren der providerspezifischen Anpassungen, inkl. modules/provider_specific/aggregator_mapping.xml ...")
if include_providerspecific_modules:
Expand All @@ -67,9 +67,10 @@
if os.path.isdir("dist/modules/provider_specific/__pycache__"):
rmtree("dist/modules/provider_specific/__pycache__")
else:
copyfile("modules/provider_specific/aggregator_mapping.xml",
"dist/modules/provider_specific/aggregator_mapping.xml")
copyfile("modules/provider_specific/aggregator_mapping.xml", "dist/modules/provider_specific/aggregator_mapping.xml")
copyfile("modules/provider_specific/provider_script_sets_mapping.xml", "dist/modules/provider_specific/provider_script_sets_mapping.xml")
copyfile("modules/provider_specific/provider_script_template.py", "dist/modules/provider_specific/provider_script_template.py")
copytree("modules/provider_specific/common", "dist/modules/provider_specific/common")

logger.info("Kopieren des Templates zur METS/MODS-Generierung ...")
copyfile("modules/ead2mets/mets_template.xml", "dist/modules/ead2mets/mets_template.xml")
Expand Down
25 changes: 17 additions & 8 deletions gui_components/get_module_metadata.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from lxml import etree
import os, json
from loguru import logger
import os, json, inspect
import importlib.util


def fetch_providerspecific_modules():
Expand All @@ -26,14 +28,21 @@ def get_module_description(provider_module, provider_isil):
# Ermitteln der Modul-Beschreibung zur Darstellung in der GUI

module_desc = ""

module_path = "modules/provider_specific/{}/{}".format(provider_isil.replace("-", "_"), provider_module)
try:
module_metadata_path = "modules/provider_specific/{}/metadata/{}.xml".format(provider_isil.replace("-", "_"), provider_module[:-3])
module_metadata_in = etree.parse(module_metadata_path)
findlist = module_metadata_in.findall("//description")
module_desc = findlist[0].text
except OSError:
pass
spec = importlib.util.spec_from_file_location("parse_xml_content", module_path)
provider_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(provider_module)
module_docstring = inspect.getdoc(provider_module.parse_xml_content)

if module_docstring is not None:
module_desc = module_docstring
except OSError as e:
logger.debug("Beschreibung für Modul {} konnte nicht ausgelesen werden: {}".format(module_path, e))
except ModuleNotFoundError as e:
logger.debug("Beschreibung für Modul {} konnte nicht ausgelesen werden, da die dort verwendete Bibliothek '{}' nicht installiert ist.".format(module_path, e.name))
except ImportError as e:
logger.debug("Beschreibung für Modul {} konnte nicht ausgelesen werden, da das Python-Modul '{}' nicht importiert werden kann.".format(module_path, e.name))

return module_desc

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'provider_scripts_configure_maintenance_function_dialog.ui'
#
# Created by: PyQt5 UI code generator 5.15.2
#
# WARNING: Any manual changes made to this file will be lost when pyuic5 is
# run again. Do not edit this file unless you know what you are doing.


from PyQt5 import QtCore, QtGui, QtWidgets


class Ui_providerScriptsConfigureMaintenanceFunctionDialog(object):
def setupUi(self, providerScriptsConfigureMaintenanceFunctionDialog):
providerScriptsConfigureMaintenanceFunctionDialog.setObjectName("providerScriptsConfigureMaintenanceFunctionDialog")
providerScriptsConfigureMaintenanceFunctionDialog.resize(337, 177)
providerScriptsConfigureMaintenanceFunctionDialog.setModal(True)
self.gridLayout = QtWidgets.QGridLayout(providerScriptsConfigureMaintenanceFunctionDialog)
self.gridLayout.setObjectName("gridLayout")
self.groupBox = QtWidgets.QGroupBox(providerScriptsConfigureMaintenanceFunctionDialog)
self.groupBox.setObjectName("groupBox")
self.gridLayout_2 = QtWidgets.QGridLayout(self.groupBox)
self.gridLayout_2.setObjectName("gridLayout_2")
self.comboBox_choose_maintenance_function = QtWidgets.QComboBox(self.groupBox)
self.comboBox_choose_maintenance_function.setObjectName("comboBox_choose_maintenance_function")
self.comboBox_choose_maintenance_function.addItem("")
self.comboBox_choose_maintenance_function.addItem("")
self.comboBox_choose_maintenance_function.addItem("")
self.comboBox_choose_maintenance_function.addItem("")
self.gridLayout_2.addWidget(self.comboBox_choose_maintenance_function, 0, 0, 1, 1)
self.frame = QtWidgets.QFrame(self.groupBox)
self.frame.setStyleSheet("background-color: rgba(164, 219, 255, 235);")
self.frame.setFrameShape(QtWidgets.QFrame.NoFrame)
self.frame.setFrameShadow(QtWidgets.QFrame.Raised)
self.frame.setObjectName("frame")
self.gridLayout_3 = QtWidgets.QGridLayout(self.frame)
self.gridLayout_3.setObjectName("gridLayout_3")
self.label_maintenance_function_info = QtWidgets.QLabel(self.frame)
self.label_maintenance_function_info.setScaledContents(True)
self.label_maintenance_function_info.setWordWrap(True)
self.label_maintenance_function_info.setObjectName("label_maintenance_function_info")
self.gridLayout_3.addWidget(self.label_maintenance_function_info, 0, 0, 1, 1)
self.gridLayout_2.addWidget(self.frame, 1, 0, 1, 1)
self.gridLayout.addWidget(self.groupBox, 0, 0, 1, 1)
self.buttonBox_providerscripts_maintenance_configuration = QtWidgets.QDialogButtonBox(providerScriptsConfigureMaintenanceFunctionDialog)
self.buttonBox_providerscripts_maintenance_configuration.setOrientation(QtCore.Qt.Horizontal)
self.buttonBox_providerscripts_maintenance_configuration.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok)
self.buttonBox_providerscripts_maintenance_configuration.setObjectName("buttonBox_providerscripts_maintenance_configuration")
self.gridLayout.addWidget(self.buttonBox_providerscripts_maintenance_configuration, 1, 0, 1, 1)

self.retranslateUi(providerScriptsConfigureMaintenanceFunctionDialog)
self.buttonBox_providerscripts_maintenance_configuration.accepted.connect(providerScriptsConfigureMaintenanceFunctionDialog.accept)
self.buttonBox_providerscripts_maintenance_configuration.rejected.connect(providerScriptsConfigureMaintenanceFunctionDialog.reject)
QtCore.QMetaObject.connectSlotsByName(providerScriptsConfigureMaintenanceFunctionDialog)

def retranslateUi(self, providerScriptsConfigureMaintenanceFunctionDialog):
_translate = QtCore.QCoreApplication.translate
providerScriptsConfigureMaintenanceFunctionDialog.setWindowTitle(_translate("providerScriptsConfigureMaintenanceFunctionDialog", "Vorprozessierungsschritt konfigurieren"))
self.groupBox.setTitle(_translate("providerScriptsConfigureMaintenanceFunctionDialog", "Art der Vorprozessierung wählen"))
self.comboBox_choose_maintenance_function.setItemText(0, _translate("providerScriptsConfigureMaintenanceFunctionDialog", "Rechte/Lizenz-Anreicherung"))
self.comboBox_choose_maintenance_function.setItemText(1, _translate("providerScriptsConfigureMaintenanceFunctionDialog", "DDB-2017-Vorprozessierung"))
self.comboBox_choose_maintenance_function.setItemText(2, _translate("providerScriptsConfigureMaintenanceFunctionDialog", "Aggregator-Anreicherung"))
self.comboBox_choose_maintenance_function.setItemText(3, _translate("providerScriptsConfigureMaintenanceFunctionDialog", "Mapping-Definition anwenden"))
self.label_maintenance_function_info.setText(_translate("providerScriptsConfigureMaintenanceFunctionDialog", "Die gewählte Vorprozessierung kann im Menü \"Tools\" des Hauptfensters weiter konfiguriert werden."))


if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
providerScriptsConfigureMaintenanceFunctionDialog = QtWidgets.QDialog()
ui = Ui_providerScriptsConfigureMaintenanceFunctionDialog()
ui.setupUi(providerScriptsConfigureMaintenanceFunctionDialog)
providerScriptsConfigureMaintenanceFunctionDialog.show()
sys.exit(app.exec_())
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>providerScriptsConfigureMaintenanceFunctionDialog</class>
<widget class="QDialog" name="providerScriptsConfigureMaintenanceFunctionDialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>337</width>
<height>177</height>
</rect>
</property>
<property name="windowTitle">
<string>Vorprozessierungsschritt konfigurieren</string>
</property>
<property name="modal">
<bool>true</bool>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Art der Vorprozessierung wählen</string>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QComboBox" name="comboBox_choose_maintenance_function">
<item>
<property name="text">
<string>Rechte/Lizenz-Anreicherung</string>
</property>
</item>
<item>
<property name="text">
<string>DDB-2017-Vorprozessierung</string>
</property>
</item>
<item>
<property name="text">
<string>Aggregator-Anreicherung</string>
</property>
</item>
<item>
<property name="text">
<string>Mapping-Definition anwenden</string>
</property>
</item>
</widget>
</item>
<item row="1" column="0">
<widget class="QFrame" name="frame">
<property name="styleSheet">
<string notr="true">background-color: rgba(164, 219, 255, 235);</string>
</property>
<property name="frameShape">
<enum>QFrame::NoFrame</enum>
</property>
<property name="frameShadow">
<enum>QFrame::Raised</enum>
</property>
<layout class="QGridLayout" name="gridLayout_3">
<item row="0" column="0">
<widget class="QLabel" name="label_maintenance_function_info">
<property name="text">
<string>Die gewählte Vorprozessierung kann im Menü &quot;Tools&quot; des Hauptfensters weiter konfiguriert werden.</string>
</property>
<property name="scaledContents">
<bool>true</bool>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
</item>
<item row="1" column="0">
<widget class="QDialogButtonBox" name="buttonBox_providerscripts_maintenance_configuration">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections>
<connection>
<sender>buttonBox_providerscripts_maintenance_configuration</sender>
<signal>accepted()</signal>
<receiver>providerScriptsConfigureMaintenanceFunctionDialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>248</x>
<y>254</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox_providerscripts_maintenance_configuration</sender>
<signal>rejected()</signal>
<receiver>providerScriptsConfigureMaintenanceFunctionDialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>316</x>
<y>260</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>
Loading

0 comments on commit 9436fea

Please sign in to comment.