Skip to content

Commit

Permalink
Provides DPV-DGA extension; fixes GDPR rights html
Browse files Browse the repository at this point in the history 
The DPV-DGA extension is added to the path /dpv/dpv-dga. Implementations
for RDFS+SKOS and OWL are not included. See #62.

The GDPR HTML did not correctly display associated rights for the legal
bases in its table. This has been fixed.
  • Loading branch information
@coolharsh55
coolharsh55 committed Sep 20, 2023
1 parent eb69b62 commit 63b96e2
Show file tree
Hide file tree
Showing 84 changed files with 24,293 additions and 7,030 deletions.
250 changes: 130 additions & 120 deletions documentation-generator/001_download_vocab_in_csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,75 +16,75 @@
# Format of this data structure is
# (DOCUMENT_NAME, ((DOCUMENT_ID, SHEET_NAME)+)
#
# ('base', '1x5Wfl-2Xp22R89lhNwNpYP0xN0zfQLFlNe3On6pwNM4',
# (
# 'Namespaces',
# 'Namespaces_Other',
# 'BaseOntology',
# 'BaseOntology_properties',
# )),
# ('dpv-pd','1SI6gZh9-dq1rf_etfrlYHj0QZwq9Vd25f_OHPX5hbSQ',
# (
# 'PersonalData',
# 'PersonalData_properties',
# 'dpv-pd',
# )),
# ('purpose_processing', '1ePg6BU2Zp9fiSDuEnKuVi6dIRrFLEVdatbVxjHRk-8s',
# (
# 'Purpose',
# 'Purpose_properties',
# 'Processing',
# 'Processing_properties',
# 'ProcessingContext',
# 'ProcessingContext_properties',
# 'ProcessingScale',
# 'ProcessingScale_properties',
# )),
# ('context_status', '1VPQW1DanprQhMwnhSqyKSGbEXdTmLHdc6UjpWJhyLMA',
# (
# 'Context',
# 'Context_properties',
# 'Status',
# 'Status_properties',
# )),
# ('toms', '16d0_k6ueoXxXRTgecih9Ny7NpeXYF8icm4QX99cPYJA',
# (
# 'TechnicalOrganisationalMeasure',
# 'TechnicalOrganisationalMeasure_properties',
# 'TechnicalMeasure',
# 'OrganisationalMeasure',
# )),
# ('entities', '1g6zLqVt5FlNlgsXq_NW2W9INv3KdGEFjJCyOd03UmOg',
# (
# 'Entities',
# 'Entities_properties',
# 'Entities_Authority',
# 'Entities_Authority_properties',
# 'Entities_LegalRole',
# 'Entities_LegalRole_properties',
# 'Entities_Organisation',
# 'Entities_DataSubject',
# 'Entities_DataSubject_properties',
# )),
# ('location_jurisdiction', '19exhY34jq6VDApRp2abHD-br6rpm6Q7BOP7H_pm5sKM',
# (
# 'Jurisdiction',
# 'Jurisdiction_properties',
# 'legal_properties',
# 'legal_Locations',
# 'legal_Laws',
# 'legal_Authorities',
# 'legal_EU_EEA',
# 'legal_EU_Adequacy',
# )),
# ('legal_basis', '13Ub4LXHruocffYnd7JKCMvzi1MYv3Gy61d3UmQBhARc',
# (
# 'LegalBasis',
# 'LegalBasis_properties',
# 'ConsentTypes',
# 'ConsentStatus',
# 'Consent_properties',
# )),
('base', '1x5Wfl-2Xp22R89lhNwNpYP0xN0zfQLFlNe3On6pwNM4',
(
'Namespaces',
'Namespaces_Other',
'BaseOntology',
'BaseOntology_properties',
)),
('dpv-pd','1SI6gZh9-dq1rf_etfrlYHj0QZwq9Vd25f_OHPX5hbSQ',
(
'PersonalData',
'PersonalData_properties',
'dpv-pd',
)),
('purpose_processing', '1ePg6BU2Zp9fiSDuEnKuVi6dIRrFLEVdatbVxjHRk-8s',
(
'Purpose',
'Purpose_properties',
'Processing',
'Processing_properties',
'ProcessingContext',
'ProcessingContext_properties',
'ProcessingScale',
'ProcessingScale_properties',
)),
('context_status', '1VPQW1DanprQhMwnhSqyKSGbEXdTmLHdc6UjpWJhyLMA',
(
'Context',
'Context_properties',
'Status',
'Status_properties',
)),
('toms', '16d0_k6ueoXxXRTgecih9Ny7NpeXYF8icm4QX99cPYJA',
(
'TechnicalOrganisationalMeasure',
'TechnicalOrganisationalMeasure_properties',
'TechnicalMeasure',
'OrganisationalMeasure',
)),
('entities', '1g6zLqVt5FlNlgsXq_NW2W9INv3KdGEFjJCyOd03UmOg',
(
'Entities',
'Entities_properties',
'Entities_Authority',
'Entities_Authority_properties',
'Entities_LegalRole',
'Entities_LegalRole_properties',
'Entities_Organisation',
'Entities_DataSubject',
'Entities_DataSubject_properties',
)),
('location_jurisdiction', '19exhY34jq6VDApRp2abHD-br6rpm6Q7BOP7H_pm5sKM',
(
'Jurisdiction',
'Jurisdiction_properties',
'legal_properties',
'legal_Locations',
'legal_Laws',
'legal_Authorities',
'legal_EU_EEA',
'legal_EU_Adequacy',
)),
('legal_basis', '13Ub4LXHruocffYnd7JKCMvzi1MYv3Gy61d3UmQBhARc',
(
'LegalBasis',
'LegalBasis_properties',
'ConsentTypes',
'ConsentStatus',
'Consent_properties',
)),
('gdpr', '1lDJZpl0UND8Bm_4iWKVQtgmMUz0YwP2R63CgP7Gro-U',
(
'GDPR_LegalBasis',
Expand All @@ -97,57 +97,67 @@
'GDPR_DPIA_properties',
'GDPR_compliance'
)),
# ('dpv-tech', '1GVmF4c7b-9xMSs0TyT45kXoCLLUVs8bbW34tfcozbuA',
# (
# 'tech-core',
# 'tech-core-properties',
# 'tech-data',
# 'tech-ops',
# 'tech-security',
# 'tech-surveillance',
# 'tech-provision',
# 'tech-provision-properties',
# 'tech-actors',
# 'tech-actors-properties',
# 'tech-comms',
# 'tech-comms-properties',
# 'tech-tools',
# 'tech_algorithms',
# )),
# ('risk', '1y8r3Vk-_Gi1MqbyAM6Ot4DoNDJpa2ZVhCyCyFQkGBy0',
# (
# 'Risk',
# 'Risk_properties',
# 'RiskConsequences',
# 'RiskLevels',
# 'RiskMatrix',
# 'RiskControls',
# 'RiskAssessmentTechniques',
# 'RiskManagement',
# 'RiskMethodology',
# 'Justifications',
# )),
# ('rights', '1XW-L6rGWbgGGp62q8eA22SWvh4wUWK5BpC0zfD6wAxM',
# (
# 'Rights',
# 'Rights_properties',
# 'EUFundamentalRights',
# )),
# ('rules', '1SDmlzSo1Ax_35v754Jzx4oFGKvGo5nyNtEAL0vSBbM0',
# (
# 'Rules',
# 'Rules_properties',
# )),
# ('standards', '1z-qaB2m6lD1ROmPVf9yhfG05D68Z7H4glYLERj6ZCRk',
# (
# 'Standards_ISO',
# )),
# ('ucr', '1__STWvOEZRc1u2J-8teOYjLpnTPlZ80_ebTytrUlWgQ',
# (
# 'UseCase',
# 'Requirement',
# 'Example',
# )),
('dga', '1wKsf0Vqr0Gg1C91MqshtI5tjGXmQvXu4p4xF0yK0KaA',
(
'DGA_LegalBasis',
'DGA_LegalRights',
'DGA_Services',
'DGA_Registers',
'DGA_TOMs',
'DGA_entities',
'DGA_properties',
)),
('dpv-tech', '1GVmF4c7b-9xMSs0TyT45kXoCLLUVs8bbW34tfcozbuA',
(
'tech-core',
'tech-core-properties',
'tech-data',
'tech-ops',
'tech-security',
'tech-surveillance',
'tech-provision',
'tech-provision-properties',
'tech-actors',
'tech-actors-properties',
'tech-comms',
'tech-comms-properties',
'tech-tools',
'tech_algorithms',
)),
('risk', '1y8r3Vk-_Gi1MqbyAM6Ot4DoNDJpa2ZVhCyCyFQkGBy0',
(
'Risk',
'Risk_properties',
'RiskConsequences',
'RiskLevels',
'RiskMatrix',
'RiskControls',
'RiskAssessmentTechniques',
'RiskManagement',
'RiskMethodology',
'Justifications',
)),
('rights', '1XW-L6rGWbgGGp62q8eA22SWvh4wUWK5BpC0zfD6wAxM',
(
'Rights',
'Rights_properties',
'EUFundamentalRights',
)),
('rules', '1SDmlzSo1Ax_35v754Jzx4oFGKvGo5nyNtEAL0vSBbM0',
(
'Rules',
'Rules_properties',
)),
('standards', '1z-qaB2m6lD1ROmPVf9yhfG05D68Z7H4glYLERj6ZCRk',
(
'Standards_ISO',
)),
('ucr', '1__STWvOEZRc1u2J-8teOYjLpnTPlZ80_ebTytrUlWgQ',
(
'UseCase',
'Requirement',
'Example',
)),
)


Expand Down
83 changes: 83 additions & 0 deletions documentation-generator/002_parse_csv_to_rdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
EXPORT_DPV_MODULE_PATH = '../dpv/modules'
EXPORT_DPV_GDPR_PATH = '../dpv-gdpr'
EXPORT_DPV_GDPR_MODULE_PATH = '../dpv-gdpr/modules'
EXPORT_DPV_DGA_PATH = '../dpv-dga'
EXPORT_DPV_DGA_MODULE_PATH = '../dpv-dga/modules'
EXPORT_DPV_PD_PATH = '../dpv-pd'
EXPORT_DPV_LEGAL_PATH = '../dpv-legal'
EXPORT_DPV_LEGAL_MODULE_PATH = '../dpv-legal/modules'
Expand Down Expand Up @@ -140,6 +142,7 @@ def add_common_triples_for_all_terms(term, graph):
graph.add((BASE[f'{term.term}'], SKOS.note, Literal(term.skos_note, lang='en')))
# rdfs:isDefinedBy
if term.skos_scopeNote:
DEBUG(term.skos_scopeNote)
links = [l.strip() for l in term.skos_scopeNote.replace('(','').replace(')','').split(',')]
link_iterator = iter(links)
for label in link_iterator:
Expand Down Expand Up @@ -209,6 +212,7 @@ def add_triples_for_classes(classes, graph):
graph.add((BASE[f'{cls.term}'], RDF.type, DPV.Concept))
# rdfs:subClassOf
if cls.dpv_isSubTypeOf:
DEBUG(cls.dpv_isSubTypeOf)
parents = [p.strip() for p in cls.dpv_isSubTypeOf.split(',')]
for parent in parents:
if parent == "dpv:Concept":
Expand Down Expand Up @@ -607,6 +611,85 @@ def serialize_graph(graph, filepath):

##############################################################################

# DPV-DGA #


DPV_DGA_CSV_FILES = {
'legal_basis': {
'classes': f'{IMPORT_CSV_PATH}/DGA_LegalBasis.csv',
},
'legal_rights': {
'classes': f'{IMPORT_CSV_PATH}/DGA_LegalRights.csv',
},
'services': {
'classes': f'{IMPORT_CSV_PATH}/DGA_Services.csv',
},
'registers': {
'classes': f'{IMPORT_CSV_PATH}/DGA_Registers.csv',
},
'toms': {
'classes': f'{IMPORT_CSV_PATH}/DGA_TOMs.csv',
},
'entities': {
'classes': f'{IMPORT_CSV_PATH}/DGA_entities.csv',
'properties': f'{IMPORT_CSV_PATH}/DGA_properties.csv',
},
}

BASE = NAMESPACES['dpv-dga']
DPV_DGA_GRAPH = Graph()
proposed_terms = {}
DPV_DGA_GRAPH.add((BASE[''], RDF.type, SKOS.ConceptScheme))

for name, module in DPV_DGA_CSV_FILES.items():
graph = Graph()
proposed = []
DEBUG('------')
DEBUG(f'Processing {name} module')
for prefix, namespace in NAMESPACES.items():
graph.namespace_manager.bind(prefix, namespace)
if 'classes' in module:
classes = extract_terms_from_csv(module['classes'], DPV_Class)
DEBUG(f'there are {len(classes)} classes in {name}')
returnval = add_triples_for_classes(classes, graph)
if returnval:
proposed.extend(returnval)
if 'properties' in module:
properties = extract_terms_from_csv(module['properties'], DPV_Property)
DEBUG(f'there are {len(properties)} properties in {name}')
returnval = add_triples_for_properties(properties, graph)
if returnval:
proposed.extend(returnval)
if proposed:
proposed_terms[name] = proposed
# add collection representing concepts
graph.add((BASE[f'{name.title()}Concepts'], RDF.type, SKOS.Collection))
graph.add((BASE[f'{name.title()}Concepts'], SKOS.prefLabel, Literal(f'{name.title()} Concepts', datatype=XSD.string)))
for concept, _, _ in graph.triples((None, RDF.type, SKOS.Concept)):
graph.add((BASE[f'{name.title()}Concepts'], SKOS.member, concept))
DPV_DGA_GRAPH.add((concept, SKOS.inScheme, DPV_DGA['']))
# serialize
serialize_graph(graph, f'{EXPORT_DPV_DGA_MODULE_PATH}/{name}')
if 'topconcept' in module:
DPV_DGA_GRAPH.add((BASE[''], SKOS.hasTopConcept, module['topconcept']))
DPV_DGA_GRAPH += graph

if proposed_terms:
with open(f'{EXPORT_DPV_DGA_PATH}/proposed.json', 'w') as fd:
json.dump(proposed_terms, fd)
DEBUG(f'exported proposed terms to {EXPORT_DPV_DGA_PATH}/proposed.json')
else:
DEBUG('no proposed terms in DPV-DGA')
graph = Graph()
graph.parse('ontology_metadata/dpv-dga.ttl', format='turtle')
DPV_DGA_GRAPH += graph

for prefix, namespace in NAMESPACES.items():
DPV_DGA_GRAPH.namespace_manager.bind(prefix, namespace)
serialize_graph(DPV_DGA_GRAPH, f'{EXPORT_DPV_DGA_PATH}/dpv-dga')

##############################################################################

# DPV-PD #
# dpv-gdpr is the exact same as dpv in terms of requirements and structure
# except that the namespace is different
Expand Down
Loading

0 comments on commit 63b96e2

Please sign in to comment.