From cdf96f2955ba1c8139cb7224548669d81b72e831 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 25 Oct 2023 16:21:25 +0100 Subject: [PATCH 01/11] Copied in purpose from specification document --- src/docs/about.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/docs/about.md b/src/docs/about.md index 1e96779..654910d 100644 --- a/src/docs/about.md +++ b/src/docs/about.md @@ -1,3 +1,29 @@ # UK Cross-Government Metadata Exchange Model -A metadata model for describing data assets for exchanging between UK government organisations. +A metadata model for describing data assets for exchanging between UK government organisations. A primary use case for this model is populating the Cross-Government Data Marketplace with details of data assets provided by UK Government organisations. + +## Purpose + +This metadata model is focused on the essential attributes needed to describe a critical data asset in the context of a cross-government data share and to facilitate data discoverability in the Government Data Marketplace. + +The metadata model is based in [DCAT Working Draft v3](https://www.w3.org/TR/vocab-dcat-3/) itself an extension of [Dublin Core](https://www.dublincore.org/specifications/dublin-core/dcmi-terms/) which is a metadata standard already approved by the [Open Standards Board](https://www.gov.uk/government/groups/open-standards-board) to [describe data shared in government](https://www.gov.uk/government/publications/recommended-open-standards-for-government/using-metadata-to-describe-data-shared-within-government). It extends the attributes under the [existing guidance](https://www.gov.uk/guidance/record-information-about-data-sets-you-share-with-others) and once it is baselined will subsequently update the latter. + +It is designed to add clarification and context to Pillar 1 of the Cross-Government Metadata Approach and endorsed for implementation by the Data Standards Authority Steering Board. Its objective is to present and describe all of the metadata elements that are to be included in a critical data set to be onboarded into the Government Data Catalogue. + +Subsequent iterations will provide more comprehensive descriptions and cover areas which may have additional specific requirements (such as the Health and Geospatial domains) with accompanying guidance to be developed in the form of a playbook for use by relevant DDaT professionals across government and wider public sector, where applicable. + +While this set of attributes is specific to the Government Data Marketplace, it is envisaged that other Data Catalogues, Marketplaces and/or Data Portals will be able to apply this specification within their individual context with little or no modifications, to ensure better interoperability between government information systems. + +### Data Catalogue Vocabulary (DCAT) + +DCAT is an RDF vocabulary designed to facilitate interoperability between data catalogues published on the Web and [already in use in data.gov.uk](https://guidance.data.gov.uk/publish_and_manage_data/harvest_or_add_data/harvest_data/dcat/#accepted-dcat-and-data-json-fields). + +DCAT enables a publisher to describe datasets and data services in a catalogue using a standard model and vocabulary that facilitates the consumption and aggregation of metadata from multiple catalogues. This can increase the discoverability of datasets and data services. It also makes it possible to have a decentralised approach to publishing data catalogues and makes federated search for datasets across catalogues in multiple sites possible using the same query mechanism and structure. + +The namespace for DCAT is [`http://www.w3.org/ns/dcat#`](http://www.w3.org/ns/dcat#). However, it should be noted that DCAT makes extensive use of terms from other vocabularies, in particular Dublin Core. DCAT itself defines a minimal set of classes and properties of its own. + +A __Dataset__ in DCAT is defined as a _"collection of data, published or curated by a single agent, and available for access or download in one or more representations.”_ A dataset is a conceptual entity which can be represented by one or more distributions that serialise the dataset for transfer. Distributions of a dataset can be provided via data services. + +A __Data Service__ in DCAT is defined as a _"collection of operations that provides access to one or more datasets or data processing functions."_ + +A __Distribution__ in DCAT is defined as a _"specific representation of a dataset. A dataset might be available in multiple serializations that may differ in various ways, including natural language, media-type or format, schematic organization, temporal and spatial resolution, level of detail or profiles (which might specify any or all of the above)."_ From 2a41091de1c0ec155e497f740039219813918b1f Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 25 Oct 2023 16:58:03 +0100 Subject: [PATCH 02/11] Add cross-reference to about page --- src/model/uk_cross_government_metadata_exchange_model.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/model/uk_cross_government_metadata_exchange_model.yaml b/src/model/uk_cross_government_metadata_exchange_model.yaml index 5fa4857..492317f 100644 --- a/src/model/uk_cross_government_metadata_exchange_model.yaml +++ b/src/model/uk_cross_government_metadata_exchange_model.yaml @@ -2,7 +2,10 @@ id: https://w3id.org/co-cddo/ukgov-metadata-exchange-model/ name: uk-cross-government-metadata-exchange-model title: UK Cross-Government Metadata Exchange Model -description: A metadata model for describing data assets for exchanging between UK government organisations. +description: | + A metadata model for describing data assets for exchanging between UK government organisations. + + For more details relating to the development of this metadata model, please see the [about](about) page. license: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ see_also: - https://co-cddo.github.io/ukgov-metadata-exchange-model From ace9bc5da7bc80ed3f4da356f9a4bcb0d3a2b551 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 25 Oct 2023 16:58:41 +0100 Subject: [PATCH 03/11] Add details of MIWG to a Governance section --- src/docs/about.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/docs/about.md b/src/docs/about.md index 654910d..5d15add 100644 --- a/src/docs/about.md +++ b/src/docs/about.md @@ -27,3 +27,14 @@ A __Dataset__ in DCAT is defined as a _"collection of data, published or curated A __Data Service__ in DCAT is defined as a _"collection of operations that provides access to one or more datasets or data processing functions."_ A __Distribution__ in DCAT is defined as a _"specific representation of a dataset. A dataset might be available in multiple serializations that may differ in various ways, including natural language, media-type or format, schematic organization, temporal and spatial resolution, level of detail or profiles (which might specify any or all of the above)."_ + +## Governance + +### Metadata Implementation Working Group + +The Cross-Government Metadata Exchange Model has been discussed and agreed by the Cross-Government Metadata Implementation Working Group. The working group is coordinated by the [Central Digital and Data Office](https://www.gov.uk/government/organisations/central-digital-and-data-office), and consists of representatives from Government organisations. + +The goals of the Metadata Implementation Working Group are +- To guide the implementation of the Cross-Government Metadata Approach and adoption of the underlying metadata standards via the Metadata Requirements Specification (MRS), +- To ensure that core/essential metadata is described consistently and available to potential users of data assets within the Government Data Catalogue, +- To collaborate in developing guidance and share lessons from implementing data catalogues in conformance with Metadata Management best practice. From 0b2cc0d0e5c30e1fdea448dc9b5d7b4e277cc021 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 11:23:17 +0100 Subject: [PATCH 04/11] Revised governance text --- src/docs/about.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/docs/about.md b/src/docs/about.md index 5d15add..fd2c646 100644 --- a/src/docs/about.md +++ b/src/docs/about.md @@ -30,11 +30,5 @@ A __Distribution__ in DCAT is defined as a _"specific representation of a datase ## Governance -### Metadata Implementation Working Group +The Cross-Government Metadata Exchange Model has been developed and agreed by the [Cross-Government Metadata Implementation Working Group](miwg.md). The working group is coordinated by the [Central Digital and Data Office](https://www.gov.uk/government/organisations/central-digital-and-data-office), and consists of data professionals in Government Digital and Data functions from across central and local government, devolved administrations, and the wider public sector. The outputs of the working group are reported through the Data Quality Hub and Data Standards Authority Peer Review Group as well as the Data Standards Authority Steering Board. -The Cross-Government Metadata Exchange Model has been discussed and agreed by the Cross-Government Metadata Implementation Working Group. The working group is coordinated by the [Central Digital and Data Office](https://www.gov.uk/government/organisations/central-digital-and-data-office), and consists of representatives from Government organisations. - -The goals of the Metadata Implementation Working Group are -- To guide the implementation of the Cross-Government Metadata Approach and adoption of the underlying metadata standards via the Metadata Requirements Specification (MRS), -- To ensure that core/essential metadata is described consistently and available to potential users of data assets within the Government Data Catalogue, -- To collaborate in developing guidance and share lessons from implementing data catalogues in conformance with Metadata Management best practice. From 99c7fe3a8cef87d647acf6eb5f147933f73c5043 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 11:23:28 +0100 Subject: [PATCH 05/11] Added page about MIWG --- src/docs/miwg.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/docs/miwg.md diff --git a/src/docs/miwg.md b/src/docs/miwg.md new file mode 100644 index 0000000..6651a27 --- /dev/null +++ b/src/docs/miwg.md @@ -0,0 +1,53 @@ +# Metadata Implementation Working Group + +## Purpose + +1. To guide the implementation of the Cross-Government Metadata Implementation Approach and adoption of the underlying metadata standards via the Metadata Exchange Model Specification, +1. To ensure that core/essential metadata is described consistently and available to potential users of data assets within the Government Data Marketplace, +1. To collaborate in developing guidance and share lessons from implementing data catalogues in conformance with Metadata Management best practice. + +## Audience and Scope +- Data professionals in Government Digital and Data functions, such as Data Architects/Modellers, Data Engineers and those in governance roles that broadly map to “Data Owners and Stewards”, +- People across central/local government, as well as the devolved administrations and the wider public sector, who have a role in owning, developing and/or maintaining metadata for a data asset. + +## Goal +To ensure metadata within the Government Data Marketplace is sufficient to: +- FIND: enable discovery of the data asset, +- USE: facilitate an informed decision by the customer as to whether a data asset is fit for purpose and meets their business need, and +- ACCESS: enable access to the data by ensuring the request is channelled to the right team(s) and by describing the access requirements and restrictions of the data source. + +## Desired Outcomes +Through sharing experience, the Working Group aims to enhance the metadata produced across government, and to make it available via the Government Data Catalogue as it develops, and thus improve the discoverability of data assets. We will do this by: + +1. __Gaining and sharing knowledge__ on the implementation of the Metadata Exchange Model Specification, collaborating to tackle any common issues that may arise +1. __Enrich__ the Metadata Exchange Model Specification based on members’ experience of implementation, and, following adoption of essential components of the specification, develop further aspects of the wider Cross-Government Metadata Implementation Approach to improve Metadata Management capability +1. __Advise__ on development of a Playbook to support Cross-Government Data Sharing focusing initially on practical guidance to adopt the Metadata Exchange Model Specification +1. __Assisting__ organisations with the onboarding of their critical data asset metadata into the Government Data Marketplace + +## Time Commitment +- Meetings are convened on a monthly basis. Frequency is adjusted according to workslate. +- Each session is 1 hour, with approximately an additional hour of preparation time required. + +## Key Deliverables +- Work through the content of the Metadata Exchange Model Specification to validate and agree core set of attributes essential to support a cross-government data share +- Identify any extensions to the core DCAT profile which may be required for different use cases, scenarios that organisations may need to cater for specifically, e.g. geospatial, health data sets, etc +- Feed metadata requirements to the development of the underlying metamodel that will support the delivery of the Government Data Marketplace +- Contribute to the action plan for metadata for Phase IV of the CDDO Portfolio roadmap + +## Preparation for initial and subsequent meeting sessions +Members’ are asked to: +- share their experiences of using DCAT, which elements of the specification they may have already adopted, and any areas that are more challenging to implement. +- complete in advance of the first meeting a short summary of their position on the above to facilitate this discussion. + +In parallel: +- Members will also be given regular updates on progress of the Government Data Marketplace development. + +## Governance +- Output of the WG will be reported through the DQ Hub and DSA Peer Review Group (PRG) as well as the DSA Steering Board (also to be used in case of any escalation being required) +- Where relevant, an update will also be communicated through existing channels to the: + 1. CDO Council + 1. Data and Technology Architecture Design Authority (DTADA) and the + 1. Government Data Architecture Community (GDAC) + +## Comms and Collaboration space +- Meetings are held via MS Teams, with collaboration workspaces in KHub.net and UK Gov Slack From 88f2e7e53f0d82d97ce38aef13994b895fa04790 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 11:39:13 +0100 Subject: [PATCH 06/11] Dev and contributions text --- src/docs/about.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/docs/about.md b/src/docs/about.md index fd2c646..155ff66 100644 --- a/src/docs/about.md +++ b/src/docs/about.md @@ -32,3 +32,9 @@ A __Distribution__ in DCAT is defined as a _"specific representation of a datase The Cross-Government Metadata Exchange Model has been developed and agreed by the [Cross-Government Metadata Implementation Working Group](miwg.md). The working group is coordinated by the [Central Digital and Data Office](https://www.gov.uk/government/organisations/central-digital-and-data-office), and consists of data professionals in Government Digital and Data functions from across central and local government, devolved administrations, and the wider public sector. The outputs of the working group are reported through the Data Quality Hub and Data Standards Authority Peer Review Group as well as the Data Standards Authority Steering Board. +### Development + +These pages are hosted on GitHub. The pages are generated using the [LinkML](https://linkml.io/) framework and the code is managed on [GitHub](https://github.com/co-cddo/ukgov-metadata-exchange-model/). For full details see the [README file](https://github.com/co-cddo/ukgov-metadata-exchange-model#readme). + +Contributions to the model are welcomed. If you spot something wrong or want to make suggestions for improvements to the model, then please [create an issue on GitHub](https://github.com/co-cddo/ukgov-metadata-exchange-model/issues/new/choose) (_You will need a GitHub account to be able to do this_). Please select the appropriate issue type, e.g. Bug report or Feature request, and complete the template as best you can. + From 1ab781d59807fdb89063382ddf6bb2e92e5a4d0f Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 11:41:39 +0100 Subject: [PATCH 07/11] Promote heading --- src/docs/about.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/about.md b/src/docs/about.md index 155ff66..fe6aef5 100644 --- a/src/docs/about.md +++ b/src/docs/about.md @@ -32,7 +32,7 @@ A __Distribution__ in DCAT is defined as a _"specific representation of a datase The Cross-Government Metadata Exchange Model has been developed and agreed by the [Cross-Government Metadata Implementation Working Group](miwg.md). The working group is coordinated by the [Central Digital and Data Office](https://www.gov.uk/government/organisations/central-digital-and-data-office), and consists of data professionals in Government Digital and Data functions from across central and local government, devolved administrations, and the wider public sector. The outputs of the working group are reported through the Data Quality Hub and Data Standards Authority Peer Review Group as well as the Data Standards Authority Steering Board. -### Development +## Development These pages are hosted on GitHub. The pages are generated using the [LinkML](https://linkml.io/) framework and the code is managed on [GitHub](https://github.com/co-cddo/ukgov-metadata-exchange-model/). For full details see the [README file](https://github.com/co-cddo/ukgov-metadata-exchange-model#readme). From 63b21414387cb3afa3798a4358b3375ac810952c Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 14:45:00 +0100 Subject: [PATCH 08/11] Fix formatting --- src/docs/miwg.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/docs/miwg.md b/src/docs/miwg.md index 6651a27..88efb13 100644 --- a/src/docs/miwg.md +++ b/src/docs/miwg.md @@ -12,6 +12,7 @@ ## Goal To ensure metadata within the Government Data Marketplace is sufficient to: + - FIND: enable discovery of the data asset, - USE: facilitate an informed decision by the customer as to whether a data asset is fit for purpose and meets their business need, and - ACCESS: enable access to the data by ensuring the request is channelled to the right team(s) and by describing the access requirements and restrictions of the data source. @@ -36,10 +37,12 @@ Through sharing experience, the Working Group aims to enhance the metadata produ ## Preparation for initial and subsequent meeting sessions Members’ are asked to: + - share their experiences of using DCAT, which elements of the specification they may have already adopted, and any areas that are more challenging to implement. - complete in advance of the first meeting a short summary of their position on the above to facilitate this discussion. In parallel: + - Members will also be given regular updates on progress of the Government Data Marketplace development. ## Governance From d7185ca900ec06cc32c97bb8f395a8b053370434 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 14:47:53 +0100 Subject: [PATCH 09/11] Add link to guidance --- src/model/uk_cross_government_metadata_exchange_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/model/uk_cross_government_metadata_exchange_model.yaml b/src/model/uk_cross_government_metadata_exchange_model.yaml index 492317f..c4df701 100644 --- a/src/model/uk_cross_government_metadata_exchange_model.yaml +++ b/src/model/uk_cross_government_metadata_exchange_model.yaml @@ -3,7 +3,7 @@ id: https://w3id.org/co-cddo/ukgov-metadata-exchange-model/ name: uk-cross-government-metadata-exchange-model title: UK Cross-Government Metadata Exchange Model description: | - A metadata model for describing data assets for exchanging between UK government organisations. + A metadata model for describing data assets for exchanging between UK government organisations. This is a realisation of the [UK government guidance](https://www.gov.uk/government/publications/recommended-open-standards-for-government/using-metadata-to-describe-data-assets-in-a-data-catalogue) to adopt the [DCAT](https://www.w3.org/TR/vocab-dcat-3/) vocabulary for describing data. For more details relating to the development of this metadata model, please see the [about](about) page. license: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ From 3bdb011eeb6dbdb2e181acf8e6692137d339b3aa Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 15:21:09 +0100 Subject: [PATCH 10/11] Remove subset section from index page --- src/docs/templates/index.md.jinja2 | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/docs/templates/index.md.jinja2 b/src/docs/templates/index.md.jinja2 index 74cb853..01555ce 100644 --- a/src/docs/templates/index.md.jinja2 +++ b/src/docs/templates/index.md.jinja2 @@ -56,11 +56,3 @@ Name: {{ schema.name }} {% for t in gen.all_type_objects()|sort(attribute=sort_by) -%} | {{gen.link(t)}} | {{t.description|enshorten}} | {% endfor %} - -## Subsets - -| Subset | Description | -| --- | --- | -{% for ss in schemaview.all_subsets().values()|sort(attribute='name') -%} -| {{gen.link(ss)}} | {{ss.description|enshorten}} | -{% endfor %} \ No newline at end of file From 20b238ed2000d63fd3f9daf5fa8c259a22c63616 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 26 Oct 2023 15:22:06 +0100 Subject: [PATCH 11/11] Update schema URI and hide from index page --- src/docs/templates/index.md.jinja2 | 4 +--- src/model/uk_cross_government_metadata_exchange_model.yaml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/docs/templates/index.md.jinja2 b/src/docs/templates/index.md.jinja2 index 01555ce..1d677f1 100644 --- a/src/docs/templates/index.md.jinja2 +++ b/src/docs/templates/index.md.jinja2 @@ -6,9 +6,7 @@ {% if schema.description %}{{ schema.description }}{% endif %} -URI: {{ schema.id }} - -Name: {{ schema.name }} + {% if include_top_level_diagram %} diff --git a/src/model/uk_cross_government_metadata_exchange_model.yaml b/src/model/uk_cross_government_metadata_exchange_model.yaml index c4df701..c81987a 100644 --- a/src/model/uk_cross_government_metadata_exchange_model.yaml +++ b/src/model/uk_cross_government_metadata_exchange_model.yaml @@ -1,5 +1,5 @@ --- -id: https://w3id.org/co-cddo/ukgov-metadata-exchange-model/ +id: https://github.com/co-cddo/ukgov-metadata-exchange-model/blob/main/src/model/uk_cross_government_metadata_exchange_model.yaml name: uk-cross-government-metadata-exchange-model title: UK Cross-Government Metadata Exchange Model description: |