Skip to content

Commit

Permalink
Updated signe laval config to use a custom file
Browse files Browse the repository at this point in the history
  • Loading branch information
@sahalali
sahalali committed Oct 21, 2024
1 parent 4c7cf0a commit 4a797c2
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 1,231 deletions.
54 changes: 39 additions & 15 deletions .github/workflows/export-cms-entities.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ on:
source:
required: true
type: string
event_mapping_file:
custom_config_file:
required: false
type: string
secrets:
Expand Down Expand Up @@ -76,25 +76,30 @@ jobs:
- name: Download configuration files for transformation
id: step-5
run: |
event_url=${{ inputs.event_mapping_file || 'cms-events-configuration.json' }}
curl "https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/${event_url}" >> event-transform-config.json &&
run: |
curl "https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-events-configuration.json" >> event-transform-config.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-places-configuration.json' >> place-transform-config.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-organizations-configuration.json' >> organization-transform-config.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-people-configuration.json' >> person-transform-config.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-taxonomies-configuration.json' >> taxonomy-transform-config.json
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-taxonomies-configuration.json' >> taxonomy-transform-config.json
- name: Download custom configuration file
id: step-6
if: ${{ inputs.custom_config_file }}
run: |
custom_url=${{ inputs.custom_config_file }}
curl "https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/${custom_url}" >> custom-transform-config.json
- name: Download column addition files for transformation
id: step-6
id: step-7
run: |
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/event.json' >> event-columns.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/place.json' >> place-columns.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/organization.json' >> organization-columns.json &&
curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/person.json' >> person-columns.json
- name: Transform configurations to include column additions
id: step-7
id: step-8
shell: bash
run: |
entities=("event" "place" "organization" "person")
Expand All @@ -110,7 +115,7 @@ jobs:
- name: Transform events in JSON to RDF
id: step-8
id: step-9
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform events.json \
Expand All @@ -119,7 +124,7 @@ jobs:
-f json >> events.ttl
- name: Transform places in JSON to RDF
id: step-9
id: step-10
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform places.json \
Expand All @@ -128,7 +133,7 @@ jobs:
-f json >> places.ttl
- name: Transform organizations in JSON to RDF
id: step-10
id: step-11
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform organizations.json \
Expand All @@ -137,7 +142,7 @@ jobs:
-f json >> organizations.ttl
- name: Transform people in JSON to RDF
id: step-11
id: step-12
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform people.json \
Expand All @@ -146,16 +151,26 @@ jobs:
-f json >> people.ttl
- name: Transform taxonomies in JSON to RDF
id: step-12
id: step-13
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform '/__w/artsdata-planet-footlight/artsdata-planet-footlight/flatten-taxonomies.json' \
-u http://localhost:7333 \
--configurations taxonomy-transform-config.json \
-f json >> taxonomies.ttl
- name: Transform custom fields in JSON to RDF
if: ${{ inputs.custom_config_file }}
id: step-14
run: |
/opt/ontorefine/dist/bin/ontorefine-cli \
transform events.json \
-u http://localhost:7333 \
--configurations custom-transform-config.json \
-f json >> custom.ttl
- name: Concatenate the files
id: step-13
id: step-15
run : |
cat \
events.ttl \
Expand All @@ -165,8 +180,17 @@ jobs:
taxonomies.ttl \
> ${{inputs.source}}.ttl
- name: Concatenate custom fields
id: step-16
if: ${{ inputs.custom_config_file }}
run: |
cat \
${{inputs.source}}.ttl \
custom.ttl \
> ${{inputs.source}}.ttl
- name: Upload entities in RDF to S3
id: step-14
id: step-17
uses: keithweaver/[email protected]
with:
command: cp
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/export-signe-laval-entities-to-artsdata.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
uses: ./.github/workflows/export-cms-entities.yml
with:
source: signe-laval
event_mapping_file: "cms-events-signe-laval-configuration.json"
custom_config_file: "cms-custom-signe-laval-configuration.json"
secrets:
PUBLISHER_URI_GREGORY: ${{ secrets.PUBLISHER_URI_GREGORY }}
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
Expand Down
120 changes: 120 additions & 0 deletions ontotext-refine/cms-custom-signe-laval-configuration.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
{
"importOptions": [
{
"recordPath": [
"_",
"data",
"_"
],
"limit": -1,
"trimStrings": false,
"guessCellValueTypes": false,
"storeEmptyStrings": false,
"includeFileSources": false,
"includeArchiveFileName": false,
"disableAutoPreview": false,
"projectName": "cli-transform-events.json-2024-08-02",
"projectTags": [],
"fileSource": "events.json",
"archiveFileName": null
}
],
"operations": [
{
"op": "core/fill-down",
"engineConfig": {
"facets": [],
"mode": "record-based"
},
"columnName": "_ - id",
"description": "Fill down cells in column _ - id"
},
{
"op": "core/column-addition-by-fetching-urls",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"baseColumnName": "_ - id",
"urlExpression": "grel:\"https://api.footlight.io/events/\" + value",
"onError": "set-to-blank",
"newColumnName": "eventType",
"columnInsertIndex": 1,
"delay": 5000,
"cacheResponses": true,
"httpHeadersJson": [
{
"name": "authorization",
"value": ""
},
{
"name": "user-agent",
"value": "OpenRefine [1]"
},
{
"name": "accept",
"value": "*/*"
}
],
"description": "Create column eventType at index 1 by fetching URLs based on column _ - id using expression grel:\"https://api.footlight.io/events/\" + value"
},
{
"op": "mapping-editor/save-rdf-mapping",
"mapping": {
"baseIRI": "http://lod.footlight.io/resource/",
"namespaces": {
"xsd": "http://www.w3.org/2001/XMLSchema#",
"schema": "http://schema.org/",
"skos": "http://www.w3.org/2004/02/skos/core#",
"footlight": "http://lod.footlight.io/resource/"
},
"subjectMappings": [
{
"subject": {
"valueSource": {
"source": "column",
"columnName": "_ - id"
},
"transformation": {
"language": "prefix",
"expression": "footlight"
}
},
"typeMappings": [],
"propertyMappings": [
{
"property": {
"valueSource": {
"source": "constant",
"constant": "additionalType"
},
"transformation": {
"language": "prefix",
"expression": "schema"
}
},
"values": [
{
"valueSource": {
"source": "row_index"
},
"transformation": {
"language": "grel",
"expression": "forEach(parseJson(cells[\"eventType\"].value).data[\"63bd80901c6b6c005aad6f4b\"], v, \"http://lod.footlight.io/resource/\" + v.id)"
},
"valueType": {
"type": "iri",
"typeMappings": [],
"propertyMappings": []
}
}
]
}
]
}
]
},
"description": "Save RDF Mapping"
}
]
}
Loading

0 comments on commit 4a797c2

Please sign in to comment.