Updated workflows to include a column addition step

culturecreates · Oct 8, 2024 · 31ade91 · 31ade91
1 parent cee6810
commit 31ade91
Show file tree

Hide file tree

Showing 17 changed files with 567 additions and 12,502 deletions.
diff --git a/.github/workflows/export-cms-entities.yml b/.github/workflows/export-cms-entities.yml
@@ -51,7 +51,7 @@ jobs:
     steps:
       - name: Install requirements
         id: step-1
-        run: apk update && apk add curl && apk add util-linux
+        run: apk update && apk add curl && apk add util-linux && apk add jq
 
       - name: Run Onto Refine server
         id: step-2
@@ -80,9 +80,31 @@ jobs:
           curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-people-configuration.json' >> person-transform-config.json &&
           curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/cms-taxonomies-configuration.json' >> taxonomy-transform-config.json
 
+      - name: Download column addition files for transformation
+        id: step-6
+        run: |
+          curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/event.json' >> event-columns.json &&
+          curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/place.json' >> place-columns.json &&
+          curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/organization.json' >> organization-columns.json &&
+          curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-footlight/main/ontotext-refine/column-additions/person.json' >> person-columns.json
+
+      - name: Transform configurations to include column additions
+        id: step-7
+        run: |
+          entities=("event" "place" "organization" "person")
+          
+          for entity in "${entities[@]}"; do
+            config_file="${entity}-transform-config.json"
+            column_file="${entity}-columns.json"
+            tmp_file=$(mktemp)
+            
+            jq '.operations |= (input + .)' "$config_file" "$column_file" > "$tmp_file"
+            mv "$tmp_file" "$config_file"
+          done
+
 
       - name: Transform events in JSON  to RDF
-        id: step-6
+        id: step-8
         run: |
           /opt/ontorefine/dist/bin/ontorefine-cli \
           transform events.json \
@@ -91,7 +113,7 @@ jobs:
           -f json >> events.ttl
 
       - name: Transform places in JSON to RDF
-        id: step-7
+        id: step-9
         run: |
           /opt/ontorefine/dist/bin/ontorefine-cli \
           transform places.json \
@@ -100,7 +122,7 @@ jobs:
           -f json >> places.ttl
 
       - name: Transform organizations in JSON to RDF
-        id: step-8
+        id: step-10
         run: |
           /opt/ontorefine/dist/bin/ontorefine-cli \
           transform organizations.json \
@@ -109,7 +131,7 @@ jobs:
           -f json >> organizations.ttl
 
       - name: Transform people in JSON to RDF
-        id: step-9
+        id: step-11
         run: |
           /opt/ontorefine/dist/bin/ontorefine-cli \
           transform people.json \
@@ -118,7 +140,7 @@ jobs:
           -f json >> people.ttl
 
       - name: Transform taxonomies in JSON to RDF
-        id: step-10
+        id: step-12
         run: |
           /opt/ontorefine/dist/bin/ontorefine-cli \
           transform '/__w/artsdata-planet-footlight/artsdata-planet-footlight/flatten-taxonomies.json' \
@@ -127,7 +149,7 @@ jobs:
           -f json >> taxonomies.ttl
 
       - name: Concatenate the files
-        id: step-11
+        id: step-13
         run : |
           cat \
           events.ttl \
@@ -138,7 +160,7 @@ jobs:
           > ${{inputs.source}}.ttl
 
       - name: Upload entities in RDF to S3
-        id: step-12
+        id: step-14
         uses: keithweaver/[email protected]
         with:
             command: cp

diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+*.ttl
+updated*configuration.json
diff --git a/event-entities.ttl b/event-entities.ttl
diff --git a/ontotext-refine/cms-events-configuration.json b/ontotext-refine/cms-events-configuration.json
@@ -20,149 +20,6 @@
     }
   ],
   "operations": [
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - name - en",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - name - en at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - name - fr",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - name - fr at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - location - _ - url",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - location - _ - url at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - location - _ - name - en",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - location - _ - name - en at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - location - _ - name - fr",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - location - _ - name - fr at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - description - en",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - description - en at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - description - fr",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - description - fr at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - organizers - _ - name - en",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - organizers - _ - name - en at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - organizers - _ - name - fr",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - organizers - _ - name - fr at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - additionalType - _ - name - en",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - additionalType - _ - name - en at index 1 based on column _ - id using expression set-to-blank"
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - additionalType - _ - name - fr",
-      "columnInsertIndex": 1,
-      "description": "Create column _ - additionalType - _ - name - fr at index 1 based on column _ - id using expression set-to-blank"
-    },
     {
       "op": "core/fill-down",
       "engineConfig": {

diff --git a/ontotext-refine/cms-organizations-configuration.json b/ontotext-refine/cms-organizations-configuration.json
@@ -29,102 +29,6 @@
       "columnName": "_ - id",
       "description": "Fill down cells in column _ - id"
     },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - name - fr",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - name - en",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - description - en",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - description - fr",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - image - original",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - location - name - en",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - location - name - fr",
-      "columnInsertIndex": 1
-    },
-    {
-      "op": "core/column-addition",
-      "engineConfig": {
-        "facets": [],
-        "mode": "row-based"
-      },
-      "baseColumnName": "_ - id",
-      "expression": "set-to-blank",
-      "onError": "set-to-blank",
-      "newColumnName": "_ - url",
-      "columnInsertIndex": 1
-    },
     {
       "op": "mapping-editor/save-rdf-mapping",
       "mapping": {