From c7ff3b85592bb5c3e62387c5cb1c8b7f49147135 Mon Sep 17 00:00:00 2001 From: kwahlin Date: Thu, 16 Nov 2023 16:03:12 +0100 Subject: [PATCH 1/2] Move release bound script to globalchanges-1.33.sh --- librisworks/run.sh | 13 +++---------- ...ove-illustrativecontent-to-instance.groovy | 19 ------------------- whelktool/globalchanges-1.33.sh | 1 + ...ove-illustrativecontent-to-instance.groovy | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 29 deletions(-) delete mode 100644 librisworks/scripts/lxl-4221-move-illustrativecontent-to-instance.groovy create mode 100644 whelktool/scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy diff --git a/librisworks/run.sh b/librisworks/run.sh index d258e1f4f8..bed301cfba 100755 --- a/librisworks/run.sh +++ b/librisworks/run.sh @@ -39,10 +39,9 @@ NO_ANONYMOUS_TRANSLATIONS=$CLUSTERS_DIR/5-no-anonymous-translations mkdir -p $CLUSTERS_DIR $NORMALIZATIONS_DIR $MERGED_WORKS_DIR $ALL $MERGED $TITLES $SWEDISH_FICTION $NO_ANONYMOUS_TRANSLATIONS LANGUAGE_IN_TITLE=$NORMALIZATIONS_DIR/1-titles-with-language -ILL_CONTENT=$NORMALIZATIONS_DIR/2-illustrative-content -DEDUPLICATE_CONTRIBUTIONS=$NORMALIZATIONS_DIR/3-deduplicate-contributions -ADD_MISSING_CONTRIBUTION_DATA=$NORMALIZATIONS_DIR/4-add-missing-contribution-data -ROLES_TO_INSTANCE=$NORMALIZATIONS_DIR/5-roles-to-instance +DEDUPLICATE_CONTRIBUTIONS=$NORMALIZATIONS_DIR/2-deduplicate-contributions +ADD_MISSING_CONTRIBUTION_DATA=$NORMALIZATIONS_DIR/3-add-missing-contribution-data +ROLES_TO_INSTANCE=$NORMALIZATIONS_DIR/4-roles-to-instance # Clustering TODO: run only on recently updated records after first run echo "Finding new clusters..." @@ -95,12 +94,6 @@ time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDIS $ARGS --report $LANGUAGE_IN_TITLE $SCRIPTS_DIR/language-in-work-title.groovy 2>/dev/null echo "$(count_lines $LANGUAGE_IN_TITLE/MODIFIED.txt) records affected, report in $LANGUAGE_IN_TITLE" -echo -echo "Moving illustrativeContent to instance..." -time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION/$CLUSTER_TSV -jar $JAR_FILE \ - $ARGS --report $ILL_CONTENT $SCRIPTS_DIR/lxl-4221-move-illustrativecontent-to-instance.groovy 2>/dev/null -echo "$(count_lines $ILL_CONTENT/MODIFIED.txt) records affected, report in $ILL_CONTENT" - echo echo "Merging contribution objects with same agent..." time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION/$CLUSTER_TSV -jar $JAR_FILE \ diff --git a/librisworks/scripts/lxl-4221-move-illustrativecontent-to-instance.groovy b/librisworks/scripts/lxl-4221-move-illustrativecontent-to-instance.groovy deleted file mode 100644 index a44ea82fea..0000000000 --- a/librisworks/scripts/lxl-4221-move-illustrativecontent-to-instance.groovy +++ /dev/null @@ -1,19 +0,0 @@ -def ids = new File(System.getProperty('clusters')) - .readLines() - .collect { it.split('\t').collect { it.trim()} } - .flatten() - -selectByIds(ids) { - def instance = it.graph[1] - def work = instance.instanceOf - - if (!work || work['@id']) return - - def illContent = work.remove('illustrativeContent') - - if (illContent) { - instance['illustrativeContent'] = (asList(instance['illustrativeContent']) + asList(illContent)).unique() - - it.scheduleSave() - } -} \ No newline at end of file diff --git a/whelktool/globalchanges-1.33.sh b/whelktool/globalchanges-1.33.sh index 5d1011e1d2..44495ef1c0 100644 --- a/whelktool/globalchanges-1.33.sh +++ b/whelktool/globalchanges-1.33.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euxo pipefail time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/2023/08/lxl-4243-move-out-solitary-contentType-from-hasPart.groovy +time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2023/05/gf-cleanup.groovy time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2020/08/lxl-3294-move-bearer-like-gfs-from-work-to-instance.groovy time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/2023/10/elib-unspecified-contributor.groovy diff --git a/whelktool/scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy b/whelktool/scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy new file mode 100644 index 0000000000..ebca8f7133 --- /dev/null +++ b/whelktool/scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy @@ -0,0 +1,15 @@ +def where = """ + collection = 'bib' + and deleted = false + and data#>>'{@graph,1,instanceOf,@type}' = 'Text' + and data#>'{@graph,1,instanceOf, illustrativeContent}' is not null +""" + +selectBySqlWhere(where) { + def instance = it.graph[1] + def work = instance.instanceOf + + instance['illustrativeContent'] = (asList(instance['illustrativeContent']) + asList(work.remove('illustrativeContent'))).unique() + + it.scheduleSave() +} \ No newline at end of file From d5cb185a91b8e7deaf1c2fdda5b6842f18ea7a5b Mon Sep 17 00:00:00 2001 From: kwahlin Date: Fri, 17 Nov 2023 08:45:13 +0100 Subject: [PATCH 2/2] Remove not release bound scripts and add roles-to-instance script --- whelktool/globalchanges-1.33.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/whelktool/globalchanges-1.33.sh b/whelktool/globalchanges-1.33.sh index 44495ef1c0..34638dc44e 100644 --- a/whelktool/globalchanges-1.33.sh +++ b/whelktool/globalchanges-1.33.sh @@ -2,6 +2,5 @@ set -euxo pipefail time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/2023/08/lxl-4243-move-out-solitary-contentType-from-hasPart.groovy time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2023/07/lxl-4221-move-illustrativecontent-to-instance.groovy -time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2023/05/gf-cleanup.groovy -time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/cleanups/2020/08/lxl-3294-move-bearer-like-gfs-from-work-to-instance.groovy time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/2023/10/elib-unspecified-contributor.groovy +time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar build/libs/whelktool.jar --report reports/$ENV-$(date +%Y%m%d-%H%M%S) --skip-index scripts/2023/05/lxl-2512-move-contribution-by-relator-domain/script.groovy