From 96b1bd11c184d785416d0bf3b38c333fced390ef Mon Sep 17 00:00:00 2001 From: Omar Ahmad Date: Thu, 20 Feb 2025 10:38:19 +0000 Subject: [PATCH] Fixing CODEARTIFACT_DOWNLOAD_URL to be able to retrieve both CI and Release artifacts (#4354) * Fixing CODEARTIFACT_DOWNLOAD_URL to be able to retrieve both CI and Release artifacts * Fix dev CI compile error * fix testExportMetadata via CYPHER 5 prefix * Fixing environment variables NEO4J_DOCKER_CE_OVERRIDE and NEO4J_DOCKER_EE_OVERRIDE to point to correct repository --------- Co-authored-by: vga91 --- .github/workflows/CI.yaml | 69 +++++++++++-------- .../main/java/apoc/cypher/CypherExtended.java | 8 ++- .../test/java/apoc/systemdb/SystemDbTest.java | 21 +++++- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 7ab8ba32bb..0bd9490a02 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -7,7 +7,7 @@ on: branches: [ "dev" ] env: - CODEARTIFACT_DOWNLOAD_URL: ${{ secrets.CODEARTIFACT_DOWNLOAD_URL }} + CODEARTIFACT_BUILD_SERVICE_LIVE_URL: ${{ secrets.CODEARTIFACT_BUILD_SERVICE_LIVE_URL }} CODEARTIFACT_USERNAME: ${{ secrets.CODEARTIFACT_USERNAME }} ECR_NEO4J_DOCKER_URL: ${{ secrets.ECR_NEO4J_DOCKER_URL }} BRANCH_NAME: ${{ github.event.pull_request.base.ref }} @@ -45,28 +45,32 @@ jobs: - uses: ./.github/actions/setup-jdk - uses: ./.github/actions/setup-gradle-cache - - name: Determine latest neo4j CI version and docker images + - name: Determine CODEARTIFACT_DOWNLOAD_URL, NEO4JVERSION and Docker Image environment variables run: | + echo "Current branch BRANCH_NAME=$BRANCH_NAME" neo4j_version_base=$(grep -e "neo4jVersion = .*" build.gradle | cut -d '=' -f 2 | tr -d \'\" | tr -d ' ') - echo "neo4j_version_base=$neo4j_version_base" - NEO4JVERSION=`aws --no-cli-pager codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --status Published --query "versions[?starts_with(version, '$neo4j_version_base')].version | [0]" --output json | sed 's/"//g'` - if [ -z "$NEO4JVERSION" ] || [ "$NEO4JVERSION" = "null" ]; then - NEO4JVERSION="$neo4j_version_base" - fi - echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV" - echo "Found NEO4JVERSION=$NEO4JVERSION" - if [[ $BRANCH_NAME =~ ^5.[0-9]+$ ]]; then - NEO4J_DOCKER_EE_OVERRIDE="neo4j:$neo4j_version_base-enterprise" - NEO4J_DOCKER_CE_OVERRIDE="neo4j:$neo4j_version_base" + if [[ $BRANCH_NAME == "dev" ]]; then + echo "Running on dev branch and so pointing to ci-live repository in AWS CodeArtifact for CI artifacts" + CODEARTIFACT_DOWNLOAD_URL="$CODEARTIFACT_BUILD_SERVICE_LIVE_URL/maven/ci-live" + NEO4JVERSION=`aws --no-cli-pager codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --status Published --query "versions[?starts_with(version, '$neo4j_version_base')].version | [0]" --output json | sed 's/"//g'` + NEO4J_DOCKER_CE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-community-debian-nightly" + NEO4J_DOCKER_EE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-enterprise-debian-nightly" else - NEO4J_DOCKER_CE_OVERRIDE="$ECR_NEO4J_DOCKER_URL:$neo4j_version_base-community-debian-nightly" - NEO4J_DOCKER_EE_OVERRIDE="$ECR_NEO4J_DOCKER_URL:$neo4j_version_base-enterprise-debian-nightly" + echo "Running on branch $BRANCH_NAME and so pointing to release-live repository in AWS CodeArtifact for release artifacts" + CODEARTIFACT_DOWNLOAD_URL="$CODEARTIFACT_BUILD_SERVICE_LIVE_URL/maven/release-live" + NEO4JVERSION=$neo4j_version_base + NEO4J_DOCKER_CE_OVERRIDE="neo4j:$neo4j_version_base" + NEO4J_DOCKER_EE_OVERRIDE="neo4j:$neo4j_version_base-enterprise" fi - echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV" + echo "CODEARTIFACT_DOWNLOAD_URL=$CODEARTIFACT_DOWNLOAD_URL" >> "$GITHUB_ENV" + + echo "Found NEO4JVERSION=$NEO4JVERSION" + echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" echo "Found NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" + + echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV" echo "NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" >> "$GITHUB_ENV" - echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" - echo "Current branch BRANCH_NAME=$BRANCH_NAME" + echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV" - name: Compile Java run: | @@ -123,25 +127,32 @@ jobs: ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - - name: Determine latest neo4j CI version and docker images + - name: Determine CODEARTIFACT_DOWNLOAD_URL, NEO4JVERSION and Docker Image environment variables run: | + echo "Current branch BRANCH_NAME=$BRANCH_NAME" neo4j_version_base=$(grep -e "neo4jVersion = .*" build.gradle | cut -d '=' -f 2 | tr -d \'\" | tr -d ' ') - echo "neo4j_version_base=$neo4j_version_base" - NEO4JVERSION=`aws codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --query "versions[?starts_with(version,'$neo4j_version_base')] | [0].version" | tr -d '" '` - echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV" - echo "Found NEO4JVERSION=$NEO4JVERSION" - if [[ $BRANCH_NAME =~ ^5.[0-9]+$ ]]; then - NEO4J_DOCKER_EE_OVERRIDE="neo4j:$neo4j_version_base-enterprise" - NEO4J_DOCKER_CE_OVERRIDE="neo4j:$neo4j_version_base" - else + if [[ $BRANCH_NAME == "dev" ]]; then + echo "Running on dev branch and so pointing to ci-live repository in AWS CodeArtifact for CI artifacts" + CODEARTIFACT_DOWNLOAD_URL="$CODEARTIFACT_BUILD_SERVICE_LIVE_URL/maven/ci-live" + NEO4JVERSION=`aws --no-cli-pager codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --status Published --query "versions[?starts_with(version, '$neo4j_version_base')].version | [0]" --output json | sed 's/"//g'` NEO4J_DOCKER_CE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-community-debian-nightly" NEO4J_DOCKER_EE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-enterprise-debian-nightly" + else + echo "Running on branch $BRANCH_NAME and so pointing to release-live repository in AWS CodeArtifact for release artifacts" + CODEARTIFACT_DOWNLOAD_URL="$CODEARTIFACT_BUILD_SERVICE_LIVE_URL/maven/release-live" + NEO4JVERSION=$neo4j_version_base + NEO4J_DOCKER_CE_OVERRIDE="neo4j:$neo4j_version_base" + NEO4J_DOCKER_EE_OVERRIDE="neo4j:$neo4j_version_base-enterprise" fi - echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV" + echo "CODEARTIFACT_DOWNLOAD_URL=$CODEARTIFACT_DOWNLOAD_URL" >> "$GITHUB_ENV" + + echo "Found NEO4JVERSION=$NEO4JVERSION" + echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" echo "Found NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" + + echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV" echo "NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" >> "$GITHUB_ENV" - echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" - echo "Current branch BRANCH_NAME=$BRANCH_NAME" + echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV" - name: Init gradle run: | diff --git a/extended/src/main/java/apoc/cypher/CypherExtended.java b/extended/src/main/java/apoc/cypher/CypherExtended.java index 6a02e63efb..14a42d0be1 100644 --- a/extended/src/main/java/apoc/cypher/CypherExtended.java +++ b/extended/src/main/java/apoc/cypher/CypherExtended.java @@ -18,6 +18,7 @@ import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.internal.kernel.api.procs.ProcedureCallContext; import org.neo4j.logging.Log; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -84,6 +85,9 @@ public class CypherExtended { @Context public URLAccessChecker urlAccessChecker; + @Context + public ProcedureCallContext procedureCallContext; + @Procedure(name = "apoc.cypher.runFile", mode = WRITE) @Description("apoc.cypher.runFile(file or url,[{statistics:true,timeout:10,parameters:{}}]) - runs each statement in the file, all semicolon separated - currently no schema operations") public Stream runFile(@Name("file") String fileName, @Name(value = "config",defaultValue = "{}") Map config) { @@ -386,7 +390,7 @@ public static String compiled(String fragment) { @Procedure @Description("apoc.cypher.parallel(fragment, `paramMap`, `keyList`) yield value - executes fragments in parallel through a list defined in `paramMap` with a key `keyList`") public Stream parallel(@Name("fragment") String fragment, @Name("params") Map params, @Name("parallelizeOn") String key) { - if (params == null) return runCypherQuery(tx, fragment, params); + if (params == null) return runCypherQuery(tx, fragment, params, procedureCallContext); if (key == null || !params.containsKey(key)) throw new RuntimeException("Can't parallelize on key " + key + " available keys " + params.keySet()); Object value = params.get(key); @@ -473,7 +477,7 @@ public Map parallelParams(@Name("params") Map pa @Procedure @Description("apoc.cypher.parallel2(fragment, `paramMap`, `keyList`) yield value - executes fragments in parallel batches through a list defined in `paramMap` with a key `keyList`") public Stream parallel2(@Name("fragment") String fragment, @Name("params") Map params, @Name("parallelizeOn") String key) { - if (params == null) return runCypherQuery(tx, fragment, params); + if (params == null) return runCypherQuery(tx, fragment, params, procedureCallContext); if (StringUtils.isEmpty(key) || !params.containsKey(key)) throw new RuntimeException("Can't parallelize on key " + key + " available keys " + params.keySet() + ". Note that parallelizeOn parameter must be not empty"); Object value = params.get(key); diff --git a/extended/src/test/java/apoc/systemdb/SystemDbTest.java b/extended/src/test/java/apoc/systemdb/SystemDbTest.java index 7ea00ae4e3..2d4dffc085 100644 --- a/extended/src/test/java/apoc/systemdb/SystemDbTest.java +++ b/extended/src/test/java/apoc/systemdb/SystemDbTest.java @@ -37,6 +37,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static apoc.ApocConfig.APOC_TRIGGER_ENABLED; import static apoc.ApocConfig.apocConfig; @@ -122,8 +123,8 @@ public void testWriteStatements() { @Test public void testExportMetadata() { // We test triggers - final String triggerOne = "CALL apoc.trigger.add('firstTrigger', 'RETURN $alpha', {phase:\"after\"}, {params: {alpha:1}});"; - final String triggerTwo = "CALL apoc.trigger.add('beta', 'RETURN 1', {}, {params: {}});"; + final String triggerOne = "CALL apoc.trigger.add('firstTrigger', 'CYPHER 5 RETURN $alpha', {phase:\"after\"}, {params: {alpha:1}});"; + final String triggerTwo = "CALL apoc.trigger.add('beta', 'CYPHER 5 RETURN 1', {}, {params: {}});"; // In this case we paused to test that it will be exported as paused final String pauseTrigger = "CALL apoc.trigger.pause('beta');"; db.executeTransactionally(triggerOne); @@ -154,7 +155,13 @@ public void testExportMetadata() { assertEquals(Set.of(constraintForUuid), readFileLines("metadata.Uuid.schema.neo4j.cypher", directory)); assertEquals(Set.of(uuidStatement), readFileLines("metadata.Uuid.neo4j.cypher", directory)); - assertEquals(Set.of(triggerOne, triggerTwo, pauseTrigger), readFileLines("metadata.Trigger.neo4j.cypher", directory)); + Set expectedTriggers = Stream.of(triggerOne, triggerTwo, pauseTrigger) + .map(SystemDbTest::removeDuplicatedWhitespaces) + .collect(Collectors.toSet()); + Set actualTriggers = readFileLines("metadata.Trigger.neo4j.cypher", directory).stream() + .map(SystemDbTest::removeDuplicatedWhitespaces) + .collect(Collectors.toSet()); + assertEquals(expectedTriggers, actualTriggers); final String declareProcedureOutput = "CALL apoc.custom.declareProcedure('declareBar(one = 2 :: INTEGER, two = 3 :: INTEGER) :: (sum :: INTEGER)', 'RETURN $one + $two as sum', 'READ', '');"; assertEquals(Set.of(declareProcedureOutput), readFileLines("metadata.CypherProcedure.neo4j.cypher", directory)); final String declareFunctionOutput = "CALL apoc.custom.declareFunction('declareFoo(input :: INTEGER | FLOAT) :: INTEGER', 'RETURN $input as answer', false, '');"; @@ -178,6 +185,14 @@ public void testExportMetadata() { assertEquals(Set.of(uuidStatement), readFileLines("custom.Uuid.neo4j.cypher", directory)); } + /** + * Remove possible multiple whitespaces, e.g. "CYPHER 5 ", + * since is handled by Core and not modifiable from Extended + */ + private static String removeDuplicatedWhitespaces(String val) { + return val.replaceAll("\\s+", " "); + } + @Test public void testExportMetadataWithTypeInputOrFloat() { final String declareFunctionWithNumber = "CALL apoc.custom.declareFunction('declareFooNumber(input :: INTEGER | FLOAT) :: INTEGER', 'RETURN $input as answer', false, '');";