diff --git a/README.md b/README.md index c660d82..1402bf7 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This library provides a uniform configuration facade for tools running imports to Neo4j. In particular, it offers: - - a user-friendly configuration surface, backed by a JSON schema + - a user-friendly configuration surface (in JSON or YAML), called import specification, backed by a JSON schema - the Java equivalent of the import specification, a.k.a. `org.neo4j.importer.v1.ImportSpecification` - validation plugins (soon) - pre-processing plugins (soon) diff --git a/pom.xml b/pom.xml index bbc3090..4f1ac29 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,13 @@ + + com.fasterxml.jackson + jackson-bom + 2.16.1 + pom + import + org.junit junit-bom @@ -54,6 +61,10 @@ + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + com.networknt json-schema-validator @@ -215,7 +226,6 @@ - licensing diff --git a/src/main/java/org/neo4j/importer/v1/ImportSpecificationDeserializer.java b/src/main/java/org/neo4j/importer/v1/ImportSpecificationDeserializer.java index 26418f6..64f1c56 100644 --- a/src/main/java/org/neo4j/importer/v1/ImportSpecificationDeserializer.java +++ b/src/main/java/org/neo4j/importer/v1/ImportSpecificationDeserializer.java @@ -17,13 +17,13 @@ package org.neo4j.importer.v1; import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import java.io.IOException; import java.io.Reader; public class ImportSpecificationDeserializer { - private static final JsonMapper MAPPER = JsonMapper.builder() + private static final YAMLMapper MAPPER = YAMLMapper.builder() .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .build(); diff --git a/src/test/java/org/neo4j/importer/v1/e2e/AdminImportIT.java b/src/test/java/org/neo4j/importer/v1/e2e/AdminImportIT.java index 8b0a21d..246dc3c 100644 --- a/src/test/java/org/neo4j/importer/v1/e2e/AdminImportIT.java +++ b/src/test/java/org/neo4j/importer/v1/e2e/AdminImportIT.java @@ -37,7 +37,8 @@ import java.util.function.Function; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.driver.AuthTokens; import org.neo4j.driver.Driver; import org.neo4j.driver.GraphDatabase; @@ -75,7 +76,8 @@ public class AdminImportIT { private static final String TARGET_DATABASE = "northwind"; private static final String JDBC_POSTGRES_URL = - "jdbc:tc:postgresql:15.5-alpine:///%s?TC_INITSCRIPT=e2e/northwind_pg_dump.sql".formatted(TARGET_DATABASE); + "jdbc:tc:postgresql:15.5-alpine:///%s?TC_INITSCRIPT=e2e/admin-import/northwind_pg_dump.sql" + .formatted(TARGET_DATABASE); private Driver neo4jDriver; @@ -95,9 +97,11 @@ void cleanUp() { * This is **not** production-ready. * This only serves as a proof that the spec format is descriptive enough to run neo4j-admin imports. */ - @Test - void runs_import() throws Exception { - var importSpec = read("/e2e/spec.json", ImportSpecificationDeserializer::deserialize); + @ParameterizedTest + @ValueSource(strings = {"json", "yaml"}) + void runs_import(String extension) throws Exception { + var importSpec = + read("/e2e/admin-import/spec.%s".formatted(extension), ImportSpecificationDeserializer::deserialize); Map sources = importSpec.getSources().stream().collect(toMap(Source::getName, Function.identity())); File csvFolder = csvFolderPathFor("/e2e/admin-import"); @@ -118,26 +122,30 @@ void runs_import() throws Exception { Neo4jAdmin.writeData(csvFolder, relationshipTarget, nodeTargets, SourceExecutor.read(source)); } // note: custom query targets are ignored for now + var targetNeo4jDatabase = "%s-from-%s".formatted(TARGET_DATABASE, extension); - Neo4jAdmin.executeImport(NEO4J, neo4jDriver, importSpec, TARGET_DATABASE); + Neo4jAdmin.executeImport(NEO4J, neo4jDriver, importSpec, targetNeo4jDatabase); var productCount = neo4jDriver .executableQuery("MATCH (p:Product) RETURN count(p) AS count") - .withConfig(QueryConfig.builder().withDatabase(TARGET_DATABASE).build()) + .withConfig( + QueryConfig.builder().withDatabase(targetNeo4jDatabase).build()) .execute() .records(); assertThat(productCount).hasSize(1); assertThat(productCount.getFirst().get("count").asLong()).isEqualTo(77L); var categoryCount = neo4jDriver .executableQuery("MATCH (c:Category) RETURN count(c) AS count") - .withConfig(QueryConfig.builder().withDatabase(TARGET_DATABASE).build()) + .withConfig( + QueryConfig.builder().withDatabase(targetNeo4jDatabase).build()) .execute() .records(); assertThat(categoryCount).hasSize(1); assertThat(categoryCount.getFirst().get("count").asLong()).isEqualTo(8L); var productInCategoryCount = neo4jDriver .executableQuery("MATCH (:Product)-[btc:BELONGS_TO_CATEGORY]->(:Category) RETURN count(btc) AS count") - .withConfig(QueryConfig.builder().withDatabase(TARGET_DATABASE).build()) + .withConfig( + QueryConfig.builder().withDatabase(targetNeo4jDatabase).build()) .execute() .records(); assertThat(productInCategoryCount).hasSize(1); diff --git a/src/test/resources/e2e/admin-import/.gitkeep b/src/test/resources/e2e/admin-import/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/e2e/northwind_pg_dump.sql b/src/test/resources/e2e/admin-import/northwind_pg_dump.sql similarity index 100% rename from src/test/resources/e2e/northwind_pg_dump.sql rename to src/test/resources/e2e/admin-import/northwind_pg_dump.sql diff --git a/src/test/resources/e2e/spec.json b/src/test/resources/e2e/admin-import/spec.json similarity index 100% rename from src/test/resources/e2e/spec.json rename to src/test/resources/e2e/admin-import/spec.json diff --git a/src/test/resources/e2e/admin-import/spec.yaml b/src/test/resources/e2e/admin-import/spec.yaml new file mode 100644 index 0000000..538f922 --- /dev/null +++ b/src/test/resources/e2e/admin-import/spec.yaml @@ -0,0 +1,51 @@ +sources: + - name: "products" + type: "jdbc" + data_source: "northwind" + query: "SELECT productname, unitprice FROM products ORDER BY productname ASC" + - name: "categories" + type: "jdbc" + data_source: "northwind" + query: "SELECT categoryname, description FROM categories ORDER BY categoryname ASC" + - name: "products_in_categories" + type: "jdbc" + data_source: "northwind" + query: "SELECT p.productname AS productname, c.categoryname AS categoryname FROM products p NATURAL JOIN categories c ORDER BY p.productname ASC " +targets: + nodes: + - source: "products" + name: "product_nodes" + labels: + - "Product" + properties: + - source_field: "productname" + target_property: "name" + - source_field: "unitprice" + target_property: "unitPrice" + schema: + key_constraints: + - name: "name_key_constraint" + label: "Product" + properties: + - "name" + - source: "categories" + name: "category_nodes" + labels: + - "Category" + properties: + - source_field: "categoryname" + target_property: "name" + - source_field: "description" + target_property: "description" + schema: + key_constraints: + - name: "name_key_constraint" + label: "Category" + properties: + - "name" + relationships: + - source: "products_in_categories" + name: "product_in_category_relationships" + type: "BELONGS_TO_CATEGORY" + start_node_reference: "product_nodes" + end_node_reference: "category_nodes"