diff --git a/.github/codecov.yml b/.github/codecov.yml index 44dc178714..f833cc78b7 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -9,7 +9,7 @@ coverage: patch: default: target: 80% - informational: false + informational: true component_management: individual_components: @@ -32,7 +32,7 @@ component_management: informational: true - type: patch target: 80% - informational: false + informational: true - component_id: spanner-import-export name: spanner-import-export paths: diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java index 770c915f0e..0fb0f6a89b 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java @@ -285,10 +285,6 @@ public Table toTable(String tableName, Schema schema) { if (Boolean.parseBoolean(stored)) { column.stored(); } - String hidden = f.getProp(HIDDEN); - if (Boolean.parseBoolean(hidden)) { - column.isHidden(true); - } } else { boolean nullable = false; Schema avroType = f.schema(); @@ -306,6 +302,10 @@ public Table toTable(String tableName, Schema schema) { String defaultExpression = f.getProp(DEFAULT_EXPRESSION); column.parseType(sqlType).notNull(!nullable).defaultExpression(defaultExpression); } + String hidden = f.getProp(HIDDEN); + if (Boolean.parseBoolean(hidden)) { + column.isHidden(true); + } String placementKey = f.getProp(SPANNER_PLACEMENT_KEY); if (placementKey != null) { column.isPlacementKey(Boolean.parseBoolean(placementKey)); diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java index ecee02ec40..8bf2e59af1 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java @@ -152,6 +152,7 @@ public Collection convert(Ddl ddl) { for (Column cm : table.columns()) { SchemaBuilder.FieldBuilder fieldBuilder = fieldsAssembler.name(cm.name()); fieldBuilder.prop(SQL_TYPE, cm.typeString()); + fieldBuilder.prop(HIDDEN, Boolean.toString(cm.isHidden())); for (int i = 0; i < cm.columnOptions().size(); i++) { fieldBuilder.prop(SPANNER_OPTION + i, cm.columnOptions().get(i)); } @@ -162,7 +163,6 @@ public Collection convert(Ddl ddl) { fieldBuilder.prop(NOT_NULL, Boolean.toString(cm.notNull())); fieldBuilder.prop(GENERATION_EXPRESSION, cm.generationExpression()); fieldBuilder.prop(STORED, Boolean.toString(cm.isStored())); - fieldBuilder.prop(HIDDEN, Boolean.toString(cm.isHidden())); // Make the type null to allow us not export the generated column values, // which are semantically logical entities. fieldBuilder.type(SchemaBuilder.builder().nullType()).withDefault(null); diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java index c30a18ca95..0e0be9895c 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java @@ -171,6 +171,11 @@ public void simple() { + " \"name\" : \"timestamp\"," + " \"type\" : [ \"null\", {\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}]" + " }, {" + + " \"name\" : \"HiddenColumn\"," + + " \"type\" : [ \"null\", \"long\" ]," + + " \"sqlType\":\"INT64\"," + + " \"hidden\" : \"true\"" + + " }, {" + " \"name\" : \"MyTokens\"," + " \"type\" : \"null\"," + " \"default\" : null," @@ -243,6 +248,7 @@ public void simple() { + " `float32` FLOAT32," + " `float64` FLOAT64," + " `timestamp` TIMESTAMP," + + " `HiddenColumn` INT64 HIDDEN," + " `MyTokens` TOKENLIST AS ((TOKENIZE_FULLTEXT(MyData))) HIDDEN," + " `Embeddings` ARRAY(vector_length=>128)," + " CONSTRAINT `ck` CHECK(`first_name` != 'last_name')," diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java index 206ffb7026..41036bf096 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java @@ -126,6 +126,11 @@ public void simple() { .type(Type.array(Type.float32())) .arrayLength(Integer.valueOf(128)) .endColumn() + .column("HiddenColumn") + .type(Type.string()) + .max() + .isHidden(true) + .endColumn() .primaryKey() .asc("id") .asc("gen_id") @@ -160,7 +165,7 @@ public void simple() { List fields = avroSchema.getFields(); - assertThat(fields, hasSize(7)); + assertThat(fields, hasSize(8)); assertThat(fields.get(0).name(), equalTo("id")); // Not null @@ -221,6 +226,14 @@ public void simple() { assertThat(fields.get(6).getProp(NOT_NULL), equalTo(null)); assertThat(fields.get(6).getProp(STORED), equalTo(null)); + assertThat(fields.get(7).name(), equalTo("HiddenColumn")); + assertThat(fields.get(7).schema(), equalTo(nullableUnion(Schema.Type.STRING))); + assertThat(fields.get(7).getProp(SQL_TYPE), equalTo("STRING(MAX)")); + assertThat(fields.get(7).getProp(NOT_NULL), equalTo(null)); + assertThat(fields.get(7).getProp(GENERATION_EXPRESSION), equalTo(null)); + assertThat(fields.get(7).getProp(STORED), equalTo(null)); + assertThat(fields.get(7).getProp(HIDDEN), equalTo("true")); + // spanner pk assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_0"), equalTo("`id` ASC")); assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_1"), equalTo("`gen_id` ASC")); diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java index db8b7132cc..aa1c862e24 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java @@ -96,6 +96,11 @@ public void simple() { .generatedAs("CONCAT(first_name, ' ', last_name)") .stored() .endColumn() + .column("HiddenColumn") + .type(Type.string()) + .max() + .isHidden(true) + .endColumn() .primaryKey() .asc("id") .asc("gen_id") @@ -133,6 +138,7 @@ public void simple() { + " `first_name` STRING(10) DEFAULT ('John')," + " `last_name` STRING(MAX)," + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED," + + " `HiddenColumn` STRING(MAX) HIDDEN," + " CONSTRAINT `ck` CHECK (`first_name` != `last_name`)," + " ) PRIMARY KEY (`id` ASC, `gen_id` ASC)" + " CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)" @@ -152,6 +158,7 @@ public void simple() { + " `first_name` STRING(10) DEFAULT ('John')," + " `last_name` STRING(MAX)," + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED," + + " `HiddenColumn` STRING(MAX) HIDDEN," + " CONSTRAINT `ck` CHECK (`first_name` != `last_name`)," + " ) PRIMARY KEY (`id` ASC, `gen_id` ASC)")); assertThat( diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java index 6dcc555006..62113ae473 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java @@ -156,6 +156,7 @@ public void tableWithAllTypes() throws Exception { + " `arr_proto_field_2` ARRAY<`com.google.cloud.teleport.spanner.tests.Order`>," + " `arr_nested_enum` ARRAY<`com.google.cloud.teleport.spanner.tests.Order.PaymentMode`>," + " `arr_enum_field` ARRAY<`com.google.cloud.teleport.spanner.tests.TestEnum`>," + + " `hidden_column` STRING(MAX) HIDDEN," + " ) PRIMARY KEY (`first_name` ASC, `last_name` DESC, `id` ASC)"; FileDescriptorSet.Builder fileDescriptorSetBuilder = FileDescriptorSet.newBuilder(); @@ -174,7 +175,7 @@ public void tableWithAllTypes() throws Exception { assertThat(ddl.table("aLlTYPeS"), notNullValue()); Table table = ddl.table("alltypes"); - assertThat(table.columns(), hasSize(28)); + assertThat(table.columns(), hasSize(29)); // Check case sensitiveness. assertThat(table.column("first_name"), notNullValue()); @@ -231,6 +232,8 @@ public void tableWithAllTypes() throws Exception { assertThat( table.column("arr_enum_field").type(), equalTo(Type.array(Type.protoEnum("com.google.cloud.teleport.spanner.tests.TestEnum")))); + assertThat(table.column("hidden_column").type(), equalTo(Type.string())); + assertThat(table.column("hidden_column").isHidden(), is(true)); // Check not-null. assertThat(table.column("first_name").notNull(), is(false)); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java index d65c83a2a0..e59dec5e24 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java @@ -171,7 +171,7 @@ public void migrationTestWithRenameAndDropColumn() { } @Test - public void migrationTestWithSyntheticPK() { + public void migrationTestWithSyntheticPKAndExtraColumn() { // Construct a ChainedConditionCheck with 2 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/mysql-session.json b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/mysql-session.json index 10027be24f..11e7f0fa51 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/mysql-session.json +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/mysql-session.json @@ -1,29 +1,23 @@ { "SessionName": "NewSession", "EditorName": "", - "DatabaseType": "mysqldump", + "DatabaseType": "mysql", "DatabaseName": "mysql-schema.sql", "Dialect": "google_standard_sql", "Notes": null, "Tags": null, - "SpSchema": - { - "t15515": - { + "SpSchema": { + "t15515": { "Name": "Category", - "ColIds": - [ + "ColIds": [ "c15516", "c15517" ], "ShardIdColumn": "", - "ColDefs": - { - "c15516": - { + "ColDefs": { + "c15516": { "Name": "category_id", - "T": - { + "T": { "Name": "INT64", "Len": 0, "IsArray": false @@ -31,17 +25,14 @@ "NotNull": true, "Comment": "From: category_id tinyint(4)", "Id": "c15516", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } }, - "c15517": - { + "c15517": { "Name": "full_name", - "T": - { + "T": { "Name": "STRING", "Len": 25, "IsArray": false @@ -49,15 +40,13 @@ "NotNull": false, "Comment": "From: name varchar(25)", "Id": "c15517", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } } }, - "PrimaryKeys": - [ + "PrimaryKeys": [ { "ColId": "c15516", "Desc": false, @@ -66,28 +55,42 @@ ], "ForeignKeys": null, "Indexes": null, - "ParentId": "", + "ParentTable": { + "Id": "", + "OnDelete": "" + }, "Comment": "Spanner schema for source table Category", "Id": "t15515" }, - "t15519": - { + "t15519": { "Name": "Books", - "ColIds": - [ + "ColIds": [ "c15520", "c15521", "c15522", - "c15523" + "c15523", + "c1" ], "ShardIdColumn": "", - "ColDefs": - { - "c15520": - { + "ColDefs": { + "c1": { + "Name": "extraCol1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c1", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15520": { "Name": "id", - "T": - { + "T": { "Name": "INT64", "Len": 0, "IsArray": false @@ -95,17 +98,14 @@ "NotNull": true, "Comment": "From: id int(11)", "Id": "c15520", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } }, - "c15521": - { + "c15521": { "Name": "title", - "T": - { + "T": { "Name": "STRING", "Len": 200, "IsArray": false @@ -113,17 +113,14 @@ "NotNull": false, "Comment": "From: title varchar(200)", "Id": "c15521", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } }, - "c15522": - { + "c15522": { "Name": "author_id", - "T": - { + "T": { "Name": "INT64", "Len": 0, "IsArray": false @@ -131,17 +128,14 @@ "NotNull": false, "Comment": "From: author_id int(11)", "Id": "c15522", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } }, - "c15523": - { + "c15523": { "Name": "synth_id", - "T": - { + "T": { "Name": "STRING", "Len": 50, "IsArray": false @@ -149,15 +143,13 @@ "NotNull": false, "Comment": "", "Id": "c15523", - "AutoGen": - { + "AutoGen": { "Name": "", "GenerationType": "" } } }, - "PrimaryKeys": - [ + "PrimaryKeys": [ { "ColId": "c15523", "Desc": false, @@ -166,48 +158,41 @@ ], "ForeignKeys": null, "Indexes": null, - "ParentId": "", + "ParentTable": { + "Id": "", + "OnDelete": "" + }, "Comment": "Spanner schema for source table Books", "Id": "t15519" } }, - "SyntheticPKeys": - { - "t15519": - { + "SyntheticPKeys": { + "t15519": { "ColId": "c15523", "Sequence": 0 } }, - "SrcSchema": - { - "t15515": - { + "SrcSchema": { + "t15515": { "Name": "Category", "Schema": "", - "ColIds": - [ + "ColIds": [ "c15516", "c15517", "c15518" ], - "ColDefs": - { - "c15516": - { + "ColDefs": { + "c15516": { "Name": "category_id", - "Type": - { + "Type": { "Name": "tinyint", - "Mods": - [ + "Mods": [ 4 ], "ArrayBounds": null }, "NotNull": true, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -215,23 +200,23 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15516" + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } }, - "c15517": - { + "c15517": { "Name": "name", - "Type": - { + "Type": { "Name": "varchar", - "Mods": - [ + "Mods": [ 25 ], "ArrayBounds": null }, "NotNull": false, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -239,20 +224,21 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15517" + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } }, - "c15518": - { + "c15518": { "Name": "last_update", - "Type": - { + "Type": { "Name": "timestamp", "Mods": null, "ArrayBounds": null }, "NotNull": false, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -260,11 +246,14 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15518" + "Id": "c15518", + "AutoGen": { + "Name": "", + "GenerationType": "" + } } }, - "PrimaryKeys": - [ + "PrimaryKeys": [ { "ColId": "c15516", "Desc": false, @@ -275,33 +264,26 @@ "Indexes": null, "Id": "t15515" }, - "t15519": - { + "t15519": { "Name": "Books", "Schema": "", - "ColIds": - [ + "ColIds": [ "c15520", "c15521", "c15522" ], - "ColDefs": - { - "c15520": - { + "ColDefs": { + "c15520": { "Name": "id", - "Type": - { + "Type": { "Name": "int", - "Mods": - [ + "Mods": [ 11 ], "ArrayBounds": null }, "NotNull": true, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -309,23 +291,23 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15520" + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } }, - "c15521": - { + "c15521": { "Name": "title", - "Type": - { + "Type": { "Name": "varchar", - "Mods": - [ + "Mods": [ 200 ], "ArrayBounds": null }, "NotNull": false, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -333,23 +315,23 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15521" + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } }, - "c15522": - { + "c15522": { "Name": "author_id", - "Type": - { + "Type": { "Name": "int", - "Mods": - [ + "Mods": [ 11 ], "ArrayBounds": null }, "NotNull": false, - "Ignored": - { + "Ignored": { "Check": false, "Identity": false, "Default": false, @@ -357,7 +339,11 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c15522" + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } } }, "PrimaryKeys": null, @@ -366,53 +352,41 @@ "Id": "t15519" } }, - "SchemaIssues": - { - "t15515": - { - "ColumnLevelIssues": - { - "c15516": - [ + "SchemaIssues": { + "t15515": { + "ColumnLevelIssues": { + "c15516": [ 14 ], - "c15517": - [] + "c15517": [] }, "TableLevelIssues": null }, - "t15519": - { - "ColumnLevelIssues": - { - "c15520": - [ + "t15519": { + "ColumnLevelIssues": { + "c15520": [ 14 ], - "c15521": - [], - "c15522": - [ + "c15521": [], + "c15522": [ 14 ], - "c15523": - [ + "c15523": [ 2 ] }, "TableLevelIssues": null } }, - "Location": - {}, + "Location": {}, "TimezoneOffset": "+00:00", "SpDialect": "google_standard_sql", - "UniquePKey": - {}, - "Rules": - [], + "UniquePKey": {}, + "Rules": [], "IsSharded": false, "SpRegion": "", "ResourceValidation": false, - "UI": false + "UI": false, + "SpSequences": {}, + "SrcSequences": {} } \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/spanner-schema.sql index 8cb851b9c8..59baac723d 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSessionIT/spanner-schema.sql @@ -8,4 +8,5 @@ CREATE TABLE Books ( title STRING(200), author_id INT64, synth_id STRING(50), -) PRIMARY KEY (synth_id); \ No newline at end of file + extraCol1 INT64, +) PRIMARY KEY(synth_id); diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-end-to-end/terraform.tfvars b/v2/datastream-to-spanner/terraform/samples/mysql-end-to-end/terraform.tfvars index ebaa572700..2c5df5570d 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-end-to-end/terraform.tfvars +++ b/v2/datastream-to-spanner/terraform/samples/mysql-end-to-end/terraform.tfvars @@ -14,6 +14,7 @@ datastream_params = { target_gcs_bucket_name = "live-migration" # Or provide a custom bucket name pubsub_topic_name = "live-migration" # Or provide a custom topic name stream_id = "mysql-stream" # Or provide a custom stream ID + enable_backfill = true # This should always be enabled unless using sourcedb-to-spanner template for bulk migrations. max_concurrent_cdc_tasks = 50 # Adjust as needed max_concurrent_backfill_tasks = 50 # Adjust as needed mysql_host = "" diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-end-to-end/terraform.tfvars b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-end-to-end/terraform.tfvars index 2305d5bd3e..a157ee4224 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-end-to-end/terraform.tfvars +++ b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-end-to-end/terraform.tfvars @@ -9,6 +9,7 @@ common_params = { datastream_params = { stream_prefix_path = "" # Prefix for Datastream stream IDs (e.g., "data") + enable_backfill = true # This should always be enabled unless using sourcedb-to-spanner template for bulk migrations. max_concurrent_cdc_tasks = "" # Maximum concurrent CDC tasks (e.g., 5) max_concurrent_backfill_tasks = "" # Maximum concurrent backfill tasks (e.g., 15) diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/main.tf b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/main.tf index f2e712e3bf..5b13d7f7cc 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/main.tf +++ b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/main.tf @@ -188,7 +188,14 @@ resource "google_datastream_stream" "mysql_to_gcs" { location = var.common_params.region display_name = "${var.shard_list[count.index].shard_id != null ? var.shard_list[count.index].shard_id : random_pet.migration_id[count.index].id}-${var.shard_list[count.index].datastream_params.stream_id}" desired_state = "RUNNING" - backfill_all { + dynamic "backfill_all" { + for_each = var.common_params.datastream_params.enable_backfill ? [1] : [] + content {} + } + + dynamic "backfill_none" { + for_each = var.common_params.datastream_params.enable_backfill ? [] : [1] + content {} } source_config { @@ -249,6 +256,7 @@ resource "google_project_iam_member" "live_migration_roles" { } # Dataflow Flex Template Job (for CDC to Spanner) resource "google_dataflow_flex_template_job" "live_migration_job" { + count = var.common_params.dataflow_params.skip_dataflow ? 0 : 1 depends_on = [ google_project_service.enabled_apis, google_project_iam_member.live_migration_roles ] # Launch the template once the stream is created. diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/outputs.tf b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/outputs.tf index d6be22af5f..65ca75e4cf 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/outputs.tf +++ b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/outputs.tf @@ -14,7 +14,7 @@ output "resource_ids" { gcs_bucket = google_storage_bucket.datastream_bucket.name pubsub_topic = google_pubsub_topic.datastream_topic.name pubsub_subscription = google_pubsub_subscription.datastream_subscription.name - dataflow_job = google_dataflow_flex_template_job.live_migration_job.job_id + dataflow_job = var.common_params.dataflow_params.skip_dataflow ? "" : google_dataflow_flex_template_job.live_migration_job[0].job_id } ) @@ -46,7 +46,7 @@ output "resource_urls" { gcs_bucket = "https://console.cloud.google.com/storage/browser/${google_storage_bucket.datastream_bucket.name}?project=${var.common_params.project}" pubsub_topic = "https://console.cloud.google.com/cloudpubsub/topic/detail/${google_pubsub_topic.datastream_topic.name}?project=${var.common_params.project}" pubsub_subscription = "https://console.cloud.google.com/cloudpubsub/subscription/detail/${google_pubsub_subscription.datastream_subscription.name}?project=${var.common_params.project}" - dataflow_job = "https://console.cloud.google.com/dataflow/jobs/${var.common_params.region}/${google_dataflow_flex_template_job.live_migration_job.job_id}?project=${var.common_params.project}" + dataflow_job = var.common_params.dataflow_params.skip_dataflow ? "" : "https://console.cloud.google.com/dataflow/jobs/${var.common_params.region}/${google_dataflow_flex_template_job.live_migration_job[0].job_id}?project=${var.common_params.project}" }) depends_on = [ diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/terraform.tfvars b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/terraform.tfvars index 29cfefdca8..4f76e9724c 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/terraform.tfvars +++ b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/terraform.tfvars @@ -9,6 +9,7 @@ common_params = { datastream_params = { stream_prefix_path = "" # Prefix for Datastream stream IDs (e.g., "data") + enable_backfill = true # This should always be enabled unless using sourcedb-to-spanner template for bulk migrations. max_concurrent_cdc_tasks = "" # Maximum concurrent CDC tasks (e.g., 5) max_concurrent_backfill_tasks = "" # Maximum concurrent backfill tasks (e.g., 15) private_connectivity_id = "" # If using Private Service Connect @@ -33,6 +34,7 @@ common_params = { } dataflow_params = { + skip_dataflow = false template_params = { shadow_table_prefix = "" # Prefix for shadow tables (e.g., "shadow_") create_shadow_tables = "" # Whether to create shadow tables in Spanner diff --git a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/variables.tf b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/variables.tf index a640882c5e..6e84713190 100644 --- a/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/variables.tf +++ b/v2/datastream-to-spanner/terraform/samples/mysql-sharded-single-df-job/variables.tf @@ -14,6 +14,7 @@ variable "common_params" { target_connection_profile_id = optional(string, "target-gcs") gcs_root_path = optional(string, "/") source_type = optional(string, "mysql") + enable_backfill = optional(bool, true) max_concurrent_cdc_tasks = optional(number, 5) max_concurrent_backfill_tasks = optional(number, 20) private_connectivity_id = optional(string) @@ -28,6 +29,7 @@ variable "common_params" { })) }) dataflow_params = object({ + skip_dataflow = optional(bool, false) template_params = object({ shadow_table_prefix = optional(string) create_shadow_tables = optional(bool) diff --git a/v2/datastream-to-spanner/terraform/samples/postgresql-end-to-end/terraform.tfvars b/v2/datastream-to-spanner/terraform/samples/postgresql-end-to-end/terraform.tfvars index dd818340bc..a2be056a82 100644 --- a/v2/datastream-to-spanner/terraform/samples/postgresql-end-to-end/terraform.tfvars +++ b/v2/datastream-to-spanner/terraform/samples/postgresql-end-to-end/terraform.tfvars @@ -14,6 +14,7 @@ datastream_params = { target_gcs_bucket_name = "live-migration" # Or provide a custom bucket name pubsub_topic_name = "live-migration" # Or provide a custom topic name stream_id = "postgresql-stream" # Or provide a custom stream ID + enable_backfill = true # This should always be enabled unless using sourcedb-to-spanner template for bulk migrations. max_concurrent_cdc_tasks = 50 # Adjust as needed max_concurrent_backfill_tasks = 50 # Adjust as needed postgresql_host = "" diff --git a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSources.java b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSources.java index 207985491f..913767b7e8 100644 --- a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSources.java +++ b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSources.java @@ -22,26 +22,26 @@ public class CsvSources { public static CSVFormat toCsvFormat(TextFormat format) { switch (format) { case EXCEL: - return CSVFormat.EXCEL; + return CSVFormat.EXCEL.withNullString(""); case INFORMIX: - return CSVFormat.INFORMIX_UNLOAD_CSV; + return CSVFormat.INFORMIX_UNLOAD_CSV.withNullString(""); case MONGO: - return CSVFormat.MONGODB_CSV; + return CSVFormat.MONGODB_CSV.withNullString(""); case MONGO_TSV: - return CSVFormat.MONGODB_TSV; + return CSVFormat.MONGODB_TSV.withNullString(""); case MYSQL: - return CSVFormat.MYSQL; + return CSVFormat.MYSQL.withNullString(""); case ORACLE: - return CSVFormat.ORACLE; + return CSVFormat.ORACLE.withNullString(""); case POSTGRES: - return CSVFormat.POSTGRESQL_TEXT; + return CSVFormat.POSTGRESQL_TEXT.withNullString(""); case POSTGRESQL_CSV: - return CSVFormat.POSTGRESQL_CSV; + return CSVFormat.POSTGRESQL_CSV.withNullString(""); case RFC4180: - return CSVFormat.RFC4180; + return CSVFormat.RFC4180.withNullString(""); case DEFAULT: default: - return CSVFormat.DEFAULT; + return CSVFormat.DEFAULT.withNullString(""); } } } diff --git a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapper.java b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapper.java index 885ec093e2..e0f111d1ef 100644 --- a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapper.java +++ b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapper.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.neo4j.importer.v1.ImportSpecification; import org.neo4j.importer.v1.ImportSpecificationDeserializer; @@ -29,6 +30,8 @@ import org.neo4j.importer.v1.validation.SpecificationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.error.YAMLException; /** * Helper class for parsing import specification files, accepts file URI as entry point. Delegates @@ -38,19 +41,41 @@ public class JobSpecMapper { private static final Logger LOG = LoggerFactory.getLogger(JobSpecMapper.class); public static ImportSpecification parse(String jobSpecUri, OptionsParams options) { - var json = fetchContent(jobSpecUri); - var spec = new JSONObject(json); + String content = fetchContent(jobSpecUri); + JSONObject spec = getJsonObject(content); + if (!spec.has("version")) { return parseLegacyJobSpec(options, spec); } + try { // TODO: interpolate runtime tokens into new spec elements - return ImportSpecificationDeserializer.deserialize(new StringReader(json)); + return ImportSpecificationDeserializer.deserialize(new StringReader(content)); } catch (SpecificationException e) { throw validationFailure(e); } } + private static JSONObject getJsonObject(String content) { + try { + return new JSONObject(content); + } catch (JSONException jsonException) { + Yaml yaml = new Yaml(); + try { + Map yamlMap = yaml.load(content); + return new JSONObject(yamlMap); + } catch (YAMLException yamlException) { + throw new IllegalArgumentException( + "Parsing failed: content is neither valid JSON nor valid YAML." + + "\nJSON parse error: " + + jsonException.getMessage() + + "\nYAML parse error: " + + yamlException.getMessage(), + yamlException); + } + } + } + private static String fetchContent(String jobSpecUri) { try { return FileSystemUtils.getPathContents(jobSpecUri); diff --git a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/options/Neo4jFlexTemplateOptions.java b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/options/Neo4jFlexTemplateOptions.java index 4b620b0b3e..e5ad4b5aea 100644 --- a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/options/Neo4jFlexTemplateOptions.java +++ b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/options/Neo4jFlexTemplateOptions.java @@ -30,7 +30,7 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { order = 1, description = "Path to the job specification file", helpText = - "The path to the job specification file, which contains the configuration for source and target metadata.") + "The path to the job specification file, which contains the JSON description of data sources, Neo4j targets and actions.") @Validation.Required String getJobSpecUri(); @@ -39,8 +39,9 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { @TemplateParameter.GcsReadFile( order = 2, optional = true, - description = "Path to the Neo4j connection metadata", - helpText = "The path to the Neo4j connection metadata JSON file.") + description = + "The path to the Neo4j connection JSON file, which is hosted on Google Cloud Storage.", + helpText = "The path to the Neo4j connection JSON file.") @Validation.Required(groups = "connection") String getNeo4jConnectionUri(); @@ -49,9 +50,10 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 3, optional = true, - description = "Secret ID for the Neo4j connection metadata", + description = + "The secret ID for the Neo4j connection information. This value is encoded in JSON, using the same schema as the `neo4jConnectionUri`.", helpText = - "The secret ID for the Neo4j connection metadata. This is an alternative to the GCS path option.") + "The secret ID for the Neo4j connection metadata. You can use this value as an alternative to the `neo4jConnectionUri`.") @Validation.Required(groups = "connection") String getNeo4jConnectionSecretId(); @@ -60,9 +62,10 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 4, optional = true, - description = "Options JSON", - helpText = "Options JSON. Use runtime tokens.", - example = "{token1:value1,token2:value2}") + description = + "A JSON object that defines named values interpolated into the specification URIs and queries. Also known as runtime tokens.", + helpText = "A JSON object that is also called runtime tokens", + example = "{token1:value1,token2:value2}. Spec can refer to $token1 and $token2.") @Default.String("") String getOptionsJson(); @@ -71,8 +74,8 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 5, optional = true, - description = "Query SQL", - helpText = "Override SQL query.") + description = "The SQL query override for all BigQuery sources.", + helpText = "SQL query override.") @Default.String("") String getReadQuery(); @@ -81,8 +84,8 @@ public interface Neo4jFlexTemplateOptions extends CommonTemplateOptions { @TemplateParameter.GcsReadFile( order = 6, optional = true, - description = "Path to Text File", - helpText = "Override text file pattern", + description = "The text file path override for all text sources.", + helpText = "The text file path override", example = "gs://your-bucket/path/*.json") @Default.String("") String getInputFilePattern(); diff --git a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/utils/BeamUtils.java b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/utils/BeamUtils.java index 50a7e5f945..573879d91e 100644 --- a/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/utils/BeamUtils.java +++ b/v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/utils/BeamUtils.java @@ -150,7 +150,7 @@ public static Schema toBeamSchema( public static Schema textToBeamSchema(List fields) { return new Schema( fields.stream() - .map(field -> Schema.Field.of(field, FieldType.STRING)) + .map(field -> Schema.Field.of(field, FieldType.STRING).withNullable(true)) .collect(Collectors.toList())); } diff --git a/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSourcesTest.java b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSourcesTest.java new file mode 100644 index 0000000000..1f28e9cb9b --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/CsvSourcesTest.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.neo4j.model.helpers; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.teleport.v2.neo4j.model.sources.TextFormat; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class CsvSourcesTest { + + @Parameterized.Parameter(0) + public TextFormat textFormat; + + @Parameterized.Parameter(1) + public char delimiter; + + @Parameterized.Parameters(name = "{0}") + public static List testParameters() { + return Arrays.asList( + new Object[][] { + {TextFormat.EXCEL, ','}, + {TextFormat.INFORMIX, ','}, + {TextFormat.MONGO, ','}, + {TextFormat.MONGO_TSV, '\t'}, + {TextFormat.MYSQL, '\t'}, + {TextFormat.ORACLE, ','}, + {TextFormat.POSTGRES, '\t'}, + {TextFormat.POSTGRESQL_CSV, ','}, + {TextFormat.RFC4180, ','}, + {TextFormat.DEFAULT, ','} + }); + } + + @Test + public void shouldParseEmptyColumnsAsNullValues() throws IOException { + String line = "1" + delimiter; + CSVFormat csvFormat = CsvSources.toCsvFormat(textFormat); + + try (CSVParser csvParser = CSVParser.parse(line, csvFormat)) { + List csvRecords = csvParser.getRecords(); + assertThat(csvRecords).hasSize(1); + + CSVRecord csvRecord = csvRecords.get(0); + assertThat(csvRecord).hasSize(2); + assertThat(csvRecord.get(0)).isEqualTo("1"); + assertThat(csvRecord.get(1)).isEqualTo(null); + } + } +} diff --git a/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapperTest.java b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapperTest.java new file mode 100644 index 0000000000..42ee4f11ac --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/model/helpers/JobSpecMapperTest.java @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.neo4j.model.helpers; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.cloud.teleport.v2.neo4j.model.job.OptionsParams; +import java.util.List; +import org.junit.Test; + +public class JobSpecMapperTest { + + private static final String SPEC_PATH = "src/test/resources/testing-specs/job-spec-mapper-test"; + + @Test + public void parses_valid_json_legacy_spec() { + var importSpecification = + JobSpecMapper.parse(SPEC_PATH + "/valid-json-legacy-spec.json", new OptionsParams()); + + var sources = importSpecification.getSources(); + assertThat(sources).hasSize(1); + assertThat(sources.get(0).getType()).isEqualTo("text"); + assertThat(sources.get(0).getName()).isEqualTo("source_csv"); + + var targets = importSpecification.getTargets(); + + var nodes = targets.getNodes(); + assertThat(nodes).hasSize(2); + + var sourceNode = nodes.get(0); + assertThat(sourceNode.getSource()).isEqualTo("source_csv"); + assertThat(sourceNode.getName()).isEqualTo("Source CSV rel file-source"); + assertThat(sourceNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(sourceNode.getLabels()).isEqualTo(List.of("Source")); + assertThat(sourceNode.getProperties()).hasSize(1); + assertThat(sourceNode.getProperties().get(0).getSourceField()).isEqualTo("source"); + assertThat(sourceNode.getProperties().get(0).getTargetProperty()).isEqualTo("src_id"); + assertThat(sourceNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("Source CSV rel file-source-Source-node-single-key-for-src_id"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("Source"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("src_id"); + + var targetNode = nodes.get(1); + assertThat(targetNode.getSource()).isEqualTo("source_csv"); + assertThat(targetNode.getName()).isEqualTo("Source CSV rel file-target"); + assertThat(targetNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(targetNode.getLabels()).isEqualTo(List.of("Target")); + assertThat(targetNode.getProperties()).hasSize(1); + assertThat(targetNode.getProperties().get(0).getSourceField()).isEqualTo("target"); + assertThat(targetNode.getProperties().get(0).getTargetProperty()).isEqualTo("tgt_id"); + assertThat(targetNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("Source CSV rel file-target-Target-node-single-key-for-tgt_id"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("Target"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("tgt_id"); + + var relationships = targets.getRelationships(); + assertThat(relationships).hasSize(1); + + var relationship = relationships.get(0); + assertThat(relationship.getName()).isEqualTo("Source CSV rel file"); + assertThat(relationship.getSource()).isEqualTo("source_csv"); + assertThat(relationship.getType()).isEqualTo("LINKS"); + assertThat(relationship.getWriteMode().name()).isEqualTo("CREATE"); + assertThat(relationship.getNodeMatchMode().name()).isEqualTo("MERGE"); + assertThat(relationship.getStartNodeReference()).isEqualTo("Source CSV rel file-source"); + assertThat(relationship.getEndNodeReference()).isEqualTo("Source CSV rel file-target"); + assertThat(relationship.getProperties()).hasSize(1); + assertThat(relationship.getProperties().get(0).getSourceField()).isEqualTo("timestamp"); + assertThat(relationship.getProperties().get(0).getTargetProperty()).isEqualTo("ts"); + } + + @Test + public void parses_valid_json_import_spec() { + var importSpecification = + JobSpecMapper.parse(SPEC_PATH + "/valid-json-import-spec.json", new OptionsParams()); + + var sources = importSpecification.getSources(); + assertThat(sources).hasSize(2); + assertThat(sources.get(0).getType()).isEqualTo("bigquery"); + assertThat(sources.get(0).getName()).isEqualTo("a-source"); + assertThat(sources.get(1).getType()).isEqualTo("bigquery"); + assertThat(sources.get(1).getName()).isEqualTo("b-source"); + + var targets = importSpecification.getTargets(); + + var nodes = targets.getNodes(); + assertThat(nodes).hasSize(2); + + var sourceNode = nodes.get(0); + assertThat(sourceNode.getSource()).isEqualTo("a-source"); + assertThat(sourceNode.getName()).isEqualTo("a-node-target"); + assertThat(sourceNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(sourceNode.getLabels()).isEqualTo(List.of("LabelA")); + assertThat(sourceNode.getProperties()).hasSize(2); + assertThat(sourceNode.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(sourceNode.getProperties().get(0).getTargetProperty()).isEqualTo("property1"); + assertThat(sourceNode.getProperties().get(1).getSourceField()).isEqualTo("field_2"); + assertThat(sourceNode.getProperties().get(1).getTargetProperty()).isEqualTo("property2"); + assertThat(sourceNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("LabelA key constraint"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("LabelA"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("property1"); + + var targetNode = nodes.get(1); + assertThat(targetNode.getSource()).isEqualTo("b-source"); + assertThat(targetNode.getName()).isEqualTo("b-node-target"); + assertThat(targetNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(targetNode.getLabels()).isEqualTo(List.of("LabelB")); + assertThat(targetNode.getProperties()).hasSize(2); + assertThat(targetNode.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(targetNode.getProperties().get(0).getTargetProperty()).isEqualTo("property1"); + assertThat(targetNode.getProperties().get(1).getSourceField()).isEqualTo("field_2"); + assertThat(targetNode.getProperties().get(1).getTargetProperty()).isEqualTo("property2"); + assertThat(targetNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("LabelB key constraint"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("LabelB"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("property1"); + + var relationships = targets.getRelationships(); + assertThat(relationships).hasSize(1); + + var relationship = relationships.get(0); + assertThat(relationship.getName()).isEqualTo("a-target"); + assertThat(relationship.getSource()).isEqualTo("a-source"); + assertThat(relationship.getType()).isEqualTo("TYPE"); + assertThat(relationship.getWriteMode().name()).isEqualTo("CREATE"); + assertThat(relationship.getNodeMatchMode().name()).isEqualTo("MERGE"); + assertThat(relationship.getStartNodeReference()).isEqualTo("a-node-target"); + assertThat(relationship.getEndNodeReference()).isEqualTo("b-node-target"); + assertThat(relationship.getProperties()).hasSize(1); + assertThat(relationship.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(relationship.getProperties().get(0).getTargetProperty()).isEqualTo("id"); + assertThat(relationship.getSchema().getKeyConstraints()).hasSize(1); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("rel key constraint"); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("id"); + } + + @Test + public void parses_valid_yaml_import_spec() { + var importSpecification = + JobSpecMapper.parse(SPEC_PATH + "/valid-yaml-import-spec.yaml", new OptionsParams()); + + var sources = importSpecification.getSources(); + assertThat(sources).hasSize(2); + assertThat(sources.get(0).getType()).isEqualTo("bigquery"); + assertThat(sources.get(0).getName()).isEqualTo("a-source"); + assertThat(sources.get(1).getType()).isEqualTo("bigquery"); + assertThat(sources.get(1).getName()).isEqualTo("b-source"); + + var targets = importSpecification.getTargets(); + + var nodes = targets.getNodes(); + assertThat(nodes).hasSize(2); + + var sourceNode = nodes.get(0); + assertThat(sourceNode.getSource()).isEqualTo("a-source"); + assertThat(sourceNode.getName()).isEqualTo("a-node-target"); + assertThat(sourceNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(sourceNode.getLabels()).isEqualTo(List.of("LabelA")); + assertThat(sourceNode.getProperties()).hasSize(2); + assertThat(sourceNode.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(sourceNode.getProperties().get(0).getTargetProperty()).isEqualTo("property1"); + assertThat(sourceNode.getProperties().get(1).getSourceField()).isEqualTo("field_2"); + assertThat(sourceNode.getProperties().get(1).getTargetProperty()).isEqualTo("property2"); + assertThat(sourceNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("LabelA key constraint"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("LabelA"); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(sourceNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("property1"); + + var targetNode = nodes.get(1); + assertThat(targetNode.getSource()).isEqualTo("b-source"); + assertThat(targetNode.getName()).isEqualTo("b-node-target"); + assertThat(targetNode.getWriteMode().name()).isEqualTo("MERGE"); + assertThat(targetNode.getLabels()).isEqualTo(List.of("LabelB")); + assertThat(targetNode.getProperties()).hasSize(2); + assertThat(targetNode.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(targetNode.getProperties().get(0).getTargetProperty()).isEqualTo("property1"); + assertThat(targetNode.getProperties().get(1).getSourceField()).isEqualTo("field_2"); + assertThat(targetNode.getProperties().get(1).getTargetProperty()).isEqualTo("property2"); + assertThat(targetNode.getSchema().getKeyConstraints()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("LabelB key constraint"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getLabel()).isEqualTo("LabelB"); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(targetNode.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("property1"); + + var relationships = targets.getRelationships(); + assertThat(relationships).hasSize(1); + + var relationship = relationships.get(0); + assertThat(relationship.getName()).isEqualTo("a-target"); + assertThat(relationship.getSource()).isEqualTo("a-source"); + assertThat(relationship.getType()).isEqualTo("TYPE"); + assertThat(relationship.getWriteMode().name()).isEqualTo("CREATE"); + assertThat(relationship.getNodeMatchMode().name()).isEqualTo("MERGE"); + assertThat(relationship.getStartNodeReference()).isEqualTo("a-node-target"); + assertThat(relationship.getEndNodeReference()).isEqualTo("b-node-target"); + assertThat(relationship.getProperties()).hasSize(1); + assertThat(relationship.getProperties().get(0).getSourceField()).isEqualTo("field_1"); + assertThat(relationship.getProperties().get(0).getTargetProperty()).isEqualTo("id"); + assertThat(relationship.getSchema().getKeyConstraints()).hasSize(1); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getName()) + .isEqualTo("rel key constraint"); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getProperties()).hasSize(1); + assertThat(relationship.getSchema().getKeyConstraints().get(0).getProperties().get(0)) + .isEqualTo("id"); + } + + @Test + public void throws_exception_invalid_json() { + assertThrows( + "Parsing failed: content is neither valid JSON nor valid YAML.\n" + + "JSON parse error: Unterminated string at 21 [character 0 line 3]\n" + + "YAML parse error: while parsing a flow mapping\n" + + " in 'string', line 1, column 1:\n" + + " {\n" + + " ^\n" + + "expected ',' or '}', but got \n" + + " in 'string', line 3, column 6:\n" + + " \"sources\": [\n" + + " ^", + IllegalArgumentException.class, + () -> JobSpecMapper.parse(SPEC_PATH + "/invalid-json.json", new OptionsParams())); + } + + @Test + public void throws_exception_invalid_yaml() { + assertThrows( + "Parsing failed: content is neither valid JSON nor valid YAML.\n" + + "JSON parse error: A JSONObject text must begin with '{' at 1 [character 2 line 1]\n" + + "YAML parse error: while parsing a block mapping\n" + + " in 'string', line 1, column 1:\n" + + " version: \"1\n" + + " ^\n" + + "expected , but found ''\n" + + " in 'string', line 5, column 13:\n" + + " query: \"SELECT field_1, field_2 FROM my. ... \n" + + " ^", + IllegalArgumentException.class, + () -> JobSpecMapper.parse(SPEC_PATH + "/invalid-yaml.yaml", new OptionsParams())); + } + + @Test + public void throws_exception_valid_json_wrong_format_legacy_spec() { + assertThrows( + "Unable to process Neo4j job specification", + RuntimeException.class, + () -> + JobSpecMapper.parse( + SPEC_PATH + "/valid-json-wrong-format-legacy-spec.json", new OptionsParams())); + } + + @Test + public void throws_exception_valid_json_wrong_format_import_spec() { + assertThrows( + "Unable to process Neo4j job specification", + RuntimeException.class, + () -> + JobSpecMapper.parse( + SPEC_PATH + "/valid-json-wrong-format-import-spec.json", new OptionsParams())); + } + + @Test + public void throws_exception_valid_yaml_wrong_format_import_spec() { + assertThrows( + "Unable to process Neo4j job specification", + RuntimeException.class, + () -> + JobSpecMapper.parse( + SPEC_PATH + "/valid-yaml-wrong-format-import-spec.yaml", new OptionsParams())); + } +} diff --git a/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/templates/DataConversionIT.java b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/templates/DataConversionIT.java index 43465f21e1..3621f8fd1e 100644 --- a/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/templates/DataConversionIT.java +++ b/v2/googlecloud-to-neo4j/src/test/java/com/google/cloud/teleport/v2/neo4j/templates/DataConversionIT.java @@ -43,12 +43,14 @@ import java.util.stream.Collectors; import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.TemplateTestBase; import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; import org.apache.beam.it.neo4j.Neo4jResourceManager; +import org.apache.beam.it.neo4j.conditions.Neo4jQueryCheck; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -302,6 +304,46 @@ public void supportsMappedTypesForExternalCsv() throws Exception { .meetsConditions(); } + @Test + public void treatsEmptyCsvStringsAsNulls() throws IOException { + gcsClient.createArtifact( + "external.csv", contentOf("/testing-specs/data-conversion/empty-strings.csv")); + gcsClient.createArtifact( + "spec.json", contentOf("/testing-specs/data-conversion/empty-strings-spec.json")); + gcsClient.createArtifact( + "neo4j.json", + String.format( + "{\n" + + " \"server_url\": \"%s\",\n" + + " \"database\": \"%s\",\n" + + " \"auth_type\": \"basic\",\n" + + " \"username\": \"neo4j\",\n" + + " \"pwd\": \"%s\"\n" + + "}", + neo4jClient.getUri(), neo4jClient.getDatabaseName(), neo4jClient.getAdminPassword())); + + LaunchConfig.Builder options = + LaunchConfig.builder(testName, specPath) + .addParameter("jobSpecUri", getGcsPath("spec.json")) + .addParameter("neo4jConnectionUri", getGcsPath("neo4j.json")) + .addParameter( + "optionsJson", + String.format("{\"externalcsvuri\": \"%s\"}", getGcsPath("external.csv"))); + + LaunchInfo info = launchTemplate(options); + + assertThatPipeline(info).isRunning(); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(info), + Neo4jQueryCheck.builder(neo4jClient) + .setQuery("MATCH (n:Node) RETURN n {.*} as properties") + .setExpectedResult(List.of(Map.of("properties", Map.of("int64", "1")))) + .build()); + assertThatResult(result).meetsConditions(); + } + @SuppressWarnings("rawtypes") @NotNull private Supplier[] generateChecks(Map expectedRow) { diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings-spec.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings-spec.json new file mode 100644 index 0000000000..42efba53e8 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings-spec.json @@ -0,0 +1,28 @@ +{ + "source": { + "type": "text", + "name": "data", + "ordered_field_names": "int64,datetime", + "uri": "$externalcsvuri" + }, + "targets": [ + { + "node": { + "name": "Node", + "source": "data", + "mode": "append", + "mappings": { + "label": "\"Node\"", + "properties": { + "keys": [ + "int64" + ], + "dates": [ + "datetime" + ] + } + } + } + } + ] +} diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings.csv b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings.csv new file mode 100644 index 0000000000..3b88900682 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/data-conversion/empty-strings.csv @@ -0,0 +1 @@ +1, \ No newline at end of file diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-json.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-json.json new file mode 100644 index 0000000000..43e1f9431b --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-json.json @@ -0,0 +1,3 @@ +{ + "version": "1 +} diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-yaml.yaml b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-yaml.yaml new file mode 100644 index 0000000000..1b560b7d58 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/invalid-yaml.yaml @@ -0,0 +1 @@ +version: "1 diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-import-spec.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-import-spec.json new file mode 100644 index 0000000000..7c3eec8b94 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-import-spec.json @@ -0,0 +1,102 @@ +{ + "version": "1", + "sources": [ + { + "type": "bigquery", + "name": "a-source", + "query": "SELECT field_1, field_2 FROM my.table" + }, + { + "type": "bigquery", + "name": "b-source", + "query": "SELECT field_1, field_2 FROM my.table" + } + ], + "targets": { + "nodes": [ + { + "source": "a-source", + "name": "a-node-target", + "write_mode": "merge", + "labels": [ + "LabelA" + ], + "properties": [ + { + "source_field": "field_1", + "target_property": "property1" + }, + { + "source_field": "field_2", + "target_property": "property2" + } + ], + "schema": { + "key_constraints": [ + { + "name": "LabelA key constraint", + "label": "LabelA", + "properties": [ + "property1" + ] + } + ] + } + }, + { + "source": "b-source", + "name": "b-node-target", + "write_mode": "merge", + "labels": [ + "LabelB" + ], + "properties": [ + { + "source_field": "field_1", + "target_property": "property1" + }, + { + "source_field": "field_2", + "target_property": "property2" + } + ], + "schema": { + "key_constraints": [ + { + "name": "LabelB key constraint", + "label": "LabelB", + "properties": [ + "property1" + ] + } + ] + } + } + ], + "relationships": [ + { + "name": "a-target", + "source": "a-source", + "type": "TYPE", + "write_mode": "create", + "node_match_mode": "merge", + "start_node_reference": "a-node-target", + "end_node_reference": "b-node-target", + "properties": [ + { + "source_field": "field_1", + "target_property": "id" + } + ], + "schema": { + "key_constraints": [ + { + "name": "rel key constraint", + "properties": ["id"] + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-legacy-spec.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-legacy-spec.json new file mode 100644 index 0000000000..345617d5ea --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-legacy-spec.json @@ -0,0 +1,51 @@ +{ + "config": { + "node_write_batch_size": 5000, + "edge_write_batch_size": 1000 + }, + "sources": [ + { + "type": "text", + "format": "EXCEL", + "name": "source_csv", + "uri": "gs://some/rels.csv", + "delimiter": ",", + "ordered_field_names": "source,source_ts,target,target_ts,rel_id_1,rel_id_2,timestamp" + } + ], + "targets": [ + { + "edge": { + "source": "source_csv", + "name": "Source CSV rel file", + "mode": "append", + "edge_nodes_match_mode": "merge", + "transform": { + "group": true + }, + "mappings": { + "type": "\"LINKS\"", + "source": { + "key": { + "source": "src_id" + }, + "label": "\"Source\"" + }, + "target": { + "key": { + "target": "tgt_id" + }, + "label": "\"Target\"" + }, + "properties": { + "longs": [ + { + "timestamp": "ts" + } + ] + } + } + } + } + ] +} diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-import-spec.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-import-spec.json new file mode 100644 index 0000000000..bba9e22e61 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-import-spec.json @@ -0,0 +1,104 @@ +{ + "version": "1", + "sources": [ + { + "type": "bigquery", + "name": "a-source", + "query": "SELECT field_1, field_2, non_existent_field FROM my.table" + }, + { + "type": "not_supported_type", + "name": "b-source", + "query": "SELECT field_1, field_2 FROM my.table" + } + ], + "targets": { + "nodes": [ + { + "source": "unknown-source", + "name": "a-node-target", + "write_mode": "unknown_mode", + "labels": [ + "LabelA", + "LabelA" + ], + "properties": [ + { + "source_field": "nonexistent_field", + "target_property": "property1" + }, + { + "source_field": "field_2" + } + ], + "schema": { + "key_constraints": [ + { + "name": "LabelA key constraint", + "label": "LabelA", + "properties": [ + "non_existent_property" + ] + } + ] + } + }, + { + "source": "b-source", + "name": "b-node-target", + "write_mode": "merge", + "labels": [ + "LabelB" + ], + "properties": [ + { + "source_field": "field_1", + "target_property": "property1" + }, + { + "source_field": "field_2", + "target_property": "property2" + } + ], + "schema": { + "key_constraints": [ + { + "name": "LabelB key constraint", + "label": "LabelX", + "properties": [ + "property1" + ] + } + ] + } + } + ], + "relationships": [ + { + "name": "a-target", + "source": "a-source", + "type": "UNSUPPORTED_TYPE", + "write_mode": "append", + "node_match_mode": "unknown_match_mode", + "start_node_reference": "missing-node-target", + "end_node_reference": "b-node-target", + "properties": [ + { + "source_field": "nonexistent_field", + "target_property": "id" + } + ], + "schema": { + "key_constraints": [ + { + "name": "rel key constraint", + "properties": [ + "nonexistent_id" + ] + } + ] + } + } + ] + } +} diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-legacy-spec.json b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-legacy-spec.json new file mode 100644 index 0000000000..518d76b75f --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-json-wrong-format-legacy-spec.json @@ -0,0 +1,51 @@ +{ + "config": { + "node_write_batch_size": 5000, + "edge_write_batch_size": 1000 + }, + "sources": [ + { + "type": "text", + "format": "EXCEL", + "name": "source_csv", + "uri": "gs://some/rels.csv", + "delimiter": ",", + "ordered_field_names": "source,source_ts,target,target_ts,rel_id_1,rel_id_2,timestamp" + } + ], + "targets": [ + { + "edge": { + "source": "source_csv", + "name": "Source CSV rel file", + "mode": "merge", + "edge_nodes_match_mode": "create", + "transform": { + "group": true + }, + "mappings": { + "name": "LINK", + "source": { + "name": { + "source": "source_id" + }, + "label": "SrcLabel" + }, + "target": { + "key": { + "target": "target_id" + }, + "label": "TgtLabel" + }, + "properties": { + "longs": [ + { + "invalid_timestamp": "ts" + } + ] + } + } + } + } + ] +} diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-import-spec.yaml b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-import-spec.yaml new file mode 100644 index 0000000000..8b7eb494da --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-import-spec.yaml @@ -0,0 +1,61 @@ +version: "1" +sources: + - type: bigquery + name: a-source + query: SELECT field_1, field_2 FROM my.table + - type: bigquery + name: b-source + query: SELECT field_1, field_2 FROM my.table + +targets: + nodes: + - source: a-source + name: a-node-target + write_mode: merge + labels: + - LabelA + properties: + - source_field: field_1 + target_property: property1 + - source_field: field_2 + target_property: property2 + schema: + key_constraints: + - name: LabelA key constraint + label: LabelA + properties: + - property1 + + - source: b-source + name: b-node-target + write_mode: merge + labels: + - LabelB + properties: + - source_field: field_1 + target_property: property1 + - source_field: field_2 + target_property: property2 + schema: + key_constraints: + - name: LabelB key constraint + label: LabelB + properties: + - property1 + + relationships: + - name: a-target + source: a-source + type: TYPE + write_mode: create + node_match_mode: merge + start_node_reference: a-node-target + end_node_reference: b-node-target + properties: + - source_field: field_1 + target_property: id + schema: + key_constraints: + - name: rel key constraint + properties: + - id diff --git a/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-wrong-format-import-spec.yaml b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-wrong-format-import-spec.yaml new file mode 100644 index 0000000000..e9f97e4c67 --- /dev/null +++ b/v2/googlecloud-to-neo4j/src/test/resources/testing-specs/job-spec-mapper-test/valid-yaml-wrong-format-import-spec.yaml @@ -0,0 +1,61 @@ +version: "1" +sources: + - type: bigquery + name: a-source + query: SELECT field_1, field_2, non_existent_field FROM my.table + - type: not_supported_type + name: b-source + query: SELECT field_1, field_2 FROM my.table + +targets: + nodes: + - source: unknown-source + name: a-node-target + write_mode: unknown_mode + labels: + - LabelA + - LabelA + properties: + - source_field: nonexistent_field + target_property: property1 + - source_field: field_2 + schema: + key_constraints: + - name: LabelA key constraint + label: LabelA + properties: + - non_existent_property + + - source: b-source + name: b-node-target + write_mode: merge + labels: + - LabelB + properties: + - source_field: field_1 + target_property: property1 + - source_field: field_2 + target_property: property2 + schema: + key_constraints: + - name: LabelB key constraint + label: LabelX + properties: + - property1 + + relationships: + - name: a-target + source: a-source + type: UNSUPPORTED_TYPE + write_mode: append + node_match_mode: unknown_match_mode + start_node_reference: missing-node-target + end_node_reference: b-node-target + properties: + - source_field: nonexistent_field + target_property: id + schema: + key_constraints: + - name: rel key constraint + properties: + - nonexistent_id diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/JdbcToBigQuery.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/JdbcToBigQuery.java index 14da8ea49a..ce34c3e0ac 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/JdbcToBigQuery.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/JdbcToBigQuery.java @@ -64,8 +64,7 @@ preview = true, requirements = { "The JDBC drivers for the relational database must be available.", - "The BigQuery table must exist before pipeline execution.", - "The BigQuery table must have a compatible schema.", + "If BigQuery table already exist before pipeline execution, it must have a compatible schema.", "The relational database must be accessible from the subnet where Dataflow runs." }) public class JdbcToBigQuery { diff --git a/v2/mysql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/MySQLToBigQuery.java b/v2/mysql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/MySQLToBigQuery.java index 693aa67f27..73c8e9e9da 100644 --- a/v2/mysql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/MySQLToBigQuery.java +++ b/v2/mysql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/MySQLToBigQuery.java @@ -42,8 +42,7 @@ "https://cloud.google.com/dataflow/docs/guides/templates/provided/mysql-to-bigquery", contactInformation = "https://cloud.google.com/support", requirements = { - "The BigQuery table must exist before pipeline execution.", - "The BigQuery table must have a compatible schema.", + "If BigQuery table already exist before pipeline execution, it must have a compatible schema.", "The MySQL database must be accessible from the subnetwork where Dataflow runs." }) public class MySQLToBigQuery extends JdbcToBigQuery {} diff --git a/v2/oracle-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/OracleToBigQuery.java b/v2/oracle-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/OracleToBigQuery.java index d4e171b3ba..fb62ddc237 100644 --- a/v2/oracle-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/OracleToBigQuery.java +++ b/v2/oracle-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/OracleToBigQuery.java @@ -42,8 +42,7 @@ "https://cloud.google.com/dataflow/docs/guides/templates/provided/oracle-to-bigquery", contactInformation = "https://cloud.google.com/support", requirements = { - "The BigQuery table must exist before pipeline execution.", - "The BigQuery table must have a compatible schema.", + "If BigQuery table already exist before pipeline execution, it must have a compatible schema.", "The Oracle database must be accessible from the subnetwork where Dataflow runs." }) public class OracleToBigQuery extends JdbcToBigQuery {} diff --git a/v2/postgresql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PostgreSQLToBigQuery.java b/v2/postgresql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PostgreSQLToBigQuery.java index b5fca5e54f..c5a3bdbaf6 100644 --- a/v2/postgresql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PostgreSQLToBigQuery.java +++ b/v2/postgresql-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PostgreSQLToBigQuery.java @@ -42,8 +42,7 @@ "https://cloud.google.com/dataflow/docs/guides/templates/provided/postgresql-to-bigquery", contactInformation = "https://cloud.google.com/support", requirements = { - "The BigQuery table must exist before pipeline execution.", - "The BigQuery table must have a compatible schema.", + "If BigQuery table already exist before pipeline execution, it must have a compatible schema.", "The PostgreSQL database must be accessible from the subnetwork where Dataflow runs.", }) public class PostgreSQLToBigQuery extends JdbcToBigQuery {} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/OptionsToConfigBuilder.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/OptionsToConfigBuilder.java index 6b22a6ed6b..7c8ddb5b7c 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/OptionsToConfigBuilder.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/OptionsToConfigBuilder.java @@ -19,6 +19,7 @@ import static com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.JdbcIOWrapperConfig.builderWithPostgreSQLDefaults; import com.google.cloud.teleport.v2.source.reader.auth.dbauth.LocalCredentialsProvider; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.JdbcIOWrapperConfig; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.SQLDialect; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.defaults.MySqlConfigDefaults; @@ -130,7 +131,7 @@ public static JdbcIOWrapperConfig getJdbcIOWrapperConfig( if (sourceDbURL == null) { sourceDbURL = "jdbc:postgresql://" + host + ":" + port + "/" + dbName; } - sourceDbURL = sourceDbURL + "?currentSchema=" + sourceSchemaReference.namespace(); + sourceDbURL = sourceDbURL + "?currentSchema=" + sourceSchemaReference.jdbc().namespace(); if (StringUtils.isNotBlank(connectionProperties)) { sourceDbURL = sourceDbURL + "&" + connectionProperties; } @@ -236,9 +237,10 @@ private static JdbcIOWrapperConfig.Builder builderWithDefaultsFor(SQLDialect dia return builderWithMySqlDefaults(); } + // TODO(vardhanvthigle): Standardize for Css. private static SourceSchemaReference sourceSchemaReferenceFrom( SQLDialect dialect, String dbName, String namespace) { - SourceSchemaReference.Builder builder = SourceSchemaReference.builder(); + JdbcSchemaReference.Builder builder = JdbcSchemaReference.builder(); // Namespaces are not supported for MySQL if (dialect == SQLDialect.POSTGRESQL) { if (StringUtils.isBlank(namespace)) { @@ -247,7 +249,7 @@ private static SourceSchemaReference sourceSchemaReferenceFrom( builder.setNamespace(namespace); } } - return builder.setDbName(dbName).build(); + return SourceSchemaReference.ofJdbc(builder.setDbName(dbName).build()); } private OptionsToConfigBuilder() {} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/IoWrapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/IoWrapper.java index a4d90671d3..ccdc9fe1be 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/IoWrapper.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/IoWrapper.java @@ -23,8 +23,12 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; +/** IO Wrapper Interface for adding new IO sources. */ public interface IoWrapper { + + /** Get a list of reader transforms. */ ImmutableMap>> getTableReaders(); + /** Discover source schema. */ SourceSchema discoverTableSchema(); } diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSource.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSource.java new file mode 100644 index 0000000000..6d0f90f61c --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSource.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.io.Serializable; +import java.net.InetSocketAddress; +import java.util.List; + +/** + * Encapsulates details of a Cassandra Cluster. Cassandra Cluster can connect to multiple KeySpaces, + * just like a Mysql instance can have multiple databases. + */ +@AutoValue +public abstract class CassandraDataSource implements Serializable { + + /** Name of the Cassandra Cluster. */ + public abstract String clusterName(); + + /** Contact points for connecting to a Cassandra Cluster. */ + public abstract ImmutableList contactPoints(); + + public static Builder builder() { + return new AutoValue_CassandraDataSource.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setClusterName(String value); + + public abstract Builder setContactPoints(ImmutableList value); + + public Builder setContactPoints(List value) { + return setContactPoints(ImmutableList.copyOf(value)); + } + + public abstract CassandraDataSource build(); + } +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapper.java new file mode 100644 index 0000000000..852b8e415d --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapper.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper; + +import com.google.cloud.teleport.v2.source.reader.io.IoWrapper; +import com.google.cloud.teleport.v2.source.reader.io.row.SourceRow; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchema; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceTableReference; +import com.google.common.collect.ImmutableMap; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; + +/** IOWrapper for Cassandra Source. */ +public class CassandraIoWrapper implements IoWrapper { + + /** Get a list of reader transforms for Cassandra source. */ + @Override + public ImmutableMap>> + getTableReaders() { + // TODO(vardhanvthigle) + return null; + } + + /** Discover source schema for Cassandra. */ + @Override + public SourceSchema discoverTableSchema() { + // TODO(vardhanvthigle) + return null; + } +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/package-info.java new file mode 100644 index 0000000000..f9c871a51d --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** IoWrapper for cassandra source. */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/package-info.java new file mode 100644 index 0000000000..b7e9338a24 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Cassandra source for reader. */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReference.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReference.java new file mode 100644 index 0000000000..53296eff35 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReference.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.schema; + +import com.google.auto.value.AutoValue; +import com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper.CassandraDataSource; +import java.io.Serializable; + +@AutoValue +public abstract class CassandraSchemaReference implements Serializable { + + /** + * Name of the Cassandra KeySpace. This is equivalent of the JDBC database name. + * + *

Note that Cassandra also has a clusterName, which is at the level of MySQL instance and + * hence encapsulated in {@link CassandraDataSource DataSource} + */ + public abstract String keyspaceName(); + + public static Builder builder() { + return new AutoValue_CassandraSchemaReference.Builder(); + } + + /** + * Returns a stable unique name to be used in PTransforms. + * + * @return name of the {@link CassandraSchemaReference} + */ + public String getName() { + return "KeySpace." + keyspaceName(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setKeyspaceName(String value); + + public abstract CassandraSchemaReference build(); + } +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/package-info.java new file mode 100644 index 0000000000..2db780ed5e --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Schema Discovery and Mapping for Cassandra Source. */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.schema; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSource.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSource.java new file mode 100644 index 0000000000..2707bcd750 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSource.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.datasource; + +import com.google.auto.value.AutoOneOf; +import com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper.CassandraDataSource; + +@AutoOneOf(DataSource.Kind.class) +public abstract class DataSource { + public enum Kind { + JDBC, + CASSANDRA + }; + + public abstract Kind getKind(); + + public abstract javax.sql.DataSource jdbc(); + + public abstract CassandraDataSource cassandra(); + + public static DataSource ofJdbc(javax.sql.DataSource dataSource) { + return AutoOneOf_DataSource.jdbc(dataSource); + } + + public static DataSource ofCassandra(CassandraDataSource dataSource) { + return AutoOneOf_DataSource.cassandra(dataSource); + } +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/package-info.java new file mode 100644 index 0000000000..cfcab79b5a --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/datasource/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Enclosing datasource for various reader sources. */ +package com.google.cloud.teleport.v2.source.reader.io.datasource; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReference.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReference.java new file mode 100644 index 0000000000..a658ad2353 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReference.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.jdbc; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import javax.annotation.Nullable; + +/** Value class to enclose the database name and PG namespace. */ +@AutoValue +public abstract class JdbcSchemaReference implements Serializable { + + /** Name of the JDBC Database. */ + public abstract String dbName(); + + /** NameSpace if needed for PG source. Null for all other sources. */ + @Nullable + public abstract String namespace(); + + public static JdbcSchemaReference.Builder builder() { + return new AutoValue_JdbcSchemaReference.Builder(); + } + + /** + * Returns a stable unique name to be used in PTransforms. + * + * @return name of the {@link JdbcSchemaReference}. + */ + public String getName() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("Db.").append(this.dbName()); + if (this.namespace() != null) { + stringBuilder.append(".Namespace.").append(this.namespace()); + } + return stringBuilder.toString(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract JdbcSchemaReference.Builder setDbName(String value); + + public abstract JdbcSchemaReference.Builder setNamespace(String value); + + public abstract JdbcSchemaReference build(); + } +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapter.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapter.java index 272cb871bd..6f24ebe5e8 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapter.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapter.java @@ -15,8 +15,19 @@ */ package com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; +import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.UniformSplitterDBAdapter; import com.google.cloud.teleport.v2.source.reader.io.schema.RetriableSchemaDiscovery; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference.Kind; +import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; /** * Interface to support various dialects of JDBC databases. @@ -24,4 +35,80 @@ *

Note:As a prt of M2 effort, this interface will expose more mehtods than just extending * {@link RetriableSchemaDiscovery}. */ -public interface DialectAdapter extends RetriableSchemaDiscovery, UniformSplitterDBAdapter {} +public interface DialectAdapter extends RetriableSchemaDiscovery, UniformSplitterDBAdapter { + + /** + * Discover Tables to migrate. This method could be used to auto infer tables to migrate if not + * passed via options. + * + * @param dataSource Provider for JDBC connection. + * @param sourceSchemaReference Source database name and (optionally namespace) + * @return The list of table names for the given database. + * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. + * @throws RetriableSchemaDiscoveryException - Retriable exception during Schema Discovery. + *

Note: + *

The Implementations must log every exception and generate metrics as appropriate. + */ + default ImmutableList discoverTables( + DataSource dataSource, SourceSchemaReference sourceSchemaReference) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { + Preconditions.checkArgument(sourceSchemaReference.getKind() == Kind.JDBC); + return discoverTables(dataSource.jdbc(), sourceSchemaReference.jdbc()); + } + + ImmutableList discoverTables( + javax.sql.DataSource dataSource, JdbcSchemaReference sourceSchemaReference) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException; + + /** + * Discover the schema of tables to migrate. + * + * @param dataSource Provider for JDBC connection. + * @param schemaReference Source database name and (optionally namespace) + * @param tables Tables to migrate. + * @return The discovered schema. + * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. + * @throws RetriableSchemaDiscoveryException - Retriable exception during Schema Discovery. + *

Note: + *

The Implementations must log every exception and generate metrics as appropriate. + */ + default ImmutableMap> discoverTableSchema( + DataSource dataSource, SourceSchemaReference schemaReference, ImmutableList tables) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { + Preconditions.checkArgument(schemaReference.getKind() == Kind.JDBC); + return discoverTableSchema(dataSource.jdbc(), schemaReference.jdbc(), tables); + } + + ImmutableMap> discoverTableSchema( + javax.sql.DataSource dataSource, + JdbcSchemaReference schemaReference, + ImmutableList tables) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException; + + /** + * Discover the indexes of tables to migrate. + * + * @param dataSource Provider for JDBC connection. + * @param sourceSchemaReference Source database name and (optionally namespace) + * @param tables Tables to migrate. + * @return The discovered indexes. + * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. + * @throws RetriableSchemaDiscoveryException - Retriable exception during Schema Discovery. + *

Note: + *

The Implementations must log every exception and generate metrics as appropriate. + */ + default ImmutableMap> discoverTableIndexes( + DataSource dataSource, + SourceSchemaReference sourceSchemaReference, + ImmutableList tables) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { + Preconditions.checkArgument(sourceSchemaReference.getKind() == Kind.JDBC); + return discoverTableIndexes(dataSource.jdbc(), sourceSchemaReference.jdbc(), tables); + } + + ImmutableMap> discoverTableIndexes( + javax.sql.DataSource dataSource, + JdbcSchemaReference sourceSchemaReference, + ImmutableList tables) + throws SchemaDiscoveryException, RetriableSchemaDiscoveryException; +} diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapter.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapter.java index bd745eee44..fb0a15508c 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapter.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapter.java @@ -24,13 +24,13 @@ import com.google.cloud.teleport.v2.constants.MetricCounters; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.DialectAdapter; import com.google.cloud.teleport.v2.source.reader.io.jdbc.rowmapper.JdbcSourceRowMapper; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.stringmapper.CollationReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo.IndexType; -import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -119,7 +119,7 @@ public MysqlDialectAdapter(MySqlVersion mySqlVersion) { */ @Override public ImmutableList discoverTables( - DataSource dataSource, SourceSchemaReference sourceSchemaReference) + DataSource dataSource, JdbcSchemaReference sourceSchemaReference) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info(String.format("Discovering tables for DataSource: %s", dataSource)); @@ -184,12 +184,12 @@ public ImmutableList discoverTables( @Override public ImmutableMap> discoverTableSchema( DataSource dataSource, - SourceSchemaReference sourceSchemaReference, + JdbcSchemaReference sourceSchemaReference, ImmutableList tables) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info( String.format( - "Discovering table schema for Datasource: %s, SourceSchemaReference: %s, tables: %s", + "Discovering table schema for Datasource: %s, JdbcSchemaReference: %s, tables: %s", dataSource, sourceSchemaReference, tables)); String discoveryQuery = getSchemaDiscoveryQuery(sourceSchemaReference); @@ -229,7 +229,7 @@ public ImmutableMap> discoverTabl tableSchemaBuilder.build(); logger.info( String.format( - "Discovered table schema for Datasource: %s, SourceSchemaReference: %s, tables: %s, schema: %s", + "Discovered table schema for Datasource: %s, JdbcSchemaReference: %s, tables: %s, schema: %s", dataSource, sourceSchemaReference, tables, tableSchema)); return tableSchema; @@ -248,12 +248,12 @@ public ImmutableMap> discoverTabl @Override public ImmutableMap> discoverTableIndexes( DataSource dataSource, - SourceSchemaReference sourceSchemaReference, + JdbcSchemaReference sourceSchemaReference, ImmutableList tables) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info( String.format( - "Discovering Indexes for DataSource: %s, SourceSchemaReference: %s, Tables: %s", + "Discovering Indexes for DataSource: %s, JdbcSchemaReference: %s, Tables: %s", dataSource, sourceSchemaReference, tables)); String discoveryQuery = getIndexDiscoveryQuery(sourceSchemaReference); ImmutableMap.Builder> tableIndexesBuilder = @@ -292,12 +292,12 @@ public ImmutableMap> discoverTableI tableIndexesBuilder.build(); logger.info( String.format( - "Discovered Indexes for DataSource: %s, SourceSchemaReference: %s, Tables: %s.\nIndexes: %s", + "Discovered Indexes for DataSource: %s, JdbcSchemaReference: %s, Tables: %s.\nIndexes: %s", dataSource, sourceSchemaReference, tables, tableIndexes)); return tableIndexes; } - protected static String getSchemaDiscoveryQuery(SourceSchemaReference sourceSchemaReference) { + protected static String getSchemaDiscoveryQuery(JdbcSchemaReference sourceSchemaReference) { return "SELECT " + String.join(",", InformationSchemaCols.colList()) + " FROM INFORMATION_SCHEMA.Columns WHERE TABLE_SCHEMA = " @@ -315,7 +315,7 @@ protected static String getSchemaDiscoveryQuery(SourceSchemaReference sourceSche * @param sourceSchemaReference * @return */ - protected static String getIndexDiscoveryQuery(SourceSchemaReference sourceSchemaReference) { + protected static String getIndexDiscoveryQuery(JdbcSchemaReference sourceSchemaReference) { return "SELECT *" + " FROM INFORMATION_SCHEMA.STATISTICS stats" + " JOIN " @@ -382,7 +382,7 @@ private ImmutableMap getTableCols( /** * Get the PadSpace attribute from {@link ResultSet} for index discovery query {@link - * #getIndexDiscoveryQuery(SourceSchemaReference)}. This method takes care of the fact that older + * #getIndexDiscoveryQuery(JdbcSchemaReference)}. This method takes care of the fact that older * versions of MySQL notably Mysql5.7 don't have a {@link * InformationSchemaStatsCols#PAD_SPACE_COL} column and default to PAD SPACE comparisons. */ diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapter.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapter.java index eb31f63648..34b7080fe4 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapter.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapter.java @@ -24,12 +24,12 @@ import com.google.cloud.teleport.v2.constants.MetricCounters; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.DialectAdapter; import com.google.cloud.teleport.v2.source.reader.io.jdbc.rowmapper.JdbcSourceRowMapper; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.stringmapper.CollationReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo; -import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -100,7 +100,7 @@ public PostgreSQLDialectAdapter(PostgreSQLVersion version) { */ @Override public ImmutableList discoverTables( - DataSource dataSource, SourceSchemaReference sourceSchemaReference) + DataSource dataSource, JdbcSchemaReference sourceSchemaReference) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info("Discovering tables for DataSource: {}", dataSource); @@ -158,11 +158,11 @@ public ImmutableList discoverTables( @Override public ImmutableMap> discoverTableSchema( DataSource dataSource, - SourceSchemaReference sourceSchemaReference, + JdbcSchemaReference sourceSchemaReference, ImmutableList tables) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info( - "Discovering table schema for Datasource: {}, SourceSchemaReference: {}, tables: {}", + "Discovering table schema for Datasource: {}, JdbcSchemaReference: {}, tables: {}", dataSource, sourceSchemaReference, tables); @@ -251,7 +251,7 @@ public ImmutableMap> discoverTabl ImmutableMap> tableSchema = tableSchemaBuilder.build(); logger.info( - "Discovered table schema for Datasource: {}, SourceSchemaReference: {}, tables: {}, schema: {}", + "Discovered table schema for Datasource: {}, JdbcSchemaReference: {}, tables: {}, schema: {}", dataSource, sourceSchemaReference, tables, @@ -274,11 +274,11 @@ public ImmutableMap> discoverTabl @Override public ImmutableMap> discoverTableIndexes( DataSource dataSource, - SourceSchemaReference sourceSchemaReference, + JdbcSchemaReference sourceSchemaReference, ImmutableList tables) throws SchemaDiscoveryException, RetriableSchemaDiscoveryException { logger.info( - "Discovering Indexes for DataSource: {}, SourceSchemaReference: {}, Tables: {}", + "Discovering Indexes for DataSource: {}, JdbcSchemaReference: {}, Tables: {}", dataSource, sourceSchemaReference, tables); @@ -387,7 +387,7 @@ public ImmutableMap> discoverTableI ImmutableMap> tableIndexes = tableIndexesBuilder.build(); logger.info( - "Discovered Indexes for DataSource: {}, SourceSchemaReference: {}, Tables: {}.\nIndexes: {}", + "Discovered Indexes for DataSource: {}, JdbcSchemaReference: {}, Tables: {}.\nIndexes: {}", dataSource, sourceSchemaReference, tables, diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapper.java index 53d2672951..2fe9c626e0 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapper.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapper.java @@ -18,6 +18,7 @@ import static com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo.INDEX_TYPE_TO_CLASS; import com.google.cloud.teleport.v2.source.reader.io.IoWrapper; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; import com.google.cloud.teleport.v2.source.reader.io.exception.SuitableIndexNotFoundException; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.JdbcIOWrapperConfig; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.TableConfig; @@ -41,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import javax.sql.DataSource; import org.apache.beam.sdk.io.jdbc.JdbcIO; import org.apache.beam.sdk.io.jdbc.JdbcIO.DataSourceConfiguration; import org.apache.beam.sdk.io.jdbc.JdbcIO.ReadWithPartitions; @@ -80,15 +80,16 @@ public final class JdbcIoWrapper implements IoWrapper { public static JdbcIoWrapper of(JdbcIOWrapperConfig config) throws SuitableIndexNotFoundException { DataSourceConfiguration dataSourceConfiguration = getDataSourceConfiguration(config); - DataSource dataSource = dataSourceConfiguration.buildDatasource(); + javax.sql.DataSource dataSource = dataSourceConfiguration.buildDatasource(); setDataSourceLoginTimeout((BasicDataSource) dataSource, config); SchemaDiscovery schemaDiscovery = new SchemaDiscoveryImpl(config.dialectAdapter(), config.schemaDiscoveryBackOff()); ImmutableList tableConfigs = - autoInferTableConfigs(config, schemaDiscovery, dataSource); - SourceSchema sourceSchema = getSourceSchema(config, schemaDiscovery, dataSource, tableConfigs); + autoInferTableConfigs(config, schemaDiscovery, DataSource.ofJdbc(dataSource)); + SourceSchema sourceSchema = + getSourceSchema(config, schemaDiscovery, DataSource.ofJdbc(dataSource), tableConfigs); ImmutableMap>> tableReaders = buildTableReaders(config, tableConfigs, dataSourceConfiguration, sourceSchema); return new JdbcIoWrapper(tableReaders, sourceSchema); diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java index f58ce213aa..47807677ce 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java @@ -15,8 +15,11 @@ */ package com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + import com.google.auto.value.AutoValue; import com.google.cloud.teleport.v2.source.reader.auth.dbauth.DbAuth; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.DialectAdapter; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.defaults.MySqlConfigDefaults; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.defaults.PostgreSQLConfigDefaults; @@ -24,6 +27,7 @@ import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.range.Range; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.transforms.ReadWithUniformPartitions; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference.Kind; import com.google.cloud.teleport.v2.source.reader.io.schema.typemapping.UnifiedTypeMapper.MapperType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -46,9 +50,13 @@ public abstract class JdbcIOWrapperConfig { /** Source URL. */ public abstract String sourceDbURL(); - /** {@link SourceSchemaReference}. */ + /** {@link SoucreSchemaReference}. */ public abstract SourceSchemaReference sourceSchemaReference(); + public JdbcSchemaReference jdbcSourceSchemaReference() { + return sourceSchemaReference().jdbc(); + } + /** List of Tables to migrate. Auto-inferred if emtpy. */ public abstract ImmutableList tables(); @@ -301,6 +309,10 @@ public abstract static class Builder { public abstract Builder setSourceSchemaReference(SourceSchemaReference value); + public Builder setSourceSchemaReference(JdbcSchemaReference value) { + return setSourceSchemaReference(SourceSchemaReference.ofJdbc(value)); + } + public abstract Builder setTables(ImmutableList value); public abstract Builder setTableVsPartitionColumns( @@ -357,6 +369,12 @@ public abstract Builder setAdditionalOperationsOnRanges( public abstract Builder setMaxConnections(Long value); - public abstract JdbcIOWrapperConfig build(); + public abstract JdbcIOWrapperConfig autoBuild(); + + public JdbcIOWrapperConfig build() { + JdbcIOWrapperConfig config = autoBuild(); + checkState(config.sourceSchemaReference().getKind() == Kind.JDBC); + return config; + } } } diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/jdbc/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/jdbc/package-info.java new file mode 100644 index 0000000000..1ad4e99e1e --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/jdbc/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** IoWrapper for jdbc sources. */ +package com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.jdbc; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/package-info.java index 473e81c656..986e835205 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/package-info.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/package-info.java @@ -14,5 +14,5 @@ * the License. */ -/** IoWrapper for jdbc sources. */ +/** IoWrapper for various sources. */ package com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/package-info.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/package-info.java new file mode 100644 index 0000000000..19ace808dc --- /dev/null +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Jdbc Source for reader. */ +package com.google.cloud.teleport.v2.source.reader.io.jdbc; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/RetriableSchemaDiscovery.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/RetriableSchemaDiscovery.java index 204e26ba2d..764e016756 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/RetriableSchemaDiscovery.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/RetriableSchemaDiscovery.java @@ -15,12 +15,12 @@ */ package com.google.cloud.teleport.v2.source.reader.io.schema; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import javax.sql.DataSource; /** * Discover the schema of tables to migrate. Implementation must adapt to requirements of the source @@ -39,7 +39,7 @@ public interface RetriableSchemaDiscovery { * Discover Tables to migrate. This method could be used to auto infer tables to migrate if not * passed via options. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @return The list of table names for the given database. * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. @@ -54,7 +54,7 @@ ImmutableList discoverTables( /** * Discover the schema of tables to migrate. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param schemaReference Source database name and (optionally namespace) * @param tables Tables to migrate. * @return The discovered schema. @@ -70,7 +70,7 @@ ImmutableMap> discoverTableSchema /** * Discover the indexes of tables to migrate. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @param tables Tables to migrate. * @return The discovered indexes. diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscovery.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscovery.java index 71053b479b..a4479d690f 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscovery.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscovery.java @@ -15,11 +15,11 @@ */ package com.google.cloud.teleport.v2.source.reader.io.schema; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import javax.sql.DataSource; /** * Discover Schema of Source Tables. The main Reader code must use the {@link SchemaDiscovery} @@ -31,7 +31,7 @@ public interface SchemaDiscovery { * Discover Tables to migrate. This method could be used to auto infer tables to migrate if not * passed via options. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @return The list of table names for the given database. * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. @@ -46,7 +46,7 @@ ImmutableList discoverTables( /** * Discover the schema of tables to migrate. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @param tables Tables to migrate. * @return The discovered schema. @@ -64,7 +64,7 @@ ImmutableMap> discoverTableSchema /** * Discover the indexes of tables to migrate. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @param tables Tables to migrate. * @return The discovered indexes. diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImpl.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImpl.java index 0c4df2875b..288035912b 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImpl.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImpl.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.source.reader.io.schema; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryRetriesExhaustedException; @@ -22,7 +23,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import javax.sql.DataSource; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.FluentBackoff; @@ -48,7 +48,7 @@ public SchemaDiscoveryImpl( * Discover Tables to migrate. This method could be used to auto infer tables to migrate if not * passed via options. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @return The list of table names for the given database. * @throws SchemaDiscoveryException - Fatal exception during Schema Discovery. @@ -67,7 +67,7 @@ public ImmutableList discoverTables( /** * Discover the schema of tables to migrate. * - * @param dataSource - Provider for JDBC connection. + * @param dataSource - Provider for source connection. * @param sourceSchemaReference - Source database name and (optionally namespace) * @param tables - Tables to migrate. * @return - The discovered schema @@ -88,7 +88,7 @@ public ImmutableMap> discoverTabl /** * Discover the indexes of tables to migrate. * - * @param dataSource Provider for JDBC connection. + * @param dataSource Provider for source connection. * @param sourceSchemaReference Source database name and (optionally namespace) * @param tables Tables to migrate. * @return The discovered indexes. diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReference.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReference.java index 59ddfdd051..9d4f52c44f 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReference.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReference.java @@ -15,21 +15,45 @@ */ package com.google.cloud.teleport.v2.source.reader.io.schema; -import com.google.auto.value.AutoValue; +import com.google.auto.value.AutoOneOf; +import com.google.cloud.teleport.v2.source.reader.io.cassandra.schema.CassandraSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import java.io.Serializable; -import javax.annotation.Nullable; -/** Value class to enclose the database name and PG namespace. */ -@AutoValue +/** + * Different sources employ varying styles for referencing a schema. For instance, a Postgres source + * uses a database name and namespace, while a Cassandra source uses a cluster name and keyspace. In + * the below Value class we enclose the schema reference. Note: In most of the cases an interface + * would be a better choice to a `OneOf` pattern. In this particular case though, it's not possible + * to have a common minimal interface that encodes getters and setters for properties like nameSpace + * or ClusterName. If we go that route, we would either have to compriamise on readability (like + * level1, level2 reference etc), or have a bloated interface. Hence, a oneOf of independent (but + * conceptually related) classes suits a better pattern to encode this information. + * + * @see: autoOneOf + */ +@AutoOneOf(SourceSchemaReference.Kind.class) public abstract class SourceSchemaReference implements Serializable { - public abstract String dbName(); + public enum Kind { + JDBC, + CASSANDRA + }; + + public abstract Kind getKind(); - @Nullable - public abstract String namespace(); + public abstract JdbcSchemaReference jdbc(); + + public abstract CassandraSchemaReference cassandra(); + + public static SourceSchemaReference ofJdbc(JdbcSchemaReference jdbcSchemaReference) { + return AutoOneOf_SourceSchemaReference.jdbc(jdbcSchemaReference); + } - public static Builder builder() { - return new AutoValue_SourceSchemaReference.Builder().setNamespace(null); + public static SourceSchemaReference ofCassandra( + CassandraSchemaReference cassandraSchemaReference) { + return AutoOneOf_SourceSchemaReference.cassandra(cassandraSchemaReference); } /** @@ -38,21 +62,13 @@ public static Builder builder() { * @return name of the {@link SourceTableReference} */ public String getName() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("Db.").append(this.dbName()); - if (this.namespace() != null) { - stringBuilder.append(".Namespace.").append(this.namespace()); + switch (this.getKind()) { + case JDBC: + return this.jdbc().getName(); + case CASSANDRA: + return this.cassandra().getName(); + default: + throw new IllegalStateException("name not implemented for kind " + this.getKind()); } - return stringBuilder.toString(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setDbName(String value); - - public abstract Builder setNamespace(String value); - - public abstract SourceSchemaReference build(); } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/FakeReader.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/FakeReader.java index 8a333704bc..97900d1b69 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/FakeReader.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/FakeReader.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.source.reader; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchema; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.transform.ReaderTransform; @@ -32,7 +33,8 @@ public class FakeReader implements Reader { FakeReader(int rowCount, int tableCount) { this.rowCountPerTable = rowCount; this.tableCount = tableCount; - this.sourceSchemaReference = SourceSchemaReference.builder().setDbName(this.dbName).build(); + this.sourceSchemaReference = + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName(this.dbName).build()); this.readerTransformTestUtils = new ReaderTransformTestUtils( this.rowCountPerTable, this.tableCount, this.sourceSchemaReference); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSourceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSourceTest.java new file mode 100644 index 0000000000..fe2d1d48d1 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraDataSourceTest.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper; + +import static com.google.common.truth.Truth.assertThat; + +import java.net.InetSocketAddress; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; + +/** Test class for {@link CassandraDataSource}. */ +@RunWith(MockitoJUnitRunner.class) +public class CassandraDataSourceTest { + @Test + public void testCassandraDataSoureBasic() { + String testCluster = "testCluster"; + String testHost = "testHost"; + int testPort = 9042; + CassandraDataSource cassandraDataSource = + CassandraDataSource.builder() + .setClusterName(testCluster) + .setContactPoints(List.of(new InetSocketAddress(testHost, testPort))) + .build(); + assertThat(cassandraDataSource.clusterName()).isEqualTo(testCluster); + assertThat(cassandraDataSource.contactPoints()) + .isEqualTo(ImmutableList.of(new InetSocketAddress(testHost, testPort))); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapperTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapperTest.java new file mode 100644 index 0000000000..972080419d --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIoWrapperTest.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +/** Test class for {@link CassandraIoWrapper}. */ +@RunWith(MockitoJUnitRunner.class) +public class CassandraIoWrapperTest { + @Test + public void testCassandraIoWrapperBasic() { + // Todo(vardhanvthigle) + assertThat((new CassandraIoWrapper()).getTableReaders()).isNull(); + assertThat((new CassandraIoWrapper()).discoverTableSchema()).isNull(); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReferenceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReferenceTest.java new file mode 100644 index 0000000000..8b07c35cde --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/cassandra/schema/CassandraSchemaReferenceTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.cassandra.schema; + +import static com.google.common.truth.Truth.assertThat; + +import junit.framework.TestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +/** Test class for {@link CassandraSchemaReference}. */ +@RunWith(MockitoJUnitRunner.class) +public class CassandraSchemaReferenceTest extends TestCase { + + @Test + public void testCassandraSchemaReferenceBasic() { + String testKeySpace = "testKeySpace"; + CassandraSchemaReference ref = + CassandraSchemaReference.builder().setKeyspaceName(testKeySpace).build(); + assertThat(ref.keyspaceName()).isEqualTo(testKeySpace); + assertThat(ref.getName()).isEqualTo("KeySpace." + testKeySpace); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSourceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSourceTest.java new file mode 100644 index 0000000000..88ab089c57 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/datasource/DataSourceTest.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.datasource; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper.CassandraDataSource; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.Kind; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +/** Test class for {@link DataSource}. */ +@RunWith(MockitoJUnitRunner.class) +public class DataSourceTest { + @Mock javax.sql.DataSource mockJdbcDataSource; + @Mock CassandraDataSource mockCassandraDataSource; + + @Test + public void testDataSourceBasic() { + assertThat(DataSource.ofJdbc(mockJdbcDataSource).getKind()).isEqualTo(Kind.JDBC); + assertThat(DataSource.ofJdbc(mockJdbcDataSource).jdbc()).isEqualTo(mockJdbcDataSource); + assertThat(DataSource.ofCassandra(mockCassandraDataSource).cassandra()) + .isEqualTo(mockCassandraDataSource); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReferenceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReferenceTest.java new file mode 100644 index 0000000000..de750067b4 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/JdbcSchemaReferenceTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.jdbc; + +import static com.google.common.truth.Truth.assertThat; + +import junit.framework.TestCase; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +/** Test class for {@link JdbcSchemaReference}. */ +@RunWith(MockitoJUnitRunner.class) +public class JdbcSchemaReferenceTest extends TestCase { + + @Test + public void testDbNameWithNullNamespaceBuilds() { + final String testDB = "testDb"; + JdbcSchemaReference ref = JdbcSchemaReference.builder().setDbName(testDB).build(); + assertThat(ref.namespace()).isNull(); + assertThat(ref.dbName()).isEqualTo(testDB); + assertThat(ref.getName()).isEqualTo("Db." + testDB); + } + + @Test + public void testDbNameWithNamespaceBuilds() { + final String testDB = "testDb"; + final String testNamespace = "testNamespace"; + JdbcSchemaReference ref = + JdbcSchemaReference.builder().setDbName(testDB).setNamespace(testNamespace).build(); + assertThat(ref.dbName()).isEqualTo(testDB); + assertThat(ref.namespace()).isEqualTo(testNamespace); + assertThat(ref.getName()).isEqualTo("Db." + testDB + ".Namespace." + testNamespace); + } + + @Test + public void testNullDbNameThrowsIllegalStateException() { + // As dbName is a required property, we expect "java.lang.IllegalStateException" + Assert.assertThrows( + java.lang.IllegalStateException.class, () -> JdbcSchemaReference.builder().build()); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapterTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapterTest.java new file mode 100644 index 0000000000..8a5180f0cb --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/DialectAdapterTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter; + +import static org.junit.Assert.assertThrows; + +import com.google.cloud.teleport.v2.source.reader.io.cassandra.iowrapper.CassandraDataSource; +import com.google.cloud.teleport.v2.source.reader.io.cassandra.schema.CassandraSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.mysql.MysqlDialectAdapter; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.mysql.MysqlDialectAdapter.MySqlVersion; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DialectAdapterTest { + @Mock CassandraSchemaReference mockCassandraSchemaReference; + @Mock CassandraDataSource mockCassandraDataSource; + + @Test + public void testMismatchedSource() { + assertThrows( + IllegalArgumentException.class, + () -> + new MysqlDialectAdapter(MySqlVersion.DEFAULT) + .discoverTableSchema( + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofCassandra( + mockCassandraDataSource), + SourceSchemaReference.ofCassandra(mockCassandraSchemaReference), + ImmutableList.of())); + + assertThrows( + IllegalArgumentException.class, + () -> + new MysqlDialectAdapter(MySqlVersion.DEFAULT) + .discoverTables( + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofCassandra( + mockCassandraDataSource), + SourceSchemaReference.ofCassandra(mockCassandraSchemaReference))); + assertThrows( + IllegalArgumentException.class, + () -> + new MysqlDialectAdapter(MySqlVersion.DEFAULT) + .discoverTableIndexes( + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofCassandra( + mockCassandraDataSource), + SourceSchemaReference.ofCassandra(mockCassandraSchemaReference), + ImmutableList.of())); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapterTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapterTest.java index 7857e70818..74bd9beea3 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapterTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/mysql/MysqlDialectAdapterTest.java @@ -27,6 +27,7 @@ import com.google.auto.value.AutoValue; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.mysql.MysqlDialectAdapter.InformationSchemaCols; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.mysql.MysqlDialectAdapter.InformationSchemaStatsCols; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.mysql.MysqlDialectAdapter.MySqlVersion; @@ -65,8 +66,8 @@ public class MysqlDialectAdapterTest { @Test public void testDiscoverTableSchema() throws SQLException, RetriableSchemaDiscoveryException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); final ResultSet mockResultSet = getMockInfoSchemaRs(); when(mockDataSource.getConnection()).thenReturn(mockConnection); @@ -77,15 +78,18 @@ public void testDiscoverTableSchema() throws SQLException, RetriableSchemaDiscov assertThat( new MysqlDialectAdapter(MySqlVersion.DEFAULT) .discoverTableSchema( - mockDataSource, sourceSchemaReference, ImmutableList.of(testTable))) + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofJdbc( + mockDataSource), + SourceSchemaReference.ofJdbc(sourceSchemaReference), + ImmutableList.of(testTable))) .isEqualTo(getExpectedColumnMapping(testTable)); } @Test public void testDiscoverTableSchemaGetConnectionException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) @@ -109,8 +113,8 @@ public void testDiscoverTableSchemaGetConnectionException() throws SQLException @Test public void testDiscoverTableSchemaPrepareStatementException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockConnection.prepareStatement(anyString())).thenThrow(new SQLException("test")); when(mockDataSource.getConnection()).thenReturn(mockConnection); @@ -126,8 +130,8 @@ public void testDiscoverTableSchemaPrepareStatementException() throws SQLExcepti @Test public void testDiscoverTableSchemaSetStringException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()).thenReturn(mockConnection); when(mockConnection.prepareStatement(anyString())).thenReturn(mockPreparedStatement); @@ -145,8 +149,8 @@ public void testDiscoverTableSchemaSetStringException() throws SQLException { public void testDiscoverTableSchemaExecuteQueryException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()).thenReturn(mockConnection); when(mockConnection.prepareStatement(anyString())).thenReturn(mockPreparedStatement); @@ -165,8 +169,8 @@ public void testDiscoverTableSchemaExecuteQueryException() throws SQLException { public void testDiscoverTableSchemaRsException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); when(mockDataSource.getConnection()).thenReturn(mockConnection); @@ -187,7 +191,7 @@ public void testDiscoverTableSchemaRsException() throws SQLException { public void testGetSchemaDiscoveryQuery() { assertThat( MysqlDialectAdapter.getSchemaDiscoveryQuery( - SourceSchemaReference.builder().setDbName("testDB").build())) + JdbcSchemaReference.builder().setDbName("testDB").build())) .isEqualTo( "SELECT COLUMN_NAME,COLUMN_TYPE,CHARACTER_MAXIMUM_LENGTH,NUMERIC_PRECISION,NUMERIC_SCALE FROM INFORMATION_SCHEMA.Columns WHERE TABLE_SCHEMA = 'testDB' AND TABLE_NAME = ?"); } @@ -197,8 +201,8 @@ public void testDiscoverTablesBasic() throws SQLException, RetriableSchemaDiscov ImmutableList testTables = ImmutableList.of("testTable1", "testTable2", "testTable3", "testTable4"); - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); when(mockDataSource.getConnection()).thenReturn(mockConnection); when(mockConnection.createStatement()).thenReturn(mockStatement); @@ -219,7 +223,10 @@ public void testDiscoverTablesBasic() throws SQLException, RetriableSchemaDiscov ImmutableList tables = new MysqlDialectAdapter(MySqlVersion.DEFAULT) - .discoverTables(mockDataSource, sourceSchemaReference); + .discoverTables( + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofJdbc( + mockDataSource), + SourceSchemaReference.ofJdbc(sourceSchemaReference)); assertThat(tables).isEqualTo(testTables); } @@ -227,8 +234,8 @@ public void testDiscoverTablesBasic() throws SQLException, RetriableSchemaDiscov @Test public void testDiscoverTablesGetConnectionException() throws SQLException { - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) @@ -250,8 +257,8 @@ public void testDiscoverTablesGetConnectionException() throws SQLException { @Test public void testDiscoverTablesRsException() throws SQLException { - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); when(mockDataSource.getConnection()).thenReturn(mockConnection); when(mockConnection.createStatement()).thenReturn(mockStatement); @@ -343,8 +350,8 @@ public void testDiscoverIndexesBasic() throws SQLException, RetriableSchemaDisco .setStringMaxLength(255) .build()); - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); ResultSetMetaData mockMetadata = mock(ResultSetMetaData.class); @@ -364,7 +371,11 @@ public void testDiscoverIndexesBasic() throws SQLException, RetriableSchemaDisco ImmutableMap> discoveredIndexes = new MysqlDialectAdapter(MySqlVersion.DEFAULT) - .discoverTableIndexes(mockDataSource, sourceSchemaReference, testTables); + .discoverTableIndexes( + com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource.ofJdbc( + mockDataSource), + SourceSchemaReference.ofJdbc(sourceSchemaReference), + testTables); assertThat(discoveredIndexes) .isEqualTo(ImmutableMap.of(testTables.get(0), expectedSourceColumnIndexInfos)); @@ -418,8 +429,8 @@ public void testDiscoverIndexes5_7() throws SQLException, RetriableSchemaDiscove .setStringMaxLength(100) .build()); - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); ResultSetMetaData mockMetadata = mock(ResultSetMetaData.class); when(mockResultSet.getMetaData()).thenReturn(mockMetadata); @@ -545,7 +556,7 @@ private static void wireMockResultSet( public void testGetIndexDiscoveryQuery() { assertThat( MysqlDialectAdapter.getIndexDiscoveryQuery( - SourceSchemaReference.builder().setDbName("testDB").build())) + JdbcSchemaReference.builder().setDbName("testDB").build())) .isEqualTo( "SELECT * FROM INFORMATION_SCHEMA.STATISTICS stats JOIN INFORMATION_SCHEMA.COLUMNS cols ON stats.table_schema = cols.table_schema AND stats.table_name = cols.table_name AND stats.column_name = cols.column_name LEFT JOIN INFORMATION_SCHEMA.COLLATIONS collations ON cols.COLLATION_NAME = collations.COLLATION_NAME WHERE stats.TABLE_SCHEMA = 'testDB' AND stats.TABLE_NAME = ?"); } @@ -553,8 +564,8 @@ public void testGetIndexDiscoveryQuery() { @Test public void testDiscoverTableIndexesGetConnectionException() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) @@ -581,8 +592,8 @@ public void testDiscoverIndexesSqlExceptions() ImmutableList testTables = ImmutableList.of("testTable1"); long exceptionCount = 0; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); ResultSet mockResultSet = mock(ResultSet.class); when(mockDataSource.getConnection()).thenReturn(mockConnection); when(mockConnection.prepareStatement(anyString())) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapterTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapterTest.java index 55c842dbe9..a9b6800e86 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapterTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/dialectadapter/postgresql/PostgreSQLDialectAdapterTest.java @@ -22,12 +22,12 @@ import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.ResourceUtils; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.postgresql.PostgreSQLDialectAdapter.PostgreSQLVersion; import com.google.cloud.teleport.v2.source.reader.io.jdbc.uniformsplitter.stringmapper.CollationReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo.IndexType; -import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -57,13 +57,13 @@ public class PostgreSQLDialectAdapterTest { @Mock ResultSet mockResultSet; - private SourceSchemaReference sourceSchemaReference; + private JdbcSchemaReference sourceSchemaReference; private PostgreSQLDialectAdapter adapter; @Before public void setUp() throws Exception { sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").setNamespace("public").build(); + JdbcSchemaReference.builder().setDbName("testDB").setNamespace("public").build(); adapter = new PostgreSQLDialectAdapter(PostgreSQLVersion.DEFAULT); } @@ -82,8 +82,8 @@ public void testDiscoverTablesReturnsSchemaAndTableNames() @Test public void testDiscoverTableExceptions() throws SQLException { - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) @@ -158,8 +158,8 @@ public void testDiscoverTableSchema() throws SQLException, RetriableSchemaDiscov @Test public void testDiscoverTableSchemaExceptions() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) @@ -277,8 +277,8 @@ public void testDiscoverTableIndexes() throws SQLException, RetriableSchemaDisco @Test public void testDiscoverTableIndexesExceptions() throws SQLException { final String testTable = "testTable"; - final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + final JdbcSchemaReference sourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); when(mockDataSource.getConnection()) .thenThrow(new SQLTransientConnectionException("test")) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcDataSourceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcDataSourceTest.java index afed23cded..735690d451 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcDataSourceTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcDataSourceTest.java @@ -18,10 +18,10 @@ import static com.google.common.truth.Truth.assertThat; import com.google.cloud.teleport.v2.source.reader.auth.dbauth.LocalCredentialsProvider; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.DialectAdapter; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.JdbcIOWrapperConfig; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.defaults.MySqlConfigDefaults; -import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.common.collect.ImmutableList; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -45,8 +45,8 @@ public class JdbcDataSourceTest { @Test public void testJdbcDataSourceBasic() throws IOException, ClassNotFoundException { - SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + JdbcSchemaReference testSourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); JdbcIOWrapperConfig jdbcIOWrapperConfig = JdbcIOWrapperConfig.builderWithMySqlDefaults() @@ -90,8 +90,8 @@ public void testJdbcDataSourceBasic() throws IOException, ClassNotFoundException @Test public void testJdbcDataSourceSerDe() throws IOException, ClassNotFoundException { - SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + JdbcSchemaReference testSourceSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); JdbcIOWrapperConfig jdbcIOWrapperConfig = JdbcIOWrapperConfig.builderWithMySqlDefaults() diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapperTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapperTest.java index df7f215785..1c8335b4de 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapperTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/JdbcIoWrapperTest.java @@ -28,6 +28,7 @@ import com.google.cloud.teleport.v2.source.reader.auth.dbauth.LocalCredentialsProvider; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SuitableIndexNotFoundException; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.dialectadapter.DialectAdapter; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.JdbcIOWrapperConfig; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.SQLDialect; @@ -80,11 +81,12 @@ public void initDerby() throws SQLException, ClassNotFoundException { @Test public void testJdbcIoWrapperBasic() throws RetriableSchemaDiscoveryException { SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); String testCol = "ID"; SourceColumnType testColType = new SourceColumnType("INTEGER", new Long[] {}, null); - when(mockDialectAdapter.discoverTables(any(), any())).thenReturn(ImmutableList.of("testTable")); - when(mockDialectAdapter.discoverTableIndexes(any(), any(), any())) + when(mockDialectAdapter.discoverTables(any(), (SourceSchemaReference) any())) + .thenReturn(ImmutableList.of("testTable")); + when(mockDialectAdapter.discoverTableIndexes(any(), (SourceSchemaReference) any(), any())) .thenReturn( ImmutableMap.of( "testTable", @@ -98,7 +100,7 @@ public void testJdbcIoWrapperBasic() throws RetriableSchemaDiscoveryException { .setIsUnique(true) .setOrdinalPosition(1) .build()))); - when(mockDialectAdapter.discoverTableSchema(any(), any(), any())) + when(mockDialectAdapter.discoverTableSchema(any(), (SourceSchemaReference) any(), any())) .thenReturn(ImmutableMap.of("testTable", ImmutableMap.of(testCol, testColType))); JdbcIoWrapper jdbcIoWrapper = JdbcIoWrapper.of( @@ -130,11 +132,12 @@ public void testJdbcIoWrapperBasic() throws RetriableSchemaDiscoveryException { @Test public void testJdbcIoWrapperWithoutInference() throws RetriableSchemaDiscoveryException { SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); String testCol = "ID"; SourceColumnType testColType = new SourceColumnType("INTEGER", new Long[] {}, null); - when(mockDialectAdapter.discoverTables(any(), any())).thenReturn(ImmutableList.of("testTable")); - when(mockDialectAdapter.discoverTableIndexes(any(), any(), any())) + when(mockDialectAdapter.discoverTables(any(), (SourceSchemaReference) any())) + .thenReturn(ImmutableList.of("testTable")); + when(mockDialectAdapter.discoverTableIndexes(any(), (SourceSchemaReference) any(), any())) .thenReturn( ImmutableMap.of( "testTable", @@ -148,7 +151,7 @@ public void testJdbcIoWrapperWithoutInference() throws RetriableSchemaDiscoveryE .setIsUnique(true) .setOrdinalPosition(1) .build()))); - when(mockDialectAdapter.discoverTableSchema(any(), any(), any())) + when(mockDialectAdapter.discoverTableSchema(any(), (SourceSchemaReference) any(), any())) .thenReturn(ImmutableMap.of("testTable", ImmutableMap.of(testCol, testColType))); JdbcIoWrapper jdbcIoWrapper = JdbcIoWrapper.of( @@ -181,11 +184,12 @@ public void testJdbcIoWrapperWithoutInference() throws RetriableSchemaDiscoveryE @Test public void testJdbcIoWrapperNoIndexException() throws RetriableSchemaDiscoveryException { SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); String testCol = "ID"; SourceColumnType testColType = new SourceColumnType("INTEGER", new Long[] {}, null); - when(mockDialectAdapter.discoverTables(any(), any())).thenReturn(ImmutableList.of("testTable")); - when(mockDialectAdapter.discoverTableIndexes(any(), any(), any())) + when(mockDialectAdapter.discoverTables(any(), (SourceSchemaReference) any())) + .thenReturn(ImmutableList.of("testTable")); + when(mockDialectAdapter.discoverTableIndexes(any(), (SourceSchemaReference) any(), any())) .thenReturn(ImmutableMap.of(/* No Index on testTable */ )) .thenReturn( ImmutableMap.of( @@ -244,11 +248,12 @@ public void testJdbcIoWrapperNoIndexException() throws RetriableSchemaDiscoveryE public void testJdbcIoWrapperDifferentTables() throws RetriableSchemaDiscoveryException { // Test to check what happens if config passes tables not present in source SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); String testCol = "ID"; SourceColumnType testColType = new SourceColumnType("INTEGER", new Long[] {}, null); - when(mockDialectAdapter.discoverTables(any(), any())).thenReturn(ImmutableList.of("testTable")); - when(mockDialectAdapter.discoverTableIndexes(any(), any(), any())) + when(mockDialectAdapter.discoverTables(any(), (SourceSchemaReference) any())) + .thenReturn(ImmutableList.of("testTable")); + when(mockDialectAdapter.discoverTableIndexes(any(), (SourceSchemaReference) any(), any())) .thenReturn( ImmutableMap.of( "testTable", @@ -262,7 +267,7 @@ public void testJdbcIoWrapperDifferentTables() throws RetriableSchemaDiscoveryEx .setIsUnique(true) .setOrdinalPosition(1) .build()))); - when(mockDialectAdapter.discoverTableSchema(any(), any(), any())) + when(mockDialectAdapter.discoverTableSchema(any(), (SourceSchemaReference) any(), any())) .thenReturn(ImmutableMap.of("testTable", ImmutableMap.of(testCol, testColType))); JdbcIoWrapper jdbcIoWrapper = JdbcIoWrapper.of( @@ -317,8 +322,9 @@ public void testReadWithUniformPartitionFeatureFlag() throws RetriableSchemaDisc String testCol = "ID"; SourceColumnType testColType = new SourceColumnType("INTEGER", new Long[] {}, null); - when(mockDialectAdapter.discoverTables(any(), any())).thenReturn(ImmutableList.of("testTable")); - when(mockDialectAdapter.discoverTableIndexes(any(), any(), any())) + when(mockDialectAdapter.discoverTables(any(), (SourceSchemaReference) any())) + .thenReturn(ImmutableList.of("testTable")); + when(mockDialectAdapter.discoverTableIndexes(any(), (SourceSchemaReference) any(), any())) .thenReturn( ImmutableMap.of( "testTable", @@ -332,11 +338,11 @@ public void testReadWithUniformPartitionFeatureFlag() throws RetriableSchemaDisc .setIsUnique(true) .setOrdinalPosition(1) .build()))); - when(mockDialectAdapter.discoverTableSchema(any(), any(), any())) + when(mockDialectAdapter.discoverTableSchema(any(), (SourceSchemaReference) any(), any())) .thenReturn(ImmutableMap.of("testTable", ImmutableMap.of(testCol, testColType))); SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); JdbcIOWrapperConfig configWithFeatureEnabled = JdbcIOWrapperConfig.builderWithMySqlDefaults() @@ -392,7 +398,7 @@ public void testLoginTimeout() throws RetriableSchemaDiscoveryException { .addConnectionProperty("loginTimeout", String.valueOf(testLoginTimeoutSeconds)); SourceSchemaReference testSourceSchemaReference = - SourceSchemaReference.builder().setDbName("testDB").build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName("testDB").build()); JdbcIOWrapperConfig config = JdbcIOWrapperConfig.builderWithMySqlDefaults() diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImplTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImplTest.java index e84ee66e6a..59490fe984 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImplTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaDiscoveryImplTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.cloud.teleport.v2.source.reader.io.datasource.DataSource; import com.google.cloud.teleport.v2.source.reader.io.exception.RetriableSchemaDiscoveryException; import com.google.cloud.teleport.v2.source.reader.io.exception.SchemaDiscoveryRetriesExhaustedException; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceColumnIndexInfo.IndexType; @@ -31,7 +32,6 @@ import java.io.IOException; import java.sql.SQLException; import java.sql.SQLTransientConnectionException; -import javax.sql.DataSource; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.FluentBackoff; import org.joda.time.Duration; @@ -46,6 +46,7 @@ public class SchemaDiscoveryImplTest { @Mock RetriableSchemaDiscovery mockRetriableSchemaDiscovery; @Mock DataSource mockDataSource; + @Mock javax.sql.DataSource mockJdbcDataSource; @Mock SourceSchemaReference mockSourceSchemaReference; @@ -53,6 +54,7 @@ public class SchemaDiscoveryImplTest { public void testSchemaDiscoveryImpl() throws RetriableSchemaDiscoveryException { final int testRetryCount = 2; final int expectedCallsCount = testRetryCount + 1; + when(mockDataSource.jdbc()).thenReturn(mockJdbcDataSource); when(mockRetriableSchemaDiscovery.discoverTableSchema( mockDataSource, mockSourceSchemaReference, ImmutableList.of())) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())) @@ -80,6 +82,7 @@ public void testSchemaDiscoveryImplThrowsRetriesExhausted() when(mockFluentBackoff.backoff()).thenReturn(mockBackoff); when(mockBackoff.nextBackOffMillis()).thenThrow(new IOException("test")); + when(mockDataSource.jdbc()).thenReturn(mockJdbcDataSource); when(mockRetriableSchemaDiscovery.discoverTableSchema( mockDataSource, mockSourceSchemaReference, ImmutableList.of())) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())); @@ -100,6 +103,7 @@ public void testSchemaDiscoveryImplHandlesIOException() final int testRetryCount = 2; final int expectedCallsCount = testRetryCount + 1; + when(mockDataSource.jdbc()).thenReturn(mockJdbcDataSource); when(mockRetriableSchemaDiscovery.discoverTableSchema( mockDataSource, mockSourceSchemaReference, ImmutableList.of())) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())); @@ -124,6 +128,7 @@ public void testTableDiscoveryImpl() throws RetriableSchemaDiscoveryException { final int testRetryCount = 2; final int expectedCallsCount = testRetryCount + 1; final ImmutableList testTables = ImmutableList.of("testTable1", "testTable2"); + when(mockDataSource.jdbc()).thenReturn(mockJdbcDataSource); when(mockRetriableSchemaDiscovery.discoverTables(mockDataSource, mockSourceSchemaReference)) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())) @@ -157,6 +162,7 @@ public void testIndexDiscoveryImpl() throws RetriableSchemaDiscoveryException { .setOrdinalPosition(1) .setIndexType(IndexType.NUMERIC) .build())); + when(mockDataSource.jdbc()).thenReturn(mockJdbcDataSource); when(mockRetriableSchemaDiscovery.discoverTableIndexes( mockDataSource, mockSourceSchemaReference, ImmutableList.of("testTable1"))) .thenThrow(new RetriableSchemaDiscoveryException(new SQLTransientConnectionException())) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaTestUtils.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaTestUtils.java index 03fced202e..ce2de13a96 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaTestUtils.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SchemaTestUtils.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.source.reader.io.schema; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.SQLDialect; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnType; @@ -25,7 +26,8 @@ public class SchemaTestUtils { static final String TEST_FIELD_NAME_2 = "lastName"; public static SourceSchemaReference generateSchemaReference(String namespace, String dbName) { - return SourceSchemaReference.builder().setNamespace(namespace).setDbName(dbName).build(); + return SourceSchemaReference.ofJdbc( + JdbcSchemaReference.builder().setNamespace(namespace).setDbName(dbName).build()); } public static SourceTableSchema generateTestTableSchema(String tableName) { diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReferenceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReferenceTest.java index 10ce623616..a12ffbf17a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReferenceTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaReferenceTest.java @@ -17,40 +17,31 @@ import static com.google.common.truth.Truth.assertThat; -import junit.framework.TestCase; -import org.junit.Assert; +import com.google.cloud.teleport.v2.source.reader.io.cassandra.schema.CassandraSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; +import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference.Kind; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; -/** Test class for {@link SourceSchemaReference}. */ @RunWith(MockitoJUnitRunner.class) -public class SourceSchemaReferenceTest extends TestCase { - - @Test - public void testDbNameWithNullNamespaceBuilds() { - final String testDB = "testDb"; - SourceSchemaReference ref = SourceSchemaReference.builder().setDbName(testDB).build(); - assertThat(ref.namespace()).isNull(); - assertThat(ref.dbName()).isEqualTo(testDB); - assertThat(ref.getName()).isEqualTo("Db." + testDB); - } - - @Test - public void testDbNameWithNamespaceBuilds() { - final String testDB = "testDb"; - final String testNamespace = "testNamespace"; - SourceSchemaReference ref = - SourceSchemaReference.builder().setDbName(testDB).setNamespace(testNamespace).build(); - assertThat(ref.dbName()).isEqualTo(testDB); - assertThat(ref.namespace()).isEqualTo(testNamespace); - assertThat(ref.getName()).isEqualTo("Db." + testDB + ".Namespace." + testNamespace); - } - +public class SourceSchemaReferenceTest { @Test - public void testNullDbNameThrowsIllegalStateException() { - // As dbName is a required property, we expect "java.lang.IllegalStateException" - Assert.assertThrows( - java.lang.IllegalStateException.class, () -> SourceSchemaReference.builder().build()); + public void testSourceSchemaReferenceBasic() { + JdbcSchemaReference jdbcSchemaReference = + JdbcSchemaReference.builder().setDbName("testDB").build(); + CassandraSchemaReference cassandraSchemaReference = + CassandraSchemaReference.builder().setKeyspaceName("testKeySpace").build(); + assertThat(SourceSchemaReference.ofJdbc(jdbcSchemaReference).getName()) + .isEqualTo(jdbcSchemaReference.getName()); + assertThat(SourceSchemaReference.ofJdbc(jdbcSchemaReference).getKind()).isEqualTo(Kind.JDBC); + assertThat(SourceSchemaReference.ofJdbc(jdbcSchemaReference).jdbc()) + .isEqualTo(jdbcSchemaReference); + assertThat(SourceSchemaReference.ofCassandra(cassandraSchemaReference).getKind()) + .isEqualTo(Kind.CASSANDRA); + assertThat(SourceSchemaReference.ofCassandra(cassandraSchemaReference).getName()) + .isEqualTo(cassandraSchemaReference.getName()); + assertThat(SourceSchemaReference.ofCassandra(cassandraSchemaReference).cassandra()) + .isEqualTo(cassandraSchemaReference); } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaTest.java index 0782381404..33ae864273 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceSchemaTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import junit.framework.TestCase; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,10 +34,12 @@ public void testJdbcSourceSchemaBuilds() { SourceSchema testSchema = SourceSchema.builder() - .setSchemaReference(SourceSchemaReference.builder().setDbName(testDb).build()) + .setSchemaReference( + SourceSchemaReference.ofJdbc( + JdbcSchemaReference.builder().setDbName(testDb).build())) .addTableSchema(SchemaTestUtils.generateTestTableSchema(testTable)) .build(); - assertThat(testSchema.schemaReference().dbName()).isEqualTo(testDb); + assertThat(testSchema.schemaReference().jdbc().dbName()).isEqualTo(testDb); assertThat(testSchema.tableSchemas()).hasSize(1); assertThat(testSchema.tableSchemas().get(0).avroSchema().getFields()).hasSize(2); } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceTableReferenceTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceTableReferenceTest.java index 731057f95b..9cc01ffb60 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceTableReferenceTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/schema/SourceTableReferenceTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; @@ -31,12 +32,14 @@ public void testSourceTableReferenceBuilds() { final String testTableUUID = "10a03145-47bd-4d6c-8168-3eab0f9f847b"; SourceTableReference ref = SourceTableReference.builder() - .setSourceSchemaReference(SourceSchemaReference.builder().setDbName(testDB).build()) + .setSourceSchemaReference( + SourceSchemaReference.ofJdbc( + JdbcSchemaReference.builder().setDbName(testDB).build())) .setSourceTableName(testTable) .setSourceTableSchemaUUID(testTableUUID) .build(); - assertThat(ref.sourceSchemaReference().namespace()).isNull(); - assertThat(ref.sourceSchemaReference().dbName()).isEqualTo(testDB); + assertThat(ref.sourceSchemaReference().jdbc().namespace()).isNull(); + assertThat(ref.sourceSchemaReference().jdbc().dbName()).isEqualTo(testDB); assertThat(ref.sourceTableName()).isEqualTo(testTable); assertThat(ref.sourceTableSchemaUUID()).isEqualTo(testTableUUID); assertThat(ref.getName()).isEqualTo("Db.testDb.Table.testTable"); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/AccumulatingTableReaderTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/AccumulatingTableReaderTest.java index dbd710bd4d..3d0bf9f00b 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/AccumulatingTableReaderTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/AccumulatingTableReaderTest.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.source.reader.io.transform; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.row.SourceRow; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceTableReference; @@ -39,7 +40,7 @@ public class AccumulatingTableReaderTest implements Serializable { public void testAccumulatingTableReader() { final String dbName = "testDB"; final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName(dbName).build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName(dbName).build()); final long rowCountPerTable = 10; final long tableCount = 1; final TupleTag sourceRowTupleTag = new TupleTag<>(); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/ReaderTransformTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/ReaderTransformTest.java index b4e506d898..368c38c36b 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/ReaderTransformTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/source/reader/io/transform/ReaderTransformTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.cloud.teleport.v2.source.reader.io.jdbc.JdbcSchemaReference; import com.google.cloud.teleport.v2.source.reader.io.schema.SourceSchemaReference; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,7 +31,7 @@ public class ReaderTransformTest { public void testReaderTransformBuilds() { final String dbName = "testDB"; final SourceSchemaReference sourceSchemaReference = - SourceSchemaReference.builder().setDbName(dbName).build(); + SourceSchemaReference.ofJdbc(JdbcSchemaReference.builder().setDbName(dbName).build()); final long rowCountPerTable = 10; final long tableCount = 1; ReaderTransformTestUtils readerTransformTestUtils = diff --git a/v2/spanner-to-sourcedb/pom.xml b/v2/spanner-to-sourcedb/pom.xml index 841405fb9a..ef8cda1668 100644 --- a/v2/spanner-to-sourcedb/pom.xml +++ b/v2/spanner-to-sourcedb/pom.xml @@ -89,6 +89,18 @@ ${project.version} test + + com.datastax.oss + java-driver-core + 4.16.0 + compile + + + org.slf4j + slf4j-api + 2.0.11 + compile + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraConnection.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraConnection.java new file mode 100644 index 0000000000..41956c17ef --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraConnection.java @@ -0,0 +1,45 @@ +package com.google.cloud.teleport.v2.templates.utils; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.auth.ProgrammaticPlainTextAuthProvider; + +import java.net.InetSocketAddress; + +public class CassandraConnection { + private static CassandraConnection instance; + private CqlSession session; + + // Private constructor to enforce Singleton + private CassandraConnection(String host, int port, String datacenter, String username, String password) { + try { + session = CqlSession.builder() + .addContactPoint(new InetSocketAddress(host, port)) + .withLocalDatacenter(datacenter) + // .withAuthProvider(new ProgrammaticPlainTextAuthProvider(username, password)) // Use if authentication is enabled + .build(); + session.execute("USE mykeyspace;"); + + System.out.println("Connected to Cassandra!"); + } catch (Exception e) { + throw new RuntimeException("Failed to connect to Cassandra", e); + } + } + public static synchronized CassandraConnection getInstance(String host, int port, String datacenter, String username, String password) { + if (instance == null) { + instance = new CassandraConnection(host, port, datacenter, username, password); + } + return instance; + } + + // Method to get the CqlSession + public CqlSession getSession() { + return session; + } + + // Close the session when done + public void close() { + if (session != null) { + session.close(); + System.out.println("Cassandra connection closed."); + } + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraDao.java new file mode 100644 index 0000000000..a62552f677 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/CassandraDao.java @@ -0,0 +1,44 @@ +package com.google.cloud.teleport.v2.templates.utils; + +import java.io.Serializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.datastax.oss.driver.api.core.CqlSession; +/** Writes data to MySQL. */ +public class CassandraDao implements Serializable { + private static final Logger LOG = LoggerFactory.getLogger(CassandraDao.class); + String cassandraHost; + String cassandraUser; + String cassandraPasswd; + int port; + String datacenter; + + public CassandraDao(String cassandraHost, int cassandraPort, String dataCenter, String cassandraUser, String cassandraPasswd) { + this.cassandraHost = cassandraHost; + this.port = cassandraPort; + this.cassandraUser = cassandraUser; + this.cassandraPasswd = cassandraPasswd; + this.datacenter = dataCenter; + } + + // writes to database + public void write(String cassandraStatement) { + CassandraConnection cassandraConnection = null; + CqlSession session = null; + + try { + cassandraConnection = CassandraConnection.getInstance(this.cassandraHost , this.port, this.datacenter, this.cassandraUser,this.cassandraPasswd); + session = cassandraConnection.getSession(); + session.execute(cassandraStatement); + } catch (Exception e) { + System.err.println("An error occurred while interacting with Cassandra:"); + e.printStackTrace(); + } finally { + // Ensure the connection is closed in the finally block + if (cassandraConnection != null) { + cassandraConnection.close(); + } + } + } +} + diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java index 718d97b184..0a97e21d00 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java @@ -49,7 +49,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Integration test for {@link SpannerToSourceDb} Flex template. */ +/** + * Integration test for {@link SpannerToSourceDb} Flex template for basic run including new spanner + * tables and column rename use-case. + */ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerToSourceDb.class) @RunWith(JUnit4.class) @@ -134,7 +137,7 @@ public static void cleanUp() throws IOException { } @Test - public void spannerToSourceDbBasic() throws InterruptedException { + public void spannerToSourceDbBasic() throws InterruptedException, IOException { assertThatPipeline(jobInfo).isRunning(); // Write row in Spanner writeRowInSpanner(); @@ -144,16 +147,20 @@ public void spannerToSourceDbBasic() throws InterruptedException { private void writeRowInSpanner() { // Write a single record to Spanner - Mutation m = + Mutation m1 = Mutation.newInsertOrUpdateBuilder("Users") .set("id") .to(1) - .set("name") + .set("full_name") .to("FF") .set("from") .to("AA") .build(); - spannerResourceManager.write(m); + spannerResourceManager.write(m1); + + Mutation m2 = + Mutation.newInsertOrUpdateBuilder("Users2").set("id").to(2).set("name").to("B").build(); + spannerResourceManager.write(m2); // Write a single record to Spanner for the given logical shard // Add the record with the transaction tag as txBy= @@ -171,14 +178,14 @@ private void writeRowInSpanner() { .run( (TransactionCallable) transaction -> { - Mutation m2 = + Mutation m3 = Mutation.newInsertOrUpdateBuilder("Users") .set("id") .to(2) - .set("name") + .set("full_name") .to("GG") .build(); - transaction.buffer(m2); + transaction.buffer(m3); return null; }); } diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/session.json index 48749eebf6..6584d15d49 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/session.json +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/session.json @@ -28,7 +28,7 @@ "Id": "c142" }, "c143": { - "Name": "name", + "Name": "full_name", "T": { "Name": "STRING", "Len": 25, diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/spanner-schema.sql index 820882cacb..d0bb0aebec 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/spanner-schema.sql +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbIT/spanner-schema.sql @@ -1,9 +1,14 @@ CREATE TABLE IF NOT EXISTS Users ( id INT64 NOT NULL, - name STRING(25), + full_name STRING(25), `from` STRING(25) ) PRIMARY KEY(id); +CREATE TABLE IF NOT EXISTS Users2 ( + id INT64 NOT NULL, + name STRING(25), + ) PRIMARY KEY(id); + CREATE CHANGE STREAM allstream FOR ALL OPTIONS ( value_capture_type = 'NEW_ROW', diff --git a/v2/sqlserver-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/SQLServerToBigQuery.java b/v2/sqlserver-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/SQLServerToBigQuery.java index 018f22e8fd..f0a9a692c3 100644 --- a/v2/sqlserver-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/SQLServerToBigQuery.java +++ b/v2/sqlserver-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/SQLServerToBigQuery.java @@ -42,8 +42,7 @@ "https://cloud.google.com/dataflow/docs/guides/templates/provided/sqlserver-to-bigquery", contactInformation = "https://cloud.google.com/support", requirements = { - "The BigQuery table must exist before pipeline execution.", - "The BigQuery table must have a compatible schema.", + "If BigQuery table already exist before pipeline execution, it must have a compatible schema.", "The SQL Server database must be accessible from the subnetwork where Dataflow runs.", }) public class SQLServerToBigQuery extends JdbcToBigQuery {}