Merge branch 'GoogleCloudPlatform:main' into pr/2000

ollionorg · Nov 20, 2024 · 3e375ac · 3e375ac
2 parents f561fd8 + b04de34
commit 3e375ac
Show file tree

Hide file tree

Showing 84 changed files with 2,121 additions and 395 deletions.
diff --git a/.github/codecov.yml b/.github/codecov.yml
@@ -9,7 +9,7 @@ coverage:
     patch:
       default:
         target: 80%
-        informational: false
+        informational: true
 
 component_management:
   individual_components:
@@ -32,7 +32,7 @@ component_management:
       informational: true
     - type: patch
       target: 80%
-      informational: false
+      informational: true
   - component_id: spanner-import-export
     name: spanner-import-export
     paths:

diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java
@@ -285,10 +285,6 @@ public Table toTable(String tableName, Schema schema) {
         if (Boolean.parseBoolean(stored)) {
           column.stored();
         }
-        String hidden = f.getProp(HIDDEN);
-        if (Boolean.parseBoolean(hidden)) {
-          column.isHidden(true);
-        }
       } else {
         boolean nullable = false;
         Schema avroType = f.schema();
@@ -306,6 +302,10 @@ public Table toTable(String tableName, Schema schema) {
         String defaultExpression = f.getProp(DEFAULT_EXPRESSION);
         column.parseType(sqlType).notNull(!nullable).defaultExpression(defaultExpression);
       }
+      String hidden = f.getProp(HIDDEN);
+      if (Boolean.parseBoolean(hidden)) {
+        column.isHidden(true);
+      }
       String placementKey = f.getProp(SPANNER_PLACEMENT_KEY);
       if (placementKey != null) {
         column.isPlacementKey(Boolean.parseBoolean(placementKey));

diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java
@@ -152,6 +152,7 @@ public Collection<Schema> convert(Ddl ddl) {
       for (Column cm : table.columns()) {
         SchemaBuilder.FieldBuilder<Schema> fieldBuilder = fieldsAssembler.name(cm.name());
         fieldBuilder.prop(SQL_TYPE, cm.typeString());
+        fieldBuilder.prop(HIDDEN, Boolean.toString(cm.isHidden()));
         for (int i = 0; i < cm.columnOptions().size(); i++) {
           fieldBuilder.prop(SPANNER_OPTION + i, cm.columnOptions().get(i));
         }
@@ -162,7 +163,6 @@ public Collection<Schema> convert(Ddl ddl) {
           fieldBuilder.prop(NOT_NULL, Boolean.toString(cm.notNull()));
           fieldBuilder.prop(GENERATION_EXPRESSION, cm.generationExpression());
           fieldBuilder.prop(STORED, Boolean.toString(cm.isStored()));
-          fieldBuilder.prop(HIDDEN, Boolean.toString(cm.isHidden()));
           // Make the type null to allow us not export the generated column values,
           // which are semantically logical entities.
           fieldBuilder.type(SchemaBuilder.builder().nullType()).withDefault(null);

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java
@@ -171,6 +171,11 @@ public void simple() {
             + "    \"name\" : \"timestamp\","
             + "    \"type\" : [ \"null\", {\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}]"
             + "  }, {"
+            + "    \"name\" : \"HiddenColumn\","
+            + "    \"type\" : [ \"null\", \"long\" ],"
+            + "    \"sqlType\":\"INT64\","
+            + "    \"hidden\" : \"true\""
+            + "  }, {"
             + "    \"name\" : \"MyTokens\","
             + "    \"type\" : \"null\","
             + "    \"default\" : null,"
@@ -243,6 +248,7 @@ public void simple() {
                 + " `float32`         FLOAT32,"
                 + " `float64`         FLOAT64,"
                 + " `timestamp`       TIMESTAMP,"
+                + " `HiddenColumn`    INT64 HIDDEN,"
                 + " `MyTokens`                              TOKENLIST AS ((TOKENIZE_FULLTEXT(MyData))) HIDDEN,"
                 + " `Embeddings`                            ARRAY<FLOAT32>(vector_length=>128),"
                 + " CONSTRAINT `ck` CHECK(`first_name` != 'last_name'),"

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java
@@ -126,6 +126,11 @@ public void simple() {
             .type(Type.array(Type.float32()))
             .arrayLength(Integer.valueOf(128))
             .endColumn()
+            .column("HiddenColumn")
+            .type(Type.string())
+            .max()
+            .isHidden(true)
+            .endColumn()
             .primaryKey()
             .asc("id")
             .asc("gen_id")
@@ -160,7 +165,7 @@ public void simple() {
 
     List<Schema.Field> fields = avroSchema.getFields();
 
-    assertThat(fields, hasSize(7));
+    assertThat(fields, hasSize(8));
 
     assertThat(fields.get(0).name(), equalTo("id"));
     // Not null
@@ -221,6 +226,14 @@ public void simple() {
     assertThat(fields.get(6).getProp(NOT_NULL), equalTo(null));
     assertThat(fields.get(6).getProp(STORED), equalTo(null));
 
+    assertThat(fields.get(7).name(), equalTo("HiddenColumn"));
+    assertThat(fields.get(7).schema(), equalTo(nullableUnion(Schema.Type.STRING)));
+    assertThat(fields.get(7).getProp(SQL_TYPE), equalTo("STRING(MAX)"));
+    assertThat(fields.get(7).getProp(NOT_NULL), equalTo(null));
+    assertThat(fields.get(7).getProp(GENERATION_EXPRESSION), equalTo(null));
+    assertThat(fields.get(7).getProp(STORED), equalTo(null));
+    assertThat(fields.get(7).getProp(HIDDEN), equalTo("true"));
+
     // spanner pk
     assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_0"), equalTo("`id` ASC"));
     assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_1"), equalTo("`gen_id` ASC"));

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/DdlTest.java
@@ -96,6 +96,11 @@ public void simple() {
         .generatedAs("CONCAT(first_name, ' ', last_name)")
         .stored()
         .endColumn()
+        .column("HiddenColumn")
+        .type(Type.string())
+        .max()
+        .isHidden(true)
+        .endColumn()
         .primaryKey()
         .asc("id")
         .asc("gen_id")
@@ -133,6 +138,7 @@ public void simple() {
                 + " `first_name` STRING(10) DEFAULT ('John'),"
                 + " `last_name` STRING(MAX),"
                 + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED,"
+                + " `HiddenColumn`                          STRING(MAX) HIDDEN,"
                 + " CONSTRAINT `ck` CHECK (`first_name` != `last_name`),"
                 + " ) PRIMARY KEY (`id` ASC, `gen_id` ASC)"
                 + " CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"
@@ -152,6 +158,7 @@ public void simple() {
                 + " `first_name` STRING(10) DEFAULT ('John'),"
                 + " `last_name` STRING(MAX),"
                 + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED,"
+                + " `HiddenColumn`                          STRING(MAX) HIDDEN,"
                 + " CONSTRAINT `ck` CHECK (`first_name` != `last_name`),"
                 + " ) PRIMARY KEY (`id` ASC, `gen_id` ASC)"));
     assertThat(

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerIT.java
@@ -156,6 +156,7 @@ public void tableWithAllTypes() throws Exception {
             + " `arr_proto_field_2`     ARRAY<`com.google.cloud.teleport.spanner.tests.Order`>,"
             + " `arr_nested_enum`       ARRAY<`com.google.cloud.teleport.spanner.tests.Order.PaymentMode`>,"
             + " `arr_enum_field`        ARRAY<`com.google.cloud.teleport.spanner.tests.TestEnum`>,"
+            + " `hidden_column`         STRING(MAX) HIDDEN,"
             + " ) PRIMARY KEY (`first_name` ASC, `last_name` DESC, `id` ASC)";
 
     FileDescriptorSet.Builder fileDescriptorSetBuilder = FileDescriptorSet.newBuilder();
@@ -174,7 +175,7 @@ public void tableWithAllTypes() throws Exception {
     assertThat(ddl.table("aLlTYPeS"), notNullValue());
 
     Table table = ddl.table("alltypes");
-    assertThat(table.columns(), hasSize(28));
+    assertThat(table.columns(), hasSize(29));
 
     // Check case sensitiveness.
     assertThat(table.column("first_name"), notNullValue());
@@ -231,6 +232,8 @@ public void tableWithAllTypes() throws Exception {
     assertThat(
         table.column("arr_enum_field").type(),
         equalTo(Type.array(Type.protoEnum("com.google.cloud.teleport.spanner.tests.TestEnum"))));
+    assertThat(table.column("hidden_column").type(), equalTo(Type.string()));
+    assertThat(table.column("hidden_column").isHidden(), is(true));
 
     // Check not-null.
     assertThat(table.column("first_name").notNull(), is(false));

diff --git a/...er/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java b/...er/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSessionIT.java
@@ -171,7 +171,7 @@ public void migrationTestWithRenameAndDropColumn() {
   }
 
   @Test
-  public void migrationTestWithSyntheticPK() {
+  public void migrationTestWithSyntheticPKAndExtraColumn() {
     // Construct a ChainedConditionCheck with 2 stages.
     // 1. Send initial wave of events
     // 2. Wait on Spanner to have events