Merge pull request #42 from wal-kh/master

Add new proto message option to output the bq schema based on the field number
GoogleCloudPlatform · Feb 29, 2024 · e994945 · e994945
2 parents 8edab4b + 76ad96b
commit e994945
Show file tree

Hide file tree

Showing 6 changed files with 269 additions and 30 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -21,8 +21,6 @@ jobs:
       uses: actions/checkout@v2
     - name: Install protoc
       uses: arduino/setup-protoc@v1
-    - name: Install protoc-gen-go
-      run: go get -u github.com/golang/protobuf/protoc-gen-go
 
     # Formatting, go mod tidy, and re-generate proto extension code
     - name: Run go fmt on all modules

diff --git a/bq_table.proto b/bq_table.proto
@@ -56,4 +56,11 @@ message BigQueryMessageOptions {
   // or "<field name>:RECORD:<protobuf type>" for message types.
   // "NULLABLE" by default, different mode may be set via optional suffix ":<mode>"
   repeated string extra_fields = 3;
+
+  // If set will output the schema file order based on the provided value.
+  enum FieldOrder {
+    FIELD_ORDER_UNSPECIFIED = 0;
+    FIELD_ORDER_BY_NUMBER = 1;
+  }
+  FieldOrder output_field_order = 4;
 }
diff --git a/pkg/converter/convert.go b/pkg/converter/convert.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"path"
+	"sort"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/protos"
@@ -274,7 +275,14 @@ func convertMessageType(
 	}
 
 	parentMessages[msg] = true
-	for fieldIndex, fieldDesc := range msg.GetField() {
+	fields := msg.GetField()
+	// Sort fields by the field numbers if the option is set.
+	if opts.GetOutputFieldOrder() == protos.BigQueryMessageOptions_FIELD_ORDER_BY_NUMBER {
+		sort.Slice(fields, func(i, j int) bool {
+			return fields[i].GetNumber() < fields[j].GetNumber()
+		})
+	}
+	for fieldIndex, fieldDesc := range fields {
 		fieldCommentPath := fmt.Sprintf("%s.%d.%d", path, fieldPath, fieldIndex)
 		field, err := convertField(curPkg, fieldDesc, opts, parentMessages, comments, fieldCommentPath)
 		if err != nil {

diff --git a/pkg/converter/plugin_test.go b/pkg/converter/plugin_test.go
@@ -504,3 +504,154 @@ func TestExtraFields(t *testing.T) {
 			]`,
 		})
 }
+
+func TestOrderSchemaByFieldNumber(t *testing.T) {
+	testConvert(t, `
+			file_to_generate: "foo.proto"
+			proto_file <
+				name: "foo.proto"
+				package: "example_package.nested"
+				message_type <
+					name: "FooProto"
+					field < name: "first" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "third" number: 3 type: TYPE_INT32 label: LABEL_REQUIRED >
+					field < name: "second" number: 2 type: TYPE_INT32 label: LABEL_REPEATED >
+					options < [gen_bq_schema.bigquery_opts] <
+						table_name: "foo_table"
+						output_field_order: 1
+						>
+					>
+				>
+			>
+		`,
+		map[string]string{
+			"example_package/nested/foo_table.schema": `[
+				{ "name": "first", "type": "INTEGER", "mode": "NULLABLE" },
+				{ "name": "second", "type": "INTEGER", "mode": "REPEATED" },
+				{ "name": "third", "type": "INTEGER", "mode": "REQUIRED" }
+			]`,
+		})
+}
+
+func TestNestedOrderSchemaByFieldNumber(t *testing.T) {
+	testConvert(t, `
+			file_to_generate: "foo.proto"
+			proto_file <
+				name: "foo.proto"
+				package: "example_package"
+				message_type <
+					name: "FooProto"
+					field < name: "f_1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "f_3" number: 3 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field <
+							name: "bar"
+							number: 2
+							type: TYPE_MESSAGE
+							label: LABEL_OPTIONAL
+              type_name: "BarProto"
+						>
+					options < [gen_bq_schema.bigquery_opts] <
+						table_name: "foo_table"
+						output_field_order: 1
+						>
+					>
+				>
+				message_type <
+					name: "BarProto"
+					field < name: "b_2" number: 2 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_3" number: 3 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					options < [gen_bq_schema.bigquery_opts] <
+					output_field_order: 1
+					>
+					>
+				>
+				>
+		`,
+		map[string]string{
+			"example_package/foo_table.schema": `[
+				{ "name": "f_1", "type": "INTEGER", "mode": "NULLABLE" },
+				{
+					"name": "bar", "type": "RECORD", "mode": "NULLABLE",
+					"fields": [
+						{ "name": "b_1", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_2", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_3", "type": "INTEGER", "mode": "NULLABLE" }
+					]
+				},
+				{ "name": "f_3", "type": "INTEGER", "mode": "NULLABLE" }
+			]`,
+		})
+}
+
+func TestMultipleMessageOrderByFieldNumber(t *testing.T) {
+	testConvert(t, `
+			file_to_generate: "foo.proto"
+			proto_file <
+				name: "foo.proto"
+				package: "example_package"
+				message_type <
+					name: "FooProto"
+					field < name: "f_1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "f_3" number: 4 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field <
+							name: "bar_ordered"
+							number: 2
+							type: TYPE_MESSAGE
+							label: LABEL_OPTIONAL
+              type_name: "BarOrderedProto"
+						>
+						field <
+							name: "bar_unordered"
+							number: 3
+							type: TYPE_MESSAGE
+							label: LABEL_OPTIONAL
+              type_name: "BarUnOrderedProto"
+						>
+					options < [gen_bq_schema.bigquery_opts] <
+						table_name: "foo_table"
+						output_field_order: 1
+						>
+					>
+				>
+				message_type <
+					name: "BarOrderedProto"
+					field < name: "b_2" number: 2 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_3" number: 3 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					options < [gen_bq_schema.bigquery_opts] <
+						output_field_order: 1
+					>
+					>
+				>
+				message_type <
+					name: "BarUnOrderedProto"
+					field < name: "b_2" number: 2 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_3" number: 3 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "b_1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+				>
+				>
+		`,
+		map[string]string{
+			"example_package/foo_table.schema": `[
+				{ "name": "f_1", "type": "INTEGER", "mode": "NULLABLE" },
+				{
+					"name": "bar_ordered", "type": "RECORD", "mode": "NULLABLE",
+					"fields": [
+						{ "name": "b_1", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_2", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_3", "type": "INTEGER", "mode": "NULLABLE" }
+					]
+				},
+				{
+					"name": "bar_unordered", "type": "RECORD", "mode": "NULLABLE",
+					"fields": [
+						{ "name": "b_2", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_3", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "b_1", "type": "INTEGER", "mode": "NULLABLE" }
+					]
+				},
+				{ "name": "f_3", "type": "INTEGER", "mode": "NULLABLE" }
+			]`,
+		})
+}
diff --git a/protos/bq_field.pb.go b/protos/bq_field.pb.go