Skip to content

Commit

Permalink
Fix calling gRPC stub if proto-scheme contains nested enum
Browse files Browse the repository at this point in the history
Calling of gRPC stub fails if the stub contains proto-scheme with nested
enum. Regardless of whether message with the nested enum is used or not.
  • Loading branch information
ashashev authored and danslapman committed Nov 25, 2023
1 parent 769b949 commit 96f3a7d
Show file tree
Hide file tree
Showing 7 changed files with 147 additions and 146 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ object GrpcExractor {

def addSchemaToRegistry(schema: GrpcRootMessage, registry: DynamicSchema.Builder): Unit =
schema match {
case GrpcMessageSchema(name, fields, oneofs, nested) =>
case GrpcMessageSchema(name, fields, oneofs, nested, nestedEnums) =>
val builder = MessageDefinition
.newBuilder(name)
fields.foreach {
Expand All @@ -66,6 +66,19 @@ object GrpcExractor {
}
builder.addMessageDefinition(nestedBuilder.build())
}
nestedEnums.getOrElse(List.empty).foreach { nst =>
val nestedBuilder = EnumDefinition.newBuilder(nst.name)
nst.values.foreach { case (name, number) =>
ignore(
nestedBuilder
.addValue(
name.asString,
number.asInt
)
)
}
builder.addEnumDefinition(nestedBuilder.build())
}

ignore(registry.addMessageDefinition(builder.build()))
case GrpcEnumSchema(name, values) =>
Expand Down Expand Up @@ -119,12 +132,7 @@ object GrpcExractor {
def toGrpcProtoDefinition: GrpcProtoDefinition = {
val descriptor: DescriptorProtos.FileDescriptorProto = dynamicSchema.getFileDescriptorSet.getFile(0)
val namespace = descriptor.hasPackage.option(descriptor.getPackage)
val enums = descriptor.getEnumTypeList.asScala.map { enum =>
GrpcEnumSchema(
enum.getName,
enum.getValueList.asScala.map(i => (i.getName.coerce[FieldName], i.getNumber.coerce[FieldNumber])).toMap
)
}.toList
val enums = descriptor.getEnumTypeList.asScala.map(enum2enumScheme).toList
val messages = descriptor.getMessageTypeList.asScala
.filter(!_.getOptions.getMapEntry)
.map(message2messageSchema)
Expand All @@ -137,11 +145,18 @@ object GrpcExractor {
}
}

private def enum2enumScheme(enumProto: DescriptorProtos.EnumDescriptorProto): GrpcEnumSchema =
GrpcEnumSchema(
enumProto.getName,
enumProto.getValueList.asScala.map(i => (i.getName.coerce[FieldName], i.getNumber.coerce[FieldNumber])).toMap
)

private def message2messageSchema(message: DescriptorProtos.DescriptorProto): GrpcMessageSchema = {
val (fields, oneofs) = message.getFieldList.asScala.toList
.partition(f => !f.hasOneofIndex || isProto3OptionalField(f, message.getOneofDeclList.asScala.map(_.getName).toSet))

val nested = message.getNestedTypeList.asScala.toList
val nestedEnums = message.getEnumTypeList().asScala.toList
val nested = message.getNestedTypeList.asScala.toList

GrpcMessageSchema(
message.getName,
Expand Down Expand Up @@ -176,6 +191,10 @@ object GrpcExractor {
.map(message2messageSchema) match {
case Nil => None
case list => Some(list)
},
nestedEnums.map(enum2enumScheme) match {
case Nil => None
case list => Some(list)
}
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ case class GrpcMessageSchema(
name: String,
fields: List[GrpcField],
oneofs: Option[List[GrpcOneOfSchema]] = None,
nested: Option[List[GrpcMessageSchema]] = None
nested: Option[List[GrpcMessageSchema]] = None,
nestedEnums: Option[List[GrpcEnumSchema]] = None,
) extends GrpcRootMessage

object GrpcMessageSchema {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ object GrpcStub {
definition.schemas.find(_.name == name)
)
rootFields <- rootMessage match {
case GrpcMessageSchema(_, fields, oneofs, _) =>
case GrpcMessageSchema(_, fields, oneofs, _, _) =>
ZIO.succeed(fields ++ oneofs.map(_.flatMap(_.options)).getOrElse(List.empty))
case GrpcEnumSchema(_, _) =>
ZIO.fail(ValidationError(Vector(s"Enum cannot be a root message, but '$className' is")))
Expand Down Expand Up @@ -85,7 +85,7 @@ object GrpcStub {
definition.schemas.find(_.name == field.typeName) match {
case Some(message) =>
message match {
case GrpcMessageSchema(_, fs, oneofs, _) =>
case GrpcMessageSchema(_, fs, oneofs, _, _) =>
fields.set(fs ++ oneofs.map(_.flatMap(_.options)).getOrElse(List.empty))
case GrpcEnumSchema(_, _) => fields.set(List.empty)
}
Expand Down
9 changes: 8 additions & 1 deletion backend/mockingbird/src/test/resources/nested.proto
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ message GetStocksRequest {
}

message GetStocksResponse {
enum StockKinds {
SK_UNKNOWN = 0;
SK_GROWTH = 1;
SK_INCOME = 2;
SK_VALUE = 3;
SK_BLUE_CHIP = 4;
}
message Stock {
int64 quantity = 1;
string name = 2;
Expand All @@ -20,4 +27,4 @@ message GetStocksResponse {

service StockService {
rpc GetStocks(GetStocksRequest) returns (GetStocksResponse);
}
}
Original file line number Diff line number Diff line change
@@ -1,90 +1,79 @@
package ru.tinkoff.tcb.protobuf

import java.io.File
import java.io.FileInputStream
import java.nio.file.Files
import java.nio.file.Paths
import scala.language.postfixOps
import scala.reflect.io.Directory
import scala.sys.process.*

import com.github.os72.protobuf.dynamic.DynamicSchema
import zio.managed.*
import zio.test.*

import ru.tinkoff.tcb.mockingbird.grpc.GrpcExractor.FromDynamicSchema
import ru.tinkoff.tcb.mockingbird.grpc.GrpcExractor.FromGrpcProtoDefinition
import ru.tinkoff.tcb.mockingbird.model.GrpcEnumSchema
import ru.tinkoff.tcb.mockingbird.model.GrpcMessageSchema
import ru.tinkoff.tcb.mockingbird.model.GrpcProtoDefinition
import ru.tinkoff.tcb.mockingbird.model.GrpcRootMessage

object MappersSpec extends ZIOSpecDefault {
val allTypesInRequests = Set(
".BrandAndModelRequest",
".CarGenRequest",
".CarSearchRequest",
".CarConfigurationsRequest",
".Condition",
".CarPriceRequest",
".PreciseCarPriceRequest",
".PriceRequest",
".MemoRequest",
".MemoRequest.ReqEntry", // <-- It's type of the "req" field
)

val allTypesInNested = Set(
".GetStocksRequest",
".GetStocksResponse",
".GetStocksResponse.StockKinds",
".GetStocksResponse.Stock",
".GetStocksResponse.Stocks",
)

override def spec: Spec[TestEnvironment & Scope, Any] =
suite("Mappers suite")(
test("Mappers from DynamicSchema to GrpcProtoDefinition") {
for {
content <- Utils.getProtoDescriptionFromResource("requests.proto")
schema = DynamicSchema.parseFrom(content)
protoDefinition = schema.toGrpcProtoDefinition
} yield assertTrue(getAllTypes(protoDefinition) == allTypesInRequests)
},
test("Mappers from DynamicSchema to GrpcProtoDefinition and back are consistent") {
val managed = ZManaged.acquireReleaseWith(ZIO.attemptBlockingIO(Files.createTempDirectory("temp"))) { path =>
ZIO.attemptBlockingIO {
val dir = new Directory(path.toFile)
dir.deleteRecursively()
}.orDie
}
(for {
path <- managed
readStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(
Files.newInputStream(
Paths.get("./mockingbird/src/test/resources/requests.proto")
)
)
}
writeStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO {
Files.newOutputStream(Paths.get(s"${path.toString}/requests.proto"))
}
}
_ <- ZIO.attemptBlockingIO(writeStream.write(readStream.readAllBytes())).toManaged
_ <- ZManaged.succeed {
s"protoc --descriptor_set_out=${path.toString}/descriptor.desc --proto_path=${path.toString} requests.proto" !
}
stream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(new FileInputStream(new File(s"${path.toString}/descriptor.desc")))
}
content <- ZIO.attemptBlockingIO(stream.readAllBytes()).toManaged
for {
content <- Utils.getProtoDescriptionFromResource("requests.proto")
schema = DynamicSchema.parseFrom(content)
protoDefinition = schema.toGrpcProtoDefinition
protoDefinitionAgain = protoDefinition.toDynamicSchema.toGrpcProtoDefinition
} yield assertTrue(protoDefinition == protoDefinitionAgain)).useNow
} yield assertTrue(protoDefinition == protoDefinitionAgain)
},
test("Mappers from nested DynamicSchema to GrpcProtoDefinition and back are consistent") {
val managed = ZManaged.acquireReleaseWith(ZIO.attemptBlockingIO(Files.createTempDirectory("temp"))) { path =>
ZIO.attemptBlockingIO {
val dir = new Directory(path.toFile)
dir.deleteRecursively()
}.orDie
}
(for {
path <- managed
readStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(
Files.newInputStream(
Paths.get("./mockingbird/src/test/resources/nested.proto")
)
)
}
writeStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO {
Files.newOutputStream(Paths.get(s"${path.toString}/nested.proto"))
}
}
_ <- ZIO.attemptBlockingIO(writeStream.write(readStream.readAllBytes())).toManaged
_ <- ZManaged.succeed {
s"protoc --descriptor_set_out=${path.toString}/nested_descriptor.desc --proto_path=${path.toString} nested.proto" !
}
stream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(new FileInputStream(new File(s"${path.toString}/nested_descriptor.desc")))
}
content <- ZIO.attemptBlockingIO(stream.readAllBytes()).toManaged
for {
content <- Utils.getProtoDescriptionFromResource("nested.proto")
schema = DynamicSchema.parseFrom(content)
protoDefinition = schema.toGrpcProtoDefinition
} yield assertTrue(getAllTypes(protoDefinition) == allTypesInNested)
},
test("Mappers from nested DynamicSchema to GrpcProtoDefinition and back are consistent") {
for {
content <- Utils.getProtoDescriptionFromResource("nested.proto")
schema = DynamicSchema.parseFrom(content)
protoDefinition = schema.toGrpcProtoDefinition
protoDefinitionAgain = protoDefinition.toDynamicSchema.toGrpcProtoDefinition
} yield assertTrue(protoDefinition == protoDefinitionAgain)).useNow
} yield assertTrue(protoDefinition == protoDefinitionAgain)
}
)

def getAllTypes(pd: GrpcProtoDefinition): Set[String] =
pd.schemas.flatMap(getAllTypes("")).toSet

def getAllTypes(packageName: String)(m: GrpcRootMessage): List[String] =
m match {
case GrpcEnumSchema(name, values) => s"$packageName.$name" :: Nil
case GrpcMessageSchema(name, fields, oneofs, nested, nestedEnums) =>
val npn = s"$packageName.$name"
npn :: nested.getOrElse(Nil).flatMap(getAllTypes(npn)) ::: nestedEnums.getOrElse(Nil).flatMap(getAllTypes(npn))
}
}
Original file line number Diff line number Diff line change
@@ -1,16 +1,8 @@
package ru.tinkoff.tcb.protobuf

import java.io.File
import java.io.FileInputStream
import java.nio.file.Files
import java.nio.file.Paths
import scala.jdk.CollectionConverters.SetHasAsScala
import scala.language.postfixOps
import scala.reflect.io.Directory
import scala.sys.process.*

import com.github.os72.protobuf.dynamic.DynamicSchema
import zio.managed.*
import zio.test.*

object ProtoToDescriptorSpec extends ZIOSpecDefault {
Expand All @@ -24,71 +16,19 @@ object ProtoToDescriptorSpec extends ZIOSpecDefault {
"utp.stock_service.v1.GetStocksResponse.Stocks"
)

override def spec: Spec[TestEnvironment, Any] =
override def spec: Spec[TestEnvironment & Scope, Any] =
suite("Proto to descriptor")(
test("DynamicSchema is successfully parsed from proto file") {
val managed = ZManaged.acquireReleaseWith(ZIO.attemptBlockingIO(Files.createTempDirectory("temp"))) { path =>
ZIO.attemptBlockingIO {
val dir = new Directory(path.toFile)
dir.deleteRecursively()
}.orDie
}
(for {
path <- managed
readStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(
Files.newInputStream(
Paths.get("./mockingbird/src/test/resources/requests.proto")
)
)
}
writeStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO {
Files.newOutputStream(Paths.get(s"${path.toString}/requests.proto"))
}
}
_ <- ZIO.attemptBlockingIO(writeStream.write(readStream.readAllBytes())).toManaged
_ <- ZManaged.succeed {
s"protoc --descriptor_set_out=${path.toString}/descriptor.desc --proto_path=${path.toString} requests.proto" !
}
stream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(new FileInputStream(new File(s"${path.toString}/descriptor.desc")))
}
content <- ZIO.attemptBlockingIO(stream.readAllBytes()).toManaged
for {
content <- Utils.getProtoDescriptionFromResource("requests.proto")
schema = DynamicSchema.parseFrom(content)
} yield assertTrue(messageTypes.subsetOf(schema.getMessageTypes.asScala))).useNow
} yield assertTrue(messageTypes.subsetOf(schema.getMessageTypes.asScala))
},
test("DynamicSchema is successfully parsed from proto file with nested schema") {
val managed = ZManaged.acquireReleaseWith(ZIO.attemptBlockingIO(Files.createTempDirectory("temp"))) { path =>
ZIO.attemptBlockingIO {
val dir = new Directory(path.toFile)
dir.deleteRecursively()
}.orDie
}
(for {
path <- managed
readStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(
Files.newInputStream(
Paths.get("./mockingbird/src/test/resources/nested.proto")
)
)
}
writeStream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO {
Files.newOutputStream(Paths.get(s"${path.toString}/nested.proto"))
}
}
_ <- ZIO.attemptBlockingIO(writeStream.write(readStream.readAllBytes())).toManaged
_ <- ZManaged.succeed {
s"protoc --descriptor_set_out=${path.toString}/nested_descriptor.desc --proto_path=${path.toString} nested.proto" !
}
stream <- ZManaged.fromAutoCloseable {
ZIO.attemptBlockingIO(new FileInputStream(new File(s"${path.toString}/nested_descriptor.desc")))
}
content <- ZIO.attemptBlockingIO(stream.readAllBytes()).toManaged
for {
content <- Utils.getProtoDescriptionFromResource("nested.proto")
schema = DynamicSchema.parseFrom(content)
} yield assertTrue(nestedMessageTypes.subsetOf(schema.getMessageTypes.asScala))).useNow
} yield assertTrue(nestedMessageTypes.subsetOf(schema.getMessageTypes.asScala))
}
)
}
Loading

0 comments on commit 96f3a7d

Please sign in to comment.