Merge pull request #213 from jsternberg/vtproto

protobuf: add vtproto to supplement protobuf marshaling

bench/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v1.0.0-rc93 // indirect
+	github.com/planetscale/vtprotobuf v0.6.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	golang.org/x/sys v0.11.0 // indirect

bench/go.sum

@@ -468,6 +468,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/moby/patternmatcher v0.5.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
+	github.com/planetscale/vtprotobuf v0.6.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205
 	golang.org/x/sync v0.1.0

go.sum

@@ -18,6 +18,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

hack/dockerfiles/generated-files.Dockerfile

@@ -18,23 +18,37 @@ RUN <<EOT
   rm -f /opt/protoc/readme.md
 EOT
 
+FROM base AS protoc-libs
+WORKDIR /app
+RUN --mount=type=bind,source=go.mod,target=/app/go.mod \
+    --mount=type=bind,source=go.sum,target=/app/go.sum \
+    --mount=type=cache,target=/root/.cache \
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+  set -e
+  mkdir -p /opt/protoc
+  go mod download github.com/planetscale/vtprotobuf
+  cp -r $(go list -m -f='{{.Dir}}' github.com/planetscale/vtprotobuf)/include /opt/protoc
+EOT
+
 FROM base AS tools
 WORKDIR /app
 RUN --mount=type=bind,source=go.mod,target=/app/go.mod \
     --mount=type=bind,source=go.sum,target=/app/go.sum \
     --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod \
   go install \
+    github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto \
     google.golang.org/protobuf/cmd/protoc-gen-go
 COPY --link --from=protoc /opt/protoc /usr/local
+COPY --link --from=protoc-libs /opt/protoc /usr/local
 
 FROM tools AS generate
 RUN --mount=target=github.com/tonistiigi/fsutil \
     --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod <<EOT
   set -e
   mkdir /out
-  find github.com/tonistiigi/fsutil -name '*.proto' | xargs protoc --go_out=/out
+  find github.com/tonistiigi/fsutil -name '*.proto' | xargs protoc --go_out=/out --go-vtproto_out=/out
 EOT
 
 FROM scratch AS update

receive.go

@@ -197,7 +197,7 @@ func (r *receiver) run(ctx context.Context) error {
 		}
 		var p types.Packet
 		for {
-			p = types.Packet{Data: p.Data[:0]}
+			p.ResetVT()
 			if err := r.conn.RecvMsg(&p); err != nil {
 				return err
 			}

tools/tools.go

@@ -3,5 +3,6 @@
 package tools
 
 import (
+	_ "github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto"
 	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
 )

types/stat.go

@@ -1,40 +1,19 @@
 package types
 
-import (
-	"os"
-
-	"google.golang.org/protobuf/proto"
-)
+import "os"
 
 func (s *Stat) IsDir() bool {
 	return os.FileMode(s.Mode).IsDir()
 }
 
 func (s *Stat) Marshal() ([]byte, error) {
-	return proto.MarshalOptions{Deterministic: true}.Marshal(s)
+	return s.MarshalVTStrict()
 }
 
 func (s *Stat) Unmarshal(dAtA []byte) error {
-	return proto.UnmarshalOptions{Merge: true}.Unmarshal(dAtA, s)
+	return s.UnmarshalVT(dAtA)
 }
 
 func (s *Stat) Clone() *Stat {
-	clone := &Stat{
-		Path:     s.Path,
-		Mode:     s.Mode,
-		Uid:      s.Uid,
-		Gid:      s.Gid,
-		Size:     s.Size,
-		ModTime:  s.ModTime,
-		Linkname: s.Linkname,
-		Devmajor: s.Devmajor,
-		Devminor: s.Devminor,
-	}
-	if s.Xattrs != nil {
-		s.Xattrs = make(map[string][]byte, len(s.Xattrs))
-		for k, v := range s.Xattrs {
-			clone.Xattrs[k] = v
-		}
-	}
-	return clone
+	return s.CloneVT()
 }

types/stat.proto

@@ -4,6 +4,8 @@ package fsutil.types;
 
 option go_package = "github.com/tonistiigi/fsutil/types";
 
+import "github.com/planetscale/vtprotobuf/vtproto/ext.proto";
+
 message Stat {
   string path = 1;
   uint32 mode = 2;