From d0d6cb970eab692f345557c3dde7cc7c9b27bdbf Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Tue, 15 Oct 2024 23:26:27 -0400 Subject: [PATCH 01/21] feat(issue-33): rough out upload command --- cmd/griot/main.go | 33 ++++++++ cmd/griot/upload/upload.go | 67 +++++++++++++++ cmd/internal/command/command.go | 141 ++++++++++++++++++++++++++++++++ go.mod | 9 +- go.sum | 12 +++ 5 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 cmd/griot/main.go create mode 100644 cmd/griot/upload/upload.go create mode 100644 cmd/internal/command/command.go diff --git a/cmd/griot/main.go b/cmd/griot/main.go new file mode 100644 index 0000000..46138ea --- /dev/null +++ b/cmd/griot/main.go @@ -0,0 +1,33 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/spf13/pflag" + "github.com/z5labs/griot/cmd/griot/upload" + "github.com/z5labs/griot/cmd/internal/command" +) + +func main() { + cmd := command.New( + "griot", + command.PersistentFlags(func(fs *pflag.FlagSet) { + fs.Var(&command.DefaultMinLogLevel, "log-level", "Set the minimum log level.") + }), + command.Sub(upload.New()), + ) + + command.Run(cmd) +} diff --git a/cmd/griot/upload/upload.go b/cmd/griot/upload/upload.go new file mode 100644 index 0000000..9ceb3f3 --- /dev/null +++ b/cmd/griot/upload/upload.go @@ -0,0 +1,67 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upload + +import ( + "context" + "io" + "log/slog" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/z5labs/griot/cmd/internal/command" +) + +func New() *cobra.Command { + return command.New( + "upload", + command.Short("Upload content"), + command.Flags(func(fs *pflag.FlagSet) { + fs.String("name", "", "Provide an optional name to help identify this content later.") + fs.String("media-type", "", "Specify the content Media Type.") + fs.String("source-file", "", "Specify the content source file.") + }), + command.Handle(initUploadHandler), + ) +} + +type config struct { + command.LoggingConfig `flag:",squash"` + + Name string `flag:"name"` + MediaType string `flag:"media-type"` + SourceFile string `flag:"source-file"` +} + +type handler struct { + log *slog.Logger + + contentName string + mediaType string + filename string + out io.Writer +} + +func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) { + h := &handler{ + log: command.Logger(cfg.LoggingConfig), + } + return h, nil +} + +func (h *handler) Handle(ctx context.Context) error { + h.log.InfoContext(ctx, "hello") + return nil +} diff --git a/cmd/internal/command/command.go b/cmd/internal/command/command.go new file mode 100644 index 0000000..046aae2 --- /dev/null +++ b/cmd/internal/command/command.go @@ -0,0 +1,141 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "context" + "log/slog" + "os" + "os/signal" + + "github.com/go-viper/mapstructure/v2" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var DefaultMinLogLevel LogLevel + +func init() { + DefaultMinLogLevel.Set(slog.LevelWarn.String()) +} + +type LogLevel slog.LevelVar + +func (l *LogLevel) Set(v string) error { + return (*slog.LevelVar)(l).UnmarshalText([]byte(v)) +} + +func (l *LogLevel) String() string { + return (*slog.LevelVar)(l).Level().String() +} + +func (l *LogLevel) Type() string { + return "slog.Level" +} + +type LoggingConfig struct { + Level *LogLevel `flag:"log-level"` +} + +func Logger(cfg LoggingConfig) *slog.Logger { + return slog.New(slog.NewJSONHandler( + os.Stderr, + &slog.HandlerOptions{ + AddSource: true, + Level: (*slog.LevelVar)(cfg.Level).Level(), + }, + )) +} + +type Option func(*cobra.Command) + +func Short(desription string) Option { + return func(c *cobra.Command) { + c.Short = desription + } +} + +func Flags(f func(*pflag.FlagSet)) Option { + return func(c *cobra.Command) { + f(c.Flags()) + } +} + +func PersistentFlags(f func(*pflag.FlagSet)) Option { + return func(c *cobra.Command) { + f(c.PersistentFlags()) + } +} + +func Sub(sub *cobra.Command) Option { + return func(c *cobra.Command) { + c.AddCommand(sub) + } +} + +type Handler interface { + Handle(context.Context) error +} + +func Handle[T any](f func(context.Context, T) (Handler, error)) Option { + return func(c *cobra.Command) { + c.RunE = func(cmd *cobra.Command, args []string) error { + m := make(map[string]any) + cmd.Flags().VisitAll(func(f *pflag.Flag) { + m[f.Name] = f.Value + }) + + var cfg T + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &cfg, + TagName: "flag", + }) + if err != nil { + return err + } + + err = dec.Decode(m) + if err != nil { + return err + } + + h, err := f(cmd.Context(), cfg) + if err != nil { + return err + } + return h.Handle(cmd.Context()) + } + } +} + +func New(use string, opts ...Option) *cobra.Command { + cmd := &cobra.Command{ + Use: use, + } + for _, opt := range opts { + opt(cmd) + } + return cmd +} + +func Run(c *cobra.Command) { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer cancel() + + err := c.ExecuteContext(ctx) + if err == nil { + return + } +} diff --git a/go.mod b/go.mod index bd357f3..9ebdc91 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,11 @@ module github.com/z5labs/griot go 1.23.0 -require google.golang.org/protobuf v1.35.1 +require ( + github.com/go-viper/mapstructure/v2 v2.2.1 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + google.golang.org/protobuf v1.35.1 +) + +require github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 00c144d..70f294c 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,18 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 21691cd7b2f9c8f5340aad5c5a4412830d1ab173 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Wed, 16 Oct 2024 00:13:28 -0400 Subject: [PATCH 02/21] feat(issue-33): rough out content client --- go.mod | 9 ++++++- go.sum | 24 +++++++++++++++---- services/content/content.go | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 services/content/content.go diff --git a/go.mod b/go.mod index 9ebdc91..dbf3836 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,14 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 + go.opentelemetry.io/otel v1.31.0 google.golang.org/protobuf v1.35.1 ) -require github.com/inconshreveable/mousetrap v1.1.0 // indirect +require ( + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect +) diff --git a/go.sum b/go.sum index 70f294c..d42b4e7 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,34 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/content/content.go b/services/content/content.go new file mode 100644 index 0000000..20a4fd8 --- /dev/null +++ b/services/content/content.go @@ -0,0 +1,47 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package content +package content + +import ( + "context" + "io" + + "github.com/z5labs/griot/services/content/contentpb" + + "go.opentelemetry.io/otel" +) + +type Client struct{} + +func NewClient() *Client { + c := &Client{} + return c +} + +type UploadContentRequest struct { + Metadata *contentpb.Metadata + Body io.Reader +} + +type UploadContentResponse struct { + Id string `json:"id"` +} + +func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) (*UploadContentResponse, error) { + _, span := otel.Tracer("content").Start(ctx, "Client.UploadContent") + defer span.End() + return nil, nil +} From 0346e37312323a3de057fc8e8de066f1983cea43 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Wed, 16 Oct 2024 00:14:11 -0400 Subject: [PATCH 03/21] feat(issue-33): fully implement upload command --- cmd/griot/upload/upload.go | 87 +++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/cmd/griot/upload/upload.go b/cmd/griot/upload/upload.go index 9ceb3f3..25bd5ba 100644 --- a/cmd/griot/upload/upload.go +++ b/cmd/griot/upload/upload.go @@ -16,12 +16,22 @@ package upload import ( "context" + "crypto/sha256" + "encoding/json" + "fmt" + "hash" "io" "log/slog" + "os" + "strings" + + "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/services/content" + "github.com/z5labs/griot/services/content/contentpb" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/z5labs/griot/cmd/internal/command" + "go.opentelemetry.io/otel" ) func New() *cobra.Command { @@ -32,6 +42,7 @@ func New() *cobra.Command { fs.String("name", "", "Provide an optional name to help identify this content later.") fs.String("media-type", "", "Specify the content Media Type.") fs.String("source-file", "", "Specify the content source file.") + fs.String("hash-func", "sha256", "Specify hash function used for calculating content checksum.") }), command.Handle(initUploadHandler), ) @@ -43,6 +54,11 @@ type config struct { Name string `flag:"name"` MediaType string `flag:"media-type"` SourceFile string `flag:"source-file"` + HashFunc string `flag:"hash-func"` +} + +type uploadClient interface { + UploadContent(context.Context, *content.UploadContentRequest) (*content.UploadContentResponse, error) } type handler struct { @@ -50,18 +66,79 @@ type handler struct { contentName string mediaType string - filename string + hasher hash.Hash + src io.ReadSeeker out io.Writer + + content uploadClient } func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) { + var hasher hash.Hash + switch cfg.HashFunc { + case "sha256": + hasher = sha256.New() + default: + return nil, fmt.Errorf("unsupported hash function: %s", cfg.HashFunc) + } + + src, err := openSourceFile(cfg.SourceFile) + if err != nil { + return nil, err + } + h := &handler{ - log: command.Logger(cfg.LoggingConfig), + log: command.Logger(cfg.LoggingConfig), + contentName: cfg.Name, + mediaType: cfg.MediaType, + hasher: hasher, + src: src, + out: os.Stdout, + content: content.NewClient(), } return h, nil } func (h *handler) Handle(ctx context.Context) error { - h.log.InfoContext(ctx, "hello") - return nil + spanCtx, span := otel.Tracer("upload").Start(ctx, "handler.Handle") + defer span.End() + + _, err := io.Copy(h.hasher, h.src) + if err != nil { + h.log.ErrorContext(spanCtx, "failed to compute hash", slog.String("error", err.Error())) + return err + } + + _, err = h.src.Seek(0, 0) + if err != nil { + h.log.ErrorContext(spanCtx, "failed to seek to start of content source", slog.String("error", err.Error())) + return err + } + + req := &content.UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Name: &h.contentName, + Checksum: &contentpb.Checksum{ + HashFunc: contentpb.HashFunc_SHA256.Enum(), + Hash: h.hasher.Sum(nil), + }, + }, + Body: h.src, + } + resp, err := h.content.UploadContent(spanCtx, req) + if err != nil { + h.log.ErrorContext(spanCtx, "failed to upload content", slog.String("error", err.Error())) + return err + } + + enc := json.NewEncoder(h.out) + return enc.Encode(resp) +} + +func openSourceFile(filename string) (*os.File, error) { + filename = strings.TrimSpace(filename) + if len(filename) == 0 { + return os.Stdin, nil + } + return os.Open(filename) } From b7e1701fb6988187a8b103b2cc6f07d98074a241 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Wed, 16 Oct 2024 00:56:40 -0400 Subject: [PATCH 04/21] feat(issue-33): add some otel support --- cmd/griot/main.go | 3 + cmd/griot/upload/upload.go | 8 +- cmd/internal/command/command.go | 134 ++++++++++++++++++++++++++++---- go.mod | 4 + go.sum | 8 ++ 5 files changed, 143 insertions(+), 14 deletions(-) diff --git a/cmd/griot/main.go b/cmd/griot/main.go index 46138ea..f58b1dd 100644 --- a/cmd/griot/main.go +++ b/cmd/griot/main.go @@ -25,6 +25,9 @@ func main() { "griot", command.PersistentFlags(func(fs *pflag.FlagSet) { fs.Var(&command.DefaultMinLogLevel, "log-level", "Set the minimum log level.") + + fs.Bool("enable-otel", false, "Enable OpenTelemetry tracing.") + fs.String("trace-out", "", "Specify output file for OpenTelemetry trace data. If not set, a temp file will be created and used.") }), command.Sub(upload.New()), ) diff --git a/cmd/griot/upload/upload.go b/cmd/griot/upload/upload.go index 25bd5ba..5eb0ec3 100644 --- a/cmd/griot/upload/upload.go +++ b/cmd/griot/upload/upload.go @@ -74,6 +74,11 @@ type handler struct { } func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) { + spanCtx, span := otel.Tracer("upload").Start(ctx, "initUploadHandler") + defer span.End() + + log := command.Logger(cfg.LoggingConfig) + var hasher hash.Hash switch cfg.HashFunc { case "sha256": @@ -84,11 +89,12 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) src, err := openSourceFile(cfg.SourceFile) if err != nil { + log.ErrorContext(spanCtx, "failed to open source file", slog.String("error", err.Error())) return nil, err } h := &handler{ - log: command.Logger(cfg.LoggingConfig), + log: log, contentName: cfg.Name, mediaType: cfg.MediaType, hasher: hasher, diff --git a/cmd/internal/command/command.go b/cmd/internal/command/command.go index 046aae2..f5b98f3 100644 --- a/cmd/internal/command/command.go +++ b/cmd/internal/command/command.go @@ -16,13 +16,21 @@ package command import ( "context" + "errors" + "io" "log/slog" "os" "os/signal" + "strings" "github.com/go-viper/mapstructure/v2" "github.com/spf13/cobra" "github.com/spf13/pflag" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) var DefaultMinLogLevel LogLevel @@ -59,6 +67,11 @@ func Logger(cfg LoggingConfig) *slog.Logger { )) } +type OTelConfig struct { + Enabled bool `flag:"enable-otel"` + TraceDestination string `flag:"trace-out"` +} + type Option func(*cobra.Command) func Short(desription string) Option { @@ -91,35 +104,125 @@ type Handler interface { func Handle[T any](f func(context.Context, T) (Handler, error)) Option { return func(c *cobra.Command) { - c.RunE = func(cmd *cobra.Command, args []string) error { - m := make(map[string]any) - cmd.Flags().VisitAll(func(f *pflag.Flag) { - m[f.Name] = f.Value - }) + var postRunHooks []func(context.Context) error - var cfg T - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - Result: &cfg, - TagName: "flag", - }) + c.PreRunE = func(cmd *cobra.Command, args []string) error { + var cfg OTelConfig + err := decodeFlags(cmd.Flags(), &cfg) + if err != nil { + return err + } + + if !cfg.Enabled { + return nil + } + + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.Baggage{}, + propagation.TraceContext{}, + )) + + out, err := openDestination(cfg.TraceDestination) + if err != nil { + return err + } + + tp, err := initTracerProvider(cmd.Context(), out) if err != nil { return err } - err = dec.Decode(m) + otel.SetTracerProvider(tp) + postRunHooks = append(postRunHooks, tp.Shutdown) + return nil + } + + c.RunE = func(cmd *cobra.Command, args []string) error { + spanCtx, span := otel.Tracer("command").Start(cmd.Context(), "cobra.Command.RunE") + defer span.End() + + var cfg T + err := decodeFlags(cmd.Flags(), &cfg) if err != nil { return err } - h, err := f(cmd.Context(), cfg) + h, err := f(spanCtx, cfg) if err != nil { return err } - return h.Handle(cmd.Context()) + return h.Handle(spanCtx) + } + + c.PostRunE = func(cmd *cobra.Command, args []string) error { + var errs []error + for _, hook := range postRunHooks { + err := hook(cmd.Context()) + if err == nil { + continue + } + errs = append(errs, err) + } + if len(errs) == 0 { + return nil + } + if len(errs) == 1 { + return errs[0] + } + return errors.Join(errs...) } } } +func decodeFlags(fs *pflag.FlagSet, v interface{}) error { + m := make(map[string]any) + fs.VisitAll(func(f *pflag.Flag) { + m[f.Name] = f.Value + }) + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: v, + TagName: "flag", + }) + if err != nil { + return err + } + return dec.Decode(m) +} + +func openDestination(filename string) (*os.File, error) { + filename = strings.TrimSpace(filename) + if len(filename) > 0 { + return os.Create(filename) + } + return os.CreateTemp("", "griot_traces_*") +} + +func initTracerProvider(ctx context.Context, out io.Writer) (*sdktrace.TracerProvider, error) { + rsc, err := resource.Detect(ctx) + if err != nil { + return nil, err + } + + exp, err := stdouttrace.New( + stdouttrace.WithWriter(out), + ) + if err != nil { + return nil, err + } + + sp := sdktrace.NewSimpleSpanProcessor( + exp, + ) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(rsc), + sdktrace.WithSpanProcessor(sp), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + return tp, nil +} + func New(use string, opts ...Option) *cobra.Command { cmd := &cobra.Command{ Use: use, @@ -138,4 +241,9 @@ func Run(c *cobra.Command) { if err == nil { return } + + log := Logger(LoggingConfig{ + Level: &DefaultMinLogLevel, + }) + log.Error("encountered unexpected error", slog.String("error", err.Error())) } diff --git a/go.mod b/go.mod index dbf3836..48e7331 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,17 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 + go.opentelemetry.io/otel/sdk v1.31.0 google.golang.org/protobuf v1.35.1 ) require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect + golang.org/x/sys v0.26.0 // indirect ) diff --git a/go.sum b/go.sum index d42b4e7..3704c78 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIx github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -23,10 +25,16 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e9ac24be3b2fd6fe554065a7ef18c7cc102fa3d4 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sun, 20 Oct 2024 18:10:46 -0400 Subject: [PATCH 05/21] refactor(issue-33): added resource specific command for content management --- cmd/griot/content/content.go | 30 ++++++++++++++++++++++++ cmd/griot/{ => content}/upload/upload.go | 22 ++++++++++------- cmd/griot/main.go | 4 ++-- 3 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 cmd/griot/content/content.go rename cmd/griot/{ => content}/upload/upload.go (90%) diff --git a/cmd/griot/content/content.go b/cmd/griot/content/content.go new file mode 100644 index 0000000..61f299a --- /dev/null +++ b/cmd/griot/content/content.go @@ -0,0 +1,30 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import ( + "github.com/z5labs/griot/cmd/griot/content/upload" + "github.com/z5labs/griot/cmd/internal/command" + + "github.com/spf13/cobra" +) + +func New() *cobra.Command { + return command.New( + "content", + command.Short("Manage content"), + command.Sub(upload.New()), + ) +} diff --git a/cmd/griot/upload/upload.go b/cmd/griot/content/upload/upload.go similarity index 90% rename from cmd/griot/upload/upload.go rename to cmd/griot/content/upload/upload.go index 5eb0ec3..9a62586 100644 --- a/cmd/griot/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -22,7 +22,9 @@ import ( "hash" "io" "log/slog" + "maps" "os" + "slices" "strings" "github.com/z5labs/griot/cmd/internal/command" @@ -42,7 +44,14 @@ func New() *cobra.Command { fs.String("name", "", "Provide an optional name to help identify this content later.") fs.String("media-type", "", "Specify the content Media Type.") fs.String("source-file", "", "Specify the content source file.") - fs.String("hash-func", "sha256", "Specify hash function used for calculating content checksum.") + fs.String( + "hash-func", + contentpb.HashFunc_SHA256.String(), + fmt.Sprintf( + "Specify hash function used for calculating content checksum. (values %s)", + strings.Join(slices.Collect(maps.Values(contentpb.HashFunc_name)), ","), + ), + ) }), command.Handle(initUploadHandler), ) @@ -122,14 +131,9 @@ func (h *handler) Handle(ctx context.Context) error { } req := &content.UploadContentRequest{ - Metadata: &contentpb.Metadata{ - Name: &h.contentName, - Checksum: &contentpb.Checksum{ - HashFunc: contentpb.HashFunc_SHA256.Enum(), - Hash: h.hasher.Sum(nil), - }, - }, - Body: h.src, + Name: h.contentName, + HashFunc: contentpb.HashFunc_SHA256, + Body: h.src, } resp, err := h.content.UploadContent(spanCtx, req) if err != nil { diff --git a/cmd/griot/main.go b/cmd/griot/main.go index f58b1dd..e55032a 100644 --- a/cmd/griot/main.go +++ b/cmd/griot/main.go @@ -16,7 +16,7 @@ package main import ( "github.com/spf13/pflag" - "github.com/z5labs/griot/cmd/griot/upload" + "github.com/z5labs/griot/cmd/griot/content" "github.com/z5labs/griot/cmd/internal/command" ) @@ -29,7 +29,7 @@ func main() { fs.Bool("enable-otel", false, "Enable OpenTelemetry tracing.") fs.String("trace-out", "", "Specify output file for OpenTelemetry trace data. If not set, a temp file will be created and used.") }), - command.Sub(upload.New()), + command.Sub(content.New()), ) command.Run(cmd) From ff6767d504c7f93d95364d050b6bcd95124946cb Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sun, 20 Oct 2024 18:11:23 -0400 Subject: [PATCH 06/21] refactor(issue-33): flatten client request --- services/content/content.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/content/content.go b/services/content/content.go index 20a4fd8..9469782 100644 --- a/services/content/content.go +++ b/services/content/content.go @@ -32,7 +32,8 @@ func NewClient() *Client { } type UploadContentRequest struct { - Metadata *contentpb.Metadata + Name string + HashFunc contentpb.HashFunc Body io.Reader } From 435cc086064c5595f1546f64a7435c6fd8b2380b Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 01:41:49 -0400 Subject: [PATCH 07/21] refactor(issue-33): use humus for cli --- cmd/internal/command/command.go | 234 ++++++++++---------------------- go.mod | 24 +++- go.sum | 58 +++++++- 3 files changed, 147 insertions(+), 169 deletions(-) diff --git a/cmd/internal/command/command.go b/cmd/internal/command/command.go index f5b98f3..193fce2 100644 --- a/cmd/internal/command/command.go +++ b/cmd/internal/command/command.go @@ -18,160 +18,126 @@ import ( "context" "errors" "io" - "log/slog" - "os" - "os/signal" - "strings" "github.com/go-viper/mapstructure/v2" "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/z5labs/humus" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -var DefaultMinLogLevel LogLevel +type Option func(*App) -func init() { - DefaultMinLogLevel.Set(slog.LevelWarn.String()) +func Short(desription string) Option { + return func(a *App) { + a.cmd.Short = desription + } } -type LogLevel slog.LevelVar - -func (l *LogLevel) Set(v string) error { - return (*slog.LevelVar)(l).UnmarshalText([]byte(v)) +func Flags(f func(*pflag.FlagSet)) Option { + return func(a *App) { + f(a.cmd.Flags()) + } } -func (l *LogLevel) String() string { - return (*slog.LevelVar)(l).Level().String() +func PersistentFlags(f func(*pflag.FlagSet)) Option { + return func(a *App) { + f(a.cmd.PersistentFlags()) + } } -func (l *LogLevel) Type() string { - return "slog.Level" +func Sub(sub *App) Option { + return func(a *App) { + a.cmd.AddCommand(sub.cmd) + } } -type LoggingConfig struct { - Level *LogLevel `flag:"log-level"` +type Validator interface { + Validate(context.Context) error } -func Logger(cfg LoggingConfig) *slog.Logger { - return slog.New(slog.NewJSONHandler( - os.Stderr, - &slog.HandlerOptions{ - AddSource: true, - Level: (*slog.LevelVar)(cfg.Level).Level(), - }, - )) -} +type ValidatorFunc func(context.Context) error -type OTelConfig struct { - Enabled bool `flag:"enable-otel"` - TraceDestination string `flag:"trace-out"` +func (f ValidatorFunc) Validate(ctx context.Context) error { + return f(ctx) } -type Option func(*cobra.Command) - -func Short(desription string) Option { - return func(c *cobra.Command) { - c.Short = desription +func ValidateAll(ctx context.Context, vs ...Validator) error { + errs := make([]error, 0, len(vs)) + for _, v := range vs { + err := v.Validate(ctx) + if err == nil { + continue + } + errs = append(errs, err) } -} - -func Flags(f func(*pflag.FlagSet)) Option { - return func(c *cobra.Command) { - f(c.Flags()) + if len(errs) == 0 { + return nil } -} - -func PersistentFlags(f func(*pflag.FlagSet)) Option { - return func(c *cobra.Command) { - f(c.PersistentFlags()) - } -} - -func Sub(sub *cobra.Command) Option { - return func(c *cobra.Command) { - c.AddCommand(sub) + if len(errs) == 1 { + return errs[0] } + return errors.Join(errs...) } type Handler interface { Handle(context.Context) error } -func Handle[T any](f func(context.Context, T) (Handler, error)) Option { - return func(c *cobra.Command) { - var postRunHooks []func(context.Context) error +func Handle[T Validator](f func(context.Context, T) (Handler, error)) Option { + return func(a *App) { + a.cmd.RunE = func(cmd *cobra.Command, args []string) error { + spanCtx, span := otel.Tracer("command").Start(cmd.Context(), "cobra.Command.RunE") + defer span.End() - c.PreRunE = func(cmd *cobra.Command, args []string) error { - var cfg OTelConfig + var cfg T err := decodeFlags(cmd.Flags(), &cfg) if err != nil { return err } - if !cfg.Enabled { - return nil - } - - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( - propagation.Baggage{}, - propagation.TraceContext{}, - )) - - out, err := openDestination(cfg.TraceDestination) + err = cfg.Validate(spanCtx) if err != nil { return err } - tp, err := initTracerProvider(cmd.Context(), out) + h, err := f(spanCtx, cfg) if err != nil { return err } - - otel.SetTracerProvider(tp) - postRunHooks = append(postRunHooks, tp.Shutdown) - return nil + return h.Handle(spanCtx) } + } +} - c.RunE = func(cmd *cobra.Command, args []string) error { - spanCtx, span := otel.Tracer("command").Start(cmd.Context(), "cobra.Command.RunE") - defer span.End() +type App struct { + cmd *cobra.Command +} - var cfg T - err := decodeFlags(cmd.Flags(), &cfg) - if err != nil { - return err - } +func NewApp(name string, opts ...Option) *App { + a := &App{ + cmd: &cobra.Command{ + Use: name, + }, + } + for _, opt := range opts { + opt(a) + } + return a +} - h, err := f(spanCtx, cfg) - if err != nil { - return err - } - return h.Handle(spanCtx) +func Run[T any](r io.Reader, f func(context.Context, T) (*App, error)) { + humus.Run(r, func(ctx context.Context, cfg T) (humus.App, error) { + app, err := f(ctx, cfg) + if err != nil { + return nil, err } + return app, nil + }) +} - c.PostRunE = func(cmd *cobra.Command, args []string) error { - var errs []error - for _, hook := range postRunHooks { - err := hook(cmd.Context()) - if err == nil { - continue - } - errs = append(errs, err) - } - if len(errs) == 0 { - return nil - } - if len(errs) == 1 { - return errs[0] - } - return errors.Join(errs...) - } - } +func (a *App) Run(ctx context.Context) error { + return a.cmd.ExecuteContext(ctx) } func decodeFlags(fs *pflag.FlagSet, v interface{}) error { @@ -189,61 +155,3 @@ func decodeFlags(fs *pflag.FlagSet, v interface{}) error { } return dec.Decode(m) } - -func openDestination(filename string) (*os.File, error) { - filename = strings.TrimSpace(filename) - if len(filename) > 0 { - return os.Create(filename) - } - return os.CreateTemp("", "griot_traces_*") -} - -func initTracerProvider(ctx context.Context, out io.Writer) (*sdktrace.TracerProvider, error) { - rsc, err := resource.Detect(ctx) - if err != nil { - return nil, err - } - - exp, err := stdouttrace.New( - stdouttrace.WithWriter(out), - ) - if err != nil { - return nil, err - } - - sp := sdktrace.NewSimpleSpanProcessor( - exp, - ) - - tp := sdktrace.NewTracerProvider( - sdktrace.WithResource(rsc), - sdktrace.WithSpanProcessor(sp), - sdktrace.WithSampler(sdktrace.AlwaysSample()), - ) - return tp, nil -} - -func New(use string, opts ...Option) *cobra.Command { - cmd := &cobra.Command{ - Use: use, - } - for _, opt := range opts { - opt(cmd) - } - return cmd -} - -func Run(c *cobra.Command) { - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) - defer cancel() - - err := c.ExecuteContext(ctx) - if err == nil { - return - } - - log := Logger(LoggingConfig{ - Level: &DefaultMinLogLevel, - }) - log.Error("encountered unexpected error", slog.String("error", err.Error())) -} diff --git a/go.mod b/go.mod index 48e7331..5e589f8 100644 --- a/go.mod +++ b/go.mod @@ -6,18 +6,38 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 + github.com/z5labs/humus v0.2.0 go.opentelemetry.io/otel v1.31.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 - go.opentelemetry.io/otel/sdk v1.31.0 google.golang.org/protobuf v1.35.1 ) require ( + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/z5labs/bedrock v0.12.0 // indirect + go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3704c78..3894464 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -12,10 +14,18 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -23,20 +33,60 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/z5labs/bedrock v0.12.0 h1:mv5EQfYXfInlO++IRqsxcPcGCbKPcWKC6oTDP9VMm4E= +github.com/z5labs/bedrock v0.12.0/go.mod h1:IdEKPA+TgqDFsSVD6L+elp9eJGmSL3xQ61bC7UjNmQA= +github.com/z5labs/humus v0.2.0 h1:v/gCV2qo3leP7PuYxwmFz8qhS8hboN163PCQtm4WMMQ= +github.com/z5labs/humus v0.2.0/go.mod h1:ypGTw5pq4tlo1yFk1vWYdrUWjqx9aGKUJ0JcpJnp8ps= +go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 h1:lU3F57OSLK5mQ1PDBVAfDDaKCPv37MrEbCfTzsF4bz0= +go.opentelemetry.io/contrib/bridges/otelslog v0.5.0/go.mod h1:I84u06zJFr8T5D73fslEUbnRBimVVSBhuVw8L8I92AU= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 h1:WYsDPt0fM4KZaMhLvY+x6TVXd85P/KNl3Ez3t+0+kGs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0/go.mod h1:vfY4arMmvljeXPNJOE0idEwuoPMjAPCWmBMmj6R5Ksw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 h1:WypxHH02KX2poqqbaadmkMYalGyy/vil4HE4PM4nRJc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0/go.mod h1:U79SV99vtvGSEBeeHnpgGJfTsnsdkWLpPN/CcHAzBSI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 h1:m0yTiGDLUvVYaTFbAvCkVYIYcvwKt3G7OLoN77NUs/8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0/go.mod h1:wBQbT4UekBfegL2nx0Xk1vBcnzyBPsIVm9hRG4fYcr4= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 h1:bZHOb8k/CwwSt0DgvgaoOhBXWNdWqFWaIsGTtg1H3KE= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0/go.mod h1:XlV163j81kDdIt5b5BXCjdqVfqJFy/LJrHA697SorvQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0 h1:IyFlqNsi8VT/nwYlLJfdM0y1gavxGpEvnf6FtVfZ6X4= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0/go.mod h1:bxiX8eUeKoAEQmbq/ecUT8UqZwCjZW52yJrXJUSozsk= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= +google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 1c4ec2a709f28d9390587e970d0fdaf624ffbd00 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 01:43:03 -0400 Subject: [PATCH 08/21] refactor(issue-33): updated after migrating to humus for command package --- cmd/griot/app/app.go | 32 +++++++++++ cmd/griot/config.yaml | 16 ++++++ cmd/griot/content/content.go | 6 +- cmd/griot/content/upload/upload.go | 91 +++++++++++++++++++++++------- cmd/griot/main.go | 22 +++----- 5 files changed, 128 insertions(+), 39 deletions(-) create mode 100644 cmd/griot/app/app.go create mode 100644 cmd/griot/config.yaml diff --git a/cmd/griot/app/app.go b/cmd/griot/app/app.go new file mode 100644 index 0000000..bcdd14c --- /dev/null +++ b/cmd/griot/app/app.go @@ -0,0 +1,32 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + + "github.com/z5labs/griot/cmd/griot/content" + "github.com/z5labs/griot/cmd/internal/command" +) + +type Config struct{} + +func Init(ctx context.Context, cfg Config) (*command.App, error) { + app := command.NewApp( + "griot", + command.Sub(content.New()), + ) + return app, nil +} diff --git a/cmd/griot/config.yaml b/cmd/griot/config.yaml new file mode 100644 index 0000000..cf5215d --- /dev/null +++ b/cmd/griot/config.yaml @@ -0,0 +1,16 @@ +# Copyright 2024 Z5Labs and Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logging: + level: {{env "LOGGING_LEVEL" | default "WARN"}} \ No newline at end of file diff --git a/cmd/griot/content/content.go b/cmd/griot/content/content.go index 61f299a..f3f9602 100644 --- a/cmd/griot/content/content.go +++ b/cmd/griot/content/content.go @@ -17,12 +17,10 @@ package content import ( "github.com/z5labs/griot/cmd/griot/content/upload" "github.com/z5labs/griot/cmd/internal/command" - - "github.com/spf13/cobra" ) -func New() *cobra.Command { - return command.New( +func New() *command.App { + return command.NewApp( "content", command.Short("Manage content"), command.Sub(upload.New()), diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 9a62586..4262f5f 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -18,11 +18,13 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "hash" "io" "log/slog" "maps" + "mime" "os" "slices" "strings" @@ -30,23 +32,44 @@ import ( "github.com/z5labs/griot/cmd/internal/command" "github.com/z5labs/griot/services/content" "github.com/z5labs/griot/services/content/contentpb" + "github.com/z5labs/humus" - "github.com/spf13/cobra" "github.com/spf13/pflag" "go.opentelemetry.io/otel" ) -func New() *cobra.Command { - return command.New( +type hashFuncFlag contentpb.HashFunc + +func (x *hashFuncFlag) Set(v string) error { + h, exists := contentpb.HashFunc_value[v] + if !exists { + return fmt.Errorf("unknown value for contentpb.HashFunc: %s", v) + } + *x = hashFuncFlag(h) + return nil +} + +func (x hashFuncFlag) String() string { + return contentpb.HashFunc(x).String() +} + +func (hashFuncFlag) Type() string { + return "contentpb.HashFunc" +} + +func New() *command.App { + defaultHashFunc := hashFuncFlag(contentpb.HashFunc_SHA256) + + return command.NewApp( "upload", command.Short("Upload content"), command.Flags(func(fs *pflag.FlagSet) { fs.String("name", "", "Provide an optional name to help identify this content later.") fs.String("media-type", "", "Specify the content Media Type.") fs.String("source-file", "", "Specify the content source file.") - fs.String( + fs.Var( + &defaultHashFunc, "hash-func", - contentpb.HashFunc_SHA256.String(), fmt.Sprintf( "Specify hash function used for calculating content checksum. (values %s)", strings.Join(slices.Collect(maps.Values(contentpb.HashFunc_name)), ","), @@ -58,12 +81,46 @@ func New() *cobra.Command { } type config struct { - command.LoggingConfig `flag:",squash"` + Name string `flag:"name"` + MediaType string `flag:"media-type"` + SourceFile string `flag:"source-file"` + HashFunc contentpb.HashFunc `flag:"hash-func"` +} + +func (c config) Validate(ctx context.Context) error { + validators := []command.Validator{ + validateMediaType(c.MediaType), + validateSourceFile(c.SourceFile), + } + + return command.ValidateAll(ctx, validators...) +} - Name string `flag:"name"` - MediaType string `flag:"media-type"` - SourceFile string `flag:"source-file"` - HashFunc string `flag:"hash-func"` +func validateMediaType(mediaType string) command.ValidatorFunc { + return func(ctx context.Context) error { + if len(mediaType) == 0 { + return errors.New("media type must be provided") + } + _, _, err := mime.ParseMediaType(mediaType) + return err + } +} + +func validateSourceFile(filename string) command.ValidatorFunc { + return func(ctx context.Context) error { + if len(filename) == 0 { + return errors.New("source file name must be provided") + } + + info, err := os.Stat(filename) + if err != nil { + return err + } + if info.IsDir() { + return errors.New("source file name must be a file and not a directory") + } + return nil + } } type uploadClient interface { @@ -86,17 +143,17 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) spanCtx, span := otel.Tracer("upload").Start(ctx, "initUploadHandler") defer span.End() - log := command.Logger(cfg.LoggingConfig) + log := humus.Logger("upload") var hasher hash.Hash switch cfg.HashFunc { - case "sha256": + case contentpb.HashFunc_SHA256: hasher = sha256.New() default: return nil, fmt.Errorf("unsupported hash function: %s", cfg.HashFunc) } - src, err := openSourceFile(cfg.SourceFile) + src, err := os.Open(cfg.SourceFile) if err != nil { log.ErrorContext(spanCtx, "failed to open source file", slog.String("error", err.Error())) return nil, err @@ -144,11 +201,3 @@ func (h *handler) Handle(ctx context.Context) error { enc := json.NewEncoder(h.out) return enc.Encode(resp) } - -func openSourceFile(filename string) (*os.File, error) { - filename = strings.TrimSpace(filename) - if len(filename) == 0 { - return os.Stdin, nil - } - return os.Open(filename) -} diff --git a/cmd/griot/main.go b/cmd/griot/main.go index e55032a..f2901e3 100644 --- a/cmd/griot/main.go +++ b/cmd/griot/main.go @@ -15,22 +15,16 @@ package main import ( - "github.com/spf13/pflag" - "github.com/z5labs/griot/cmd/griot/content" + "bytes" + _ "embed" + + "github.com/z5labs/griot/cmd/griot/app" "github.com/z5labs/griot/cmd/internal/command" ) -func main() { - cmd := command.New( - "griot", - command.PersistentFlags(func(fs *pflag.FlagSet) { - fs.Var(&command.DefaultMinLogLevel, "log-level", "Set the minimum log level.") - - fs.Bool("enable-otel", false, "Enable OpenTelemetry tracing.") - fs.String("trace-out", "", "Specify output file for OpenTelemetry trace data. If not set, a temp file will be created and used.") - }), - command.Sub(content.New()), - ) +//go:embed config.yaml +var configBytes []byte - command.Run(cmd) +func main() { + command.Run(bytes.NewReader(configBytes), app.Init) } From 783cd4d79ec61135567d181e5c6b8027b56a01be Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 22:15:23 -0400 Subject: [PATCH 09/21] refactor(issue-33): make upload flag validation errors more specific --- cmd/griot/content/upload/upload.go | 105 +++++++++++++++++++---------- cmd/internal/command/command.go | 10 ++- cmd/internal/command/error.go | 38 +++++++++++ 3 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 cmd/internal/command/error.go diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 4262f5f..53af545 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -18,7 +18,6 @@ import ( "context" "crypto/sha256" "encoding/json" - "errors" "fmt" "hash" "io" @@ -32,44 +31,24 @@ import ( "github.com/z5labs/griot/cmd/internal/command" "github.com/z5labs/griot/services/content" "github.com/z5labs/griot/services/content/contentpb" - "github.com/z5labs/humus" "github.com/spf13/pflag" + "github.com/z5labs/humus" "go.opentelemetry.io/otel" ) -type hashFuncFlag contentpb.HashFunc - -func (x *hashFuncFlag) Set(v string) error { - h, exists := contentpb.HashFunc_value[v] - if !exists { - return fmt.Errorf("unknown value for contentpb.HashFunc: %s", v) - } - *x = hashFuncFlag(h) - return nil -} - -func (x hashFuncFlag) String() string { - return contentpb.HashFunc(x).String() -} - -func (hashFuncFlag) Type() string { - return "contentpb.HashFunc" -} - -func New() *command.App { - defaultHashFunc := hashFuncFlag(contentpb.HashFunc_SHA256) - +func New(args ...string) *command.App { return command.NewApp( "upload", + command.Args(args...), command.Short("Upload content"), command.Flags(func(fs *pflag.FlagSet) { fs.String("name", "", "Provide an optional name to help identify this content later.") fs.String("media-type", "", "Specify the content Media Type.") fs.String("source-file", "", "Specify the content source file.") - fs.Var( - &defaultHashFunc, + fs.String( "hash-func", + contentpb.HashFunc_SHA256.String(), fmt.Sprintf( "Specify hash function used for calculating content checksum. (values %s)", strings.Join(slices.Collect(maps.Values(contentpb.HashFunc_name)), ","), @@ -81,16 +60,17 @@ func New() *command.App { } type config struct { - Name string `flag:"name"` - MediaType string `flag:"media-type"` - SourceFile string `flag:"source-file"` - HashFunc contentpb.HashFunc `flag:"hash-func"` + Name string `flag:"name"` + MediaType string `flag:"media-type"` + SourceFile string `flag:"source-file"` + HashFunc string `flag:"hash-func"` } func (c config) Validate(ctx context.Context) error { validators := []command.Validator{ validateMediaType(c.MediaType), validateSourceFile(c.SourceFile), + validateHashFunc(c.HashFunc), } return command.ValidateAll(ctx, validators...) @@ -99,25 +79,73 @@ func (c config) Validate(ctx context.Context) error { func validateMediaType(mediaType string) command.ValidatorFunc { return func(ctx context.Context) error { if len(mediaType) == 0 { - return errors.New("media type must be provided") + return command.InvalidFlagError{ + Name: "media-type", + Cause: command.ErrFlagRequired, + } } _, _, err := mime.ParseMediaType(mediaType) - return err + if err != nil { + return command.InvalidFlagError{ + Name: "media-type", + Cause: err, + } + } + return nil } } func validateSourceFile(filename string) command.ValidatorFunc { return func(ctx context.Context) error { if len(filename) == 0 { - return errors.New("source file name must be provided") + return command.InvalidFlagError{ + Name: "source-file", + Cause: command.ErrFlagRequired, + } } info, err := os.Stat(filename) if err != nil { - return err + return command.InvalidFlagError{ + Name: "source-file", + Cause: err, + } } if info.IsDir() { - return errors.New("source file name must be a file and not a directory") + return command.InvalidFlagError{ + Name: "source-file", + Cause: command.ErrMustBeAFile, + } + } + return nil + } +} + +type UnknownHashFuncError struct { + Value string +} + +func (e UnknownHashFuncError) Error() string { + return fmt.Sprintf("unknown hash func value: %s", e.Value) +} + +func validateHashFunc(name string) command.ValidatorFunc { + return func(ctx context.Context) error { + if len(name) == 0 { + return command.InvalidFlagError{ + Name: "hash-func", + Cause: command.ErrFlagRequired, + } + } + + _, found := contentpb.HashFunc_value[name] + if !found { + return command.InvalidFlagError{ + Name: "hash-func", + Cause: UnknownHashFuncError{ + Value: name, + }, + } } return nil } @@ -146,7 +174,12 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) log := humus.Logger("upload") var hasher hash.Hash - switch cfg.HashFunc { + hashFunc, exists := contentpb.HashFunc_value[cfg.HashFunc] + if !exists { + return nil, fmt.Errorf("unknown hash function: %s", cfg.HashFunc) + } + + switch contentpb.HashFunc(hashFunc) { case contentpb.HashFunc_SHA256: hasher = sha256.New() default: diff --git a/cmd/internal/command/command.go b/cmd/internal/command/command.go index 193fce2..3b6afde 100644 --- a/cmd/internal/command/command.go +++ b/cmd/internal/command/command.go @@ -28,6 +28,12 @@ import ( type Option func(*App) +func Args(args ...string) Option { + return func(a *App) { + a.cmd.SetArgs(args) + } +} + func Short(desription string) Option { return func(a *App) { a.cmd.Short = desription @@ -117,7 +123,9 @@ type App struct { func NewApp(name string, opts ...Option) *App { a := &App{ cmd: &cobra.Command{ - Use: name, + Use: name, + SilenceErrors: true, + SilenceUsage: true, }, } for _, opt := range opts { diff --git a/cmd/internal/command/error.go b/cmd/internal/command/error.go new file mode 100644 index 0000000..3cc76ea --- /dev/null +++ b/cmd/internal/command/error.go @@ -0,0 +1,38 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "errors" + "fmt" +) + +var ( + ErrFlagRequired = errors.New("required") + ErrMustBeAFile = errors.New("must be a file and not a directory") +) + +type InvalidFlagError struct { + Name string + Cause error +} + +func (e InvalidFlagError) Error() string { + return fmt.Sprintf("invalid flag: %s: %s", e.Name, e.Cause) +} + +func (e InvalidFlagError) Unwrap() error { + return e.Cause +} From 0e7c269c1b776f723856015f4b4d97f8fcaa4a60 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 22:16:03 -0400 Subject: [PATCH 10/21] test(issue-33): added cases for flag validation --- cmd/griot/content/upload/upload_test.go | 190 ++++++++++++++++++++++++ go.mod | 3 + 2 files changed, 193 insertions(+) create mode 100644 cmd/griot/content/upload/upload_test.go diff --git a/cmd/griot/content/upload/upload_test.go b/cmd/griot/content/upload/upload_test.go new file mode 100644 index 0000000..86f046a --- /dev/null +++ b/cmd/griot/content/upload/upload_test.go @@ -0,0 +1,190 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upload + +import ( + "context" + "mime" + "os" + "testing" + + "github.com/z5labs/griot/cmd/internal/command" + + "github.com/stretchr/testify/assert" +) + +func TestApp(t *testing.T) { + t.Run("will return an error", func(t *testing.T) { + t.Run("if the media type is not set", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "*") + if !assert.Nil(t, err) { + return + } + err = f.Close() + if !assert.Nil(t, err) { + return + } + + app := New("--source-file", f.Name()) + err = app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "media-type", iferr.Name) { + return + } + if !assert.ErrorIs(t, iferr, command.ErrFlagRequired) { + return + } + }) + + t.Run("if the media type is invalid", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "*") + if !assert.Nil(t, err) { + return + } + err = f.Close() + if !assert.Nil(t, err) { + return + } + + app := New("--media-type", "text/plain; hello=", "--source-file", f.Name()) + err = app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "media-type", iferr.Name) { + return + } + if !assert.ErrorIs(t, iferr, mime.ErrInvalidMediaParameter) { + return + } + }) + + t.Run("if the source file is not set", func(t *testing.T) { + app := New("--media-type", "text/plain") + err := app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "source-file", iferr.Name) { + return + } + if !assert.ErrorIs(t, iferr, command.ErrFlagRequired) { + return + } + }) + + t.Run("if the source file does not exist", func(t *testing.T) { + app := New("--media-type", "text/plain", "--source-file", "test.txt") + err := app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "source-file", iferr.Name) { + return + } + + var perr *os.PathError + if !assert.ErrorAs(t, err, &perr) { + return + } + }) + + t.Run("if the source file name is a directory instead of a file", func(t *testing.T) { + dir := t.TempDir() + + app := New("--media-type", "text/plain", "--source-file", dir) + err := app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "source-file", iferr.Name) { + return + } + if !assert.ErrorIs(t, iferr, command.ErrMustBeAFile) { + return + } + }) + + t.Run("if the hash func is set to empty value", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "*") + if !assert.Nil(t, err) { + return + } + err = f.Close() + if !assert.Nil(t, err) { + return + } + + app := New("--media-type", "text/plain", "--source-file", f.Name(), "--hash-func", "") + err = app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "hash-func", iferr.Name) { + return + } + if !assert.ErrorIs(t, iferr, command.ErrFlagRequired) { + return + } + }) + + t.Run("if the hash func is set to an unknown value", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "*") + if !assert.Nil(t, err) { + return + } + err = f.Close() + if !assert.Nil(t, err) { + return + } + + app := New("--media-type", "text/plain", "--source-file", f.Name(), "--hash-func", "SHA") + err = app.Run(context.Background()) + + var iferr command.InvalidFlagError + if !assert.ErrorAs(t, err, &iferr) { + return + } + if !assert.Equal(t, "hash-func", iferr.Name) { + return + } + + var uerr UnknownHashFuncError + if !assert.ErrorAs(t, iferr, &uerr) { + return + } + if !assert.NotEmpty(t, uerr.Error()) { + return + } + if !assert.Equal(t, "SHA", uerr.Value) { + return + } + }) + }) +} diff --git a/go.mod b/go.mod index 5e589f8..d935927 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.9.0 github.com/z5labs/humus v0.2.0 go.opentelemetry.io/otel v1.31.0 google.golang.org/protobuf v1.35.1 @@ -13,11 +14,13 @@ require ( require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/z5labs/bedrock v0.12.0 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 // indirect From ffbb3af2cb2680283a7627d6113f47436b05976e Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 22:18:09 -0400 Subject: [PATCH 11/21] build(issue-33): ran gazelle --- MODULE.bazel | 6 ++++ MODULE.bazel.lock | 53 +++++++++------------------- cmd/griot/BUILD.bazel | 19 ++++++++++ cmd/griot/app/BUILD.bazel | 12 +++++++ cmd/griot/content/BUILD.bazel | 12 +++++++ cmd/griot/content/upload/BUILD.bazel | 26 ++++++++++++++ cmd/internal/command/BUILD.bazel | 18 ++++++++++ services/content/BUILD.bazel | 12 +++++++ 8 files changed, 121 insertions(+), 37 deletions(-) create mode 100644 cmd/griot/BUILD.bazel create mode 100644 cmd/griot/app/BUILD.bazel create mode 100644 cmd/griot/content/BUILD.bazel create mode 100644 cmd/griot/content/upload/BUILD.bazel create mode 100644 cmd/internal/command/BUILD.bazel create mode 100644 services/content/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 1b0819b..c9bb132 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,6 +16,12 @@ go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, + "com_github_go_viper_mapstructure_v2", + "com_github_spf13_cobra", + "com_github_spf13_pflag", + "com_github_stretchr_testify", + "com_github_z5labs_humus", + "io_opentelemetry_go_otel", "org_golang_google_protobuf", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ae3766e..5f802a6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -8,8 +8,8 @@ "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.1/MODULE.bazel": "39517c00a97118e7924786cd9b6fde80016386dee741d40fd9497b80d93c9b54", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.1/source.json": "5c98d61e7ed023a391c83e22e0a1c3576b84de57e75403f47fabc3ff62d05db4", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.2/MODULE.bazel": "4fae9c767526c17460c9d6e5731b09fdcab37b1322d6454acc137ac6d311e5b6", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.2/source.json": "cc829976e1b827bf3ef5ab682177a93c486c979a67a6b02dd0888e56486ed8ad", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", @@ -71,8 +71,8 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_oci/2.0.0/MODULE.bazel": "5d5cf1932238b009f874d5a9f214bbedf5d8cec8e5028e083f1147726876572f", - "https://bcr.bazel.build/modules/rules_oci/2.0.0/source.json": "ea89dd54d1f473a6fb1a6c2d0660b741ee832d5141ac0b9c90d5c8fc9b5e3bb3", + "https://bcr.bazel.build/modules/rules_oci/2.0.1/MODULE.bazel": "b1984eceba83906786f99e7e8273754458e1c5f3d39e3b0b28f03d12be9d4099", + "https://bcr.bazel.build/modules/rules_oci/2.0.1/source.json": "70f86dc00a62cde2103e8c50b8fd1f120252f2cb735ecce8aba3842ff1b5875f", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", @@ -129,8 +129,8 @@ }, "@@aspect_bazel_lib~//lib:extensions.bzl%toolchains": { "general": { - "bzlTransitiveDigest": "yyY+Nqn2Av7FUwQTgny5DHp29J+m79HFU9SzEcD/xrU=", - "usagesDigest": "IcdarHyVD51TJw8iR0YcrrtedqMPKHkeVqw5ETfEQWU=", + "bzlTransitiveDigest": "r7IIh4EKfqAOVGQAdB5/k5x8zTeExJ6xqLXx7uQWmGY=", + "usagesDigest": "jmHsY3UbKvny8kEsuA09AutSMTF6BI3QkaMjhVG+HDo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -597,7 +597,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "CTnaZGZxcnBcmQnD3YRFsLnLu0IwUQcvRGTfRDzNiVw=", + "usagesDigest": "QZFAJ0VIiJ5mGILpZn9r+TAbKxQ8T11/C+hyok0m2lU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -613,8 +613,8 @@ }, "@@rules_oci~//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "CXFLt/nBWxjkJys1nCGLwA1kZakP/vznMoIJgOVF2Gw=", - "usagesDigest": "jRqL1tWcMEl1gPAlskh3y0oUpyut4zyUZctWXBOGF5k=", + "bzlTransitiveDigest": "EeUyMVH1i0sdiUm4wGl2mwZxFcLKlcOC4svMASxOvaE=", + "usagesDigest": "BhQLj4CiuTfyF8JN+9hyWYiuQOMvrOmEO0PrfNlCTAI=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -692,13 +692,6 @@ "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" } }, - "oci_regctl_windows_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "windows_armv6" - } - }, "oci_crane_linux_amd64": { "bzlFile": "@@rules_oci~//oci:repositories.bzl", "ruleClassName": "crane_repositories", @@ -833,13 +826,6 @@ "crane_version": "v0.18.0" } }, - "oci_regctl_linux_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, "oci_crane_linux_s390x": { "bzlFile": "@@rules_oci~//oci:repositories.bzl", "ruleClassName": "crane_repositories", @@ -848,6 +834,13 @@ "crane_version": "v0.18.0" } }, + "oci_regctl_linux_arm64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, "zstd_linux_amd64": { "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", "ruleClassName": "zstd_binary_repo", @@ -1005,13 +998,6 @@ "version": "1.7" } }, - "oci_regctl_linux_i386": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_i386" - } - }, "jq_toolchains": { "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", "ruleClassName": "jq_toolchains_repo", @@ -1026,13 +1012,6 @@ "user_repository_name": "copy_to_directory" } }, - "oci_regctl_linux_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_armv6" - } - }, "bazel_features_globals": { "bzlFile": "@@bazel_features~//private:globals_repo.bzl", "ruleClassName": "globals_repo", diff --git a/cmd/griot/BUILD.bazel b/cmd/griot/BUILD.bazel new file mode 100644 index 0000000..01cb082 --- /dev/null +++ b/cmd/griot/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "griot_lib", + srcs = ["main.go"], + embedsrcs = ["config.yaml"], + importpath = "github.com/z5labs/griot/cmd/griot", + visibility = ["//visibility:private"], + deps = [ + "//cmd/griot/app", + "//cmd/internal/command", + ], +) + +go_binary( + name = "griot", + embed = [":griot_lib"], + visibility = ["//visibility:public"], +) diff --git a/cmd/griot/app/BUILD.bazel b/cmd/griot/app/BUILD.bazel new file mode 100644 index 0000000..602d1df --- /dev/null +++ b/cmd/griot/app/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "app", + srcs = ["app.go"], + importpath = "github.com/z5labs/griot/cmd/griot/app", + visibility = ["//visibility:public"], + deps = [ + "//cmd/griot/content", + "//cmd/internal/command", + ], +) diff --git a/cmd/griot/content/BUILD.bazel b/cmd/griot/content/BUILD.bazel new file mode 100644 index 0000000..e5dcdc9 --- /dev/null +++ b/cmd/griot/content/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "content", + srcs = ["content.go"], + importpath = "github.com/z5labs/griot/cmd/griot/content", + visibility = ["//visibility:public"], + deps = [ + "//cmd/griot/content/upload", + "//cmd/internal/command", + ], +) diff --git a/cmd/griot/content/upload/BUILD.bazel b/cmd/griot/content/upload/BUILD.bazel new file mode 100644 index 0000000..d97ca10 --- /dev/null +++ b/cmd/griot/content/upload/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "upload", + srcs = ["upload.go"], + importpath = "github.com/z5labs/griot/cmd/griot/content/upload", + visibility = ["//visibility:public"], + deps = [ + "//cmd/internal/command", + "//services/content", + "//services/content/contentpb", + "@com_github_spf13_pflag//:pflag", + "@com_github_z5labs_humus//:humus", + "@io_opentelemetry_go_otel//:otel", + ], +) + +go_test( + name = "upload_test", + srcs = ["upload_test.go"], + embed = [":upload"], + deps = [ + "//cmd/internal/command", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/cmd/internal/command/BUILD.bazel b/cmd/internal/command/BUILD.bazel new file mode 100644 index 0000000..058503f --- /dev/null +++ b/cmd/internal/command/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "command", + srcs = [ + "command.go", + "error.go", + ], + importpath = "github.com/z5labs/griot/cmd/internal/command", + visibility = ["//cmd:__subpackages__"], + deps = [ + "@com_github_go_viper_mapstructure_v2//:mapstructure", + "@com_github_spf13_cobra//:cobra", + "@com_github_spf13_pflag//:pflag", + "@com_github_z5labs_humus//:humus", + "@io_opentelemetry_go_otel//:otel", + ], +) diff --git a/services/content/BUILD.bazel b/services/content/BUILD.bazel new file mode 100644 index 0000000..a6ecaa6 --- /dev/null +++ b/services/content/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "content", + srcs = ["content.go"], + importpath = "github.com/z5labs/griot/services/content", + visibility = ["//visibility:public"], + deps = [ + "//services/content/contentpb", + "@io_opentelemetry_go_otel//:otel", + ], +) From d391d10f01cbbf33d4e114e990364785c59ae9b1 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Mon, 21 Oct 2024 22:57:49 -0400 Subject: [PATCH 12/21] test(issue-33): added cases for upload command handler --- MODULE.bazel | 1 + cmd/griot/content/upload/BUILD.bazel | 3 + cmd/griot/content/upload/upload.go | 55 ++++++-- cmd/griot/content/upload/upload_test.go | 165 ++++++++++++++++++++++++ go.mod | 2 +- 5 files changed, 216 insertions(+), 10 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index c9bb132..855d580 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,6 +23,7 @@ use_repo( "com_github_z5labs_humus", "io_opentelemetry_go_otel", "org_golang_google_protobuf", + "com_github_z5labs_bedrock", ) oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") diff --git a/cmd/griot/content/upload/BUILD.bazel b/cmd/griot/content/upload/BUILD.bazel index d97ca10..10baf7b 100644 --- a/cmd/griot/content/upload/BUILD.bazel +++ b/cmd/griot/content/upload/BUILD.bazel @@ -21,6 +21,9 @@ go_test( embed = [":upload"], deps = [ "//cmd/internal/command", + "//services/content", + "//services/content/contentpb", "@com_github_stretchr_testify//assert", + "@com_github_z5labs_bedrock//pkg/noop", ], ) diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 53af545..105f03a 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -155,12 +155,26 @@ type uploadClient interface { UploadContent(context.Context, *content.UploadContentRequest) (*content.UploadContentResponse, error) } +type hasher interface { + hash.Hash + + HashFunc() contentpb.HashFunc +} + +type sha256Hasher struct { + hash.Hash +} + +func (sha256Hasher) HashFunc() contentpb.HashFunc { + return contentpb.HashFunc_SHA256 +} + type handler struct { log *slog.Logger contentName string mediaType string - hasher hash.Hash + hasher hasher src io.ReadSeeker out io.Writer @@ -173,15 +187,17 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) log := humus.Logger("upload") - var hasher hash.Hash hashFunc, exists := contentpb.HashFunc_value[cfg.HashFunc] if !exists { - return nil, fmt.Errorf("unknown hash function: %s", cfg.HashFunc) + return nil, UnknownHashFuncError{ + Value: cfg.HashFunc, + } } + var contentHasher hasher switch contentpb.HashFunc(hashFunc) { case contentpb.HashFunc_SHA256: - hasher = sha256.New() + contentHasher = sha256Hasher{Hash: sha256.New()} default: return nil, fmt.Errorf("unsupported hash function: %s", cfg.HashFunc) } @@ -196,7 +212,7 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) log: log, contentName: cfg.Name, mediaType: cfg.MediaType, - hasher: hasher, + hasher: contentHasher, src: src, out: os.Stdout, content: content.NewClient(), @@ -204,25 +220,46 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) return h, nil } +type FailedToSeekReadBytesError struct { + BytesRead int64 + BytesSeeked int64 +} + +func (e FailedToSeekReadBytesError) Error() string { + return fmt.Sprintf("bytes read do not match bytes seeked: %d:%d", e.BytesRead, e.BytesSeeked) +} + func (h *handler) Handle(ctx context.Context) error { spanCtx, span := otel.Tracer("upload").Start(ctx, "handler.Handle") defer span.End() - _, err := io.Copy(h.hasher, h.src) + bytesRead, err := io.Copy(h.hasher, h.src) if err != nil { + span.RecordError(err) h.log.ErrorContext(spanCtx, "failed to compute hash", slog.String("error", err.Error())) return err } - _, err = h.src.Seek(0, 0) + bytesSeeked, err := h.src.Seek(0, 0) if err != nil { - h.log.ErrorContext(spanCtx, "failed to seek to start of content source", slog.String("error", err.Error())) + span.RecordError(err) + h.log.ErrorContext(spanCtx, "failed to perform seek on the source file", slog.String("error", err.Error())) + return err + } + if bytesRead != bytesSeeked { + err = FailedToSeekReadBytesError{ + BytesRead: bytesRead, + BytesSeeked: bytesSeeked, + } + + span.RecordError(err) + h.log.ErrorContext(spanCtx, "failed to seek to the start of the source file", slog.String("error", err.Error())) return err } req := &content.UploadContentRequest{ Name: h.contentName, - HashFunc: contentpb.HashFunc_SHA256, + HashFunc: h.hasher.HashFunc(), Body: h.src, } resp, err := h.content.UploadContent(spanCtx, req) diff --git a/cmd/griot/content/upload/upload_test.go b/cmd/griot/content/upload/upload_test.go index 86f046a..51145e7 100644 --- a/cmd/griot/content/upload/upload_test.go +++ b/cmd/griot/content/upload/upload_test.go @@ -16,13 +16,22 @@ package upload import ( "context" + "crypto/sha256" + "errors" + "io" + "log/slog" "mime" "os" + "path/filepath" + "strings" "testing" "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/services/content" + "github.com/z5labs/griot/services/content/contentpb" "github.com/stretchr/testify/assert" + "github.com/z5labs/bedrock/pkg/noop" ) func TestApp(t *testing.T) { @@ -188,3 +197,159 @@ func TestApp(t *testing.T) { }) }) } + +func TestInitUploadHandler(t *testing.T) { + t.Run("will return an error", func(t *testing.T) { + t.Run("if a unknown hash function is provided", func(t *testing.T) { + cfg := config{ + HashFunc: "SHA", + } + + _, err := initUploadHandler(context.Background(), cfg) + + var uerr UnknownHashFuncError + if !assert.ErrorAs(t, err, &uerr) { + return + } + if !assert.NotEmpty(t, uerr.Error()) { + return + } + if !assert.Equal(t, "SHA", uerr.Value) { + return + } + }) + + t.Run("if it fails to open the source file", func(t *testing.T) { + cfg := config{ + HashFunc: contentpb.HashFunc_SHA256.String(), + SourceFile: filepath.Join(t.TempDir(), "test.txt"), + } + + _, err := initUploadHandler(context.Background(), cfg) + + var perr *os.PathError + if !assert.ErrorAs(t, err, &perr) { + return + } + if !assert.Equal(t, "open", perr.Op) { + return + } + }) + }) +} + +type readNoopSeeker func([]byte) (int, error) + +func (f readNoopSeeker) Read(b []byte) (int, error) { + return f(b) +} + +func (f readNoopSeeker) Seek(offset int64, whence int) (int64, error) { + return 0, nil +} + +type seekNoopReader func(int64, int) (int64, error) + +func (f seekNoopReader) Read(b []byte) (int, error) { + return 0, io.EOF +} + +func (f seekNoopReader) Seek(offset int64, whence int) (int64, error) { + return f(offset, whence) +} + +type uploadClientFunc func(context.Context, *content.UploadContentRequest) (*content.UploadContentResponse, error) + +func (f uploadClientFunc) UploadContent(ctx context.Context, req *content.UploadContentRequest) (*content.UploadContentResponse, error) { + return f(ctx, req) +} + +func TestHandler_Handle(t *testing.T) { + t.Run("will return an error", func(t *testing.T) { + t.Run("if it fails to compute the content hash", func(t *testing.T) { + readErr := errors.New("read failed") + src := readNoopSeeker(func(b []byte) (int, error) { + return 0, readErr + }) + + h := &handler{ + log: slog.New(noop.LogHandler{}), + hasher: sha256Hasher{Hash: sha256.New()}, + src: src, + } + + err := h.Handle(context.Background()) + if !assert.Equal(t, readErr, err) { + return + } + }) + + t.Run("if it fails to perform seek on the source", func(t *testing.T) { + seekErr := errors.New("failed to seek") + src := seekNoopReader(func(i1 int64, i2 int) (int64, error) { + return 0, seekErr + }) + + h := &handler{ + log: slog.New(noop.LogHandler{}), + hasher: sha256Hasher{Hash: sha256.New()}, + src: src, + } + + err := h.Handle(context.Background()) + if !assert.Equal(t, seekErr, err) { + return + } + }) + + t.Run("if it fails to seek to the start of the source file", func(t *testing.T) { + bytesSeeked := int64(10) + src := seekNoopReader(func(i1 int64, i2 int) (int64, error) { + return bytesSeeked, nil + }) + + h := &handler{ + log: slog.New(noop.LogHandler{}), + hasher: sha256Hasher{Hash: sha256.New()}, + src: src, + } + + err := h.Handle(context.Background()) + + var ferr FailedToSeekReadBytesError + if !assert.ErrorAs(t, err, &ferr) { + return + } + if !assert.NotEmpty(t, ferr.Error()) { + return + } + if !assert.Equal(t, int64(0), ferr.BytesRead) { + return + } + if !assert.Equal(t, bytesSeeked, ferr.BytesSeeked) { + return + } + }) + + t.Run("if it fails to upload the content", func(t *testing.T) { + src := strings.NewReader(``) + + uploadErr := errors.New("failed to upload") + client := uploadClientFunc(func(ctx context.Context, ucr *content.UploadContentRequest) (*content.UploadContentResponse, error) { + return nil, uploadErr + }) + + h := &handler{ + log: slog.New(noop.LogHandler{}), + hasher: sha256Hasher{Hash: sha256.New()}, + src: src, + content: client, + } + + err := h.Handle(context.Background()) + if !assert.Equal(t, uploadErr, err) { + return + } + }) + }) +} diff --git a/go.mod b/go.mod index d935927..b80a373 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + github.com/z5labs/bedrock v0.12.0 github.com/z5labs/humus v0.2.0 go.opentelemetry.io/otel v1.31.0 google.golang.org/protobuf v1.35.1 @@ -21,7 +22,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/z5labs/bedrock v0.12.0 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 // indirect From f53743836e6d03f36099f962988c021542ed483d Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 12:20:12 -0400 Subject: [PATCH 13/21] refactor(issue-33): move media type proto to contentpb --- services/content/contentpb/BUILD.bazel | 1 + .../{indexpb => contentpb}/media_type.pb.go | 49 ++++++++-------- .../{indexpb => contentpb}/media_type.proto | 4 +- services/content/contentpb/metadata.pb.go | 57 ++++++++++++------- services/content/contentpb/metadata.proto | 2 + services/content/indexpb/BUILD.bazel | 1 - services/content/indexpb/index_record.pb.go | 48 ++++++++-------- services/content/indexpb/index_record.proto | 2 +- 8 files changed, 89 insertions(+), 75 deletions(-) rename services/content/{indexpb => contentpb}/media_type.pb.go (65%) rename services/content/{indexpb => contentpb}/media_type.proto (57%) diff --git a/services/content/contentpb/BUILD.bazel b/services/content/contentpb/BUILD.bazel index b0b88d2..b6298ce 100644 --- a/services/content/contentpb/BUILD.bazel +++ b/services/content/contentpb/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "checksum.pb.go", "content_id.pb.go", "hash_func.pb.go", + "media_type.pb.go", "metadata.pb.go", "upload_content_v1_response.pb.go", ], diff --git a/services/content/indexpb/media_type.pb.go b/services/content/contentpb/media_type.pb.go similarity index 65% rename from services/content/indexpb/media_type.pb.go rename to services/content/contentpb/media_type.pb.go index 9a8ed46..412ab3c 100644 --- a/services/content/indexpb/media_type.pb.go +++ b/services/content/contentpb/media_type.pb.go @@ -4,7 +4,7 @@ // protoc v5.30.0--dev // source: media_type.proto -package indexpb +package contentpb import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -93,27 +93,26 @@ var File_media_type_proto protoreflect.FileDescriptor var file_media_type_proto_rawDesc = []byte{ 0x0a, 0x10, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x12, 0x13, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, 0xe0, 0x01, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x4e, 0x0a, 0x0a, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x2e, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2e, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x35, 0x6c, 0x61, 0x62, 0x73, 0x2f, - 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x3b, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x62, 0x08, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x70, 0xe8, 0x07, + 0x74, 0x6f, 0x12, 0x0d, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x22, 0xda, 0x01, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, + 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x48, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, + 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x69, 0x6f, + 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x1a, + 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x3e, + 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x35, 0x6c, + 0x61, 0x62, 0x73, 0x2f, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x70, 0x62, 0x3b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x62, 0x08, + 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x70, 0xe8, 0x07, } var ( @@ -130,11 +129,11 @@ func file_media_type_proto_rawDescGZIP() []byte { var file_media_type_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_media_type_proto_goTypes = []any{ - (*MediaType)(nil), // 0: griot.content.index.MediaType - nil, // 1: griot.content.index.MediaType.ParametersEntry + (*MediaType)(nil), // 0: griot.content.MediaType + nil, // 1: griot.content.MediaType.ParametersEntry } var file_media_type_proto_depIdxs = []int32{ - 1, // 0: griot.content.index.MediaType.parameters:type_name -> griot.content.index.MediaType.ParametersEntry + 1, // 0: griot.content.MediaType.parameters:type_name -> griot.content.MediaType.ParametersEntry 1, // [1:1] is the sub-list for method output_type 1, // [1:1] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name diff --git a/services/content/indexpb/media_type.proto b/services/content/contentpb/media_type.proto similarity index 57% rename from services/content/indexpb/media_type.proto rename to services/content/contentpb/media_type.proto index 47f66d5..e565183 100644 --- a/services/content/indexpb/media_type.proto +++ b/services/content/contentpb/media_type.proto @@ -1,8 +1,8 @@ edition = "2023"; -package griot.content.index; +package griot.content; -option go_package = "github.com/z5labs/griot/services/content/indexpb;indexpb"; +option go_package = "github.com/z5labs/griot/services/content/contentpb;contentpb"; message MediaType { string type = 1; diff --git a/services/content/contentpb/metadata.pb.go b/services/content/contentpb/metadata.pb.go index 8d55639..cdf8693 100644 --- a/services/content/contentpb/metadata.pb.go +++ b/services/content/contentpb/metadata.pb.go @@ -25,8 +25,9 @@ type Metadata struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Checksum *Checksum `protobuf:"bytes,1,opt,name=checksum" json:"checksum,omitempty"` - Name *string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` + Checksum *Checksum `protobuf:"bytes,1,opt,name=checksum" json:"checksum,omitempty"` + Name *string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` + MediaType *MediaType `protobuf:"bytes,3,opt,name=media_type,json=mediaType" json:"media_type,omitempty"` } func (x *Metadata) Reset() { @@ -73,23 +74,34 @@ func (x *Metadata) GetName() string { return "" } +func (x *Metadata) GetMediaType() *MediaType { + if x != nil { + return x.MediaType + } + return nil +} + var File_metadata_proto protoreflect.FileDescriptor var file_metadata_proto_rawDesc = []byte{ 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x1a, - 0x0e, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0x53, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, 0x0a, 0x08, 0x63, - 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x52, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x7a, 0x35, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x3b, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x70, 0x62, 0x62, 0x08, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x70, 0xe8, - 0x07, + 0x0e, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x10, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x8c, 0x01, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, + 0x0a, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x52, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, + 0x73, 0x75, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, + 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x64, 0x69, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, + 0x35, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x3b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x70, 0x62, + 0x62, 0x08, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x70, 0xe8, 0x07, } var ( @@ -106,16 +118,18 @@ func file_metadata_proto_rawDescGZIP() []byte { var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_metadata_proto_goTypes = []any{ - (*Metadata)(nil), // 0: griot.content.Metadata - (*Checksum)(nil), // 1: griot.content.Checksum + (*Metadata)(nil), // 0: griot.content.Metadata + (*Checksum)(nil), // 1: griot.content.Checksum + (*MediaType)(nil), // 2: griot.content.MediaType } var file_metadata_proto_depIdxs = []int32{ 1, // 0: griot.content.Metadata.checksum:type_name -> griot.content.Checksum - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 2, // 1: griot.content.Metadata.media_type:type_name -> griot.content.MediaType + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_metadata_proto_init() } @@ -124,6 +138,7 @@ func file_metadata_proto_init() { return } file_checksum_proto_init() + file_media_type_proto_init() type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/services/content/contentpb/metadata.proto b/services/content/contentpb/metadata.proto index 5a9aee8..79e95bc 100644 --- a/services/content/contentpb/metadata.proto +++ b/services/content/contentpb/metadata.proto @@ -5,8 +5,10 @@ package griot.content; option go_package = "github.com/z5labs/griot/services/content/contentpb;contentpb"; import "checksum.proto"; +import "media_type.proto"; message Metadata { Checksum checksum = 1; string name = 2; + griot.content.MediaType media_type = 3; } \ No newline at end of file diff --git a/services/content/indexpb/BUILD.bazel b/services/content/indexpb/BUILD.bazel index 39cdedd..a727734 100644 --- a/services/content/indexpb/BUILD.bazel +++ b/services/content/indexpb/BUILD.bazel @@ -5,7 +5,6 @@ go_library( srcs = [ "content_size.pb.go", "index_record.pb.go", - "media_type.pb.go", "unit_of_information.pb.go", ], importpath = "github.com/z5labs/griot/services/content/indexpb", diff --git a/services/content/indexpb/index_record.pb.go b/services/content/indexpb/index_record.pb.go index a99554a..af40514 100644 --- a/services/content/indexpb/index_record.pb.go +++ b/services/content/indexpb/index_record.pb.go @@ -27,7 +27,7 @@ type Record struct { unknownFields protoimpl.UnknownFields ContentId *contentpb.ContentId `protobuf:"bytes,1,opt,name=content_id,json=contentId" json:"content_id,omitempty"` - ContentType *MediaType `protobuf:"bytes,2,opt,name=content_type,json=contentType" json:"content_type,omitempty"` + ContentType *contentpb.MediaType `protobuf:"bytes,2,opt,name=content_type,json=contentType" json:"content_type,omitempty"` ContentName *string `protobuf:"bytes,3,opt,name=content_name,json=contentName" json:"content_name,omitempty"` ContentSize *ContentSize `protobuf:"bytes,4,opt,name=content_size,json=contentSize" json:"content_size,omitempty"` CheckSums []*contentpb.Checksum `protobuf:"bytes,5,rep,name=check_sums,json=checkSums" json:"check_sums,omitempty"` @@ -70,7 +70,7 @@ func (x *Record) GetContentId() *contentpb.ContentId { return nil } -func (x *Record) GetContentType() *MediaType { +func (x *Record) GetContentType() *contentpb.MediaType { if x != nil { return x.ContentType } @@ -108,30 +108,29 @@ var file_index_record_proto_rawDesc = []byte{ 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0xa4, 0x02, 0x0a, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x37, 0x0a, 0x0a, + 0x6f, 0x22, 0x9e, 0x02, 0x0a, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x41, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x3b, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, + 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x64, 0x69, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x69, 0x6e, 0x64, 0x65, - 0x78, 0x2e, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x0c, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, - 0x69, 0x7a, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x69, 0x7a, 0x65, - 0x12, 0x36, 0x0a, 0x0a, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x73, 0x75, 0x6d, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x52, 0x09, 0x63, - 0x68, 0x65, 0x63, 0x6b, 0x53, 0x75, 0x6d, 0x73, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x35, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x67, 0x72, - 0x69, 0x6f, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x3b, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x70, 0x62, 0x62, 0x08, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x70, 0xe8, - 0x07, + 0x78, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x0b, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x63, 0x68, + 0x65, 0x63, 0x6b, 0x5f, 0x73, 0x75, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x75, + 0x6d, 0x73, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x7a, 0x35, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x67, 0x72, 0x69, 0x6f, 0x74, 0x2f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2f, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x3b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x62, 0x08, + 0x65, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x70, 0xe8, 0x07, } var ( @@ -150,13 +149,13 @@ var file_index_record_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_index_record_proto_goTypes = []any{ (*Record)(nil), // 0: griot.content.index.Record (*contentpb.ContentId)(nil), // 1: griot.content.ContentId - (*MediaType)(nil), // 2: griot.content.index.MediaType + (*contentpb.MediaType)(nil), // 2: griot.content.MediaType (*ContentSize)(nil), // 3: griot.content.index.ContentSize (*contentpb.Checksum)(nil), // 4: griot.content.Checksum } var file_index_record_proto_depIdxs = []int32{ 1, // 0: griot.content.index.Record.content_id:type_name -> griot.content.ContentId - 2, // 1: griot.content.index.Record.content_type:type_name -> griot.content.index.MediaType + 2, // 1: griot.content.index.Record.content_type:type_name -> griot.content.MediaType 3, // 2: griot.content.index.Record.content_size:type_name -> griot.content.index.ContentSize 4, // 3: griot.content.index.Record.check_sums:type_name -> griot.content.Checksum 4, // [4:4] is the sub-list for method output_type @@ -171,7 +170,6 @@ func file_index_record_proto_init() { if File_index_record_proto != nil { return } - file_media_type_proto_init() file_content_size_proto_init() type x struct{} out := protoimpl.TypeBuilder{ diff --git a/services/content/indexpb/index_record.proto b/services/content/indexpb/index_record.proto index 4c3d718..164910a 100644 --- a/services/content/indexpb/index_record.proto +++ b/services/content/indexpb/index_record.proto @@ -11,7 +11,7 @@ import "content_size.proto"; message Record { ContentId content_id = 1; - MediaType content_type = 2; + griot.content.MediaType content_type = 2; string content_name = 3; ContentSize content_size = 4; From 925d58a0ce787682289420da63fa5e21166db548 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 12:36:17 -0400 Subject: [PATCH 14/21] feat(issue-33): fully implement upload content client functionality --- MODULE.bazel | 5 +- cmd/griot/content/upload/BUILD.bazel | 1 + cmd/griot/content/upload/upload.go | 25 +++- go.mod | 9 +- go.sum | 28 ++++ services/content/BUILD.bazel | 5 + services/content/content.go | 194 +++++++++++++++++++++++++-- 7 files changed, 253 insertions(+), 14 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 855d580..9425d3f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,10 +20,13 @@ use_repo( "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_stretchr_testify", + "com_github_z5labs_bedrock", "com_github_z5labs_humus", + "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp", "io_opentelemetry_go_otel", + "io_opentelemetry_go_otel_metric", "org_golang_google_protobuf", - "com_github_z5labs_bedrock", + "org_golang_x_sync", ) oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") diff --git a/cmd/griot/content/upload/BUILD.bazel b/cmd/griot/content/upload/BUILD.bazel index 10baf7b..89b2c75 100644 --- a/cmd/griot/content/upload/BUILD.bazel +++ b/cmd/griot/content/upload/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//services/content/contentpb", "@com_github_spf13_pflag//:pflag", "@com_github_z5labs_humus//:humus", + "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:otelhttp", "@io_opentelemetry_go_otel//:otel", ], ) diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 105f03a..3c54efb 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -24,6 +24,7 @@ import ( "log/slog" "maps" "mime" + "net/http" "os" "slices" "strings" @@ -34,6 +35,7 @@ import ( "github.com/spf13/pflag" "github.com/z5labs/humus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" ) @@ -208,6 +210,10 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) return nil, err } + hc := &http.Client{ + Transport: otelhttp.NewTransport(http.DefaultTransport), + } + h := &handler{ log: log, contentName: cfg.Name, @@ -215,7 +221,7 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) hasher: contentHasher, src: src, out: os.Stdout, - content: content.NewClient(), + content: content.NewClient(hc), } return h, nil } @@ -257,10 +263,21 @@ func (h *handler) Handle(ctx context.Context) error { return err } + mediaType, params, _ := mime.ParseMediaType(h.mediaType) + req := &content.UploadContentRequest{ - Name: h.contentName, - HashFunc: h.hasher.HashFunc(), - Body: h.src, + Metadata: &contentpb.Metadata{ + Name: &h.contentName, + MediaType: &contentpb.MediaType{ + Type: &mediaType, + Parameters: params, + }, + Checksum: &contentpb.Checksum{ + HashFunc: h.hasher.HashFunc().Enum(), + Hash: h.hasher.Sum(nil), + }, + }, + Body: h.src, } resp, err := h.content.UploadContent(spanCtx, req) if err != nil { diff --git a/go.mod b/go.mod index b80a373..40eac6f 100644 --- a/go.mod +++ b/go.mod @@ -9,19 +9,26 @@ require ( github.com/stretchr/testify v1.9.0 github.com/z5labs/bedrock v0.12.0 github.com/z5labs/humus v0.2.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/metric v1.31.0 + golang.org/x/sync v0.8.0 google.golang.org/protobuf v1.35.1 ) require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/swaggest/jsonschema-go v0.3.72 // indirect + github.com/swaggest/openapi-go v0.2.54 // indirect + github.com/swaggest/refl v1.3.0 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 // indirect @@ -30,7 +37,6 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 // indirect go.opentelemetry.io/otel/log v0.6.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect @@ -42,5 +48,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3894464..ccf873e 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,14 @@ +github.com/bool64/dev v0.2.35 h1:M17TLsO/pV2J7PYI/gpe3Ua26ETkzZGb+dC06eoMqlk= +github.com/bool64/dev v0.2.35/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -16,6 +22,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -27,18 +35,34 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggest/assertjson v1.9.0 h1:dKu0BfJkIxv/xe//mkCrK5yZbs79jL7OVf9Ija7o2xQ= +github.com/swaggest/assertjson v1.9.0/go.mod h1:b+ZKX2VRiUjxfUIal0HDN85W0nHPAYUbYH5WkkSsFsU= +github.com/swaggest/jsonschema-go v0.3.72 h1:IHaGlR1bdBUBPfhe4tfacN2TGAPKENEGiNyNzvnVHv4= +github.com/swaggest/jsonschema-go v0.3.72/go.mod h1:OrGyEoVqpfSFJ4Am4V/FQcQ3mlEC1vVeleA+5ggbVW4= +github.com/swaggest/openapi-go v0.2.54 h1:WnFKIHAgR2RIOiYys3qvSuYmsFd2a17MIoC9Tcvog5c= +github.com/swaggest/openapi-go v0.2.54/go.mod h1:2Q7NpuG9NgpGeTaNOo852GSR6cCzSP4IznA9DNdUTQw= +github.com/swaggest/refl v1.3.0 h1:PEUWIku+ZznYfsoyheF97ypSduvMApYyGkYF3nabS0I= +github.com/swaggest/refl v1.3.0/go.mod h1:3Ujvbmh1pfSbDYjC6JGG7nMgPvpG0ehQL4iNonnLNbg= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/z5labs/bedrock v0.12.0 h1:mv5EQfYXfInlO++IRqsxcPcGCbKPcWKC6oTDP9VMm4E= github.com/z5labs/bedrock v0.12.0/go.mod h1:IdEKPA+TgqDFsSVD6L+elp9eJGmSL3xQ61bC7UjNmQA= github.com/z5labs/humus v0.2.0 h1:v/gCV2qo3leP7PuYxwmFz8qhS8hboN163PCQtm4WMMQ= github.com/z5labs/humus v0.2.0/go.mod h1:ypGTw5pq4tlo1yFk1vWYdrUWjqx9aGKUJ0JcpJnp8ps= go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 h1:lU3F57OSLK5mQ1PDBVAfDDaKCPv37MrEbCfTzsF4bz0= go.opentelemetry.io/contrib/bridges/otelslog v0.5.0/go.mod h1:I84u06zJFr8T5D73fslEUbnRBimVVSBhuVw8L8I92AU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.6.0 h1:WYsDPt0fM4KZaMhLvY+x6TVXd85P/KNl3Ez3t+0+kGs= @@ -73,6 +97,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= @@ -88,5 +114,7 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/content/BUILD.bazel b/services/content/BUILD.bazel index a6ecaa6..388680a 100644 --- a/services/content/BUILD.bazel +++ b/services/content/BUILD.bazel @@ -7,6 +7,11 @@ go_library( visibility = ["//visibility:public"], deps = [ "//services/content/contentpb", + "@com_github_z5labs_humus//rest", "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_metric//:metric", + "@org_golang_google_protobuf//proto", + "@org_golang_x_sync//errgroup", ], ) diff --git a/services/content/content.go b/services/content/content.go index 9469782..e6d00f1 100644 --- a/services/content/content.go +++ b/services/content/content.go @@ -12,28 +12,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package content +// Package content provides Content Service client and server implementations. package content import ( + "bytes" "context" + "encoding/base64" + "errors" + "fmt" "io" + "mime/multipart" + "net/http" + "net/textproto" "github.com/z5labs/griot/services/content/contentpb" + "github.com/z5labs/humus/rest" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" ) -type Client struct{} +type HttpClient interface { + Do(*http.Request) (*http.Response, error) +} + +type Client struct { + protoMarshal func(proto.Message) ([]byte, error) + http HttpClient +} -func NewClient() *Client { - c := &Client{} +func NewClient(hc HttpClient) *Client { + c := &Client{ + http: hc, + } return c } type UploadContentRequest struct { - Name string - HashFunc contentpb.HashFunc + Metadata *contentpb.Metadata Body io.Reader } @@ -42,7 +62,165 @@ type UploadContentResponse struct { } func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) (*UploadContentResponse, error) { - _, span := otel.Tracer("content").Start(ctx, "Client.UploadContent") + spanCtx, span := otel.Tracer("content").Start(ctx, "Client.UploadContent") + defer span.End() + + body, bodyWriter := io.Pipe() + + respCh := make(chan *http.Response) + eg, egctx := errgroup.WithContext(spanCtx) + eg.Go(func() error { + defer bodyWriter.Close() + + return c.writeUploadRequest(egctx, bodyWriter, req) + }) + eg.Go(func() error { + defer close(respCh) + defer body.Close() + + r, err := http.NewRequestWithContext(egctx, http.MethodPost, "", body) + if err != nil { + return err + } + r.Header.Set("Content-Type", "multipart/form-data") + + resp, err := c.http.Do(r) + if err != nil { + return err + } + select { + case <-egctx.Done(): + return egctx.Err() + case respCh <- resp: + } + return nil + }) + + err := eg.Wait() + if err != nil { + return nil, err + } + + var resp *http.Response + select { + case <-spanCtx.Done(): + return nil, spanCtx.Err() + case resp = <-respCh: + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected http status code: %d", resp.StatusCode) + } + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var uploadV1Resp contentpb.UploadContentV1Response + err = proto.Unmarshal(b, &uploadV1Resp) + if err != nil { + return nil, err + } + + uploadResp := UploadContentResponse{ + Id: uploadV1Resp.GetId().GetValue(), + } + return &uploadResp, nil +} + +func (c *Client) writeUploadRequest(ctx context.Context, w io.Writer, req *UploadContentRequest) error { + spanCtx, span := otel.Tracer("content").Start(ctx, "Client.writeUploadRequest") defer span.End() - return nil, nil + + pw := multipart.NewWriter(w) + err := c.writeMetadata(spanCtx, pw, req.Metadata) + if err != nil { + return err + } + + err = c.writeContent(spanCtx, pw, req.Metadata.Checksum.Hash, req.Body) + if err != nil { + return err + } + return nil +} + +func (c *Client) writeMetadata(ctx context.Context, pw *multipart.Writer, meta *contentpb.Metadata) error { + _, span := otel.Tracer("content").Start(ctx, "Client.writeMetadata") + defer span.End() + + b, err := c.protoMarshal(meta) + if err != nil { + return err + } + + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", `form-data; name="metadata"`) + header.Set("Content-Type", rest.ProtobufContentType) + + part, err := pw.CreatePart(header) + if err != nil { + return err + } + + n, err := io.Copy(part, bytes.NewReader(b)) + if err != nil { + return err + } + if n != int64(len(b)) { + // TODO + return errors.New("did not write all metadata bytes") + } + return nil +} + +type progressReader struct { + ctx context.Context + r io.Reader + bytesRead metric.Int64Counter +} + +func (r *progressReader) Read(b []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + } + + n, err := r.r.Read(b) + r.bytesRead.Add(r.ctx, int64(n), metric.WithAttributes( + attribute.String("griot.content.io.direction", "read"), + )) + return n, err +} + +func (c *Client) writeContent(ctx context.Context, pw *multipart.Writer, hash []byte, r io.Reader) error { + spanCtx, span := otel.Tracer("content").Start(ctx, "Client.writeContent") + defer span.End() + + bytesRead, err := otel.Meter("content").Int64Counter("griot.content.io", metric.WithUnit("By")) + if err != nil { + return err + } + + pr := &progressReader{ + ctx: spanCtx, + r: r, + bytesRead: bytesRead, + } + + filename := base64.StdEncoding.EncodeToString(hash) + + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="content"; filename=%q`, filename)) + header.Set("Content-Type", "application/octet-stream") + + part, err := pw.CreatePart(header) + if err != nil { + return err + } + + _, err = io.Copy(part, pr) + return err } From 3526fed168c5c15e22dbef21cf29a61aa43d5487 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 12:39:18 -0400 Subject: [PATCH 15/21] refactor(issue-33): just go with a single top level internal package --- cmd/griot/BUILD.bazel | 2 +- cmd/griot/app/BUILD.bazel | 2 +- cmd/griot/app/app.go | 2 +- cmd/griot/content/BUILD.bazel | 2 +- cmd/griot/content/content.go | 2 +- cmd/griot/content/upload/BUILD.bazel | 4 ++-- cmd/griot/content/upload/upload.go | 2 +- cmd/griot/content/upload/upload_test.go | 2 +- cmd/griot/main.go | 2 +- {cmd/internal => internal}/command/BUILD.bazel | 2 +- {cmd/internal => internal}/command/command.go | 0 {cmd/internal => internal}/command/error.go | 0 12 files changed, 11 insertions(+), 11 deletions(-) rename {cmd/internal => internal}/command/BUILD.bazel (87%) rename {cmd/internal => internal}/command/command.go (100%) rename {cmd/internal => internal}/command/error.go (100%) diff --git a/cmd/griot/BUILD.bazel b/cmd/griot/BUILD.bazel index 01cb082..b12a006 100644 --- a/cmd/griot/BUILD.bazel +++ b/cmd/griot/BUILD.bazel @@ -8,7 +8,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//cmd/griot/app", - "//cmd/internal/command", + "//internal/command", ], ) diff --git a/cmd/griot/app/BUILD.bazel b/cmd/griot/app/BUILD.bazel index 602d1df..0ccab02 100644 --- a/cmd/griot/app/BUILD.bazel +++ b/cmd/griot/app/BUILD.bazel @@ -7,6 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cmd/griot/content", - "//cmd/internal/command", + "//internal/command", ], ) diff --git a/cmd/griot/app/app.go b/cmd/griot/app/app.go index bcdd14c..58e9b32 100644 --- a/cmd/griot/app/app.go +++ b/cmd/griot/app/app.go @@ -18,7 +18,7 @@ import ( "context" "github.com/z5labs/griot/cmd/griot/content" - "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/internal/command" ) type Config struct{} diff --git a/cmd/griot/content/BUILD.bazel b/cmd/griot/content/BUILD.bazel index e5dcdc9..d363433 100644 --- a/cmd/griot/content/BUILD.bazel +++ b/cmd/griot/content/BUILD.bazel @@ -7,6 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cmd/griot/content/upload", - "//cmd/internal/command", + "//internal/command", ], ) diff --git a/cmd/griot/content/content.go b/cmd/griot/content/content.go index f3f9602..cb0d3e0 100644 --- a/cmd/griot/content/content.go +++ b/cmd/griot/content/content.go @@ -16,7 +16,7 @@ package content import ( "github.com/z5labs/griot/cmd/griot/content/upload" - "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/internal/command" ) func New() *command.App { diff --git a/cmd/griot/content/upload/BUILD.bazel b/cmd/griot/content/upload/BUILD.bazel index 89b2c75..c22c9f1 100644 --- a/cmd/griot/content/upload/BUILD.bazel +++ b/cmd/griot/content/upload/BUILD.bazel @@ -6,7 +6,7 @@ go_library( importpath = "github.com/z5labs/griot/cmd/griot/content/upload", visibility = ["//visibility:public"], deps = [ - "//cmd/internal/command", + "//internal/command", "//services/content", "//services/content/contentpb", "@com_github_spf13_pflag//:pflag", @@ -21,7 +21,7 @@ go_test( srcs = ["upload_test.go"], embed = [":upload"], deps = [ - "//cmd/internal/command", + "//internal/command", "//services/content", "//services/content/contentpb", "@com_github_stretchr_testify//assert", diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 3c54efb..fbcb402 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -29,7 +29,7 @@ import ( "slices" "strings" - "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/internal/command" "github.com/z5labs/griot/services/content" "github.com/z5labs/griot/services/content/contentpb" diff --git a/cmd/griot/content/upload/upload_test.go b/cmd/griot/content/upload/upload_test.go index 51145e7..528bc26 100644 --- a/cmd/griot/content/upload/upload_test.go +++ b/cmd/griot/content/upload/upload_test.go @@ -26,7 +26,7 @@ import ( "strings" "testing" - "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/internal/command" "github.com/z5labs/griot/services/content" "github.com/z5labs/griot/services/content/contentpb" diff --git a/cmd/griot/main.go b/cmd/griot/main.go index f2901e3..b1135e9 100644 --- a/cmd/griot/main.go +++ b/cmd/griot/main.go @@ -19,7 +19,7 @@ import ( _ "embed" "github.com/z5labs/griot/cmd/griot/app" - "github.com/z5labs/griot/cmd/internal/command" + "github.com/z5labs/griot/internal/command" ) //go:embed config.yaml diff --git a/cmd/internal/command/BUILD.bazel b/internal/command/BUILD.bazel similarity index 87% rename from cmd/internal/command/BUILD.bazel rename to internal/command/BUILD.bazel index 058503f..94aa634 100644 --- a/cmd/internal/command/BUILD.bazel +++ b/internal/command/BUILD.bazel @@ -6,7 +6,7 @@ go_library( "command.go", "error.go", ], - importpath = "github.com/z5labs/griot/cmd/internal/command", + importpath = "github.com/z5labs/griot/internal/command", visibility = ["//cmd:__subpackages__"], deps = [ "@com_github_go_viper_mapstructure_v2//:mapstructure", diff --git a/cmd/internal/command/command.go b/internal/command/command.go similarity index 100% rename from cmd/internal/command/command.go rename to internal/command/command.go diff --git a/cmd/internal/command/error.go b/internal/command/error.go similarity index 100% rename from cmd/internal/command/error.go rename to internal/command/error.go From 9af3a6ca0212bd5139f620f937899e9cc9f8de3d Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 13:54:04 -0400 Subject: [PATCH 16/21] test(issue-33): added cases for upload content client --- internal/ptr/BUILD.bazel | 8 + internal/ptr/ptr.go | 19 ++ services/content/BUILD.bazel | 22 +- services/content/{content.go => client.go} | 63 +++- services/content/client_example_test.go | 91 ++++++ services/content/client_test.go | 346 +++++++++++++++++++++ 6 files changed, 531 insertions(+), 18 deletions(-) create mode 100644 internal/ptr/BUILD.bazel create mode 100644 internal/ptr/ptr.go rename services/content/{content.go => client.go} (75%) create mode 100644 services/content/client_example_test.go create mode 100644 services/content/client_test.go diff --git a/internal/ptr/BUILD.bazel b/internal/ptr/BUILD.bazel new file mode 100644 index 0000000..91d4110 --- /dev/null +++ b/internal/ptr/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "ptr", + srcs = ["ptr.go"], + importpath = "github.com/z5labs/griot/internal/ptr", + visibility = ["//:__subpackages__"], +) diff --git a/internal/ptr/ptr.go b/internal/ptr/ptr.go new file mode 100644 index 0000000..29d791a --- /dev/null +++ b/internal/ptr/ptr.go @@ -0,0 +1,19 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ptr + +func Ref[T any](t T) *T { + return &t +} diff --git a/services/content/BUILD.bazel b/services/content/BUILD.bazel index 388680a..ca498d2 100644 --- a/services/content/BUILD.bazel +++ b/services/content/BUILD.bazel @@ -1,12 +1,13 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "content", - srcs = ["content.go"], + srcs = ["client.go"], importpath = "github.com/z5labs/griot/services/content", visibility = ["//visibility:public"], deps = [ "//services/content/contentpb", + "@com_github_z5labs_humus//humuspb", "@com_github_z5labs_humus//rest", "@io_opentelemetry_go_otel//:otel", "@io_opentelemetry_go_otel//attribute", @@ -15,3 +16,20 @@ go_library( "@org_golang_x_sync//errgroup", ], ) + +go_test( + name = "content_test", + srcs = [ + "client_example_test.go", + "client_test.go", + ], + embed = [":content"], + deps = [ + "//internal/ptr", + "//services/content/contentpb", + "@com_github_stretchr_testify//assert", + "@com_github_z5labs_humus//humuspb", + "@com_github_z5labs_humus//rest", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/services/content/content.go b/services/content/client.go similarity index 75% rename from services/content/content.go rename to services/content/client.go index e6d00f1..2fe3d77 100644 --- a/services/content/content.go +++ b/services/content/client.go @@ -28,6 +28,7 @@ import ( "github.com/z5labs/griot/services/content/contentpb" + "github.com/z5labs/humus/humuspb" "github.com/z5labs/humus/rest" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -41,33 +42,46 @@ type HttpClient interface { } type Client struct { - protoMarshal func(proto.Message) ([]byte, error) - http HttpClient + host string + protoMarshal func(proto.Message) ([]byte, error) + http HttpClient + protoUnmarshal func([]byte, proto.Message) error } -func NewClient(hc HttpClient) *Client { +func NewClient(hc HttpClient, host string) *Client { c := &Client{ - http: hc, + host: host, + protoMarshal: proto.Marshal, + http: hc, + protoUnmarshal: proto.Unmarshal, } return c } type UploadContentRequest struct { Metadata *contentpb.Metadata - Body io.Reader + Content io.Reader } type UploadContentResponse struct { Id string `json:"id"` } +type UnsupportedResponseContentTypeError struct { + ContentType string +} + +func (e UnsupportedResponseContentTypeError) Error() string { + return fmt.Sprintf("received unsupported response content type: %s", e.ContentType) +} + func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) (*UploadContentResponse, error) { spanCtx, span := otel.Tracer("content").Start(ctx, "Client.UploadContent") defer span.End() body, bodyWriter := io.Pipe() - respCh := make(chan *http.Response) + respCh := make(chan *http.Response, 1) eg, egctx := errgroup.WithContext(spanCtx) eg.Go(func() error { defer bodyWriter.Close() @@ -78,7 +92,7 @@ func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) ( defer close(respCh) defer body.Close() - r, err := http.NewRequestWithContext(egctx, http.MethodPost, "", body) + r, err := http.NewRequestWithContext(egctx, http.MethodPost, c.host+"/content/upload", body) if err != nil { return err } @@ -107,18 +121,31 @@ func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) ( return nil, spanCtx.Err() case resp = <-respCh: } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected http status code: %d", resp.StatusCode) - } defer resp.Body.Close() + contentType := resp.Header.Get("Content-Type") + if contentType != rest.ProtobufContentType { + return nil, UnsupportedResponseContentTypeError{ + ContentType: contentType, + } + } + b, err := io.ReadAll(resp.Body) if err != nil { return nil, err } + if resp.StatusCode != http.StatusOK { + var status humuspb.Status + err = c.protoUnmarshal(b, &status) + if err != nil { + return nil, err + } + return nil, &status + } + var uploadV1Resp contentpb.UploadContentV1Response - err = proto.Unmarshal(b, &uploadV1Resp) + err = c.protoUnmarshal(b, &uploadV1Resp) if err != nil { return nil, err } @@ -139,14 +166,18 @@ func (c *Client) writeUploadRequest(ctx context.Context, w io.Writer, req *Uploa return err } - err = c.writeContent(spanCtx, pw, req.Metadata.Checksum.Hash, req.Body) + err = c.writeContent(spanCtx, pw, req.Metadata.Checksum.Hash, req.Content) if err != nil { return err } return nil } -func (c *Client) writeMetadata(ctx context.Context, pw *multipart.Writer, meta *contentpb.Metadata) error { +type partCreater interface { + CreatePart(textproto.MIMEHeader) (io.Writer, error) +} + +func (c *Client) writeMetadata(ctx context.Context, creater partCreater, meta *contentpb.Metadata) error { _, span := otel.Tracer("content").Start(ctx, "Client.writeMetadata") defer span.End() @@ -159,7 +190,7 @@ func (c *Client) writeMetadata(ctx context.Context, pw *multipart.Writer, meta * header.Set("Content-Disposition", `form-data; name="metadata"`) header.Set("Content-Type", rest.ProtobufContentType) - part, err := pw.CreatePart(header) + part, err := creater.CreatePart(header) if err != nil { return err } @@ -195,7 +226,7 @@ func (r *progressReader) Read(b []byte) (int, error) { return n, err } -func (c *Client) writeContent(ctx context.Context, pw *multipart.Writer, hash []byte, r io.Reader) error { +func (c *Client) writeContent(ctx context.Context, creater partCreater, hash []byte, r io.Reader) error { spanCtx, span := otel.Tracer("content").Start(ctx, "Client.writeContent") defer span.End() @@ -216,7 +247,7 @@ func (c *Client) writeContent(ctx context.Context, pw *multipart.Writer, hash [] header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="content"; filename=%q`, filename)) header.Set("Content-Type", "application/octet-stream") - part, err := pw.CreatePart(header) + part, err := creater.CreatePart(header) if err != nil { return err } diff --git a/services/content/client_example_test.go b/services/content/client_example_test.go new file mode 100644 index 0000000..0604af0 --- /dev/null +++ b/services/content/client_example_test.go @@ -0,0 +1,91 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + + "github.com/z5labs/griot/internal/ptr" + "github.com/z5labs/griot/services/content/contentpb" + + "github.com/z5labs/humus/rest" + "google.golang.org/protobuf/proto" +) + +func ExampleClient_UploadContent() { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + contentId := "example-id" + b, err := proto.Marshal(&contentpb.UploadContentV1Response{ + Id: &contentpb.ContentId{ + Value: &contentId, + }, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", rest.ProtobufContentType) + w.WriteHeader(http.StatusOK) + io.Copy(w, bytes.NewReader(b)) + })) + + c := NewClient(http.DefaultClient, srv.URL) + + var content bytes.Buffer + hasher := sha256.New() + mw := io.MultiWriter(hasher, &content) + _, err := io.Copy(mw, strings.NewReader("hello world")) + if err != nil { + fmt.Println(err) + return + } + + resp, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Name: ptr.Ref("example-content"), + MediaType: &contentpb.MediaType{ + Type: ptr.Ref("text"), + Subtype: ptr.Ref("plain"), + }, + Checksum: &contentpb.Checksum{ + HashFunc: contentpb.HashFunc_SHA256.Enum(), + Hash: hasher.Sum(nil), + }, + }, + Content: &content, + }) + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(resp.Id) + //Output: example-id +} diff --git a/services/content/client_test.go b/services/content/client_test.go new file mode 100644 index 0000000..a400338 --- /dev/null +++ b/services/content/client_test.go @@ -0,0 +1,346 @@ +// Copyright 2024 Z5Labs and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/z5labs/griot/services/content/contentpb" + + "github.com/stretchr/testify/assert" + "github.com/z5labs/humus/humuspb" + "github.com/z5labs/humus/rest" + "google.golang.org/protobuf/proto" +) + +type httpClientFunc func(*http.Request) (*http.Response, error) + +func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { + return f(r) +} + +type readFunc func([]byte) (int, error) + +func (f readFunc) Read(b []byte) (int, error) { + return f(b) +} + +func TestClient_UploadContent(t *testing.T) { + t.Run("will return an error", func(t *testing.T) { + t.Run("if it fails to marshal the content metadata", func(t *testing.T) { + hc := httpClientFunc(func(r *http.Request) (*http.Response, error) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + return nil, err + }) + + c := NewClient(hc, "") + + marshalErr := errors.New("failed to marshal proto") + c.protoMarshal = func(m proto.Message) ([]byte, error) { + return nil, marshalErr + } + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{}) + if !assert.Equal(t, marshalErr, err) { + return + } + }) + + t.Run("if it fails to read the content", func(t *testing.T) { + hc := httpClientFunc(func(r *http.Request) (*http.Response, error) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + return nil, err + }) + + c := NewClient(hc, "") + + readErr := errors.New("failed to read") + content := readFunc(func(b []byte) (int, error) { + return 0, readErr + }) + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, readErr, err) { + return + } + }) + + t.Run("if it fails to do the http request", func(t *testing.T) { + httpErr := errors.New("failed to do http request") + hc := httpClientFunc(func(r *http.Request) (*http.Response, error) { + return nil, httpErr + }) + + c := NewClient(hc, "") + + content := strings.NewReader(`hello world`) + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, httpErr, err) { + return + } + }) + + t.Run("if the context is cancelled while doing http request", func(t *testing.T) { + hc := httpClientFunc(func(r *http.Request) (*http.Response, error) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + return nil, err + }) + + c := NewClient(hc, "") + + ctx, cancel := context.WithCancel(context.Background()) + content := readFunc(func(b []byte) (int, error) { + cancel() + n := copy(b, make([]byte, len(b))) + return n, nil + }) + + _, err := c.UploadContent(ctx, &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, context.Canceled, err) { + return + } + }) + + t.Run(fmt.Sprintf("if the response content type is not %s", rest.ProtobufContentType), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + })) + + hc := http.DefaultClient + + c := NewClient(hc, srv.URL) + + content := strings.NewReader("hello world") + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + + var uerr UnsupportedResponseContentTypeError + if !assert.ErrorAs(t, err, &uerr) { + return + } + if !assert.NotEmpty(t, uerr.Error()) { + return + } + if !assert.Equal(t, "application/json", uerr.ContentType) { + return + } + }) + + t.Run("if it fails to read the http response body", func(t *testing.T) { + readErr := errors.New("failed to read") + respBody := readFunc(func(b []byte) (int, error) { + return 0, readErr + }) + hc := httpClientFunc(func(r *http.Request) (*http.Response, error) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + return nil, err + } + + resp := &http.Response{ + Header: make(http.Header), + Body: io.NopCloser(respBody), + } + resp.Header.Set("Content-Type", rest.ProtobufContentType) + + return resp, nil + }) + + c := NewClient(hc, "") + + content := strings.NewReader("hello world") + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, readErr, err) { + return + } + }) + + t.Run("if it fails to unmarshal the humuspb.Status response", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + b, err := proto.Marshal(&humuspb.Status{ + Code: humuspb.Code_INTERNAL, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", rest.ProtobufContentType) + w.WriteHeader(http.StatusInternalServerError) + io.Copy(w, bytes.NewReader(b)) + })) + + hc := http.DefaultClient + + c := NewClient(hc, srv.URL) + + unmarshalErr := errors.New("failed to unmarshal") + c.protoUnmarshal = func(b []byte, m proto.Message) error { + return unmarshalErr + } + + content := strings.NewReader("hello world") + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, unmarshalErr, err) { + return + } + }) + + t.Run("if the response code is not HTTP 200", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + b, err := proto.Marshal(&humuspb.Status{ + Code: humuspb.Code_INTERNAL, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", rest.ProtobufContentType) + w.WriteHeader(http.StatusInternalServerError) + io.Copy(w, bytes.NewReader(b)) + })) + + hc := http.DefaultClient + + c := NewClient(hc, srv.URL) + + content := strings.NewReader("hello world") + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + + var status *humuspb.Status + if !assert.ErrorAs(t, err, &status) { + return + } + if !assert.Equal(t, humuspb.Code_INTERNAL, status.Code) { + return + } + }) + + t.Run("if it fails to unmarshal the contentpb.UploadContentV1Response response", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + _, err := io.Copy(io.Discard, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + b, err := proto.Marshal(&contentpb.UploadContentV1Response{}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", rest.ProtobufContentType) + w.WriteHeader(http.StatusOK) + io.Copy(w, bytes.NewReader(b)) + })) + + hc := http.DefaultClient + + c := NewClient(hc, srv.URL) + + unmarshalErr := errors.New("failed to unmarshal") + c.protoUnmarshal = func(b []byte, m proto.Message) error { + return unmarshalErr + } + + content := strings.NewReader("hello world") + + _, err := c.UploadContent(context.Background(), &UploadContentRequest{ + Metadata: &contentpb.Metadata{ + Checksum: &contentpb.Checksum{}, + }, + Content: content, + }) + if !assert.Equal(t, unmarshalErr, err) { + return + } + }) + }) +} From 21ad6e6c952e849296ba17a2fc35be6e643353e6 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 13:57:02 -0400 Subject: [PATCH 17/21] feat(issue-33): add flag for setting content host address --- cmd/griot/content/upload/upload.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index fbcb402..cfc1e16 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -45,6 +45,7 @@ func New(args ...string) *command.App { command.Args(args...), command.Short("Upload content"), command.Flags(func(fs *pflag.FlagSet) { + fs.String("content-host", "", "Specify the host for reaching griot.") fs.String("name", "", "Provide an optional name to help identify this content later.") fs.String("media-type", "", "Specify the content Media Type.") fs.String("source-file", "", "Specify the content source file.") @@ -62,6 +63,7 @@ func New(args ...string) *command.App { } type config struct { + Host string `flag:"content-host"` Name string `flag:"name"` MediaType string `flag:"media-type"` SourceFile string `flag:"source-file"` @@ -221,7 +223,7 @@ func initUploadHandler(ctx context.Context, cfg config) (command.Handler, error) hasher: contentHasher, src: src, out: os.Stdout, - content: content.NewClient(hc), + content: content.NewClient(hc, cfg.Host), } return h, nil } @@ -277,7 +279,7 @@ func (h *handler) Handle(ctx context.Context) error { Hash: h.hasher.Sum(nil), }, }, - Body: h.src, + Content: h.src, } resp, err := h.content.UploadContent(spanCtx, req) if err != nil { From d17f08a8bdf113cbdd7e20e9bf6c4109f6a95b5a Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 14:18:51 -0400 Subject: [PATCH 18/21] build(issue-33): update module lock after rebasing main --- MODULE.bazel.lock | 1290 ++++++++++++++++++++++----------------------- 1 file changed, 645 insertions(+), 645 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5f802a6..25e2ae9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -8,8 +8,8 @@ "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.2/MODULE.bazel": "4fae9c767526c17460c9d6e5731b09fdcab37b1322d6454acc137ac6d311e5b6", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.2/source.json": "cc829976e1b827bf3ef5ab682177a93c486c979a67a6b02dd0888e56486ed8ad", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.3/MODULE.bazel": "66baf724dbae7aff4787bf2245cc188d50cb08e07789769730151c0943587c14", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.3/source.json": "b290debdc0ab191a2a866b5a4e26f042c983026ff58b2e003ea634d838e3b6ae", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", @@ -102,19 +102,19 @@ "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=", - "usagesDigest": "aLmqbvowmHkkBPve05yyDNGN7oh7QE9kBADr3QIZTZs=", + "usagesDigest": "+hz7IHWN6A1oVJJWNDB6yZRG+RYhF76wAYItpAeIUIg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "local_config_apple_cc": { + "local_config_apple_cc_toolchains": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", + "ruleClassName": "_apple_cc_autoconf_toolchains", "attributes": {} }, - "local_config_apple_cc_toolchains": { + "local_config_apple_cc": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", + "ruleClassName": "_apple_cc_autoconf", "attributes": {} } }, @@ -129,62 +129,87 @@ }, "@@aspect_bazel_lib~//lib:extensions.bzl%toolchains": { "general": { - "bzlTransitiveDigest": "r7IIh4EKfqAOVGQAdB5/k5x8zTeExJ6xqLXx7uQWmGY=", - "usagesDigest": "jmHsY3UbKvny8kEsuA09AutSMTF6BI3QkaMjhVG+HDo=", + "bzlTransitiveDigest": "xOlbzeGJtMviNBabrKSUOG0zVTR0pKeFuNQdR48NeVY=", + "usagesDigest": "40dV0NikjTn+9+B2mQRmr4+Cy823T3+EkyQYkUzm3p8=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "expand_template_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "copy_directory_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", "attributes": { - "platform": "windows_amd64" + "platform": "darwin_amd64" } }, - "copy_to_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "copy_directory_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "copy_directory_freebsd_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_directory_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "copy_directory_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "copy_directory_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_platform_repo", "attributes": { "platform": "windows_amd64" } }, - "jq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "copy_directory_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", + "ruleClassName": "copy_directory_toolchains_repo", "attributes": { - "platform": "darwin_amd64", - "version": "1.7" + "user_repository_name": "copy_directory" } }, - "copy_to_directory_freebsd_amd64": { + "copy_to_directory_darwin_amd64": { "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "freebsd_amd64" + "platform": "darwin_amd64" } }, - "expand_template_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "copy_to_directory_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "linux_amd64" + "platform": "darwin_arm64" } }, - "jq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "copy_to_directory_freebsd_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "linux_arm64", - "version": "1.7" + "platform": "freebsd_amd64" } }, - "coreutils_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "copy_to_directory_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "darwin_arm64", - "version": "0.0.27" + "platform": "linux_amd64" } }, "copy_to_directory_linux_arm64": { @@ -194,76 +219,70 @@ "platform": "linux_arm64" } }, - "bsd_tar_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "copy_to_directory_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "linux_arm64" + "platform": "windows_amd64" } }, - "copy_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "copy_to_directory_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_toolchains_repo", "attributes": { - "platform": "darwin_amd64" + "user_repository_name": "copy_to_directory" } }, - "coreutils_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "jq_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { "platform": "darwin_amd64", - "version": "0.0.27" + "version": "1.7" } }, - "coreutils_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "jq_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "linux_arm64", - "version": "0.0.27" + "platform": "darwin_arm64", + "version": "1.7" } }, - "zstd_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "jq_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "linux_arm64" + "platform": "linux_amd64", + "version": "1.7" } }, - "yq_linux_s390x": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "jq_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "linux_s390x", - "version": "4.25.2" + "platform": "linux_arm64", + "version": "1.7" } }, - "yq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_host_alias_repo", - "attributes": {} - }, - "expand_template_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "jq_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "darwin_amd64" + "platform": "windows_amd64", + "version": "1.7" } }, - "copy_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } + "jq": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_host_alias_repo", + "attributes": {} }, - "jq_darwin_arm64": { + "jq_toolchains": { "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "ruleClassName": "jq_toolchains_repo", "attributes": { - "platform": "darwin_arm64", - "version": "1.7" + "user_repository_name": "jq" } }, "yq_darwin_amd64": { @@ -274,53 +293,59 @@ "version": "4.25.2" } }, - "copy_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "yq_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "platform": "linux_arm64" + "platform": "darwin_arm64", + "version": "4.25.2" } }, - "expand_template_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_toolchains_repo", + "yq_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "user_repository_name": "expand_template" + "platform": "linux_amd64", + "version": "4.25.2" } }, - "bats_assert": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", + "yq_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "sha256": "98ca3b685f8b8993e48ec057565e6e2abcc541034ed5b0e81f191505682037fd", - "urls": [ - "https://github.com/bats-core/bats-assert/archive/v2.1.0.tar.gz" - ], - "strip_prefix": "bats-assert-2.1.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"assert\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-assert\",\n visibility = [\"//visibility:public\"]\n)\n" + "platform": "linux_arm64", + "version": "4.25.2" } }, - "copy_to_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "yq_linux_s390x": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "platform": "darwin_amd64" + "platform": "linux_s390x", + "version": "4.25.2" } }, - "zstd_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "yq_linux_ppc64le": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "platform": "darwin_arm64" + "platform": "linux_ppc64le", + "version": "4.25.2" } }, - "bsd_tar_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "yq_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_platform_repo", "attributes": { - "platform": "linux_amd64" + "platform": "windows_amd64", + "version": "4.25.2" } }, + "yq": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", + "ruleClassName": "yq_host_alias_repo", + "attributes": {} + }, "yq_toolchains": { "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", "ruleClassName": "yq_toolchains_repo", @@ -328,100 +353,93 @@ "user_repository_name": "yq" } }, - "zstd_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "coreutils_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "linux_amd64" + "platform": "darwin_amd64", + "version": "0.0.27" } }, - "bats_support": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", + "coreutils_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "sha256": "7815237aafeb42ddcc1b8c698fc5808026d33317d8701d5ec2396e9634e2918f", - "urls": [ - "https://github.com/bats-core/bats-support/archive/v0.3.0.tar.gz" - ], - "strip_prefix": "bats-support-0.3.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"support\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-support\",\n visibility = [\"//visibility:public\"]\n)\n" + "platform": "darwin_arm64", + "version": "0.0.27" } }, - "bsd_tar_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "coreutils_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "windows_amd64" + "platform": "linux_amd64", + "version": "0.0.27" } }, - "jq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_host_alias_repo", - "attributes": {} - }, - "expand_template_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "coreutils_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "darwin_arm64" + "platform": "linux_arm64", + "version": "0.0.27" } }, - "bsd_tar_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "coreutils_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "darwin_arm64" + "platform": "windows_amd64", + "version": "0.0.27" } }, - "copy_to_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "coreutils_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_toolchains_repo", "attributes": { - "platform": "linux_amd64" + "user_repository_name": "coreutils" } }, - "coreutils_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "bsd_tar_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "linux_amd64", - "version": "0.0.27" + "platform": "darwin_amd64" } }, - "copy_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_toolchains_repo", + "bsd_tar_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "user_repository_name": "copy_directory" + "platform": "darwin_arm64" } }, - "yq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "bsd_tar_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "linux_amd64", - "version": "4.25.2" + "platform": "linux_amd64" } }, - "copy_to_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "bsd_tar_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "darwin_arm64" + "platform": "linux_arm64" } }, - "coreutils_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_toolchains_repo", + "bsd_tar_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "user_repository_name": "coreutils" + "platform": "windows_amd64" } }, - "copy_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "bsd_tar_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "tar_toolchains_repo", "attributes": { - "platform": "freebsd_amd64" + "user_repository_name": "bsd_tar" } }, "zstd_darwin_amd64": { @@ -431,80 +449,46 @@ "platform": "darwin_amd64" } }, - "zstd_toolchains": { + "zstd_darwin_arm64": { "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_toolchains_repo", + "ruleClassName": "zstd_binary_repo", "attributes": { - "user_repository_name": "zstd" + "platform": "darwin_arm64" } }, - "bats_file": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", + "zstd_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_binary_repo", "attributes": { - "sha256": "9b69043241f3af1c2d251f89b4fcafa5df3f05e97b89db18d7c9bdf5731bb27a", - "urls": [ - "https://github.com/bats-core/bats-file/archive/v0.4.0.tar.gz" - ], - "strip_prefix": "bats-file-0.4.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"file\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-file\",\n visibility = [\"//visibility:public\"]\n)\n" + "platform": "linux_amd64" } }, - "expand_template_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "zstd_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_binary_repo", "attributes": { "platform": "linux_arm64" } }, - "jq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "zstd_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_toolchains_repo", "attributes": { - "platform": "linux_amd64", - "version": "1.7" + "user_repository_name": "zstd" } }, - "bsd_tar_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "expand_template_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_platform_repo", "attributes": { "platform": "darwin_amd64" } }, - "bsd_tar_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "bats_toolchains": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd", - "urls": [ - "https://github.com/bats-core/bats-core/archive/v1.10.0.tar.gz" - ], - "strip_prefix": "bats-core-1.10.0", - "build_file_content": "load(\"@local_config_platform//:constraints.bzl\", \"HOST_CONSTRAINTS\")\nload(\"@aspect_bazel_lib//lib/private:bats_toolchain.bzl\", \"bats_toolchain\")\nload(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"core\",\n hardlink = \"on\",\n srcs = glob([\n \"lib/**\",\n \"libexec/**\"\n ]) + [\"bin/bats\"],\n out = \"bats-core\",\n)\n\nbats_toolchain(\n name = \"toolchain\",\n core = \":core\",\n libraries = [\"@bats_support//:support\", \"@bats_assert//:assert\", \"@bats_file//:file\"]\n)\n\ntoolchain(\n name = \"bats_toolchain\",\n exec_compatible_with = HOST_CONSTRAINTS,\n toolchain = \":toolchain\",\n toolchain_type = \"@aspect_bazel_lib//lib:bats_toolchain_type\",\n)\n" - } - }, - "yq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "4.25.2" - } - }, - "jq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "expand_template_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_platform_repo", "attributes": { - "platform": "windows_amd64", - "version": "1.7" + "platform": "darwin_arm64" } }, "expand_template_freebsd_amd64": { @@ -514,64 +498,80 @@ "platform": "freebsd_amd64" } }, - "yq_linux_ppc64le": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "expand_template_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_platform_repo", "attributes": { - "platform": "linux_ppc64le", - "version": "4.25.2" + "platform": "linux_amd64" } }, - "copy_to_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_toolchains_repo", + "expand_template_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_platform_repo", "attributes": { - "user_repository_name": "copy_to_directory" + "platform": "linux_arm64" } }, - "jq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_toolchains_repo", + "expand_template_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_platform_repo", "attributes": { - "user_repository_name": "jq" + "platform": "windows_amd64" } }, - "copy_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "expand_template_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", + "ruleClassName": "expand_template_toolchains_repo", "attributes": { - "platform": "darwin_arm64" + "user_repository_name": "expand_template" } }, - "copy_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "bats_support": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", "attributes": { - "platform": "windows_amd64" + "sha256": "7815237aafeb42ddcc1b8c698fc5808026d33317d8701d5ec2396e9634e2918f", + "urls": [ + "https://github.com/bats-core/bats-support/archive/v0.3.0.tar.gz" + ], + "strip_prefix": "bats-support-0.3.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"support\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-support\",\n visibility = [\"//visibility:public\"]\n)\n" } }, - "yq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "bats_assert": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", "attributes": { - "platform": "darwin_arm64", - "version": "4.25.2" + "sha256": "98ca3b685f8b8993e48ec057565e6e2abcc541034ed5b0e81f191505682037fd", + "urls": [ + "https://github.com/bats-core/bats-assert/archive/v2.1.0.tar.gz" + ], + "strip_prefix": "bats-assert-2.1.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"assert\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-assert\",\n visibility = [\"//visibility:public\"]\n)\n" } }, - "coreutils_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "bats_file": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", "attributes": { - "platform": "windows_amd64", - "version": "0.0.27" + "sha256": "9b69043241f3af1c2d251f89b4fcafa5df3f05e97b89db18d7c9bdf5731bb27a", + "urls": [ + "https://github.com/bats-core/bats-file/archive/v0.4.0.tar.gz" + ], + "strip_prefix": "bats-file-0.4.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"file\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-file\",\n visibility = [\"//visibility:public\"]\n)\n" } }, - "yq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "bats_toolchains": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", "attributes": { - "platform": "linux_arm64", - "version": "4.25.2" + "sha256": "a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd", + "urls": [ + "https://github.com/bats-core/bats-core/archive/v1.10.0.tar.gz" + ], + "strip_prefix": "bats-core-1.10.0", + "build_file_content": "load(\"@local_config_platform//:constraints.bzl\", \"HOST_CONSTRAINTS\")\nload(\"@aspect_bazel_lib//lib/private:bats_toolchain.bzl\", \"bats_toolchain\")\nload(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"core\",\n hardlink = \"on\",\n srcs = glob([\n \"lib/**\",\n \"libexec/**\"\n ]) + [\"bin/bats\"],\n out = \"bats-core\",\n)\n\nbats_toolchain(\n name = \"toolchain\",\n core = \":core\",\n libraries = [\"@bats_support//:support\", \"@bats_assert//:assert\", \"@bats_file//:file\"]\n)\n\ntoolchain(\n name = \"bats_toolchain\",\n exec_compatible_with = HOST_CONSTRAINTS,\n toolchain = \":toolchain\",\n toolchain_type = \"@aspect_bazel_lib//lib:bats_toolchain_type\",\n)\n" } } }, @@ -597,7 +597,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "QZFAJ0VIiJ5mGILpZn9r+TAbKxQ8T11/C+hyok0m2lU=", + "usagesDigest": "hkYK+bzahO4SEwt2YFc5tBeLQKBSxKn0RUhBni/nly0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -613,22 +613,80 @@ }, "@@rules_oci~//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "EeUyMVH1i0sdiUm4wGl2mwZxFcLKlcOC4svMASxOvaE=", - "usagesDigest": "BhQLj4CiuTfyF8JN+9hyWYiuQOMvrOmEO0PrfNlCTAI=", + "bzlTransitiveDigest": "hnn4URp35TJanShVM/sAVr9OPxb2jedn9VKMxicRg+w=", + "usagesDigest": "qE6TVNDZ9lHqcP7WRFJtzkoDrabm2wNdlWmtRSpVHBE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { + "distroless_base_linux_amd64": { + "bzlFile": "@@rules_oci~//oci/private:pull.bzl", + "ruleClassName": "oci_pull", + "attributes": { + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/base", + "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", + "platform": "linux/amd64", + "target_name": "distroless_base_linux_amd64", + "bazel_tags": [] + } + }, + "distroless_base_linux_arm64": { + "bzlFile": "@@rules_oci~//oci/private:pull.bzl", + "ruleClassName": "oci_pull", + "attributes": { + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/base", + "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", + "platform": "linux/arm64", + "target_name": "distroless_base_linux_arm64", + "bazel_tags": [] + } + }, + "distroless_base": { + "bzlFile": "@@rules_oci~//oci/private:pull.bzl", + "ruleClassName": "oci_alias", + "attributes": { + "target_name": "distroless_base", + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/base", + "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", + "platforms": { + "@@platforms//cpu:x86_64": "@distroless_base_linux_amd64", + "@@platforms//cpu:arm64": "@distroless_base_linux_arm64" + }, + "bzlmod_repository": "distroless_base", + "reproducible": true + } + }, "bazel_features_version": { "bzlFile": "@@bazel_features~//private:version_repo.bzl", "ruleClassName": "version_repo", "attributes": {} }, - "copy_to_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "bazel_features_globals": { + "bzlFile": "@@bazel_features~//private:globals_repo.bzl", + "ruleClassName": "globals_repo", "attributes": { - "platform": "windows_amd64" + "globals": { + "RunEnvironmentInfo": "5.3.0", + "DefaultInfo": "0.0.1", + "__TestingOnly_NeverAvailable": "1000000000.0.0" + } + } + }, + "bazel_skylib": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "sha256": "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz" + ] } }, "jq_darwin_amd64": { @@ -639,19 +697,20 @@ "version": "1.7" } }, - "copy_to_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "jq_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "freebsd_amd64" + "platform": "darwin_arm64", + "version": "1.7" } }, - "oci_crane_linux_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "jq_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" + "platform": "linux_amd64", + "version": "1.7" } }, "jq_linux_arm64": { @@ -662,139 +721,113 @@ "version": "1.7" } }, - "coreutils_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "0.0.27" - } - }, - "bsd_tar_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "jq_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_platform_repo", "attributes": { - "platform": "linux_arm64" + "platform": "windows_amd64", + "version": "1.7" } }, - "copy_to_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } + "jq": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_host_alias_repo", + "attributes": {} }, - "oci_regctl_toolchains": { - "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", - "ruleClassName": "toolchains_repo", + "jq_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", + "ruleClassName": "jq_toolchains_repo", "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + "user_repository_name": "jq" } }, - "oci_crane_linux_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "bsd_tar_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" + "platform": "darwin_amd64" } }, - "coreutils_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "bsd_tar_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "darwin_amd64", - "version": "0.0.27" + "platform": "darwin_arm64" } }, - "coreutils_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "bsd_tar_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "platform": "linux_arm64", - "version": "0.0.27" + "platform": "linux_amd64" } }, - "zstd_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "bsd_tar_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { "platform": "linux_arm64" } }, - "oci_crane_darwin_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "distroless_base_linux_amd64": { - "bzlFile": "@@rules_oci~//oci/private:pull.bzl", - "ruleClassName": "oci_pull", + "bsd_tar_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "bsdtar_binary_repo", "attributes": { - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/base", - "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", - "platform": "linux/amd64", - "target_name": "distroless_base_linux_amd64", - "bazel_tags": [] + "platform": "windows_amd64" } }, - "jq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "bsd_tar_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", + "ruleClassName": "tar_toolchains_repo", "attributes": { - "platform": "darwin_arm64", - "version": "1.7" + "user_repository_name": "bsd_tar" } }, - "oci_regctl_linux_s390x": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", + "coreutils_darwin_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "linux_s390x" + "platform": "darwin_amd64", + "version": "0.0.27" } }, - "oci_regctl_darwin_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", + "coreutils_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "darwin_amd64" + "platform": "darwin_arm64", + "version": "0.0.27" } }, - "oci_crane_linux_i386": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "coreutils_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" + "platform": "linux_amd64", + "version": "0.0.27" } }, - "oci_regctl_windows_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", + "coreutils_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "windows_amd64" + "platform": "linux_arm64", + "version": "0.0.27" } }, - "oci_crane_windows_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "coreutils_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_platform_repo", "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" + "platform": "windows_amd64", + "version": "0.0.27" } }, - "oci_crane_toolchains": { - "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", - "ruleClassName": "toolchains_repo", + "coreutils_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", + "ruleClassName": "coreutils_toolchains_repo", "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" + "user_repository_name": "coreutils" } }, "copy_to_directory_darwin_amd64": { @@ -804,125 +837,105 @@ "platform": "darwin_amd64" } }, - "zstd_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "copy_to_directory_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { "platform": "darwin_arm64" } }, - "bsd_tar_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "copy_to_directory_freebsd_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_to_directory_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { "platform": "linux_amd64" } }, - "oci_crane_windows_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "copy_to_directory_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" + "platform": "linux_arm64" } }, - "oci_crane_linux_s390x": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "copy_to_directory_windows_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_platform_repo", "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" + "platform": "windows_amd64" } }, - "oci_regctl_linux_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", + "copy_to_directory_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", + "ruleClassName": "copy_to_directory_toolchains_repo", "attributes": { - "platform": "linux_arm64" + "user_repository_name": "copy_to_directory" } }, - "zstd_linux_amd64": { + "zstd_darwin_amd64": { "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", "ruleClassName": "zstd_binary_repo", "attributes": { - "platform": "linux_amd64" + "platform": "darwin_amd64" } }, - "oci_regctl_darwin_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", + "zstd_darwin_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_binary_repo", "attributes": { "platform": "darwin_arm64" } }, - "bsd_tar_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_host_alias_repo", - "attributes": {} - }, - "oci_crane_darwin_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", + "zstd_linux_amd64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_binary_repo", "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" + "platform": "linux_amd64" } }, - "distroless_base": { - "bzlFile": "@@rules_oci~//oci/private:pull.bzl", - "ruleClassName": "oci_alias", + "zstd_linux_arm64": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_binary_repo", "attributes": { - "target_name": "distroless_base", - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/base", - "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_base_linux_amd64", - "@@platforms//cpu:arm64": "@distroless_base_linux_arm64" - }, - "bzlmod_repository": "distroless_base", - "reproducible": true + "platform": "linux_arm64" } }, - "bsd_tar_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "zstd_toolchains": { + "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", + "ruleClassName": "zstd_toolchains_repo", "attributes": { - "platform": "darwin_arm64" + "user_repository_name": "zstd" } }, - "copy_to_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "oci_crane_darwin_amd64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "platform": "linux_amd64" + "platform": "darwin_amd64", + "crane_version": "v0.18.0" } }, - "coreutils_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "oci_crane_darwin_arm64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "platform": "linux_amd64", - "version": "0.0.27" + "platform": "darwin_arm64", + "crane_version": "v0.18.0" } }, - "bazel_skylib": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", + "oci_crane_linux_arm64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "sha256": "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2", - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz" - ] + "platform": "linux_arm64", + "crane_version": "v0.18.0" } }, "oci_crane_linux_armv6": { @@ -933,115 +946,102 @@ "crane_version": "v0.18.0" } }, - "copy_to_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "oci_crane_linux_i386": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "platform": "darwin_arm64" + "platform": "linux_i386", + "crane_version": "v0.18.0" } }, - "coreutils_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_toolchains_repo", + "oci_crane_linux_s390x": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "user_repository_name": "coreutils" + "platform": "linux_s390x", + "crane_version": "v0.18.0" } }, - "zstd_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "oci_crane_linux_amd64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "platform": "darwin_amd64" + "platform": "linux_amd64", + "crane_version": "v0.18.0" } }, - "zstd_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_toolchains_repo", + "oci_crane_windows_armv6": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "user_repository_name": "zstd" + "platform": "windows_armv6", + "crane_version": "v0.18.0" } }, - "jq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "oci_crane_windows_amd64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "crane_repositories", "attributes": { - "platform": "linux_amd64", - "version": "1.7" + "platform": "windows_amd64", + "crane_version": "v0.18.0" } }, - "bsd_tar_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", + "oci_crane_toolchains": { + "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", + "ruleClassName": "toolchains_repo", "attributes": { - "platform": "darwin_amd64" + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" } }, - "oci_regctl_linux_amd64": { + "oci_regctl_darwin_amd64": { "bzlFile": "@@rules_oci~//oci:repositories.bzl", "ruleClassName": "regctl_repositories", "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" + "platform": "darwin_amd64" } }, - "jq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "oci_regctl_darwin_arm64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", "attributes": { - "platform": "windows_amd64", - "version": "1.7" + "platform": "darwin_arm64" } }, - "jq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_toolchains_repo", + "oci_regctl_linux_arm64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", "attributes": { - "user_repository_name": "jq" + "platform": "linux_arm64" } }, - "copy_to_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_toolchains_repo", + "oci_regctl_linux_s390x": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", "attributes": { - "user_repository_name": "copy_to_directory" + "platform": "linux_s390x" } }, - "bazel_features_globals": { - "bzlFile": "@@bazel_features~//private:globals_repo.bzl", - "ruleClassName": "globals_repo", + "oci_regctl_linux_amd64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", "attributes": { - "globals": { - "RunEnvironmentInfo": "5.3.0", - "DefaultInfo": "0.0.1", - "__TestingOnly_NeverAvailable": "1000000000.0.0" - } + "platform": "linux_amd64" } }, - "distroless_base_linux_arm64": { - "bzlFile": "@@rules_oci~//oci/private:pull.bzl", - "ruleClassName": "oci_pull", + "oci_regctl_windows_amd64": { + "bzlFile": "@@rules_oci~//oci:repositories.bzl", + "ruleClassName": "regctl_repositories", "attributes": { - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/base", - "identifier": "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", - "platform": "linux/arm64", - "target_name": "distroless_base_linux_arm64", - "bazel_tags": [] + "platform": "windows_amd64" } - }, - "coreutils_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + }, + "oci_regctl_toolchains": { + "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", + "ruleClassName": "toolchains_repo", "attributes": { - "platform": "windows_amd64", - "version": "0.0.27" + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" } } }, @@ -1086,25 +1086,25 @@ }, "@@rules_python~//python/extensions:python.bzl%python": { "general": { - "bzlTransitiveDigest": "uHZmn4VCpelMhuk7Rz8Q5oK94ttURW5KkyvXa6hRTfk=", - "usagesDigest": "fQEsnAYwqRJT7/lTBAe+NllONXk6f/Tc57oiPxLG8SI=", + "bzlTransitiveDigest": "8vDKUdCc6qEk2/YsFiPsFO1Jqgl+XPFRklapOxMAbE8=", + "usagesDigest": "abUgYqI1bdv/jc3Xu7C2SbT7mmtxAziRT/kUCFERO+A=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": { "RULES_PYTHON_BZLMOD_DEBUG": null }, "generatedRepoSpecs": { - "python_3_11_s390x-unknown-linux-gnu": { + "python_3_11_aarch64-apple-darwin": { "bzlFile": "@@rules_python~//python:repositories.bzl", "ruleClassName": "python_repository", "attributes": { - "sha256": "49520e3ff494708020f306e30b0964f079170be83e956be4504f850557378a22", + "sha256": "b042c966920cf8465385ca3522986b12d745151a72c060991088977ca36d3883", "patches": [], - "platform": "s390x-unknown-linux-gnu", + "platform": "aarch64-apple-darwin", "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz", + "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz", "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz" ], "distutils_content": "", "strip_prefix": "python", @@ -1112,40 +1112,6 @@ "ignore_root_user_error": false } }, - "python_3_11_host": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "host_toolchain", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, - "python_3_11": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "toolchain_aliases", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, "python_3_11_aarch64-unknown-linux-gnu": { "bzlFile": "@@rules_python~//python:repositories.bzl", "ruleClassName": "python_repository", @@ -1164,17 +1130,17 @@ "ignore_root_user_error": false } }, - "python_3_11_aarch64-apple-darwin": { + "python_3_11_ppc64le-unknown-linux-gnu": { "bzlFile": "@@rules_python~//python:repositories.bzl", "ruleClassName": "python_repository", "attributes": { - "sha256": "b042c966920cf8465385ca3522986b12d745151a72c060991088977ca36d3883", + "sha256": "b44e1b74afe75c7b19143413632c4386708ae229117f8f950c2094e9681d34c7", "patches": [], - "platform": "aarch64-apple-darwin", + "platform": "ppc64le-unknown-linux-gnu", "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz", + "release_filename": "20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz", "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz" + "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz" ], "distutils_content": "", "strip_prefix": "python", @@ -1182,17 +1148,17 @@ "ignore_root_user_error": false } }, - "python_3_11_ppc64le-unknown-linux-gnu": { + "python_3_11_s390x-unknown-linux-gnu": { "bzlFile": "@@rules_python~//python:repositories.bzl", "ruleClassName": "python_repository", "attributes": { - "sha256": "b44e1b74afe75c7b19143413632c4386708ae229117f8f950c2094e9681d34c7", + "sha256": "49520e3ff494708020f306e30b0964f079170be83e956be4504f850557378a22", "patches": [], - "platform": "ppc64le-unknown-linux-gnu", + "platform": "s390x-unknown-linux-gnu", "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "release_filename": "20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz", "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz" ], "distutils_content": "", "strip_prefix": "python", @@ -1218,34 +1184,6 @@ "ignore_root_user_error": false } }, - "pythons_hub": { - "bzlFile": "@@rules_python~//python/private/bzlmod:pythons_hub.bzl", - "ruleClassName": "hub_repo", - "attributes": { - "default_python_version": "3.11", - "toolchain_prefixes": [ - "_0000_python_3_11_" - ], - "toolchain_python_versions": [ - "3.11" - ], - "toolchain_set_python_version_constraints": [ - "False" - ], - "toolchain_user_repository_names": [ - "python_3_11" - ] - } - }, - "python_versions": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "multi_toolchain_aliases", - "attributes": { - "python_versions": { - "3.11": "python_3_11" - } - } - }, "python_3_11_x86_64-pc-windows-msvc": { "bzlFile": "@@rules_python~//python:repositories.bzl", "ruleClassName": "python_repository", @@ -1281,6 +1219,68 @@ "coverage_tool": "", "ignore_root_user_error": false } + }, + "python_3_11_host": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "host_toolchain", + "attributes": { + "python_version": "3.11.7", + "user_repository_name": "python_3_11", + "platforms": [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "ppc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu" + ] + } + }, + "python_3_11": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "toolchain_aliases", + "attributes": { + "python_version": "3.11.7", + "user_repository_name": "python_3_11", + "platforms": [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "ppc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu" + ] + } + }, + "pythons_hub": { + "bzlFile": "@@rules_python~//python/private/bzlmod:pythons_hub.bzl", + "ruleClassName": "hub_repo", + "attributes": { + "default_python_version": "3.11", + "toolchain_prefixes": [ + "_0000_python_3_11_" + ], + "toolchain_python_versions": [ + "3.11" + ], + "toolchain_set_python_version_constraints": [ + "False" + ], + "toolchain_user_repository_names": [ + "python_3_11" + ] + } + }, + "python_versions": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "multi_toolchain_aliases", + "attributes": { + "python_versions": { + "3.11": "python_3_11" + } + } } }, "recordedRepoMappingEntries": [ @@ -1299,18 +1299,23 @@ }, "@@rules_python~//python/private/bzlmod:internal_deps.bzl%internal_deps": { "general": { - "bzlTransitiveDigest": "k05IAnu/N6e9CnPr7q0UXa7U54cIegSInQS0KZiOV9Y=", - "usagesDigest": "4Fj9JSpEDoJSLPRSbvSTol2bTL7baZjuA3k9U7kG/1k=", + "bzlTransitiveDigest": "7yogJIhmw7i9Wq/n9sQB8N0F84220dJbw64SjOwrmQk=", + "usagesDigest": "r7vtlnQfWxEwrL+QFXux06yzeWEkq/hrcwAssoCoSLY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "pypi__wheel": { + "rules_python_internal": { + "bzlFile": "@@rules_python~//python/private:internal_config_repo.bzl", + "ruleClassName": "internal_config_repo", + "attributes": {} + }, + "pypi__build": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", - "sha256": "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", + "url": "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", + "sha256": "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } @@ -1325,137 +1330,132 @@ "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__importlib_metadata": { + "pypi__colorama": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", - "sha256": "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__pyproject_hooks": { + "pypi__importlib_metadata": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", - "sha256": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", + "url": "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", + "sha256": "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__pep517": { + "pypi__installer": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/ee/2f/ef63e64e9429111e73d3d6cbee80591672d16f2725e648ebc52096f3d323/pep517-0.13.0-py3-none-any.whl", - "sha256": "4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__packaging": { + "pypi__more_itertools": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", - "sha256": "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "url": "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", + "sha256": "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__pip_tools": { + "pypi__packaging": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", - "sha256": "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", + "url": "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", + "sha256": "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__setuptools": { + "pypi__pep517": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", - "sha256": "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", + "url": "https://files.pythonhosted.org/packages/ee/2f/ef63e64e9429111e73d3d6cbee80591672d16f2725e648ebc52096f3d323/pep517-0.13.0-py3-none-any.whl", + "sha256": "4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__zipp": { + "pypi__pip": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", - "sha256": "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", + "url": "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", + "sha256": "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__colorama": { + "pypi__pip_tools": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "url": "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", + "sha256": "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__build": { + "pypi__pyproject_hooks": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", - "sha256": "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", + "url": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "sha256": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "rules_python_internal": { - "bzlFile": "@@rules_python~//python/private:internal_config_repo.bzl", - "ruleClassName": "internal_config_repo", - "attributes": {} - }, - "pypi__pip": { + "pypi__setuptools": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", - "sha256": "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", + "url": "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", + "sha256": "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__installer": { + "pypi__tomli": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__more_itertools": { + "pypi__wheel": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", - "sha256": "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", + "url": "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", + "sha256": "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } }, - "pypi__tomli": { + "pypi__zipp": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "url": "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", + "sha256": "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", "type": "zip", "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" } From d5e1292b76328e45e97c0cce555dea3b2983fa1f Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Sat, 26 Oct 2024 14:34:05 -0400 Subject: [PATCH 19/21] fix(issue-33): make sure to close content source file --- cmd/griot/content/upload/upload.go | 3 ++- cmd/griot/content/upload/upload_test.go | 28 +++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index cfc1e16..8eed69c 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -179,7 +179,7 @@ type handler struct { contentName string mediaType string hasher hasher - src io.ReadSeeker + src io.ReadSeekCloser out io.Writer content uploadClient @@ -240,6 +240,7 @@ func (e FailedToSeekReadBytesError) Error() string { func (h *handler) Handle(ctx context.Context) error { spanCtx, span := otel.Tracer("upload").Start(ctx, "handler.Handle") defer span.End() + defer h.src.Close() bytesRead, err := io.Copy(h.hasher, h.src) if err != nil { diff --git a/cmd/griot/content/upload/upload_test.go b/cmd/griot/content/upload/upload_test.go index 528bc26..c162d1f 100644 --- a/cmd/griot/content/upload/upload_test.go +++ b/cmd/griot/content/upload/upload_test.go @@ -258,6 +258,14 @@ func (f seekNoopReader) Seek(offset int64, whence int) (int64, error) { return f(offset, whence) } +type readSeekerNopCloser struct { + io.ReadSeeker +} + +func (readSeekerNopCloser) Close() error { + return nil +} + type uploadClientFunc func(context.Context, *content.UploadContentRequest) (*content.UploadContentResponse, error) func (f uploadClientFunc) UploadContent(ctx context.Context, req *content.UploadContentRequest) (*content.UploadContentResponse, error) { @@ -275,7 +283,9 @@ func TestHandler_Handle(t *testing.T) { h := &handler{ log: slog.New(noop.LogHandler{}), hasher: sha256Hasher{Hash: sha256.New()}, - src: src, + src: readSeekerNopCloser{ + ReadSeeker: src, + }, } err := h.Handle(context.Background()) @@ -293,7 +303,9 @@ func TestHandler_Handle(t *testing.T) { h := &handler{ log: slog.New(noop.LogHandler{}), hasher: sha256Hasher{Hash: sha256.New()}, - src: src, + src: readSeekerNopCloser{ + ReadSeeker: src, + }, } err := h.Handle(context.Background()) @@ -311,7 +323,9 @@ func TestHandler_Handle(t *testing.T) { h := &handler{ log: slog.New(noop.LogHandler{}), hasher: sha256Hasher{Hash: sha256.New()}, - src: src, + src: readSeekerNopCloser{ + ReadSeeker: src, + }, } err := h.Handle(context.Background()) @@ -340,9 +354,11 @@ func TestHandler_Handle(t *testing.T) { }) h := &handler{ - log: slog.New(noop.LogHandler{}), - hasher: sha256Hasher{Hash: sha256.New()}, - src: src, + log: slog.New(noop.LogHandler{}), + hasher: sha256Hasher{Hash: sha256.New()}, + src: readSeekerNopCloser{ + ReadSeeker: src, + }, content: client, } From 9260bcfdab5d972bdbdef509b713db87b73c62ea Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Thu, 31 Oct 2024 22:34:08 -0400 Subject: [PATCH 20/21] chore(issue-33): record errors on spans --- cmd/griot/content/upload/upload.go | 2 ++ services/content/client.go | 25 +++++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/cmd/griot/content/upload/upload.go b/cmd/griot/content/upload/upload.go index 8eed69c..deddce0 100644 --- a/cmd/griot/content/upload/upload.go +++ b/cmd/griot/content/upload/upload.go @@ -282,8 +282,10 @@ func (h *handler) Handle(ctx context.Context) error { }, Content: h.src, } + resp, err := h.content.UploadContent(spanCtx, req) if err != nil { + span.RecordError(err) h.log.ErrorContext(spanCtx, "failed to upload content", slog.String("error", err.Error())) return err } diff --git a/services/content/client.go b/services/content/client.go index 2fe3d77..329de88 100644 --- a/services/content/client.go +++ b/services/content/client.go @@ -112,6 +112,7 @@ func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) ( err := eg.Wait() if err != nil { + span.RecordError(err) return nil, err } @@ -125,13 +126,16 @@ func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) ( contentType := resp.Header.Get("Content-Type") if contentType != rest.ProtobufContentType { - return nil, UnsupportedResponseContentTypeError{ + err = UnsupportedResponseContentTypeError{ ContentType: contentType, } + span.RecordError(err) + return nil, err } b, err := io.ReadAll(resp.Body) if err != nil { + span.RecordError(err) return nil, err } @@ -139,14 +143,18 @@ func (c *Client) UploadContent(ctx context.Context, req *UploadContentRequest) ( var status humuspb.Status err = c.protoUnmarshal(b, &status) if err != nil { + span.RecordError(err) return nil, err } + + span.RecordError(&status) return nil, &status } var uploadV1Resp contentpb.UploadContentV1Response err = c.protoUnmarshal(b, &uploadV1Resp) if err != nil { + span.RecordError(err) return nil, err } @@ -183,6 +191,7 @@ func (c *Client) writeMetadata(ctx context.Context, creater partCreater, meta *c b, err := c.protoMarshal(meta) if err != nil { + span.RecordError(err) return err } @@ -192,16 +201,19 @@ func (c *Client) writeMetadata(ctx context.Context, creater partCreater, meta *c part, err := creater.CreatePart(header) if err != nil { + span.RecordError(err) return err } n, err := io.Copy(part, bytes.NewReader(b)) if err != nil { + span.RecordError(err) return err } if n != int64(len(b)) { - // TODO - return errors.New("did not write all metadata bytes") + err = errors.New("did not write all metadata bytes") + span.RecordError(err) + return err } return nil } @@ -249,9 +261,14 @@ func (c *Client) writeContent(ctx context.Context, creater partCreater, hash []b part, err := creater.CreatePart(header) if err != nil { + span.RecordError(err) return err } _, err = io.Copy(part, pr) - return err + if err != nil { + span.RecordError(err) + return err + } + return nil } From 37c62a5f31c743bd2a23498c6cffc12f118a17f6 Mon Sep 17 00:00:00 2001 From: Richard Carson Derr Date: Thu, 31 Oct 2024 22:35:36 -0400 Subject: [PATCH 21/21] build(issue-33): bazel tidy --- MODULE.bazel.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 25e2ae9..11d6a07 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -614,7 +614,7 @@ "@@rules_oci~//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "hnn4URp35TJanShVM/sAVr9OPxb2jedn9VKMxicRg+w=", - "usagesDigest": "qE6TVNDZ9lHqcP7WRFJtzkoDrabm2wNdlWmtRSpVHBE=", + "usagesDigest": "4T/9WIAiMEEUB1m0pFRls1F/wPKLVrmNgCZxc+LFscM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {},