diff --git a/go/go.mod b/go/go.mod index d6cce6b85..2dfd6981f 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,6 +3,8 @@ module github.com/google/genkit/go go 1.22.0 require ( + cloud.google.com/go/aiplatform v1.60.0 + cloud.google.com/go/logging v1.9.0 cloud.google.com/go/vertexai v0.7.1 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.46.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.22.0 @@ -11,6 +13,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/invopop/jsonschema v0.12.0 + github.com/jba/slog v0.2.0 github.com/wk8/go-ordered-map/v2 v2.1.8 go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/metric v1.26.0 @@ -19,13 +22,13 @@ require ( go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 google.golang.org/api v0.177.0 + google.golang.org/protobuf v1.34.0 gopkg.in/yaml.v3 v3.0.1 ) require ( cloud.google.com/go v0.112.2 // indirect cloud.google.com/go/ai v0.3.0 // indirect - cloud.google.com/go/aiplatform v1.60.0 // indirect cloud.google.com/go/auth v0.3.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect @@ -60,6 +63,5 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.34.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go/go.sum b/go/go.sum index b989162af..5ec241394 100644 --- a/go/go.sum +++ b/go/go.sum @@ -93,6 +93,10 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/ github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jba/slog v0.1.0 h1:m7pbPxGRvFcQy4vONykm/9X+0Fx4FGEDl7A6E/C/z9Q= +github.com/jba/slog v0.1.0/go.mod h1:R9u+1Qbl7LcDnJaFNIPer+AJa3yK9eZ8SQUE4waKFiw= +github.com/jba/slog v0.2.0 h1:jI0U5NRR3EJKGsbeEVpItJNogk0c4RMeCl7vJmogCJI= +github.com/jba/slog v0.2.0/go.mod h1:0Dh7Vyz3Td68Z1OwzadfincHwr7v+PpzadrS2Jua338= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/go/plugins/googlecloud/googlecloud.go b/go/plugins/googlecloud/googlecloud.go index e88aae37d..1b2efbb78 100644 --- a/go/plugins/googlecloud/googlecloud.go +++ b/go/plugins/googlecloud/googlecloud.go @@ -20,9 +20,11 @@ package googlecloud import ( "context" + "log/slog" "os" "time" + "cloud.google.com/go/logging" mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" "github.com/google/genkit/go/genkit" @@ -40,6 +42,10 @@ type Options struct { // The interval for exporting metric data. // The default is 60 seconds. MetricInterval time.Duration + + // The minimum level at which logs will be written. + // Defaults to [slog.LevelInfo]. + LogLevel slog.Leveler } // Init initializes all telemetry in this package. @@ -59,8 +65,10 @@ func Init(ctx context.Context, projectID string, opts *Options) error { } aexp := &adjustingTraceExporter{texp} genkit.RegisterSpanProcessor(sdktrace.NewBatchSpanProcessor(aexp)) - - return setMeterProvider(projectID, opts.MetricInterval) + if err := setMeterProvider(projectID, opts.MetricInterval); err != nil { + return err + } + return setLogHandler(projectID, opts.LogLevel) } func setMeterProvider(projectID string, interval time.Duration) error { @@ -109,3 +117,13 @@ func (s adjustedSpan) Attributes() []attribute.KeyValue { } return ts } + +func setLogHandler(projectID string, level slog.Leveler) error { + c, err := logging.NewClient(context.Background(), "projects/"+projectID) + if err != nil { + return err + } + logger := c.Logger("genkit_log") + slog.SetDefault(slog.New(newHandler(level, logger.Log))) + return nil +} diff --git a/go/plugins/googlecloud/googlecloud_test.go b/go/plugins/googlecloud/googlecloud_test.go index f7ac5620b..111aa7c7d 100644 --- a/go/plugins/googlecloud/googlecloud_test.go +++ b/go/plugins/googlecloud/googlecloud_test.go @@ -19,6 +19,9 @@ package googlecloud import ( "context" "flag" + "log/slog" + "os" + "runtime" "testing" "time" @@ -30,8 +33,11 @@ import ( var projectID = flag.String("project", "", "GCP project ID") // This test is part of verifying that we can export traces to GCP. -// To verify, run the test, then visit the GCP Trace Explorer and look for the "test" -// trace, and visit the Metrics Explorer and look for the "Generic Node - test" metric. +// To verify, run the test, then: +// - visit the GCP Trace Explorer and look for the "test" trace +// - visit the Metrics Explorer and look for the "Generic Node - test" metric. +// - visit the Logging Explorer and look for the genkit_log logName, or run +// gcloud --project PROJECT_ID logging read 'logName:genkit_log' func TestGCP(t *testing.T) { if *projectID == "" { t.Skip("no -project") @@ -64,4 +70,14 @@ func TestGCP(t *testing.T) { // Allow time to sample and export. time.Sleep(2 * time.Second) }) + t.Run("logging", func(t *testing.T) { + if err := setLogHandler(*projectID, slog.LevelInfo); err != nil { + t.Fatal(err) + } + slog.Info("testing GCP logging", + "binaryName", os.Args[0], + "goVersion", runtime.Version()) + // Allow time to export. + time.Sleep(2 * time.Second) + }) } diff --git a/go/plugins/googlecloud/slog_handler.go b/go/plugins/googlecloud/slog_handler.go new file mode 100644 index 000000000..7c644b0cd --- /dev/null +++ b/go/plugins/googlecloud/slog_handler.go @@ -0,0 +1,149 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The googlecloud package supports telemetry (tracing, metrics and logging) using +// Google Cloud services. +package googlecloud + +import ( + "context" + "log/slog" + + "cloud.google.com/go/logging" + "github.com/jba/slog/withsupport" +) + +func newHandler(level slog.Leveler, f func(logging.Entry)) *handler { + if level == nil { + level = slog.LevelInfo + } + return &handler{ + level: level, + handleEntry: f, + } +} + +type handler struct { + level slog.Leveler + handleEntry func(logging.Entry) + goa *withsupport.GroupOrAttrs +} + +func (h *handler) Enabled(ctx context.Context, level slog.Level) bool { + return level >= h.level.Level() +} + +func (h *handler) WithAttrs(as []slog.Attr) slog.Handler { + h2 := *h + h2.goa = h2.goa.WithAttrs(as) + return &h2 +} + +func (h *handler) WithGroup(name string) slog.Handler { + h2 := *h + h2.goa = h2.goa.WithGroup(name) + return &h2 +} + +func (h *handler) Handle(ctx context.Context, r slog.Record) error { + h.handleEntry(h.recordToEntry(ctx, r)) + return nil +} + +func (h *handler) recordToEntry(ctx context.Context, r slog.Record) logging.Entry { + return logging.Entry{ + Timestamp: r.Time, + Severity: levelToSeverity(r.Level), + Payload: recordToMap(r, h.goa.Collect()), + Labels: map[string]string{"module": "genkit"}, + // TODO: add a monitored resource + // Resource: &monitoredres.MonitoredResource{}, + // TODO: add trace information from the context. + // Trace: "", + // SpanID: "", + // TraceSampled: false, + } +} + +func levelToSeverity(l slog.Level) logging.Severity { + switch { + case l < slog.LevelInfo: + return logging.Debug + case l == slog.LevelInfo: + return logging.Info + case l < slog.LevelWarn: + return logging.Notice + case l < slog.LevelError: + return logging.Warning + case l == slog.LevelError: + return logging.Error + case l <= slog.LevelError+4: + return logging.Critical + case l <= slog.LevelError+8: + return logging.Alert + default: + return logging.Emergency + } +} +func recordToMap(r slog.Record, goras []*withsupport.GroupOrAttrs) map[string]any { + root := map[string]any{} + root[slog.MessageKey] = r.Message + + m := root + for i, gora := range goras { + if gora.Group != "" { + if i == len(goras)-1 && r.NumAttrs() == 0 { + continue + } + m2 := map[string]any{} + m[gora.Group] = m2 + m = m2 + } else { + for _, a := range gora.Attrs { + handleAttr(a, m) + } + } + } + r.Attrs(func(a slog.Attr) bool { + handleAttr(a, m) + return true + }) + return root +} + +func handleAttr(a slog.Attr, m map[string]any) { + if a.Equal(slog.Attr{}) { + return + } + v := a.Value.Resolve() + if v.Kind() == slog.KindGroup { + gas := v.Group() + if len(gas) == 0 { + return + } + if a.Key == "" { + for _, ga := range gas { + handleAttr(ga, m) + } + } else { + gm := map[string]any{} + for _, ga := range gas { + handleAttr(ga, gm) + } + m[a.Key] = gm + } + } else { + m[a.Key] = v.Any() + } +} diff --git a/go/plugins/googlecloud/slog_handler_test.go b/go/plugins/googlecloud/slog_handler_test.go new file mode 100644 index 000000000..7feb3eb01 --- /dev/null +++ b/go/plugins/googlecloud/slog_handler_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The googlecloud package supports telemetry (tracing, metrics and logging) using +// Google Cloud services. +package googlecloud + +import ( + "log/slog" + "testing" + "testing/slogtest" + + "cloud.google.com/go/logging" +) + +func TestHandler(t *testing.T) { + var results []map[string]any + + f := func(e logging.Entry) { + results = append(results, entryToMap(e)) + } + + if err := slogtest.TestHandler(newHandler(slog.LevelInfo, f), func() []map[string]any { return results }); err != nil { + t.Fatal(err) + } +} + +func entryToMap(e logging.Entry) map[string]any { + m := map[string]any{} + if !e.Timestamp.IsZero() { + m[slog.TimeKey] = e.Timestamp + } + m[slog.LevelKey] = e.Severity + pm := e.Payload.(map[string]any) + for k, v := range pm { + m[k] = v + } + return m +}