[Go] logging for GCP

go/go.mod

@@ -3,6 +3,8 @@ module github.com/google/genkit/go
 go 1.22.0
 
 require (
+	cloud.google.com/go/aiplatform v1.60.0
+	cloud.google.com/go/logging v1.9.0
 	cloud.google.com/go/vertexai v0.7.1
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.46.0
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.22.0
@@ -11,6 +13,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/invopop/jsonschema v0.12.0
+	github.com/jba/slog v0.2.0
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	go.opentelemetry.io/otel v1.26.0
 	go.opentelemetry.io/otel/metric v1.26.0
@@ -19,13 +22,13 @@ require (
 	go.opentelemetry.io/otel/trace v1.26.0
 	golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
 	google.golang.org/api v0.177.0
+	google.golang.org/protobuf v1.34.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	cloud.google.com/go v0.112.2 // indirect
 	cloud.google.com/go/ai v0.3.0 // indirect
-	cloud.google.com/go/aiplatform v1.60.0 // indirect
 	cloud.google.com/go/auth v0.3.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
@@ -60,6 +63,5 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
 	google.golang.org/grpc v1.63.2 // indirect
-	google.golang.org/protobuf v1.34.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go/go.sum

@@ -93,6 +93,10 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/
 github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
 github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
 github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
+github.com/jba/slog v0.1.0 h1:m7pbPxGRvFcQy4vONykm/9X+0Fx4FGEDl7A6E/C/z9Q=
+github.com/jba/slog v0.1.0/go.mod h1:R9u+1Qbl7LcDnJaFNIPer+AJa3yK9eZ8SQUE4waKFiw=
+github.com/jba/slog v0.2.0 h1:jI0U5NRR3EJKGsbeEVpItJNogk0c4RMeCl7vJmogCJI=
+github.com/jba/slog v0.2.0/go.mod h1:0Dh7Vyz3Td68Z1OwzadfincHwr7v+PpzadrS2Jua338=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=

go/plugins/googlecloud/googlecloud.go

@@ -20,9 +20,11 @@ package googlecloud
 
 import (
 	"context"
+	"log/slog"
 	"os"
 	"time"
 
+	"cloud.google.com/go/logging"
 	mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
 	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
 	"github.com/google/genkit/go/genkit"
@@ -40,6 +42,10 @@ type Options struct {
 	// The interval for exporting metric data.
 	// The default is 60 seconds.
 	MetricInterval time.Duration
+
+	// The minimum level at which logs will be written.
+	// Defaults to [slog.LevelInfo].
+	LogLevel slog.Leveler
 }
 
 // Init initializes all telemetry in this package.
@@ -59,8 +65,10 @@ func Init(ctx context.Context, projectID string, opts *Options) error {
 	}
 	aexp := &adjustingTraceExporter{texp}
 	genkit.RegisterSpanProcessor(sdktrace.NewBatchSpanProcessor(aexp))
-
-	return setMeterProvider(projectID, opts.MetricInterval)
+	if err := setMeterProvider(projectID, opts.MetricInterval); err != nil {
+		return err
+	}
+	return setLogHandler(projectID, opts.LogLevel)
 }
 
 func setMeterProvider(projectID string, interval time.Duration) error {
@@ -109,3 +117,13 @@ func (s adjustedSpan) Attributes() []attribute.KeyValue {
 	}
 	return ts
 }
+
+func setLogHandler(projectID string, level slog.Leveler) error {
+	c, err := logging.NewClient(context.Background(), "projects/"+projectID)
+	if err != nil {
+		return err
+	}
+	logger := c.Logger("genkit_log")
+	slog.SetDefault(slog.New(newHandler(level, logger.Log)))
+	return nil
+}

go/plugins/googlecloud/googlecloud_test.go

@@ -19,6 +19,9 @@ package googlecloud
 import (
 	"context"
 	"flag"
+	"log/slog"
+	"os"
+	"runtime"
 	"testing"
 	"time"
 
@@ -30,8 +33,11 @@ import (
 var projectID = flag.String("project", "", "GCP project ID")
 
 // This test is part of verifying that we can export traces to GCP.
-// To verify, run the test, then visit the GCP Trace Explorer and look for the "test"
-// trace, and visit the Metrics Explorer and look for the "Generic Node - test" metric.
+// To verify, run the test, then:
+//   - visit the GCP Trace Explorer and look for the "test" trace
+//   - visit the Metrics Explorer and look for the "Generic Node - test" metric.
+//   - visit the Logging Explorer and look for the genkit_log logName, or run
+//     gcloud --project PROJECT_ID logging read 'logName:genkit_log'
 func TestGCP(t *testing.T) {
 	if *projectID == "" {
 		t.Skip("no -project")
@@ -64,4 +70,14 @@ func TestGCP(t *testing.T) {
 		// Allow time to sample and export.
 		time.Sleep(2 * time.Second)
 	})
+	t.Run("logging", func(t *testing.T) {
+		if err := setLogHandler(*projectID, slog.LevelInfo); err != nil {
+			t.Fatal(err)
+		}
+		slog.Info("testing GCP logging",
+			"binaryName", os.Args[0],
+			"goVersion", runtime.Version())
+		// Allow time to export.
+		time.Sleep(2 * time.Second)
+	})
 }

go/plugins/googlecloud/slog_handler.go

@@ -0,0 +1,149 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The googlecloud package supports telemetry (tracing, metrics and logging) using
+// Google Cloud services.
+package googlecloud
+
+import (
+	"context"
+	"log/slog"
+
+	"cloud.google.com/go/logging"
+	"github.com/jba/slog/withsupport"
+)
+
+func newHandler(level slog.Leveler, f func(logging.Entry)) *handler {
+	if level == nil {
+		level = slog.LevelInfo
+	}
+	return &handler{
+		level:       level,
+		handleEntry: f,
+	}
+}
+
+type handler struct {
+	level       slog.Leveler
+	handleEntry func(logging.Entry)
+	goa         *withsupport.GroupOrAttrs
+}
+
+func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
+	return level >= h.level.Level()
+}
+
+func (h *handler) WithAttrs(as []slog.Attr) slog.Handler {
+	h2 := *h
+	h2.goa = h2.goa.WithAttrs(as)
+	return &h2
+}
+
+func (h *handler) WithGroup(name string) slog.Handler {
+	h2 := *h
+	h2.goa = h2.goa.WithGroup(name)
+	return &h2
+}
+
+func (h *handler) Handle(ctx context.Context, r slog.Record) error {
+	h.handleEntry(h.recordToEntry(ctx, r))
+	return nil
+}
+
+func (h *handler) recordToEntry(ctx context.Context, r slog.Record) logging.Entry {
+	return logging.Entry{
+		Timestamp: r.Time,
+		Severity:  levelToSeverity(r.Level),
+		Payload:   recordToMap(r, h.goa.Collect()),
+		Labels:    map[string]string{"module": "genkit"},
+		// TODO: add a monitored resource
+		// Resource:       &monitoredres.MonitoredResource{},
+		// TODO: add trace information from the context.
+		// Trace:        "",
+		// SpanID:       "",
+		// TraceSampled: false,
+	}
+}
+
+func levelToSeverity(l slog.Level) logging.Severity {
+	switch {
+	case l < slog.LevelInfo:
+		return logging.Debug
+	case l == slog.LevelInfo:
+		return logging.Info
+	case l < slog.LevelWarn:
+		return logging.Notice
+	case l < slog.LevelError:
+		return logging.Warning
+	case l == slog.LevelError:
+		return logging.Error
+	case l <= slog.LevelError+4:
+		return logging.Critical
+	case l <= slog.LevelError+8:
+		return logging.Alert
+	default:
+		return logging.Emergency
+	}
+}
+func recordToMap(r slog.Record, goras []*withsupport.GroupOrAttrs) map[string]any {
+	root := map[string]any{}
+	root[slog.MessageKey] = r.Message
+
+	m := root
+	for i, gora := range goras {
+		if gora.Group != "" {
+			if i == len(goras)-1 && r.NumAttrs() == 0 {
+				continue
+			}
+			m2 := map[string]any{}
+			m[gora.Group] = m2
+			m = m2
+		} else {
+			for _, a := range gora.Attrs {
+				handleAttr(a, m)
+			}
+		}
+	}
+	r.Attrs(func(a slog.Attr) bool {
+		handleAttr(a, m)
+		return true
+	})
+	return root
+}
+
+func handleAttr(a slog.Attr, m map[string]any) {
+	if a.Equal(slog.Attr{}) {
+		return
+	}
+	v := a.Value.Resolve()
+	if v.Kind() == slog.KindGroup {
+		gas := v.Group()
+		if len(gas) == 0 {
+			return
+		}
+		if a.Key == "" {
+			for _, ga := range gas {
+				handleAttr(ga, m)
+			}
+		} else {
+			gm := map[string]any{}
+			for _, ga := range gas {
+				handleAttr(ga, gm)
+			}
+			m[a.Key] = gm
+		}
+	} else {
+		m[a.Key] = v.Any()
+	}
+}

go/plugins/googlecloud/slog_handler_test.go

@@ -0,0 +1,50 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The googlecloud package supports telemetry (tracing, metrics and logging) using
+// Google Cloud services.
+package googlecloud
+
+import (
+	"log/slog"
+	"testing"
+	"testing/slogtest"
+
+	"cloud.google.com/go/logging"
+)
+
+func TestHandler(t *testing.T) {
+	var results []map[string]any
+
+	f := func(e logging.Entry) {
+		results = append(results, entryToMap(e))
+	}
+
+	if err := slogtest.TestHandler(newHandler(slog.LevelInfo, f), func() []map[string]any { return results }); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func entryToMap(e logging.Entry) map[string]any {
+	m := map[string]any{}
+	if !e.Timestamp.IsZero() {
+		m[slog.TimeKey] = e.Timestamp
+	}
+	m[slog.LevelKey] = e.Severity
+	pm := e.Payload.(map[string]any)
+	for k, v := range pm {
+		m[k] = v
+	}
+	return m
+}