Skip to content

Commit

Permalink
Make monitoring optional for both configs upload & test (#946)
Browse files Browse the repository at this point in the history 
* Add e2e test for config generation

* Add monitoring

* add test monitoring client

* wip

* implement metrics upload

* Add Stackdriver monitoring and presubmit workflow

* fix go command

* fix pr script

* Unit tests for report chaining

* update upload location

* Add sleep in monitoring

* fix return error

* Address review comments

* fix metrics

* Make monitoring optional
  • Loading branch information
@smukherj1
smukherj1 authored Feb 17, 2021
1 parent 9bf7f36 commit 04ae943
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 24 deletions.
3 changes: 1 addition & 2 deletions cmd/rbe_configs_gen/rbe_configs_gen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,7 @@ func main() {
} else {
log.Printf("Config generation was successful.")
}
// Monitoring is optional for config generation and used for internal alerting by the owners
// of this repo only.
// Monitoring is optional and used for internal alerting by the owners of this repo only.
if mc != nil {
if err := mc.ReportToolchainConfigsGeneration(ctx, *monitoringDockerImage, result); err != nil {
log.Fatalf("Failed to report config result to monitoring: %v", err)
Expand Down
38 changes: 27 additions & 11 deletions cmd/rbe_configs_upload/rbe_configs_upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ import (
var (
configsTarball = flag.String("configs_tarball", "", "Path to the configs tarball generated by rbe_configs_gen to be uploaded to GCS.")
configsManifest = flag.String("configs_manifest", "", "Path to the JSON manifest generated by rbe_configs_gen.")
monitoringProjectID = flag.String("monitoring_project_id", "", "GCP Project ID where monitoring results will be reported.")
monitoringDockerImage = flag.String("monitoring_docker_image", "", "Name of the toolchain docker image to be reported as a string label to monitoring.")
enableMonitoring = flag.Bool("enable_monitoring", false, "(Optional) Enables reporting reporting results to Google Cloud Monitoring. Defaults to false.")
monitoringProjectID = flag.String("monitoring_project_id", "", "GCP Project ID where monitoring results will be reported. Required if --enable_monitoring is true.")
monitoringDockerImage = flag.String("monitoring_docker_image", "", "Name of the toolchain docker image to be reported as a string label to monitoring. Required if --enable_monitoring is true.")
)

// manifest is the metadata about the configs that'll be uploaded to GCS.
Expand Down Expand Up @@ -136,6 +137,7 @@ func printFlags() {
log.Println("rbe_configs_upload.go \\")
log.Printf("--configs_tarball=%q \\", *configsTarball)
log.Printf("--configs_manifest=%q \\", *configsManifest)
log.Printf("--enable_monitoring=%v \\", *enableMonitoring)
log.Printf("--monitoring_project_id=%q \\", *monitoringProjectID)
log.Printf("--monitoring_docker_image=%q", *monitoringDockerImage)
}
Expand Down Expand Up @@ -172,6 +174,23 @@ func uploadConfigs(ctx context.Context, containerImage string) error {
return nil
}

func initMonitoringClient(ctx context.Context) (*monitoring.Client, error) {
if !(*enableMonitoring) {
return nil, nil
}
if len(*monitoringProjectID) == 0 {
return nil, fmt.Errorf("--monitoring_project_id is required because --enable_monitoring is true")
}
if len(*monitoringDockerImage) == 0 {
return nil, fmt.Errorf("--monitoring_docker_image is required because --enable_monitoring is true")
}
c, err := monitoring.NewClient(ctx, *monitoringProjectID)
if err != nil {
return nil, fmt.Errorf("unable to initialize the monitoring client: %w", err)
}
return c, nil
}

func main() {
flag.Parse()
printFlags()
Expand All @@ -182,15 +201,9 @@ func main() {
if len(*configsManifest) == 0 {
log.Fatalf("--configs_manifest was not specified.")
}
if len(*monitoringProjectID) == 0 {
log.Fatalf("--monitoring_project_id was not specified.")
}
if len(*monitoringDockerImage) == 0 {
log.Fatalf("--monitoring_docker_image was not specified.")
}

ctx := context.Background()
mc, err := monitoring.NewClient(ctx, *monitoringProjectID)
mc, err := initMonitoringClient(ctx)
if err != nil {
log.Fatalf("Failed to initialize monitoring: %v", err)
}
Expand All @@ -203,8 +216,11 @@ func main() {
log.Printf("Configs uploaded successfully.")
}

if err := mc.ReportToolchainConfigsUpload(ctx, *monitoringDockerImage, result); err != nil {
log.Fatalf("Failed to report results to monitoring: %v", err)
// Monitoring is optional and used for internal alerting by the owners of this repo only.
if mc != nil {
if err := mc.ReportToolchainConfigsUpload(ctx, *monitoringDockerImage, result); err != nil {
log.Fatalf("Failed to report results to monitoring: %v", err)
}
}
if !result {
os.Exit(1)
Expand Down
39 changes: 28 additions & 11 deletions tests/scripts/configs_e2e/configs_e2e.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@ var (
destRoot = flag.String("dest_root", "", "Path to an empty or non-existent output directory where the Bazel Hello world repo will be set up & a Bazel build will be executed.")
rbeInstance = flag.String("rbe_instance", "", "Name of the RBE instance to test the configs on in the format projects/<GCP project ID>/instances/<RBE Instance ID>.")
timeoutSeconds = flag.Int("timeout_seconds", 0, "Number of seconds before the Bazel build run in the test is killed and a timeout failure is declared.")
monitoringProjectID = flag.String("monitoring_project_id", "", "GCP Project ID where monitoring results will be reported.")
monitoringDockerImage = flag.String("monitoring_docker_image", "", "Name of the toolchain docker image to be reported as a string label to monitoring.")
enableMonitoring = flag.Bool("enable_monitoring", false, "(Optional) Enables reporting reporting results to Google Cloud Monitoring. Defaults to false.")
monitoringProjectID = flag.String("monitoring_project_id", "", "GCP Project ID where monitoring results will be reported. Required if --enable_monitoring is true.")
monitoringDockerImage = flag.String("monitoring_docker_image", "", "Name of the toolchain docker image to be reported as a string label to monitoring. Required if --enable_monitoring is true.")

// filesToCopy are the files that'll be copied from srcRoot to destRoot.
filesToCopy = []string{
Expand Down Expand Up @@ -365,6 +366,7 @@ func printFlags() {
log.Printf("--dest_root=%q \\", *destRoot)
log.Printf("--rbe_instance=%q \\", *rbeInstance)
log.Printf("--timeout_seconds=%d \\", *timeoutSeconds)
log.Printf("--enable_monitoring=%v \\", *enableMonitoring)
log.Printf("--monitoring_project_id=%q \\", *monitoringProjectID)
log.Printf("--monitoring_docker_image=%q", *monitoringDockerImage)
}
Expand Down Expand Up @@ -397,6 +399,23 @@ func runTest(ctx context.Context) error {
return nil
}

func initMonitoringClient(ctx context.Context) (*monitoring.Client, error) {
if !(*enableMonitoring) {
return nil, nil
}
if len(*monitoringProjectID) == 0 {
return nil, fmt.Errorf("--monitoring_project_id is required because --enable_monitoring is true")
}
if len(*monitoringDockerImage) == 0 {
return nil, fmt.Errorf("--monitoring_docker_image is required because --enable_monitoring is true")
}
c, err := monitoring.NewClient(ctx, *monitoringProjectID)
if err != nil {
return nil, fmt.Errorf("unable to initialize the monitoring client: %w", err)
}
return c, nil
}

func main() {
flag.Parse()
printFlags()
Expand All @@ -422,15 +441,9 @@ func main() {
if *timeoutSeconds <= 0 {
log.Fatalf("--timeout_seconds was either not specified or negative.")
}
if len(*monitoringProjectID) == 0 {
log.Fatalf("--monitoring_project_id was not specified.")
}
if len(*monitoringDockerImage) == 0 {
log.Fatalf("--monitoring_docker_image was not specified.")
}

ctx := context.Background()
mc, err := monitoring.NewClient(ctx, *monitoringProjectID)
mc, err := initMonitoringClient(ctx)
if err != nil {
log.Fatalf("Failed to initialize monitoring: %v", err)
}
Expand All @@ -442,8 +455,12 @@ func main() {
} else {
log.Printf("Config E2E test passed.")
}
if err := mc.ReportToolchainConfigsTest(ctx, *monitoringDockerImage, result); err != nil {
log.Fatalf("Failed to report results to monitoring: %v", err)

// Monitoring is optional and used for internal alerting by the owners of this repo only.
if mc != nil {
if err := mc.ReportToolchainConfigsTest(ctx, *monitoringDockerImage, result); err != nil {
log.Fatalf("Failed to report results to monitoring: %v", err)
}
}
if !result {
os.Exit(1)
Expand Down

0 comments on commit 04ae943

Please sign in to comment.