From 7c3cb876b9729b9d07bcca1c3e40c419eda5d745 Mon Sep 17 00:00:00 2001 From: dvonthenen Date: Fri, 19 Aug 2016 11:32:52 -0700 Subject: [PATCH] Added the following: 1) benchmarks for the key/value store 2) Updated glide to add back in go-bindata for vendor 3) Helper script for building linux binaries on MAC --- build-linux.sh | 4 + core/store/store_test.go | 183 ++++++++++++++++++++++++++++++++++++--- glide.yaml | 3 + 3 files changed, 176 insertions(+), 14 deletions(-) create mode 100755 build-linux.sh diff --git a/build-linux.sh b/build-linux.sh new file mode 100755 index 0000000..9329777 --- /dev/null +++ b/build-linux.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env bash + +make deps GOARCH=amd64 GOOS=linux +make GOARCH=amd64 GOOS=linux diff --git a/core/store/store_test.go b/core/store/store_test.go index e8d77c2..f1a2127 100644 --- a/core/store/store_test.go +++ b/core/store/store_test.go @@ -2,11 +2,12 @@ package store import ( "bytes" - - log "github.com/Sirupsen/logrus" "os" + "strconv" "testing" + log "github.com/Sirupsen/logrus" + gofig "github.com/akutz/gofig" lstypes "github.com/emccode/libstorage/api/types" "github.com/emccode/polly/api/types" @@ -15,11 +16,11 @@ import ( ) const ( - libStorageConfigBaseBolt = ` + libStorageConfigBaseTestBolt = ` polly: store: type: boltdb - endpoints: /tmp/boltdb + endpoints: /tmp/boltdb-test bucket: MyBoltDb_test server: services: @@ -27,6 +28,19 @@ polly: libstorage: storage: driver: vfs +` + libStorageConfigBaseBenchBolt = ` +polly: + store: + type: boltdb + endpoints: /tmp/boltdb-bench + bucket: MyBoltDb_bench + server: + services: + vfs: + libstorage: + storage: + driver: vfs ` libStorageConfigBaseConsul = ` polly: @@ -47,31 +61,28 @@ func newVolume(service, volumeID string) *types.Volume { } func TestMain(m *testing.M) { - log.SetLevel(log.DebugLevel) + log.SetLevel(log.ErrorLevel) os.Setenv("POLLY_DEBUG", "true") config := gofig.New() - configYamlBuf := []byte(libStorageConfigBaseBolt) + configYamlBuf := []byte(libStorageConfigBaseTestBolt) if err := config.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { panic(err) } + os.Remove("/tmp/boltdb-test") + os.Remove("/tmp/boltdb-bench") + var err error ps, err = NewWithConfig(config) if err != nil { log.WithError(err).Fatal("Failed to create PollyStore") } - vol := newVolume("pollytestpkg1", "testid1") - _ = ps.RemoveVolumeMetadata(vol) - - if err := ps.EraseStore(); err != nil { - log.Fatal("Could not clear polly store") - } - m.Run() } +//Testing func TestGenerateRootKey(t *testing.T) { key, err := ps.GenerateRootKey(VolumeInternalLabelsType) assert.NoError(t, err) @@ -230,7 +241,7 @@ func TestRemoveVolumeMetadata(t *testing.T) { func TestEraseStore(t *testing.T) { myConfig := gofig.New() - configYamlBuf := []byte(libStorageConfigBaseBolt) + configYamlBuf := []byte(libStorageConfigBaseTestBolt) if err := myConfig.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { panic(err) } @@ -257,3 +268,147 @@ func TestEraseStore(t *testing.T) { assert.Len(t, ids, 0) } + +//Benchmarks +func BenchmarkGetVolumeIDNotExist(b *testing.B) { + myConfig := gofig.New() + + configYamlBuf := []byte(libStorageConfigBaseBenchBolt) + if err := myConfig.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { + log.WithError(err).Fatal("Failed to create PollyStore") + } + + psBench, err := NewWithConfig(myConfig.Scope("polly.store")) + if err != nil { + log.Fatal("Failed to create PollyStore") + } + + err = psBench.EraseStore() + if err != nil { + log.WithError(err).Fatal("Failed to reset PollyStore") + } + + volume := newVolume("pollytestpkg2", "testiddoesntexist") + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + psBench.Exists(volume) + } + }) +} + +func BenchmarkGetVolumeIDs(b *testing.B) { + myConfig := gofig.New() + + configYamlBuf := []byte(libStorageConfigBaseBenchBolt) + if err := myConfig.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { + log.WithError(err).Fatal("Failed to create PollyStore") + } + + psBench, err := NewWithConfig(myConfig.Scope("polly.store")) + if err != nil { + log.Fatal("Failed to create PollyStore") + } + + err = psBench.EraseStore() + if err != nil { + log.WithError(err).Fatal("Failed to reset PollyStore") + } + + volume := newVolume("pollytestpkg1", "testid1") + + err = psBench.SaveVolumeMetadata(volume) + if err != nil { + log.WithError(err).Fatal("Failed to SaveVolumeMetadata") + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + psBench.GetVolumeIds() + } + }) +} + +func BenchmarkNewVolumeMetadata(b *testing.B) { + myConfig := gofig.New() + + configYamlBuf := []byte(libStorageConfigBaseBenchBolt) + if err := myConfig.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { + log.WithError(err).Fatal("Failed to create PollyStore") + } + + psBench, err := NewWithConfig(myConfig.Scope("polly.store")) + if err != nil { + log.Fatal("Failed to create PollyStore") + } + + err = psBench.EraseStore() + if err != nil { + log.WithError(err).Fatal("Failed to reset PollyStore") + } + + volume := newVolume("pollytestpkg1", "testid1") + volume.Schedulers = []string{"testScheduler"} + + err = psBench.SaveVolumeMetadata(volume) + if err != nil { + log.WithError(err).Fatal("Failed to SaveVolumeMetadata") + } + + volume = newVolume("pollytestpkg1", "testid1") + _, err = psBench.SetVolumeMetadata(volume) + if err != nil { + log.WithError(err).Fatal("Failed to SaveVolumeMetadata") + } + + volume.Labels = make(map[string]string) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + volume.Labels["testkey"+strconv.Itoa(i)] = "testval" + psBench.SaveVolumeMetadata(volume) + } +} + +func BenchmarkUpdateVolumeMetadata(b *testing.B) { + myConfig := gofig.New() + + configYamlBuf := []byte(libStorageConfigBaseBenchBolt) + if err := myConfig.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { + log.WithError(err).Fatal("Failed to create PollyStore") + } + + psBench, err := NewWithConfig(myConfig.Scope("polly.store")) + if err != nil { + log.Fatal("Failed to create PollyStore") + } + + err = psBench.EraseStore() + if err != nil { + log.WithError(err).Fatal("Failed to reset PollyStore") + } + + volume := newVolume("pollytestpkg1", "testid1") + volume.Schedulers = []string{"testScheduler"} + + err = psBench.SaveVolumeMetadata(volume) + if err != nil { + log.WithError(err).Fatal("Failed to SaveVolumeMetadata") + } + + volume = newVolume("pollytestpkg1", "testid1") + _, err = psBench.SetVolumeMetadata(volume) + if err != nil { + log.WithError(err).Fatal("Failed to SaveVolumeMetadata") + } + + volume.Labels = make(map[string]string) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + volume.Labels["testkey1"] = "testval" + strconv.Itoa(i) + psBench.SaveVolumeMetadata(volume) + } +} diff --git a/glide.yaml b/glide.yaml index 992ce31..48436af 100644 --- a/glide.yaml +++ b/glide.yaml @@ -17,6 +17,9 @@ import: version: v0.1.0 - package: github.com/akutz/golf ref: v0.1.1 + - package: github.com/akutz/go-bindata + ref: feature/md5checksum + vcs: git - package: github.com/docker/libkv - package: github.com/boltdb/bolt