Storage Class & TableSpace settings for offload. (#53)

* Refactor client-server interaction facilities

* Fix for go sum

* Fix unit

* Add put TableSpace and StorageClass settings

* Client-side support to storage class and tablespace settings

* Fix error handling

cmd/client/main.go

@@ -6,26 +6,34 @@ import (
 	"net"
 	"os"
 
-	"github.com/yezzey-gp/yproxy/pkg/storage"
-
 	"github.com/spf13/cobra"
 	"github.com/yezzey-gp/yproxy/config"
 	"github.com/yezzey-gp/yproxy/pkg/client"
 	"github.com/yezzey-gp/yproxy/pkg/message"
+	"github.com/yezzey-gp/yproxy/pkg/object"
 	"github.com/yezzey-gp/yproxy/pkg/proc"
+	"github.com/yezzey-gp/yproxy/pkg/tablespace"
 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 )
 
-var cfgPath string
-var oldCfgPath string
-var logLevel string
-var decrypt bool
-var encrypt bool
-var offset uint64
-var segmentPort int
-var segmentNum int
-var confirm bool
-var garbage bool
+var (
+	cfgPath    string
+	oldCfgPath string
+	logLevel   string
+
+	decrypt bool
+	/* Put command flags */
+	encrypt      bool
+	storageClass string
+	tableSpace   string
+
+	offset uint64
+
+	segmentPort int
+	segmentNum  int
+	confirm     bool
+	garbage     bool
+)
 
 // TODOV
 func Runner(f func(net.Conn, *config.Instance, []string) error) func(*cobra.Command, []string) error {
@@ -97,7 +105,16 @@ func putFunc(con net.Conn, instanceCnf *config.Instance, args []string) error {
 	ycl := client.NewYClient(con)
 	r := proc.NewProtoReader(ycl)
 
-	msg := message.NewPutMessage(args[0], encrypt).Encode()
+	msg := message.NewPutMessageV2(args[0], encrypt, []message.PutSettings{
+		{
+			Name:  message.StorageClassSetting,
+			Value: storageClass,
+		},
+		{
+			Name:  message.TableSpaceSetting,
+			Value: tableSpace,
+		},
+	}).Encode()
 	_, err := con.Write(msg)
 	if err != nil {
 		return err
@@ -167,7 +184,7 @@ func listFunc(con net.Conn, instanceCnf *config.Instance, args []string) error {
 	r := proc.NewProtoReader(ycl)
 
 	done := false
-	res := make([]*storage.ObjectInfo, 0)
+	res := make([]*object.ObjectInfo, 0)
 	for {
 		if done {
 			break
@@ -342,6 +359,8 @@ func init() {
 	rootCmd.AddCommand(copyCmd)
 
 	putCmd.PersistentFlags().BoolVarP(&encrypt, "encrypt", "e", false, "encrypt external object before put")
+	putCmd.PersistentFlags().StringVarP(&storageClass, "storage-class", "s", "STANDARD", "storage class for message upload")
+	putCmd.PersistentFlags().StringVarP(&tableSpace, "tablespace", "t", tablespace.DefaultTableSpace, "storage class for message upload")
 	rootCmd.AddCommand(putCmd)
 
 	rootCmd.AddCommand(listCmd)

config/storage.go

@@ -12,6 +12,8 @@ type Storage struct {
 	StoragePrefix   string `json:"storage_prefix" toml:"storage_prefix" yaml:"storage_prefix"`
 	StorageBucket   string `json:"storage_bucket" toml:"storage_bucket" yaml:"storage_bucket"`
 
+	TablespaceMap map[string]string `json:"tablespace_map" toml:"tablespace_map" yaml:"tablespace_map"`
+
 	// how many concurrrent connection acquire allowed
 	StorageConcurrency int64 `json:"storage_concurrency" toml:"storage_concurrency" yaml:"storage_concurrency"`
 

go.mod

@@ -28,6 +28,7 @@ require (
 )
 
 require (
+	go.uber.org/mock v0.4.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 )

go.sum

@@ -68,6 +68,8 @@ github.com/yezzey-gp/aws-sdk-go v0.1.0 h1:as6ANEva14gKdhWPjZy6qaGR+/WhP0HN4UMzDH
 github.com/yezzey-gp/aws-sdk-go v0.1.0/go.mod h1:+gUq+WgyFOP6Eto+AcgJDDeM6tnXmOT4RJmfpuafV3Y=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

pkg/message/message_test.go

@@ -81,15 +81,15 @@ func TestPutV2Msg(t *testing.T) {
 		name     string
 		encrypt  bool
 		err      error
-		settings []message.PutSetting
+		settings []message.PutSettings
 	}
 
 	for _, tt := range []tcase{
 		{
 			"nam1",
 			true,
 			nil,
-			[]message.PutSetting{
+			[]message.PutSettings{
 				{
 					Name:  "a",
 					Value: "b",

pkg/message/object_meta_message.go

@@ -4,16 +4,16 @@ import (
 	"bytes"
 	"encoding/binary"
 
-	"github.com/yezzey-gp/yproxy/pkg/storage"
+	"github.com/yezzey-gp/yproxy/pkg/object"
 )
 
 type ObjectInfoMessage struct {
-	Content []*storage.ObjectInfo
+	Content []*object.ObjectInfo
 }
 
 var _ ProtoMessage = &ObjectInfoMessage{}
 
-func NewObjectMetaMessage(content []*storage.ObjectInfo) *ObjectInfoMessage {
+func NewObjectMetaMessage(content []*object.ObjectInfo) *ObjectInfoMessage {
 	return &ObjectInfoMessage{
 		Content: content,
 	}
@@ -44,12 +44,12 @@ func (c *ObjectInfoMessage) Encode() []byte {
 
 func (c *ObjectInfoMessage) Decode(body []byte) {
 	body = body[4:]
-	c.Content = make([]*storage.ObjectInfo, 0)
+	c.Content = make([]*object.ObjectInfo, 0)
 	for len(body) > 0 {
 		name, index := c.GetString(body)
 		size := int64(binary.BigEndian.Uint64(body[index : index+8]))
 
-		c.Content = append(c.Content, &storage.ObjectInfo{
+		c.Content = append(c.Content, &object.ObjectInfo{
 			Path: name,
 			Size: size,
 		})

pkg/message/put_message_v2.go

@@ -5,7 +5,10 @@ import (
 	"encoding/binary"
 )
 
-type PutSetting struct {
+const StorageClassSetting = "StorageClass"
+const TableSpaceSetting = "TableSpace"
+
+type PutSettings struct {
 	Name  string
 	Value string
 }
@@ -14,12 +17,12 @@ type PutMessageV2 struct {
 	Encrypt bool
 	Name    string
 
-	Settings []PutSetting
+	Settings []PutSettings
 }
 
 var _ ProtoMessage = &PutMessageV2{}
 
-func NewPutMessageV2(name string, encrypt bool, settings []PutSetting) *PutMessageV2 {
+func NewPutMessageV2(name string, encrypt bool, settings []PutSettings) *PutMessageV2 {
 	return &PutMessageV2{
 		Name:     name,
 		Encrypt:  encrypt,
@@ -90,7 +93,7 @@ func (c *PutMessageV2) Decode(body []byte) {
 
 	totalOff := 4 + off + 8
 
-	c.Settings = make([]PutSetting, settLen)
+	c.Settings = make([]PutSettings, settLen)
 
 	for i := 0; i < int(settLen); i++ {